1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "gfx_v9_4.h" 51 #include "gfx_v9_0.h" 52 53 #include "asic_reg/pwr/pwr_10_0_offset.h" 54 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 55 #include "asic_reg/gc/gc_9_0_default.h" 56 57 #define GFX9_NUM_GFX_RINGS 1 58 #define GFX9_MEC_HPD_SIZE 4096 59 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 60 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 61 62 #define mmGCEA_PROBE_MAP 0x070c 63 #define mmGCEA_PROBE_MAP_BASE_IDX 0 64 65 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 66 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 71 72 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 73 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 78 79 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 80 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 85 86 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 87 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 92 93 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 94 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 100 101 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 102 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 107 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 108 109 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 110 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin"); 118 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 119 120 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 125 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 126 127 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 128 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 129 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 130 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 131 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 132 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 133 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 134 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 135 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 136 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 137 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 138 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 139 140 enum ta_ras_gfx_subblock { 141 /*CPC*/ 142 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 143 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 144 TA_RAS_BLOCK__GFX_CPC_UCODE, 145 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 146 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 147 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 148 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 151 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 152 /* CPF*/ 153 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 154 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 155 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 156 TA_RAS_BLOCK__GFX_CPF_TAG, 157 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 158 /* CPG*/ 159 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 160 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 161 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 162 TA_RAS_BLOCK__GFX_CPG_TAG, 163 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 164 /* GDS*/ 165 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 166 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 167 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 168 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 169 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 170 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 171 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 172 /* SPI*/ 173 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 174 /* SQ*/ 175 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 176 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 177 TA_RAS_BLOCK__GFX_SQ_LDS_D, 178 TA_RAS_BLOCK__GFX_SQ_LDS_I, 179 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 180 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 181 /* SQC (3 ranges)*/ 182 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 183 /* SQC range 0*/ 184 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 185 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 186 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 187 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 188 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 189 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 190 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 191 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 192 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 193 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 195 /* SQC range 1*/ 196 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 197 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 198 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 199 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 201 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 202 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 203 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 204 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 207 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 209 /* SQC range 2*/ 210 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 211 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 212 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 213 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 215 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 217 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 218 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 221 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 223 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 224 /* TA*/ 225 TA_RAS_BLOCK__GFX_TA_INDEX_START, 226 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 227 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 228 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 229 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 230 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 231 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 232 /* TCA*/ 233 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 234 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 235 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 236 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 237 /* TCC (5 sub-ranges)*/ 238 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 239 /* TCC range 0*/ 240 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 241 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 242 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 243 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 245 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 246 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 247 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 248 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 249 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 250 /* TCC range 1*/ 251 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 252 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 253 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 254 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 255 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 256 /* TCC range 2*/ 257 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 258 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 259 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 260 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 261 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 262 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 263 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 264 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 265 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 266 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 267 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 268 /* TCC range 3*/ 269 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 270 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 271 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 272 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 274 /* TCC range 4*/ 275 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 276 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 277 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 278 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 279 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 280 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 281 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 282 /* TCI*/ 283 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 284 /* TCP*/ 285 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 286 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 287 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 288 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 289 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 290 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 291 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 292 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 293 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 294 /* TD*/ 295 TA_RAS_BLOCK__GFX_TD_INDEX_START, 296 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 297 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 298 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 299 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 300 /* EA (3 sub-ranges)*/ 301 TA_RAS_BLOCK__GFX_EA_INDEX_START, 302 /* EA range 0*/ 303 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 304 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 305 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 306 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 307 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 308 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 309 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 310 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 311 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 312 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 313 /* EA range 1*/ 314 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 315 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 316 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 317 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 318 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 319 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 320 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 321 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 322 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 323 /* EA range 2*/ 324 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 325 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 326 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 327 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 328 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 329 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 330 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 331 /* UTC VM L2 bank*/ 332 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 333 /* UTC VM walker*/ 334 TA_RAS_BLOCK__UTC_VML2_WALKER, 335 /* UTC ATC L2 2MB cache*/ 336 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 337 /* UTC ATC L2 4KB cache*/ 338 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 339 TA_RAS_BLOCK__GFX_MAX 340 }; 341 342 struct ras_gfx_subblock { 343 unsigned char *name; 344 int ta_subblock; 345 int hw_supported_error_type; 346 int sw_supported_error_type; 347 }; 348 349 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 350 [AMDGPU_RAS_BLOCK__##subblock] = { \ 351 #subblock, \ 352 TA_RAS_BLOCK__##subblock, \ 353 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 354 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 355 } 356 357 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 358 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 359 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 360 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 361 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 362 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 369 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 371 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 372 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 374 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 375 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 377 0), 378 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 380 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 382 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 386 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 388 0), 389 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 390 0, 0), 391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 392 0), 393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 394 0, 0), 395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 396 0), 397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 398 1), 399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 400 0, 0, 0), 401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 402 0), 403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 404 0), 405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 406 0), 407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 408 0), 409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 410 0), 411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 412 0, 0), 413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 414 0), 415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 416 0), 417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 418 0, 0, 0), 419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 420 0), 421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 422 0), 423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 424 0), 425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 426 0), 427 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 428 0), 429 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 430 0, 0), 431 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 432 0), 433 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 434 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 435 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 438 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 442 1), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 444 1), 445 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 446 1), 447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 448 0), 449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 450 0), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 463 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 466 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 468 0, 0), 469 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 470 0), 471 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 505 }; 506 507 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 508 { 509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 529 }; 530 531 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 532 { 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 551 }; 552 553 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 554 { 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 566 }; 567 568 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 569 { 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 594 }; 595 596 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 597 { 598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 600 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 605 }; 606 607 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 608 { 609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 628 }; 629 630 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 631 { 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 644 }; 645 646 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 647 { 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 651 }; 652 653 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 654 { 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 671 }; 672 673 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 674 { 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 688 }; 689 690 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 691 { 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 703 }; 704 705 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 706 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 707 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 708 }; 709 710 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 711 { 712 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 713 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 714 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 715 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 720 }; 721 722 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 723 { 724 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 725 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 726 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 727 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 732 }; 733 734 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v) 735 { 736 static void *scratch_reg0; 737 static void *scratch_reg1; 738 static void *scratch_reg2; 739 static void *scratch_reg3; 740 static void *spare_int; 741 static uint32_t grbm_cntl; 742 static uint32_t grbm_idx; 743 744 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 745 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 746 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 747 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 748 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 749 750 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 751 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 752 753 if (amdgpu_sriov_runtime(adev)) { 754 pr_err("shouldn't call rlcg write register during runtime\n"); 755 return; 756 } 757 758 if (offset == grbm_cntl || offset == grbm_idx) { 759 if (offset == grbm_cntl) 760 writel(v, scratch_reg2); 761 else if (offset == grbm_idx) 762 writel(v, scratch_reg3); 763 764 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 765 } else { 766 uint32_t i = 0; 767 uint32_t retries = 50000; 768 769 writel(v, scratch_reg0); 770 writel(offset | 0x80000000, scratch_reg1); 771 writel(1, spare_int); 772 for (i = 0; i < retries; i++) { 773 u32 tmp; 774 775 tmp = readl(scratch_reg1); 776 if (!(tmp & 0x80000000)) 777 break; 778 779 udelay(10); 780 } 781 if (i >= retries) 782 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 783 } 784 785 } 786 787 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 788 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 789 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 790 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 791 792 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 793 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 794 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 795 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 796 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 797 struct amdgpu_cu_info *cu_info); 798 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 799 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 800 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 801 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 802 void *ras_error_status); 803 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 804 void *inject_if); 805 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 806 807 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 808 uint64_t queue_mask) 809 { 810 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 811 amdgpu_ring_write(kiq_ring, 812 PACKET3_SET_RESOURCES_VMID_MASK(0) | 813 /* vmid_mask:0* queue_type:0 (KIQ) */ 814 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 815 amdgpu_ring_write(kiq_ring, 816 lower_32_bits(queue_mask)); /* queue mask lo */ 817 amdgpu_ring_write(kiq_ring, 818 upper_32_bits(queue_mask)); /* queue mask hi */ 819 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 820 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 821 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 822 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 823 } 824 825 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 826 struct amdgpu_ring *ring) 827 { 828 struct amdgpu_device *adev = kiq_ring->adev; 829 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 830 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 831 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 832 833 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 834 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 835 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 836 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 837 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 838 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 839 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 840 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 841 /*queue_type: normal compute queue */ 842 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 843 /* alloc format: all_on_one_pipe */ 844 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 845 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 846 /* num_queues: must be 1 */ 847 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 848 amdgpu_ring_write(kiq_ring, 849 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 850 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 851 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 852 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 853 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 854 } 855 856 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 857 struct amdgpu_ring *ring, 858 enum amdgpu_unmap_queues_action action, 859 u64 gpu_addr, u64 seq) 860 { 861 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 862 863 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 864 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 865 PACKET3_UNMAP_QUEUES_ACTION(action) | 866 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 867 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 868 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 869 amdgpu_ring_write(kiq_ring, 870 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 871 872 if (action == PREEMPT_QUEUES_NO_UNMAP) { 873 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 874 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 875 amdgpu_ring_write(kiq_ring, seq); 876 } else { 877 amdgpu_ring_write(kiq_ring, 0); 878 amdgpu_ring_write(kiq_ring, 0); 879 amdgpu_ring_write(kiq_ring, 0); 880 } 881 } 882 883 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 884 struct amdgpu_ring *ring, 885 u64 addr, 886 u64 seq) 887 { 888 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 889 890 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 891 amdgpu_ring_write(kiq_ring, 892 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 893 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 894 PACKET3_QUERY_STATUS_COMMAND(2)); 895 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 896 amdgpu_ring_write(kiq_ring, 897 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 898 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 899 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 900 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 901 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 902 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 903 } 904 905 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 906 uint16_t pasid, uint32_t flush_type, 907 bool all_hub) 908 { 909 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 910 amdgpu_ring_write(kiq_ring, 911 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 912 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 913 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 914 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 915 } 916 917 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 918 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 919 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 920 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 921 .kiq_query_status = gfx_v9_0_kiq_query_status, 922 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 923 .set_resources_size = 8, 924 .map_queues_size = 7, 925 .unmap_queues_size = 6, 926 .query_status_size = 7, 927 .invalidate_tlbs_size = 2, 928 }; 929 930 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 931 { 932 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 933 } 934 935 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 936 { 937 switch (adev->asic_type) { 938 case CHIP_VEGA10: 939 soc15_program_register_sequence(adev, 940 golden_settings_gc_9_0, 941 ARRAY_SIZE(golden_settings_gc_9_0)); 942 soc15_program_register_sequence(adev, 943 golden_settings_gc_9_0_vg10, 944 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 945 break; 946 case CHIP_VEGA12: 947 soc15_program_register_sequence(adev, 948 golden_settings_gc_9_2_1, 949 ARRAY_SIZE(golden_settings_gc_9_2_1)); 950 soc15_program_register_sequence(adev, 951 golden_settings_gc_9_2_1_vg12, 952 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 953 break; 954 case CHIP_VEGA20: 955 soc15_program_register_sequence(adev, 956 golden_settings_gc_9_0, 957 ARRAY_SIZE(golden_settings_gc_9_0)); 958 soc15_program_register_sequence(adev, 959 golden_settings_gc_9_0_vg20, 960 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 961 break; 962 case CHIP_ARCTURUS: 963 soc15_program_register_sequence(adev, 964 golden_settings_gc_9_4_1_arct, 965 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 966 break; 967 case CHIP_RAVEN: 968 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 969 ARRAY_SIZE(golden_settings_gc_9_1)); 970 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 971 soc15_program_register_sequence(adev, 972 golden_settings_gc_9_1_rv2, 973 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 974 else 975 soc15_program_register_sequence(adev, 976 golden_settings_gc_9_1_rv1, 977 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 978 break; 979 case CHIP_RENOIR: 980 soc15_program_register_sequence(adev, 981 golden_settings_gc_9_1_rn, 982 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 983 return; /* for renoir, don't need common goldensetting */ 984 default: 985 break; 986 } 987 988 if (adev->asic_type != CHIP_ARCTURUS) 989 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 990 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 991 } 992 993 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 994 { 995 adev->gfx.scratch.num_reg = 8; 996 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 997 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 998 } 999 1000 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1001 bool wc, uint32_t reg, uint32_t val) 1002 { 1003 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1004 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1005 WRITE_DATA_DST_SEL(0) | 1006 (wc ? WR_CONFIRM : 0)); 1007 amdgpu_ring_write(ring, reg); 1008 amdgpu_ring_write(ring, 0); 1009 amdgpu_ring_write(ring, val); 1010 } 1011 1012 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1013 int mem_space, int opt, uint32_t addr0, 1014 uint32_t addr1, uint32_t ref, uint32_t mask, 1015 uint32_t inv) 1016 { 1017 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1018 amdgpu_ring_write(ring, 1019 /* memory (1) or register (0) */ 1020 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1021 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1022 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1023 WAIT_REG_MEM_ENGINE(eng_sel))); 1024 1025 if (mem_space) 1026 BUG_ON(addr0 & 0x3); /* Dword align */ 1027 amdgpu_ring_write(ring, addr0); 1028 amdgpu_ring_write(ring, addr1); 1029 amdgpu_ring_write(ring, ref); 1030 amdgpu_ring_write(ring, mask); 1031 amdgpu_ring_write(ring, inv); /* poll interval */ 1032 } 1033 1034 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1035 { 1036 struct amdgpu_device *adev = ring->adev; 1037 uint32_t scratch; 1038 uint32_t tmp = 0; 1039 unsigned i; 1040 int r; 1041 1042 r = amdgpu_gfx_scratch_get(adev, &scratch); 1043 if (r) 1044 return r; 1045 1046 WREG32(scratch, 0xCAFEDEAD); 1047 r = amdgpu_ring_alloc(ring, 3); 1048 if (r) 1049 goto error_free_scratch; 1050 1051 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1052 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1053 amdgpu_ring_write(ring, 0xDEADBEEF); 1054 amdgpu_ring_commit(ring); 1055 1056 for (i = 0; i < adev->usec_timeout; i++) { 1057 tmp = RREG32(scratch); 1058 if (tmp == 0xDEADBEEF) 1059 break; 1060 udelay(1); 1061 } 1062 1063 if (i >= adev->usec_timeout) 1064 r = -ETIMEDOUT; 1065 1066 error_free_scratch: 1067 amdgpu_gfx_scratch_free(adev, scratch); 1068 return r; 1069 } 1070 1071 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1072 { 1073 struct amdgpu_device *adev = ring->adev; 1074 struct amdgpu_ib ib; 1075 struct dma_fence *f = NULL; 1076 1077 unsigned index; 1078 uint64_t gpu_addr; 1079 uint32_t tmp; 1080 long r; 1081 1082 r = amdgpu_device_wb_get(adev, &index); 1083 if (r) 1084 return r; 1085 1086 gpu_addr = adev->wb.gpu_addr + (index * 4); 1087 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1088 memset(&ib, 0, sizeof(ib)); 1089 r = amdgpu_ib_get(adev, NULL, 16, 1090 AMDGPU_IB_POOL_DIRECT, &ib); 1091 if (r) 1092 goto err1; 1093 1094 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1095 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1096 ib.ptr[2] = lower_32_bits(gpu_addr); 1097 ib.ptr[3] = upper_32_bits(gpu_addr); 1098 ib.ptr[4] = 0xDEADBEEF; 1099 ib.length_dw = 5; 1100 1101 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1102 if (r) 1103 goto err2; 1104 1105 r = dma_fence_wait_timeout(f, false, timeout); 1106 if (r == 0) { 1107 r = -ETIMEDOUT; 1108 goto err2; 1109 } else if (r < 0) { 1110 goto err2; 1111 } 1112 1113 tmp = adev->wb.wb[index]; 1114 if (tmp == 0xDEADBEEF) 1115 r = 0; 1116 else 1117 r = -EINVAL; 1118 1119 err2: 1120 amdgpu_ib_free(adev, &ib, NULL); 1121 dma_fence_put(f); 1122 err1: 1123 amdgpu_device_wb_free(adev, index); 1124 return r; 1125 } 1126 1127 1128 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1129 { 1130 release_firmware(adev->gfx.pfp_fw); 1131 adev->gfx.pfp_fw = NULL; 1132 release_firmware(adev->gfx.me_fw); 1133 adev->gfx.me_fw = NULL; 1134 release_firmware(adev->gfx.ce_fw); 1135 adev->gfx.ce_fw = NULL; 1136 release_firmware(adev->gfx.rlc_fw); 1137 adev->gfx.rlc_fw = NULL; 1138 release_firmware(adev->gfx.mec_fw); 1139 adev->gfx.mec_fw = NULL; 1140 release_firmware(adev->gfx.mec2_fw); 1141 adev->gfx.mec2_fw = NULL; 1142 1143 kfree(adev->gfx.rlc.register_list_format); 1144 } 1145 1146 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1147 { 1148 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1149 1150 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1151 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1152 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1153 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1154 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1155 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1156 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1157 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1158 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1159 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1160 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1161 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1162 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1163 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1164 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1165 } 1166 1167 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1168 { 1169 adev->gfx.me_fw_write_wait = false; 1170 adev->gfx.mec_fw_write_wait = false; 1171 1172 if ((adev->asic_type != CHIP_ARCTURUS) && 1173 ((adev->gfx.mec_fw_version < 0x000001a5) || 1174 (adev->gfx.mec_feature_version < 46) || 1175 (adev->gfx.pfp_fw_version < 0x000000b7) || 1176 (adev->gfx.pfp_feature_version < 46))) 1177 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1178 1179 switch (adev->asic_type) { 1180 case CHIP_VEGA10: 1181 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1182 (adev->gfx.me_feature_version >= 42) && 1183 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1184 (adev->gfx.pfp_feature_version >= 42)) 1185 adev->gfx.me_fw_write_wait = true; 1186 1187 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1188 (adev->gfx.mec_feature_version >= 42)) 1189 adev->gfx.mec_fw_write_wait = true; 1190 break; 1191 case CHIP_VEGA12: 1192 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1193 (adev->gfx.me_feature_version >= 44) && 1194 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1195 (adev->gfx.pfp_feature_version >= 44)) 1196 adev->gfx.me_fw_write_wait = true; 1197 1198 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1199 (adev->gfx.mec_feature_version >= 44)) 1200 adev->gfx.mec_fw_write_wait = true; 1201 break; 1202 case CHIP_VEGA20: 1203 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1204 (adev->gfx.me_feature_version >= 44) && 1205 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1206 (adev->gfx.pfp_feature_version >= 44)) 1207 adev->gfx.me_fw_write_wait = true; 1208 1209 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1210 (adev->gfx.mec_feature_version >= 44)) 1211 adev->gfx.mec_fw_write_wait = true; 1212 break; 1213 case CHIP_RAVEN: 1214 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1215 (adev->gfx.me_feature_version >= 42) && 1216 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1217 (adev->gfx.pfp_feature_version >= 42)) 1218 adev->gfx.me_fw_write_wait = true; 1219 1220 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1221 (adev->gfx.mec_feature_version >= 42)) 1222 adev->gfx.mec_fw_write_wait = true; 1223 break; 1224 default: 1225 adev->gfx.me_fw_write_wait = true; 1226 adev->gfx.mec_fw_write_wait = true; 1227 break; 1228 } 1229 } 1230 1231 struct amdgpu_gfxoff_quirk { 1232 u16 chip_vendor; 1233 u16 chip_device; 1234 u16 subsys_vendor; 1235 u16 subsys_device; 1236 u8 revision; 1237 }; 1238 1239 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1240 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1241 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1242 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1243 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1244 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1245 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1246 { 0, 0, 0, 0, 0 }, 1247 }; 1248 1249 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1250 { 1251 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1252 1253 while (p && p->chip_device != 0) { 1254 if (pdev->vendor == p->chip_vendor && 1255 pdev->device == p->chip_device && 1256 pdev->subsystem_vendor == p->subsys_vendor && 1257 pdev->subsystem_device == p->subsys_device && 1258 pdev->revision == p->revision) { 1259 return true; 1260 } 1261 ++p; 1262 } 1263 return false; 1264 } 1265 1266 static bool is_raven_kicker(struct amdgpu_device *adev) 1267 { 1268 if (adev->pm.fw_version >= 0x41e2b) 1269 return true; 1270 else 1271 return false; 1272 } 1273 1274 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1275 { 1276 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1277 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1278 1279 switch (adev->asic_type) { 1280 case CHIP_VEGA10: 1281 case CHIP_VEGA12: 1282 case CHIP_VEGA20: 1283 break; 1284 case CHIP_RAVEN: 1285 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1286 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1287 ((!is_raven_kicker(adev) && 1288 adev->gfx.rlc_fw_version < 531) || 1289 (adev->gfx.rlc_feature_version < 1) || 1290 !adev->gfx.rlc.is_rlc_v2_1)) 1291 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1292 1293 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1294 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1295 AMD_PG_SUPPORT_CP | 1296 AMD_PG_SUPPORT_RLC_SMU_HS; 1297 break; 1298 case CHIP_RENOIR: 1299 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1300 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1301 AMD_PG_SUPPORT_CP | 1302 AMD_PG_SUPPORT_RLC_SMU_HS; 1303 break; 1304 default: 1305 break; 1306 } 1307 } 1308 1309 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1310 const char *chip_name) 1311 { 1312 char fw_name[30]; 1313 int err; 1314 struct amdgpu_firmware_info *info = NULL; 1315 const struct common_firmware_header *header = NULL; 1316 const struct gfx_firmware_header_v1_0 *cp_hdr; 1317 1318 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1319 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1320 if (err) 1321 goto out; 1322 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1323 if (err) 1324 goto out; 1325 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1326 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1327 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1328 1329 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1330 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1331 if (err) 1332 goto out; 1333 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1334 if (err) 1335 goto out; 1336 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1337 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1338 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1339 1340 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1341 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1342 if (err) 1343 goto out; 1344 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1345 if (err) 1346 goto out; 1347 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1348 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1349 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1350 1351 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1352 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1353 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1354 info->fw = adev->gfx.pfp_fw; 1355 header = (const struct common_firmware_header *)info->fw->data; 1356 adev->firmware.fw_size += 1357 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1358 1359 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1360 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1361 info->fw = adev->gfx.me_fw; 1362 header = (const struct common_firmware_header *)info->fw->data; 1363 adev->firmware.fw_size += 1364 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1365 1366 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1367 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1368 info->fw = adev->gfx.ce_fw; 1369 header = (const struct common_firmware_header *)info->fw->data; 1370 adev->firmware.fw_size += 1371 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1372 } 1373 1374 out: 1375 if (err) { 1376 dev_err(adev->dev, 1377 "gfx9: Failed to load firmware \"%s\"\n", 1378 fw_name); 1379 release_firmware(adev->gfx.pfp_fw); 1380 adev->gfx.pfp_fw = NULL; 1381 release_firmware(adev->gfx.me_fw); 1382 adev->gfx.me_fw = NULL; 1383 release_firmware(adev->gfx.ce_fw); 1384 adev->gfx.ce_fw = NULL; 1385 } 1386 return err; 1387 } 1388 1389 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1390 const char *chip_name) 1391 { 1392 char fw_name[30]; 1393 int err; 1394 struct amdgpu_firmware_info *info = NULL; 1395 const struct common_firmware_header *header = NULL; 1396 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1397 unsigned int *tmp = NULL; 1398 unsigned int i = 0; 1399 uint16_t version_major; 1400 uint16_t version_minor; 1401 uint32_t smu_version; 1402 1403 /* 1404 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1405 * instead of picasso_rlc.bin. 1406 * Judgment method: 1407 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1408 * or revision >= 0xD8 && revision <= 0xDF 1409 * otherwise is PCO FP5 1410 */ 1411 if (!strcmp(chip_name, "picasso") && 1412 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1413 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1414 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1415 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1416 (smu_version >= 0x41e2b)) 1417 /** 1418 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1419 */ 1420 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1421 else 1422 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1423 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1424 if (err) 1425 goto out; 1426 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1427 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1428 1429 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1430 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1431 if (version_major == 2 && version_minor == 1) 1432 adev->gfx.rlc.is_rlc_v2_1 = true; 1433 1434 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1435 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1436 adev->gfx.rlc.save_and_restore_offset = 1437 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1438 adev->gfx.rlc.clear_state_descriptor_offset = 1439 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1440 adev->gfx.rlc.avail_scratch_ram_locations = 1441 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1442 adev->gfx.rlc.reg_restore_list_size = 1443 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1444 adev->gfx.rlc.reg_list_format_start = 1445 le32_to_cpu(rlc_hdr->reg_list_format_start); 1446 adev->gfx.rlc.reg_list_format_separate_start = 1447 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1448 adev->gfx.rlc.starting_offsets_start = 1449 le32_to_cpu(rlc_hdr->starting_offsets_start); 1450 adev->gfx.rlc.reg_list_format_size_bytes = 1451 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1452 adev->gfx.rlc.reg_list_size_bytes = 1453 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1454 adev->gfx.rlc.register_list_format = 1455 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1456 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1457 if (!adev->gfx.rlc.register_list_format) { 1458 err = -ENOMEM; 1459 goto out; 1460 } 1461 1462 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1463 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1464 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1465 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1466 1467 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1468 1469 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1470 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1471 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1472 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1473 1474 if (adev->gfx.rlc.is_rlc_v2_1) 1475 gfx_v9_0_init_rlc_ext_microcode(adev); 1476 1477 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1478 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1479 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1480 info->fw = adev->gfx.rlc_fw; 1481 header = (const struct common_firmware_header *)info->fw->data; 1482 adev->firmware.fw_size += 1483 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1484 1485 if (adev->gfx.rlc.is_rlc_v2_1 && 1486 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1487 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1488 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1489 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1490 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1491 info->fw = adev->gfx.rlc_fw; 1492 adev->firmware.fw_size += 1493 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1494 1495 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1496 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1497 info->fw = adev->gfx.rlc_fw; 1498 adev->firmware.fw_size += 1499 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1500 1501 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1502 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1503 info->fw = adev->gfx.rlc_fw; 1504 adev->firmware.fw_size += 1505 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1506 } 1507 } 1508 1509 out: 1510 if (err) { 1511 dev_err(adev->dev, 1512 "gfx9: Failed to load firmware \"%s\"\n", 1513 fw_name); 1514 release_firmware(adev->gfx.rlc_fw); 1515 adev->gfx.rlc_fw = NULL; 1516 } 1517 return err; 1518 } 1519 1520 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1521 const char *chip_name) 1522 { 1523 char fw_name[30]; 1524 int err; 1525 struct amdgpu_firmware_info *info = NULL; 1526 const struct common_firmware_header *header = NULL; 1527 const struct gfx_firmware_header_v1_0 *cp_hdr; 1528 1529 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1530 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1531 if (err) 1532 goto out; 1533 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1534 if (err) 1535 goto out; 1536 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1537 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1538 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1539 1540 1541 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1542 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1543 if (!err) { 1544 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1545 if (err) 1546 goto out; 1547 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1548 adev->gfx.mec2_fw->data; 1549 adev->gfx.mec2_fw_version = 1550 le32_to_cpu(cp_hdr->header.ucode_version); 1551 adev->gfx.mec2_feature_version = 1552 le32_to_cpu(cp_hdr->ucode_feature_version); 1553 } else { 1554 err = 0; 1555 adev->gfx.mec2_fw = NULL; 1556 } 1557 1558 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1559 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1560 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1561 info->fw = adev->gfx.mec_fw; 1562 header = (const struct common_firmware_header *)info->fw->data; 1563 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1564 adev->firmware.fw_size += 1565 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1566 1567 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1568 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1569 info->fw = adev->gfx.mec_fw; 1570 adev->firmware.fw_size += 1571 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1572 1573 if (adev->gfx.mec2_fw) { 1574 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1575 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1576 info->fw = adev->gfx.mec2_fw; 1577 header = (const struct common_firmware_header *)info->fw->data; 1578 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1579 adev->firmware.fw_size += 1580 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1581 1582 /* TODO: Determine if MEC2 JT FW loading can be removed 1583 for all GFX V9 asic and above */ 1584 if (adev->asic_type != CHIP_ARCTURUS && 1585 adev->asic_type != CHIP_RENOIR) { 1586 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1587 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1588 info->fw = adev->gfx.mec2_fw; 1589 adev->firmware.fw_size += 1590 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1591 PAGE_SIZE); 1592 } 1593 } 1594 } 1595 1596 out: 1597 gfx_v9_0_check_if_need_gfxoff(adev); 1598 gfx_v9_0_check_fw_write_wait(adev); 1599 if (err) { 1600 dev_err(adev->dev, 1601 "gfx9: Failed to load firmware \"%s\"\n", 1602 fw_name); 1603 release_firmware(adev->gfx.mec_fw); 1604 adev->gfx.mec_fw = NULL; 1605 release_firmware(adev->gfx.mec2_fw); 1606 adev->gfx.mec2_fw = NULL; 1607 } 1608 return err; 1609 } 1610 1611 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1612 { 1613 const char *chip_name; 1614 int r; 1615 1616 DRM_DEBUG("\n"); 1617 1618 switch (adev->asic_type) { 1619 case CHIP_VEGA10: 1620 chip_name = "vega10"; 1621 break; 1622 case CHIP_VEGA12: 1623 chip_name = "vega12"; 1624 break; 1625 case CHIP_VEGA20: 1626 chip_name = "vega20"; 1627 break; 1628 case CHIP_RAVEN: 1629 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1630 chip_name = "raven2"; 1631 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1632 chip_name = "picasso"; 1633 else 1634 chip_name = "raven"; 1635 break; 1636 case CHIP_ARCTURUS: 1637 chip_name = "arcturus"; 1638 break; 1639 case CHIP_RENOIR: 1640 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1641 chip_name = "renoir"; 1642 else 1643 chip_name = "green_sardine"; 1644 break; 1645 default: 1646 BUG(); 1647 } 1648 1649 /* No CPG in Arcturus */ 1650 if (adev->gfx.num_gfx_rings) { 1651 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1652 if (r) 1653 return r; 1654 } 1655 1656 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1657 if (r) 1658 return r; 1659 1660 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1661 if (r) 1662 return r; 1663 1664 return r; 1665 } 1666 1667 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1668 { 1669 u32 count = 0; 1670 const struct cs_section_def *sect = NULL; 1671 const struct cs_extent_def *ext = NULL; 1672 1673 /* begin clear state */ 1674 count += 2; 1675 /* context control state */ 1676 count += 3; 1677 1678 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1679 for (ext = sect->section; ext->extent != NULL; ++ext) { 1680 if (sect->id == SECT_CONTEXT) 1681 count += 2 + ext->reg_count; 1682 else 1683 return 0; 1684 } 1685 } 1686 1687 /* end clear state */ 1688 count += 2; 1689 /* clear state */ 1690 count += 2; 1691 1692 return count; 1693 } 1694 1695 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1696 volatile u32 *buffer) 1697 { 1698 u32 count = 0, i; 1699 const struct cs_section_def *sect = NULL; 1700 const struct cs_extent_def *ext = NULL; 1701 1702 if (adev->gfx.rlc.cs_data == NULL) 1703 return; 1704 if (buffer == NULL) 1705 return; 1706 1707 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1708 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1709 1710 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1711 buffer[count++] = cpu_to_le32(0x80000000); 1712 buffer[count++] = cpu_to_le32(0x80000000); 1713 1714 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1715 for (ext = sect->section; ext->extent != NULL; ++ext) { 1716 if (sect->id == SECT_CONTEXT) { 1717 buffer[count++] = 1718 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1719 buffer[count++] = cpu_to_le32(ext->reg_index - 1720 PACKET3_SET_CONTEXT_REG_START); 1721 for (i = 0; i < ext->reg_count; i++) 1722 buffer[count++] = cpu_to_le32(ext->extent[i]); 1723 } else { 1724 return; 1725 } 1726 } 1727 } 1728 1729 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1730 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1731 1732 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1733 buffer[count++] = cpu_to_le32(0); 1734 } 1735 1736 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1737 { 1738 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1739 uint32_t pg_always_on_cu_num = 2; 1740 uint32_t always_on_cu_num; 1741 uint32_t i, j, k; 1742 uint32_t mask, cu_bitmap, counter; 1743 1744 if (adev->flags & AMD_IS_APU) 1745 always_on_cu_num = 4; 1746 else if (adev->asic_type == CHIP_VEGA12) 1747 always_on_cu_num = 8; 1748 else 1749 always_on_cu_num = 12; 1750 1751 mutex_lock(&adev->grbm_idx_mutex); 1752 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1753 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1754 mask = 1; 1755 cu_bitmap = 0; 1756 counter = 0; 1757 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1758 1759 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1760 if (cu_info->bitmap[i][j] & mask) { 1761 if (counter == pg_always_on_cu_num) 1762 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1763 if (counter < always_on_cu_num) 1764 cu_bitmap |= mask; 1765 else 1766 break; 1767 counter++; 1768 } 1769 mask <<= 1; 1770 } 1771 1772 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1773 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1774 } 1775 } 1776 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1777 mutex_unlock(&adev->grbm_idx_mutex); 1778 } 1779 1780 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1781 { 1782 uint32_t data; 1783 1784 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1785 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1786 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1787 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1788 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1789 1790 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1791 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1792 1793 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1794 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1795 1796 mutex_lock(&adev->grbm_idx_mutex); 1797 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1798 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1799 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1800 1801 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1802 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1803 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1804 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1805 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1806 1807 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1808 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1809 data &= 0x0000FFFF; 1810 data |= 0x00C00000; 1811 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1812 1813 /* 1814 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1815 * programmed in gfx_v9_0_init_always_on_cu_mask() 1816 */ 1817 1818 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1819 * but used for RLC_LB_CNTL configuration */ 1820 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1821 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1822 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1823 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1824 mutex_unlock(&adev->grbm_idx_mutex); 1825 1826 gfx_v9_0_init_always_on_cu_mask(adev); 1827 } 1828 1829 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1830 { 1831 uint32_t data; 1832 1833 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1834 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1835 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1836 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1837 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1838 1839 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1840 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1841 1842 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1843 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1844 1845 mutex_lock(&adev->grbm_idx_mutex); 1846 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1847 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1848 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1849 1850 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1851 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1852 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1853 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1854 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1855 1856 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1857 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1858 data &= 0x0000FFFF; 1859 data |= 0x00C00000; 1860 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1861 1862 /* 1863 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1864 * programmed in gfx_v9_0_init_always_on_cu_mask() 1865 */ 1866 1867 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1868 * but used for RLC_LB_CNTL configuration */ 1869 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1870 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1871 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1872 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1873 mutex_unlock(&adev->grbm_idx_mutex); 1874 1875 gfx_v9_0_init_always_on_cu_mask(adev); 1876 } 1877 1878 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1879 { 1880 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1881 } 1882 1883 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1884 { 1885 return 5; 1886 } 1887 1888 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1889 { 1890 const struct cs_section_def *cs_data; 1891 int r; 1892 1893 adev->gfx.rlc.cs_data = gfx9_cs_data; 1894 1895 cs_data = adev->gfx.rlc.cs_data; 1896 1897 if (cs_data) { 1898 /* init clear state block */ 1899 r = amdgpu_gfx_rlc_init_csb(adev); 1900 if (r) 1901 return r; 1902 } 1903 1904 if (adev->flags & AMD_IS_APU) { 1905 /* TODO: double check the cp_table_size for RV */ 1906 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1907 r = amdgpu_gfx_rlc_init_cpt(adev); 1908 if (r) 1909 return r; 1910 } 1911 1912 switch (adev->asic_type) { 1913 case CHIP_RAVEN: 1914 gfx_v9_0_init_lbpw(adev); 1915 break; 1916 case CHIP_VEGA20: 1917 gfx_v9_4_init_lbpw(adev); 1918 break; 1919 default: 1920 break; 1921 } 1922 1923 /* init spm vmid with 0xf */ 1924 if (adev->gfx.rlc.funcs->update_spm_vmid) 1925 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1926 1927 return 0; 1928 } 1929 1930 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1931 { 1932 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1933 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1934 } 1935 1936 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1937 { 1938 int r; 1939 u32 *hpd; 1940 const __le32 *fw_data; 1941 unsigned fw_size; 1942 u32 *fw; 1943 size_t mec_hpd_size; 1944 1945 const struct gfx_firmware_header_v1_0 *mec_hdr; 1946 1947 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1948 1949 /* take ownership of the relevant compute queues */ 1950 amdgpu_gfx_compute_queue_acquire(adev); 1951 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1952 if (mec_hpd_size) { 1953 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1954 AMDGPU_GEM_DOMAIN_VRAM, 1955 &adev->gfx.mec.hpd_eop_obj, 1956 &adev->gfx.mec.hpd_eop_gpu_addr, 1957 (void **)&hpd); 1958 if (r) { 1959 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1960 gfx_v9_0_mec_fini(adev); 1961 return r; 1962 } 1963 1964 memset(hpd, 0, mec_hpd_size); 1965 1966 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1967 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1968 } 1969 1970 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1971 1972 fw_data = (const __le32 *) 1973 (adev->gfx.mec_fw->data + 1974 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1975 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1976 1977 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1978 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1979 &adev->gfx.mec.mec_fw_obj, 1980 &adev->gfx.mec.mec_fw_gpu_addr, 1981 (void **)&fw); 1982 if (r) { 1983 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1984 gfx_v9_0_mec_fini(adev); 1985 return r; 1986 } 1987 1988 memcpy(fw, fw_data, fw_size); 1989 1990 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1991 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1992 1993 return 0; 1994 } 1995 1996 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1997 { 1998 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1999 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2000 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2001 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2002 (SQ_IND_INDEX__FORCE_READ_MASK)); 2003 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2004 } 2005 2006 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2007 uint32_t wave, uint32_t thread, 2008 uint32_t regno, uint32_t num, uint32_t *out) 2009 { 2010 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2011 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2012 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2013 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2014 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2015 (SQ_IND_INDEX__FORCE_READ_MASK) | 2016 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2017 while (num--) 2018 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2019 } 2020 2021 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2022 { 2023 /* type 1 wave data */ 2024 dst[(*no_fields)++] = 1; 2025 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2026 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2027 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2028 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2029 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2030 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2031 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2032 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2033 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2034 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2035 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2036 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2037 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2038 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2039 } 2040 2041 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2042 uint32_t wave, uint32_t start, 2043 uint32_t size, uint32_t *dst) 2044 { 2045 wave_read_regs( 2046 adev, simd, wave, 0, 2047 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2048 } 2049 2050 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2051 uint32_t wave, uint32_t thread, 2052 uint32_t start, uint32_t size, 2053 uint32_t *dst) 2054 { 2055 wave_read_regs( 2056 adev, simd, wave, thread, 2057 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2058 } 2059 2060 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2061 u32 me, u32 pipe, u32 q, u32 vm) 2062 { 2063 soc15_grbm_select(adev, me, pipe, q, vm); 2064 } 2065 2066 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2067 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2068 .select_se_sh = &gfx_v9_0_select_se_sh, 2069 .read_wave_data = &gfx_v9_0_read_wave_data, 2070 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2071 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2072 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2073 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2074 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2075 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2076 }; 2077 2078 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { 2079 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2080 .select_se_sh = &gfx_v9_0_select_se_sh, 2081 .read_wave_data = &gfx_v9_0_read_wave_data, 2082 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2083 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2084 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2085 .ras_error_inject = &gfx_v9_4_ras_error_inject, 2086 .query_ras_error_count = &gfx_v9_4_query_ras_error_count, 2087 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count, 2088 .query_ras_error_status = &gfx_v9_4_query_ras_error_status, 2089 }; 2090 2091 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2092 { 2093 u32 gb_addr_config; 2094 int err; 2095 2096 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2097 2098 switch (adev->asic_type) { 2099 case CHIP_VEGA10: 2100 adev->gfx.config.max_hw_contexts = 8; 2101 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2102 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2103 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2104 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2105 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2106 break; 2107 case CHIP_VEGA12: 2108 adev->gfx.config.max_hw_contexts = 8; 2109 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2110 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2111 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2112 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2113 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2114 DRM_INFO("fix gfx.config for vega12\n"); 2115 break; 2116 case CHIP_VEGA20: 2117 adev->gfx.config.max_hw_contexts = 8; 2118 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2119 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2120 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2121 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2122 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2123 gb_addr_config &= ~0xf3e777ff; 2124 gb_addr_config |= 0x22014042; 2125 /* check vbios table if gpu info is not available */ 2126 err = amdgpu_atomfirmware_get_gfx_info(adev); 2127 if (err) 2128 return err; 2129 break; 2130 case CHIP_RAVEN: 2131 adev->gfx.config.max_hw_contexts = 8; 2132 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2133 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2134 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2135 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2136 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2137 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2138 else 2139 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2140 break; 2141 case CHIP_ARCTURUS: 2142 adev->gfx.funcs = &gfx_v9_4_gfx_funcs; 2143 adev->gfx.config.max_hw_contexts = 8; 2144 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2145 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2146 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2147 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2148 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2149 gb_addr_config &= ~0xf3e777ff; 2150 gb_addr_config |= 0x22014042; 2151 break; 2152 case CHIP_RENOIR: 2153 adev->gfx.config.max_hw_contexts = 8; 2154 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2155 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2156 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2157 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2158 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2159 gb_addr_config &= ~0xf3e777ff; 2160 gb_addr_config |= 0x22010042; 2161 break; 2162 default: 2163 BUG(); 2164 break; 2165 } 2166 2167 adev->gfx.config.gb_addr_config = gb_addr_config; 2168 2169 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2170 REG_GET_FIELD( 2171 adev->gfx.config.gb_addr_config, 2172 GB_ADDR_CONFIG, 2173 NUM_PIPES); 2174 2175 adev->gfx.config.max_tile_pipes = 2176 adev->gfx.config.gb_addr_config_fields.num_pipes; 2177 2178 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2179 REG_GET_FIELD( 2180 adev->gfx.config.gb_addr_config, 2181 GB_ADDR_CONFIG, 2182 NUM_BANKS); 2183 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2184 REG_GET_FIELD( 2185 adev->gfx.config.gb_addr_config, 2186 GB_ADDR_CONFIG, 2187 MAX_COMPRESSED_FRAGS); 2188 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2189 REG_GET_FIELD( 2190 adev->gfx.config.gb_addr_config, 2191 GB_ADDR_CONFIG, 2192 NUM_RB_PER_SE); 2193 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2194 REG_GET_FIELD( 2195 adev->gfx.config.gb_addr_config, 2196 GB_ADDR_CONFIG, 2197 NUM_SHADER_ENGINES); 2198 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2199 REG_GET_FIELD( 2200 adev->gfx.config.gb_addr_config, 2201 GB_ADDR_CONFIG, 2202 PIPE_INTERLEAVE_SIZE)); 2203 2204 return 0; 2205 } 2206 2207 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2208 int mec, int pipe, int queue) 2209 { 2210 unsigned irq_type; 2211 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2212 unsigned int hw_prio; 2213 2214 ring = &adev->gfx.compute_ring[ring_id]; 2215 2216 /* mec0 is me1 */ 2217 ring->me = mec + 1; 2218 ring->pipe = pipe; 2219 ring->queue = queue; 2220 2221 ring->ring_obj = NULL; 2222 ring->use_doorbell = true; 2223 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2224 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2225 + (ring_id * GFX9_MEC_HPD_SIZE); 2226 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2227 2228 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2229 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2230 + ring->pipe; 2231 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2232 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2233 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2234 return amdgpu_ring_init(adev, ring, 1024, 2235 &adev->gfx.eop_irq, irq_type, hw_prio); 2236 } 2237 2238 static int gfx_v9_0_sw_init(void *handle) 2239 { 2240 int i, j, k, r, ring_id; 2241 struct amdgpu_ring *ring; 2242 struct amdgpu_kiq *kiq; 2243 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2244 2245 switch (adev->asic_type) { 2246 case CHIP_VEGA10: 2247 case CHIP_VEGA12: 2248 case CHIP_VEGA20: 2249 case CHIP_RAVEN: 2250 case CHIP_ARCTURUS: 2251 case CHIP_RENOIR: 2252 adev->gfx.mec.num_mec = 2; 2253 break; 2254 default: 2255 adev->gfx.mec.num_mec = 1; 2256 break; 2257 } 2258 2259 adev->gfx.mec.num_pipe_per_mec = 4; 2260 adev->gfx.mec.num_queue_per_pipe = 8; 2261 2262 /* EOP Event */ 2263 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2264 if (r) 2265 return r; 2266 2267 /* Privileged reg */ 2268 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2269 &adev->gfx.priv_reg_irq); 2270 if (r) 2271 return r; 2272 2273 /* Privileged inst */ 2274 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2275 &adev->gfx.priv_inst_irq); 2276 if (r) 2277 return r; 2278 2279 /* ECC error */ 2280 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2281 &adev->gfx.cp_ecc_error_irq); 2282 if (r) 2283 return r; 2284 2285 /* FUE error */ 2286 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2287 &adev->gfx.cp_ecc_error_irq); 2288 if (r) 2289 return r; 2290 2291 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2292 2293 gfx_v9_0_scratch_init(adev); 2294 2295 r = gfx_v9_0_init_microcode(adev); 2296 if (r) { 2297 DRM_ERROR("Failed to load gfx firmware!\n"); 2298 return r; 2299 } 2300 2301 r = adev->gfx.rlc.funcs->init(adev); 2302 if (r) { 2303 DRM_ERROR("Failed to init rlc BOs!\n"); 2304 return r; 2305 } 2306 2307 r = gfx_v9_0_mec_init(adev); 2308 if (r) { 2309 DRM_ERROR("Failed to init MEC BOs!\n"); 2310 return r; 2311 } 2312 2313 /* set up the gfx ring */ 2314 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2315 ring = &adev->gfx.gfx_ring[i]; 2316 ring->ring_obj = NULL; 2317 if (!i) 2318 sprintf(ring->name, "gfx"); 2319 else 2320 sprintf(ring->name, "gfx_%d", i); 2321 ring->use_doorbell = true; 2322 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2323 r = amdgpu_ring_init(adev, ring, 1024, 2324 &adev->gfx.eop_irq, 2325 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2326 AMDGPU_RING_PRIO_DEFAULT); 2327 if (r) 2328 return r; 2329 } 2330 2331 /* set up the compute queues - allocate horizontally across pipes */ 2332 ring_id = 0; 2333 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2334 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2335 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2336 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2337 continue; 2338 2339 r = gfx_v9_0_compute_ring_init(adev, 2340 ring_id, 2341 i, k, j); 2342 if (r) 2343 return r; 2344 2345 ring_id++; 2346 } 2347 } 2348 } 2349 2350 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2351 if (r) { 2352 DRM_ERROR("Failed to init KIQ BOs!\n"); 2353 return r; 2354 } 2355 2356 kiq = &adev->gfx.kiq; 2357 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2358 if (r) 2359 return r; 2360 2361 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2362 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2363 if (r) 2364 return r; 2365 2366 adev->gfx.ce_ram_size = 0x8000; 2367 2368 r = gfx_v9_0_gpu_early_init(adev); 2369 if (r) 2370 return r; 2371 2372 return 0; 2373 } 2374 2375 2376 static int gfx_v9_0_sw_fini(void *handle) 2377 { 2378 int i; 2379 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2380 2381 amdgpu_gfx_ras_fini(adev); 2382 2383 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2384 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2385 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2386 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2387 2388 amdgpu_gfx_mqd_sw_fini(adev); 2389 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2390 amdgpu_gfx_kiq_fini(adev); 2391 2392 gfx_v9_0_mec_fini(adev); 2393 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2394 if (adev->flags & AMD_IS_APU) { 2395 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2396 &adev->gfx.rlc.cp_table_gpu_addr, 2397 (void **)&adev->gfx.rlc.cp_table_ptr); 2398 } 2399 gfx_v9_0_free_microcode(adev); 2400 2401 return 0; 2402 } 2403 2404 2405 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2406 { 2407 /* TODO */ 2408 } 2409 2410 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2411 u32 instance) 2412 { 2413 u32 data; 2414 2415 if (instance == 0xffffffff) 2416 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2417 else 2418 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2419 2420 if (se_num == 0xffffffff) 2421 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2422 else 2423 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2424 2425 if (sh_num == 0xffffffff) 2426 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2427 else 2428 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2429 2430 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2431 } 2432 2433 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2434 { 2435 u32 data, mask; 2436 2437 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2438 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2439 2440 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2441 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2442 2443 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2444 adev->gfx.config.max_sh_per_se); 2445 2446 return (~data) & mask; 2447 } 2448 2449 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2450 { 2451 int i, j; 2452 u32 data; 2453 u32 active_rbs = 0; 2454 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2455 adev->gfx.config.max_sh_per_se; 2456 2457 mutex_lock(&adev->grbm_idx_mutex); 2458 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2459 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2460 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2461 data = gfx_v9_0_get_rb_active_bitmap(adev); 2462 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2463 rb_bitmap_width_per_sh); 2464 } 2465 } 2466 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2467 mutex_unlock(&adev->grbm_idx_mutex); 2468 2469 adev->gfx.config.backend_enable_mask = active_rbs; 2470 adev->gfx.config.num_rbs = hweight32(active_rbs); 2471 } 2472 2473 #define DEFAULT_SH_MEM_BASES (0x6000) 2474 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2475 { 2476 int i; 2477 uint32_t sh_mem_config; 2478 uint32_t sh_mem_bases; 2479 2480 /* 2481 * Configure apertures: 2482 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2483 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2484 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2485 */ 2486 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2487 2488 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2489 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2490 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2491 2492 mutex_lock(&adev->srbm_mutex); 2493 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2494 soc15_grbm_select(adev, 0, 0, 0, i); 2495 /* CP and shaders */ 2496 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2497 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2498 } 2499 soc15_grbm_select(adev, 0, 0, 0, 0); 2500 mutex_unlock(&adev->srbm_mutex); 2501 2502 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2503 acccess. These should be enabled by FW for target VMIDs. */ 2504 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2505 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2506 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2507 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2508 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2509 } 2510 } 2511 2512 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2513 { 2514 int vmid; 2515 2516 /* 2517 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2518 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2519 * the driver can enable them for graphics. VMID0 should maintain 2520 * access so that HWS firmware can save/restore entries. 2521 */ 2522 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2523 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2524 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2525 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2526 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2527 } 2528 } 2529 2530 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2531 { 2532 uint32_t tmp; 2533 2534 switch (adev->asic_type) { 2535 case CHIP_ARCTURUS: 2536 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2537 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2538 DISABLE_BARRIER_WAITCNT, 1); 2539 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2540 break; 2541 default: 2542 break; 2543 } 2544 } 2545 2546 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2547 { 2548 u32 tmp; 2549 int i; 2550 2551 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2552 2553 gfx_v9_0_tiling_mode_table_init(adev); 2554 2555 gfx_v9_0_setup_rb(adev); 2556 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2557 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2558 2559 /* XXX SH_MEM regs */ 2560 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2561 mutex_lock(&adev->srbm_mutex); 2562 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2563 soc15_grbm_select(adev, 0, 0, 0, i); 2564 /* CP and shaders */ 2565 if (i == 0) { 2566 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2567 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2568 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2569 !!adev->gmc.noretry); 2570 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2571 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2572 } else { 2573 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2574 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2575 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2576 !!adev->gmc.noretry); 2577 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2578 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2579 (adev->gmc.private_aperture_start >> 48)); 2580 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2581 (adev->gmc.shared_aperture_start >> 48)); 2582 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2583 } 2584 } 2585 soc15_grbm_select(adev, 0, 0, 0, 0); 2586 2587 mutex_unlock(&adev->srbm_mutex); 2588 2589 gfx_v9_0_init_compute_vmid(adev); 2590 gfx_v9_0_init_gds_vmid(adev); 2591 gfx_v9_0_init_sq_config(adev); 2592 } 2593 2594 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2595 { 2596 u32 i, j, k; 2597 u32 mask; 2598 2599 mutex_lock(&adev->grbm_idx_mutex); 2600 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2601 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2602 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2603 for (k = 0; k < adev->usec_timeout; k++) { 2604 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2605 break; 2606 udelay(1); 2607 } 2608 if (k == adev->usec_timeout) { 2609 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2610 0xffffffff, 0xffffffff); 2611 mutex_unlock(&adev->grbm_idx_mutex); 2612 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2613 i, j); 2614 return; 2615 } 2616 } 2617 } 2618 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2619 mutex_unlock(&adev->grbm_idx_mutex); 2620 2621 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2622 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2623 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2624 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2625 for (k = 0; k < adev->usec_timeout; k++) { 2626 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2627 break; 2628 udelay(1); 2629 } 2630 } 2631 2632 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2633 bool enable) 2634 { 2635 u32 tmp; 2636 2637 /* don't toggle interrupts that are only applicable 2638 * to me0 pipe0 on AISCs that have me0 removed */ 2639 if (!adev->gfx.num_gfx_rings) 2640 return; 2641 2642 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2643 2644 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2645 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2646 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2647 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2648 2649 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2650 } 2651 2652 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2653 { 2654 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2655 /* csib */ 2656 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2657 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2658 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2659 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2660 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2661 adev->gfx.rlc.clear_state_size); 2662 } 2663 2664 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2665 int indirect_offset, 2666 int list_size, 2667 int *unique_indirect_regs, 2668 int unique_indirect_reg_count, 2669 int *indirect_start_offsets, 2670 int *indirect_start_offsets_count, 2671 int max_start_offsets_count) 2672 { 2673 int idx; 2674 2675 for (; indirect_offset < list_size; indirect_offset++) { 2676 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2677 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2678 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2679 2680 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2681 indirect_offset += 2; 2682 2683 /* look for the matching indice */ 2684 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2685 if (unique_indirect_regs[idx] == 2686 register_list_format[indirect_offset] || 2687 !unique_indirect_regs[idx]) 2688 break; 2689 } 2690 2691 BUG_ON(idx >= unique_indirect_reg_count); 2692 2693 if (!unique_indirect_regs[idx]) 2694 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2695 2696 indirect_offset++; 2697 } 2698 } 2699 } 2700 2701 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2702 { 2703 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2704 int unique_indirect_reg_count = 0; 2705 2706 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2707 int indirect_start_offsets_count = 0; 2708 2709 int list_size = 0; 2710 int i = 0, j = 0; 2711 u32 tmp = 0; 2712 2713 u32 *register_list_format = 2714 kmemdup(adev->gfx.rlc.register_list_format, 2715 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2716 if (!register_list_format) 2717 return -ENOMEM; 2718 2719 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2720 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2721 gfx_v9_1_parse_ind_reg_list(register_list_format, 2722 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2723 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2724 unique_indirect_regs, 2725 unique_indirect_reg_count, 2726 indirect_start_offsets, 2727 &indirect_start_offsets_count, 2728 ARRAY_SIZE(indirect_start_offsets)); 2729 2730 /* enable auto inc in case it is disabled */ 2731 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2732 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2733 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2734 2735 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2736 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2737 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2738 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2739 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2740 adev->gfx.rlc.register_restore[i]); 2741 2742 /* load indirect register */ 2743 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2744 adev->gfx.rlc.reg_list_format_start); 2745 2746 /* direct register portion */ 2747 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2748 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2749 register_list_format[i]); 2750 2751 /* indirect register portion */ 2752 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2753 if (register_list_format[i] == 0xFFFFFFFF) { 2754 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2755 continue; 2756 } 2757 2758 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2759 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2760 2761 for (j = 0; j < unique_indirect_reg_count; j++) { 2762 if (register_list_format[i] == unique_indirect_regs[j]) { 2763 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2764 break; 2765 } 2766 } 2767 2768 BUG_ON(j >= unique_indirect_reg_count); 2769 2770 i++; 2771 } 2772 2773 /* set save/restore list size */ 2774 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2775 list_size = list_size >> 1; 2776 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2777 adev->gfx.rlc.reg_restore_list_size); 2778 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2779 2780 /* write the starting offsets to RLC scratch ram */ 2781 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2782 adev->gfx.rlc.starting_offsets_start); 2783 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2784 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2785 indirect_start_offsets[i]); 2786 2787 /* load unique indirect regs*/ 2788 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2789 if (unique_indirect_regs[i] != 0) { 2790 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2791 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2792 unique_indirect_regs[i] & 0x3FFFF); 2793 2794 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2795 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2796 unique_indirect_regs[i] >> 20); 2797 } 2798 } 2799 2800 kfree(register_list_format); 2801 return 0; 2802 } 2803 2804 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2805 { 2806 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2807 } 2808 2809 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2810 bool enable) 2811 { 2812 uint32_t data = 0; 2813 uint32_t default_data = 0; 2814 2815 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2816 if (enable) { 2817 /* enable GFXIP control over CGPG */ 2818 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2819 if(default_data != data) 2820 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2821 2822 /* update status */ 2823 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2824 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2825 if(default_data != data) 2826 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2827 } else { 2828 /* restore GFXIP control over GCPG */ 2829 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2830 if(default_data != data) 2831 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2832 } 2833 } 2834 2835 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2836 { 2837 uint32_t data = 0; 2838 2839 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2840 AMD_PG_SUPPORT_GFX_SMG | 2841 AMD_PG_SUPPORT_GFX_DMG)) { 2842 /* init IDLE_POLL_COUNT = 60 */ 2843 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2844 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2845 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2846 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2847 2848 /* init RLC PG Delay */ 2849 data = 0; 2850 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2851 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2852 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2853 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2854 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2855 2856 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2857 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2858 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2859 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2860 2861 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2862 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2863 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2864 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2865 2866 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2867 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2868 2869 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2870 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2871 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2872 if (adev->asic_type != CHIP_RENOIR) 2873 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2874 } 2875 } 2876 2877 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2878 bool enable) 2879 { 2880 uint32_t data = 0; 2881 uint32_t default_data = 0; 2882 2883 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2884 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2885 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2886 enable ? 1 : 0); 2887 if (default_data != data) 2888 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2889 } 2890 2891 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2892 bool enable) 2893 { 2894 uint32_t data = 0; 2895 uint32_t default_data = 0; 2896 2897 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2898 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2899 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2900 enable ? 1 : 0); 2901 if(default_data != data) 2902 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2903 } 2904 2905 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2906 bool enable) 2907 { 2908 uint32_t data = 0; 2909 uint32_t default_data = 0; 2910 2911 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2912 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2913 CP_PG_DISABLE, 2914 enable ? 0 : 1); 2915 if(default_data != data) 2916 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2917 } 2918 2919 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2920 bool enable) 2921 { 2922 uint32_t data, default_data; 2923 2924 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2925 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2926 GFX_POWER_GATING_ENABLE, 2927 enable ? 1 : 0); 2928 if(default_data != data) 2929 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2930 } 2931 2932 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2933 bool enable) 2934 { 2935 uint32_t data, default_data; 2936 2937 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2938 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2939 GFX_PIPELINE_PG_ENABLE, 2940 enable ? 1 : 0); 2941 if(default_data != data) 2942 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2943 2944 if (!enable) 2945 /* read any GFX register to wake up GFX */ 2946 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2947 } 2948 2949 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2950 bool enable) 2951 { 2952 uint32_t data, default_data; 2953 2954 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2955 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2956 STATIC_PER_CU_PG_ENABLE, 2957 enable ? 1 : 0); 2958 if(default_data != data) 2959 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2960 } 2961 2962 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2963 bool enable) 2964 { 2965 uint32_t data, default_data; 2966 2967 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2968 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2969 DYN_PER_CU_PG_ENABLE, 2970 enable ? 1 : 0); 2971 if(default_data != data) 2972 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2973 } 2974 2975 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2976 { 2977 gfx_v9_0_init_csb(adev); 2978 2979 /* 2980 * Rlc save restore list is workable since v2_1. 2981 * And it's needed by gfxoff feature. 2982 */ 2983 if (adev->gfx.rlc.is_rlc_v2_1) { 2984 if (adev->asic_type == CHIP_VEGA12 || 2985 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 2986 gfx_v9_1_init_rlc_save_restore_list(adev); 2987 gfx_v9_0_enable_save_restore_machine(adev); 2988 } 2989 2990 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2991 AMD_PG_SUPPORT_GFX_SMG | 2992 AMD_PG_SUPPORT_GFX_DMG | 2993 AMD_PG_SUPPORT_CP | 2994 AMD_PG_SUPPORT_GDS | 2995 AMD_PG_SUPPORT_RLC_SMU_HS)) { 2996 WREG32(mmRLC_JUMP_TABLE_RESTORE, 2997 adev->gfx.rlc.cp_table_gpu_addr >> 8); 2998 gfx_v9_0_init_gfx_power_gating(adev); 2999 } 3000 } 3001 3002 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3003 { 3004 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3005 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3006 gfx_v9_0_wait_for_rlc_serdes(adev); 3007 } 3008 3009 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3010 { 3011 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3012 udelay(50); 3013 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3014 udelay(50); 3015 } 3016 3017 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3018 { 3019 #ifdef AMDGPU_RLC_DEBUG_RETRY 3020 u32 rlc_ucode_ver; 3021 #endif 3022 3023 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3024 udelay(50); 3025 3026 /* carrizo do enable cp interrupt after cp inited */ 3027 if (!(adev->flags & AMD_IS_APU)) { 3028 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3029 udelay(50); 3030 } 3031 3032 #ifdef AMDGPU_RLC_DEBUG_RETRY 3033 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3034 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3035 if(rlc_ucode_ver == 0x108) { 3036 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3037 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3038 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3039 * default is 0x9C4 to create a 100us interval */ 3040 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3041 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3042 * to disable the page fault retry interrupts, default is 3043 * 0x100 (256) */ 3044 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3045 } 3046 #endif 3047 } 3048 3049 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3050 { 3051 const struct rlc_firmware_header_v2_0 *hdr; 3052 const __le32 *fw_data; 3053 unsigned i, fw_size; 3054 3055 if (!adev->gfx.rlc_fw) 3056 return -EINVAL; 3057 3058 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3059 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3060 3061 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3062 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3063 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3064 3065 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3066 RLCG_UCODE_LOADING_START_ADDRESS); 3067 for (i = 0; i < fw_size; i++) 3068 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3069 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3070 3071 return 0; 3072 } 3073 3074 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3075 { 3076 int r; 3077 3078 if (amdgpu_sriov_vf(adev)) { 3079 gfx_v9_0_init_csb(adev); 3080 return 0; 3081 } 3082 3083 adev->gfx.rlc.funcs->stop(adev); 3084 3085 /* disable CG */ 3086 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3087 3088 gfx_v9_0_init_pg(adev); 3089 3090 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3091 /* legacy rlc firmware loading */ 3092 r = gfx_v9_0_rlc_load_microcode(adev); 3093 if (r) 3094 return r; 3095 } 3096 3097 switch (adev->asic_type) { 3098 case CHIP_RAVEN: 3099 if (amdgpu_lbpw == 0) 3100 gfx_v9_0_enable_lbpw(adev, false); 3101 else 3102 gfx_v9_0_enable_lbpw(adev, true); 3103 break; 3104 case CHIP_VEGA20: 3105 if (amdgpu_lbpw > 0) 3106 gfx_v9_0_enable_lbpw(adev, true); 3107 else 3108 gfx_v9_0_enable_lbpw(adev, false); 3109 break; 3110 default: 3111 break; 3112 } 3113 3114 adev->gfx.rlc.funcs->start(adev); 3115 3116 return 0; 3117 } 3118 3119 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3120 { 3121 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3122 3123 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3124 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3125 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3126 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3127 udelay(50); 3128 } 3129 3130 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3131 { 3132 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3133 const struct gfx_firmware_header_v1_0 *ce_hdr; 3134 const struct gfx_firmware_header_v1_0 *me_hdr; 3135 const __le32 *fw_data; 3136 unsigned i, fw_size; 3137 3138 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3139 return -EINVAL; 3140 3141 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3142 adev->gfx.pfp_fw->data; 3143 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3144 adev->gfx.ce_fw->data; 3145 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3146 adev->gfx.me_fw->data; 3147 3148 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3149 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3150 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3151 3152 gfx_v9_0_cp_gfx_enable(adev, false); 3153 3154 /* PFP */ 3155 fw_data = (const __le32 *) 3156 (adev->gfx.pfp_fw->data + 3157 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3158 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3159 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3160 for (i = 0; i < fw_size; i++) 3161 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3162 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3163 3164 /* CE */ 3165 fw_data = (const __le32 *) 3166 (adev->gfx.ce_fw->data + 3167 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3168 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3169 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3170 for (i = 0; i < fw_size; i++) 3171 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3172 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3173 3174 /* ME */ 3175 fw_data = (const __le32 *) 3176 (adev->gfx.me_fw->data + 3177 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3178 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3179 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3180 for (i = 0; i < fw_size; i++) 3181 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3182 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3183 3184 return 0; 3185 } 3186 3187 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3188 { 3189 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3190 const struct cs_section_def *sect = NULL; 3191 const struct cs_extent_def *ext = NULL; 3192 int r, i, tmp; 3193 3194 /* init the CP */ 3195 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3196 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3197 3198 gfx_v9_0_cp_gfx_enable(adev, true); 3199 3200 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3201 if (r) { 3202 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3203 return r; 3204 } 3205 3206 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3207 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3208 3209 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3210 amdgpu_ring_write(ring, 0x80000000); 3211 amdgpu_ring_write(ring, 0x80000000); 3212 3213 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3214 for (ext = sect->section; ext->extent != NULL; ++ext) { 3215 if (sect->id == SECT_CONTEXT) { 3216 amdgpu_ring_write(ring, 3217 PACKET3(PACKET3_SET_CONTEXT_REG, 3218 ext->reg_count)); 3219 amdgpu_ring_write(ring, 3220 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3221 for (i = 0; i < ext->reg_count; i++) 3222 amdgpu_ring_write(ring, ext->extent[i]); 3223 } 3224 } 3225 } 3226 3227 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3228 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3229 3230 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3231 amdgpu_ring_write(ring, 0); 3232 3233 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3234 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3235 amdgpu_ring_write(ring, 0x8000); 3236 amdgpu_ring_write(ring, 0x8000); 3237 3238 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3239 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3240 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3241 amdgpu_ring_write(ring, tmp); 3242 amdgpu_ring_write(ring, 0); 3243 3244 amdgpu_ring_commit(ring); 3245 3246 return 0; 3247 } 3248 3249 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3250 { 3251 struct amdgpu_ring *ring; 3252 u32 tmp; 3253 u32 rb_bufsz; 3254 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3255 3256 /* Set the write pointer delay */ 3257 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3258 3259 /* set the RB to use vmid 0 */ 3260 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3261 3262 /* Set ring buffer size */ 3263 ring = &adev->gfx.gfx_ring[0]; 3264 rb_bufsz = order_base_2(ring->ring_size / 8); 3265 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3266 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3267 #ifdef __BIG_ENDIAN 3268 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3269 #endif 3270 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3271 3272 /* Initialize the ring buffer's write pointers */ 3273 ring->wptr = 0; 3274 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3275 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3276 3277 /* set the wb address wether it's enabled or not */ 3278 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3279 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3280 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3281 3282 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3283 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3284 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3285 3286 mdelay(1); 3287 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3288 3289 rb_addr = ring->gpu_addr >> 8; 3290 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3291 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3292 3293 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3294 if (ring->use_doorbell) { 3295 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3296 DOORBELL_OFFSET, ring->doorbell_index); 3297 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3298 DOORBELL_EN, 1); 3299 } else { 3300 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3301 } 3302 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3303 3304 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3305 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3306 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3307 3308 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3309 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3310 3311 3312 /* start the ring */ 3313 gfx_v9_0_cp_gfx_start(adev); 3314 ring->sched.ready = true; 3315 3316 return 0; 3317 } 3318 3319 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3320 { 3321 if (enable) { 3322 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3323 } else { 3324 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3325 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3326 adev->gfx.kiq.ring.sched.ready = false; 3327 } 3328 udelay(50); 3329 } 3330 3331 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3332 { 3333 const struct gfx_firmware_header_v1_0 *mec_hdr; 3334 const __le32 *fw_data; 3335 unsigned i; 3336 u32 tmp; 3337 3338 if (!adev->gfx.mec_fw) 3339 return -EINVAL; 3340 3341 gfx_v9_0_cp_compute_enable(adev, false); 3342 3343 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3344 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3345 3346 fw_data = (const __le32 *) 3347 (adev->gfx.mec_fw->data + 3348 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3349 tmp = 0; 3350 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3351 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3352 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3353 3354 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3355 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3356 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3357 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3358 3359 /* MEC1 */ 3360 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3361 mec_hdr->jt_offset); 3362 for (i = 0; i < mec_hdr->jt_size; i++) 3363 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3364 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3365 3366 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3367 adev->gfx.mec_fw_version); 3368 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3369 3370 return 0; 3371 } 3372 3373 /* KIQ functions */ 3374 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3375 { 3376 uint32_t tmp; 3377 struct amdgpu_device *adev = ring->adev; 3378 3379 /* tell RLC which is KIQ queue */ 3380 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3381 tmp &= 0xffffff00; 3382 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3383 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3384 tmp |= 0x80; 3385 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3386 } 3387 3388 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3389 { 3390 struct amdgpu_device *adev = ring->adev; 3391 3392 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3393 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3394 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3395 mqd->cp_hqd_queue_priority = 3396 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3397 } 3398 } 3399 } 3400 3401 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3402 { 3403 struct amdgpu_device *adev = ring->adev; 3404 struct v9_mqd *mqd = ring->mqd_ptr; 3405 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3406 uint32_t tmp; 3407 3408 mqd->header = 0xC0310800; 3409 mqd->compute_pipelinestat_enable = 0x00000001; 3410 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3411 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3412 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3413 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3414 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3415 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3416 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3417 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3418 mqd->compute_misc_reserved = 0x00000003; 3419 3420 mqd->dynamic_cu_mask_addr_lo = 3421 lower_32_bits(ring->mqd_gpu_addr 3422 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3423 mqd->dynamic_cu_mask_addr_hi = 3424 upper_32_bits(ring->mqd_gpu_addr 3425 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3426 3427 eop_base_addr = ring->eop_gpu_addr >> 8; 3428 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3429 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3430 3431 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3432 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3433 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3434 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3435 3436 mqd->cp_hqd_eop_control = tmp; 3437 3438 /* enable doorbell? */ 3439 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3440 3441 if (ring->use_doorbell) { 3442 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3443 DOORBELL_OFFSET, ring->doorbell_index); 3444 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3445 DOORBELL_EN, 1); 3446 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3447 DOORBELL_SOURCE, 0); 3448 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3449 DOORBELL_HIT, 0); 3450 } else { 3451 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3452 DOORBELL_EN, 0); 3453 } 3454 3455 mqd->cp_hqd_pq_doorbell_control = tmp; 3456 3457 /* disable the queue if it's active */ 3458 ring->wptr = 0; 3459 mqd->cp_hqd_dequeue_request = 0; 3460 mqd->cp_hqd_pq_rptr = 0; 3461 mqd->cp_hqd_pq_wptr_lo = 0; 3462 mqd->cp_hqd_pq_wptr_hi = 0; 3463 3464 /* set the pointer to the MQD */ 3465 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3466 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3467 3468 /* set MQD vmid to 0 */ 3469 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3470 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3471 mqd->cp_mqd_control = tmp; 3472 3473 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3474 hqd_gpu_addr = ring->gpu_addr >> 8; 3475 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3476 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3477 3478 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3479 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3480 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3481 (order_base_2(ring->ring_size / 4) - 1)); 3482 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3483 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3484 #ifdef __BIG_ENDIAN 3485 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3486 #endif 3487 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3488 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3489 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3490 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3491 mqd->cp_hqd_pq_control = tmp; 3492 3493 /* set the wb address whether it's enabled or not */ 3494 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3495 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3496 mqd->cp_hqd_pq_rptr_report_addr_hi = 3497 upper_32_bits(wb_gpu_addr) & 0xffff; 3498 3499 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3500 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3501 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3502 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3503 3504 tmp = 0; 3505 /* enable the doorbell if requested */ 3506 if (ring->use_doorbell) { 3507 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3508 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3509 DOORBELL_OFFSET, ring->doorbell_index); 3510 3511 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3512 DOORBELL_EN, 1); 3513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3514 DOORBELL_SOURCE, 0); 3515 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3516 DOORBELL_HIT, 0); 3517 } 3518 3519 mqd->cp_hqd_pq_doorbell_control = tmp; 3520 3521 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3522 ring->wptr = 0; 3523 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3524 3525 /* set the vmid for the queue */ 3526 mqd->cp_hqd_vmid = 0; 3527 3528 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3529 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3530 mqd->cp_hqd_persistent_state = tmp; 3531 3532 /* set MIN_IB_AVAIL_SIZE */ 3533 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3534 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3535 mqd->cp_hqd_ib_control = tmp; 3536 3537 /* set static priority for a queue/ring */ 3538 gfx_v9_0_mqd_set_priority(ring, mqd); 3539 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3540 3541 /* map_queues packet doesn't need activate the queue, 3542 * so only kiq need set this field. 3543 */ 3544 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3545 mqd->cp_hqd_active = 1; 3546 3547 return 0; 3548 } 3549 3550 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3551 { 3552 struct amdgpu_device *adev = ring->adev; 3553 struct v9_mqd *mqd = ring->mqd_ptr; 3554 int j; 3555 3556 /* disable wptr polling */ 3557 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3558 3559 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3560 mqd->cp_hqd_eop_base_addr_lo); 3561 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3562 mqd->cp_hqd_eop_base_addr_hi); 3563 3564 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3565 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3566 mqd->cp_hqd_eop_control); 3567 3568 /* enable doorbell? */ 3569 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3570 mqd->cp_hqd_pq_doorbell_control); 3571 3572 /* disable the queue if it's active */ 3573 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3574 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3575 for (j = 0; j < adev->usec_timeout; j++) { 3576 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3577 break; 3578 udelay(1); 3579 } 3580 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3581 mqd->cp_hqd_dequeue_request); 3582 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3583 mqd->cp_hqd_pq_rptr); 3584 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3585 mqd->cp_hqd_pq_wptr_lo); 3586 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3587 mqd->cp_hqd_pq_wptr_hi); 3588 } 3589 3590 /* set the pointer to the MQD */ 3591 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3592 mqd->cp_mqd_base_addr_lo); 3593 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3594 mqd->cp_mqd_base_addr_hi); 3595 3596 /* set MQD vmid to 0 */ 3597 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3598 mqd->cp_mqd_control); 3599 3600 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3601 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3602 mqd->cp_hqd_pq_base_lo); 3603 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3604 mqd->cp_hqd_pq_base_hi); 3605 3606 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3607 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3608 mqd->cp_hqd_pq_control); 3609 3610 /* set the wb address whether it's enabled or not */ 3611 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3612 mqd->cp_hqd_pq_rptr_report_addr_lo); 3613 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3614 mqd->cp_hqd_pq_rptr_report_addr_hi); 3615 3616 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3617 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3618 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3620 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3621 3622 /* enable the doorbell if requested */ 3623 if (ring->use_doorbell) { 3624 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3625 (adev->doorbell_index.kiq * 2) << 2); 3626 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3627 (adev->doorbell_index.userqueue_end * 2) << 2); 3628 } 3629 3630 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3631 mqd->cp_hqd_pq_doorbell_control); 3632 3633 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3634 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3635 mqd->cp_hqd_pq_wptr_lo); 3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3637 mqd->cp_hqd_pq_wptr_hi); 3638 3639 /* set the vmid for the queue */ 3640 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3641 3642 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3643 mqd->cp_hqd_persistent_state); 3644 3645 /* activate the queue */ 3646 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3647 mqd->cp_hqd_active); 3648 3649 if (ring->use_doorbell) 3650 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3651 3652 return 0; 3653 } 3654 3655 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3656 { 3657 struct amdgpu_device *adev = ring->adev; 3658 int j; 3659 3660 /* disable the queue if it's active */ 3661 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3662 3663 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3664 3665 for (j = 0; j < adev->usec_timeout; j++) { 3666 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3667 break; 3668 udelay(1); 3669 } 3670 3671 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3672 DRM_DEBUG("KIQ dequeue request failed.\n"); 3673 3674 /* Manual disable if dequeue request times out */ 3675 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3676 } 3677 3678 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3679 0); 3680 } 3681 3682 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3683 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3685 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3687 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3688 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3689 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3690 3691 return 0; 3692 } 3693 3694 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3695 { 3696 struct amdgpu_device *adev = ring->adev; 3697 struct v9_mqd *mqd = ring->mqd_ptr; 3698 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3699 3700 gfx_v9_0_kiq_setting(ring); 3701 3702 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3703 /* reset MQD to a clean status */ 3704 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3705 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3706 3707 /* reset ring buffer */ 3708 ring->wptr = 0; 3709 amdgpu_ring_clear_ring(ring); 3710 3711 mutex_lock(&adev->srbm_mutex); 3712 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3713 gfx_v9_0_kiq_init_register(ring); 3714 soc15_grbm_select(adev, 0, 0, 0, 0); 3715 mutex_unlock(&adev->srbm_mutex); 3716 } else { 3717 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3718 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3719 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3720 mutex_lock(&adev->srbm_mutex); 3721 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3722 gfx_v9_0_mqd_init(ring); 3723 gfx_v9_0_kiq_init_register(ring); 3724 soc15_grbm_select(adev, 0, 0, 0, 0); 3725 mutex_unlock(&adev->srbm_mutex); 3726 3727 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3728 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3729 } 3730 3731 return 0; 3732 } 3733 3734 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3735 { 3736 struct amdgpu_device *adev = ring->adev; 3737 struct v9_mqd *mqd = ring->mqd_ptr; 3738 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3739 3740 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3741 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3742 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3743 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3744 mutex_lock(&adev->srbm_mutex); 3745 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3746 gfx_v9_0_mqd_init(ring); 3747 soc15_grbm_select(adev, 0, 0, 0, 0); 3748 mutex_unlock(&adev->srbm_mutex); 3749 3750 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3751 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3752 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3753 /* reset MQD to a clean status */ 3754 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3755 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3756 3757 /* reset ring buffer */ 3758 ring->wptr = 0; 3759 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3760 amdgpu_ring_clear_ring(ring); 3761 } else { 3762 amdgpu_ring_clear_ring(ring); 3763 } 3764 3765 return 0; 3766 } 3767 3768 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3769 { 3770 struct amdgpu_ring *ring; 3771 int r; 3772 3773 ring = &adev->gfx.kiq.ring; 3774 3775 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3776 if (unlikely(r != 0)) 3777 return r; 3778 3779 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3780 if (unlikely(r != 0)) 3781 return r; 3782 3783 gfx_v9_0_kiq_init_queue(ring); 3784 amdgpu_bo_kunmap(ring->mqd_obj); 3785 ring->mqd_ptr = NULL; 3786 amdgpu_bo_unreserve(ring->mqd_obj); 3787 ring->sched.ready = true; 3788 return 0; 3789 } 3790 3791 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3792 { 3793 struct amdgpu_ring *ring = NULL; 3794 int r = 0, i; 3795 3796 gfx_v9_0_cp_compute_enable(adev, true); 3797 3798 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3799 ring = &adev->gfx.compute_ring[i]; 3800 3801 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3802 if (unlikely(r != 0)) 3803 goto done; 3804 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3805 if (!r) { 3806 r = gfx_v9_0_kcq_init_queue(ring); 3807 amdgpu_bo_kunmap(ring->mqd_obj); 3808 ring->mqd_ptr = NULL; 3809 } 3810 amdgpu_bo_unreserve(ring->mqd_obj); 3811 if (r) 3812 goto done; 3813 } 3814 3815 r = amdgpu_gfx_enable_kcq(adev); 3816 done: 3817 return r; 3818 } 3819 3820 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3821 { 3822 int r, i; 3823 struct amdgpu_ring *ring; 3824 3825 if (!(adev->flags & AMD_IS_APU)) 3826 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3827 3828 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3829 if (adev->gfx.num_gfx_rings) { 3830 /* legacy firmware loading */ 3831 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3832 if (r) 3833 return r; 3834 } 3835 3836 r = gfx_v9_0_cp_compute_load_microcode(adev); 3837 if (r) 3838 return r; 3839 } 3840 3841 r = gfx_v9_0_kiq_resume(adev); 3842 if (r) 3843 return r; 3844 3845 if (adev->gfx.num_gfx_rings) { 3846 r = gfx_v9_0_cp_gfx_resume(adev); 3847 if (r) 3848 return r; 3849 } 3850 3851 r = gfx_v9_0_kcq_resume(adev); 3852 if (r) 3853 return r; 3854 3855 if (adev->gfx.num_gfx_rings) { 3856 ring = &adev->gfx.gfx_ring[0]; 3857 r = amdgpu_ring_test_helper(ring); 3858 if (r) 3859 return r; 3860 } 3861 3862 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3863 ring = &adev->gfx.compute_ring[i]; 3864 amdgpu_ring_test_helper(ring); 3865 } 3866 3867 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3868 3869 return 0; 3870 } 3871 3872 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3873 { 3874 u32 tmp; 3875 3876 if (adev->asic_type != CHIP_ARCTURUS) 3877 return; 3878 3879 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3880 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3881 adev->df.hash_status.hash_64k); 3882 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3883 adev->df.hash_status.hash_2m); 3884 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3885 adev->df.hash_status.hash_1g); 3886 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3887 } 3888 3889 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3890 { 3891 if (adev->gfx.num_gfx_rings) 3892 gfx_v9_0_cp_gfx_enable(adev, enable); 3893 gfx_v9_0_cp_compute_enable(adev, enable); 3894 } 3895 3896 static int gfx_v9_0_hw_init(void *handle) 3897 { 3898 int r; 3899 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3900 3901 if (!amdgpu_sriov_vf(adev)) 3902 gfx_v9_0_init_golden_registers(adev); 3903 3904 gfx_v9_0_constants_init(adev); 3905 3906 gfx_v9_0_init_tcp_config(adev); 3907 3908 r = adev->gfx.rlc.funcs->resume(adev); 3909 if (r) 3910 return r; 3911 3912 r = gfx_v9_0_cp_resume(adev); 3913 if (r) 3914 return r; 3915 3916 return r; 3917 } 3918 3919 static int gfx_v9_0_hw_fini(void *handle) 3920 { 3921 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3922 3923 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3924 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3925 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3926 3927 /* DF freeze and kcq disable will fail */ 3928 if (!amdgpu_ras_intr_triggered()) 3929 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3930 amdgpu_gfx_disable_kcq(adev); 3931 3932 if (amdgpu_sriov_vf(adev)) { 3933 gfx_v9_0_cp_gfx_enable(adev, false); 3934 /* must disable polling for SRIOV when hw finished, otherwise 3935 * CPC engine may still keep fetching WB address which is already 3936 * invalid after sw finished and trigger DMAR reading error in 3937 * hypervisor side. 3938 */ 3939 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3940 return 0; 3941 } 3942 3943 /* Use deinitialize sequence from CAIL when unbinding device from driver, 3944 * otherwise KIQ is hanging when binding back 3945 */ 3946 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3947 mutex_lock(&adev->srbm_mutex); 3948 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 3949 adev->gfx.kiq.ring.pipe, 3950 adev->gfx.kiq.ring.queue, 0); 3951 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 3952 soc15_grbm_select(adev, 0, 0, 0, 0); 3953 mutex_unlock(&adev->srbm_mutex); 3954 } 3955 3956 gfx_v9_0_cp_enable(adev, false); 3957 adev->gfx.rlc.funcs->stop(adev); 3958 3959 return 0; 3960 } 3961 3962 static int gfx_v9_0_suspend(void *handle) 3963 { 3964 return gfx_v9_0_hw_fini(handle); 3965 } 3966 3967 static int gfx_v9_0_resume(void *handle) 3968 { 3969 return gfx_v9_0_hw_init(handle); 3970 } 3971 3972 static bool gfx_v9_0_is_idle(void *handle) 3973 { 3974 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3975 3976 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3977 GRBM_STATUS, GUI_ACTIVE)) 3978 return false; 3979 else 3980 return true; 3981 } 3982 3983 static int gfx_v9_0_wait_for_idle(void *handle) 3984 { 3985 unsigned i; 3986 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3987 3988 for (i = 0; i < adev->usec_timeout; i++) { 3989 if (gfx_v9_0_is_idle(handle)) 3990 return 0; 3991 udelay(1); 3992 } 3993 return -ETIMEDOUT; 3994 } 3995 3996 static int gfx_v9_0_soft_reset(void *handle) 3997 { 3998 u32 grbm_soft_reset = 0; 3999 u32 tmp; 4000 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4001 4002 /* GRBM_STATUS */ 4003 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4004 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4005 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4006 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4007 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4008 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4009 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4010 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4011 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4012 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4013 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4014 } 4015 4016 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4017 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4018 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4019 } 4020 4021 /* GRBM_STATUS2 */ 4022 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4023 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4024 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4025 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4026 4027 4028 if (grbm_soft_reset) { 4029 /* stop the rlc */ 4030 adev->gfx.rlc.funcs->stop(adev); 4031 4032 if (adev->gfx.num_gfx_rings) 4033 /* Disable GFX parsing/prefetching */ 4034 gfx_v9_0_cp_gfx_enable(adev, false); 4035 4036 /* Disable MEC parsing/prefetching */ 4037 gfx_v9_0_cp_compute_enable(adev, false); 4038 4039 if (grbm_soft_reset) { 4040 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4041 tmp |= grbm_soft_reset; 4042 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4043 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4044 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4045 4046 udelay(50); 4047 4048 tmp &= ~grbm_soft_reset; 4049 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4050 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4051 } 4052 4053 /* Wait a little for things to settle down */ 4054 udelay(50); 4055 } 4056 return 0; 4057 } 4058 4059 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4060 { 4061 signed long r, cnt = 0; 4062 unsigned long flags; 4063 uint32_t seq, reg_val_offs = 0; 4064 uint64_t value = 0; 4065 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4066 struct amdgpu_ring *ring = &kiq->ring; 4067 4068 BUG_ON(!ring->funcs->emit_rreg); 4069 4070 spin_lock_irqsave(&kiq->ring_lock, flags); 4071 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4072 pr_err("critical bug! too many kiq readers\n"); 4073 goto failed_unlock; 4074 } 4075 amdgpu_ring_alloc(ring, 32); 4076 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4077 amdgpu_ring_write(ring, 9 | /* src: register*/ 4078 (5 << 8) | /* dst: memory */ 4079 (1 << 16) | /* count sel */ 4080 (1 << 20)); /* write confirm */ 4081 amdgpu_ring_write(ring, 0); 4082 amdgpu_ring_write(ring, 0); 4083 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4084 reg_val_offs * 4)); 4085 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4086 reg_val_offs * 4)); 4087 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4088 if (r) 4089 goto failed_undo; 4090 4091 amdgpu_ring_commit(ring); 4092 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4093 4094 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4095 4096 /* don't wait anymore for gpu reset case because this way may 4097 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4098 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4099 * never return if we keep waiting in virt_kiq_rreg, which cause 4100 * gpu_recover() hang there. 4101 * 4102 * also don't wait anymore for IRQ context 4103 * */ 4104 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 4105 goto failed_kiq_read; 4106 4107 might_sleep(); 4108 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4109 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4110 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4111 } 4112 4113 if (cnt > MAX_KIQ_REG_TRY) 4114 goto failed_kiq_read; 4115 4116 mb(); 4117 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4118 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4119 amdgpu_device_wb_free(adev, reg_val_offs); 4120 return value; 4121 4122 failed_undo: 4123 amdgpu_ring_undo(ring); 4124 failed_unlock: 4125 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4126 failed_kiq_read: 4127 if (reg_val_offs) 4128 amdgpu_device_wb_free(adev, reg_val_offs); 4129 pr_err("failed to read gpu clock\n"); 4130 return ~0; 4131 } 4132 4133 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4134 { 4135 uint64_t clock; 4136 4137 amdgpu_gfx_off_ctrl(adev, false); 4138 mutex_lock(&adev->gfx.gpu_clock_mutex); 4139 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4140 clock = gfx_v9_0_kiq_read_clock(adev); 4141 } else { 4142 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4143 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4144 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4145 } 4146 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4147 amdgpu_gfx_off_ctrl(adev, true); 4148 return clock; 4149 } 4150 4151 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4152 uint32_t vmid, 4153 uint32_t gds_base, uint32_t gds_size, 4154 uint32_t gws_base, uint32_t gws_size, 4155 uint32_t oa_base, uint32_t oa_size) 4156 { 4157 struct amdgpu_device *adev = ring->adev; 4158 4159 /* GDS Base */ 4160 gfx_v9_0_write_data_to_reg(ring, 0, false, 4161 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4162 gds_base); 4163 4164 /* GDS Size */ 4165 gfx_v9_0_write_data_to_reg(ring, 0, false, 4166 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4167 gds_size); 4168 4169 /* GWS */ 4170 gfx_v9_0_write_data_to_reg(ring, 0, false, 4171 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4172 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4173 4174 /* OA */ 4175 gfx_v9_0_write_data_to_reg(ring, 0, false, 4176 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4177 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4178 } 4179 4180 static const u32 vgpr_init_compute_shader[] = 4181 { 4182 0xb07c0000, 0xbe8000ff, 4183 0x000000f8, 0xbf110800, 4184 0x7e000280, 0x7e020280, 4185 0x7e040280, 0x7e060280, 4186 0x7e080280, 0x7e0a0280, 4187 0x7e0c0280, 0x7e0e0280, 4188 0x80808800, 0xbe803200, 4189 0xbf84fff5, 0xbf9c0000, 4190 0xd28c0001, 0x0001007f, 4191 0xd28d0001, 0x0002027e, 4192 0x10020288, 0xb8810904, 4193 0xb7814000, 0xd1196a01, 4194 0x00000301, 0xbe800087, 4195 0xbefc00c1, 0xd89c4000, 4196 0x00020201, 0xd89cc080, 4197 0x00040401, 0x320202ff, 4198 0x00000800, 0x80808100, 4199 0xbf84fff8, 0x7e020280, 4200 0xbf810000, 0x00000000, 4201 }; 4202 4203 static const u32 sgpr_init_compute_shader[] = 4204 { 4205 0xb07c0000, 0xbe8000ff, 4206 0x0000005f, 0xbee50080, 4207 0xbe812c65, 0xbe822c65, 4208 0xbe832c65, 0xbe842c65, 4209 0xbe852c65, 0xb77c0005, 4210 0x80808500, 0xbf84fff8, 4211 0xbe800080, 0xbf810000, 4212 }; 4213 4214 static const u32 vgpr_init_compute_shader_arcturus[] = { 4215 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4216 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4217 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4218 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4219 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4220 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4221 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4222 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4223 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4224 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4225 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4226 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4227 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4228 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4229 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4230 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4231 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4232 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4233 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4234 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4235 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4236 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4237 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4238 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4239 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4240 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4241 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4242 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4243 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4244 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4245 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4246 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4247 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4248 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4249 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4250 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4251 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4252 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4253 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4254 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4255 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4256 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4257 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4258 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4259 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4260 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4261 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4262 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4263 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4264 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4265 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4266 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4267 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4268 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4269 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4270 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4271 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4272 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4273 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4274 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4275 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4276 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4277 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4278 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4279 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4280 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4281 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4282 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4283 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4284 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4285 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4286 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4287 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4288 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4289 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4290 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4291 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4292 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4293 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4294 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4295 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4296 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4297 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4298 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4299 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4300 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4301 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4302 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4303 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4304 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4305 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4306 0xbf84fff8, 0xbf810000, 4307 }; 4308 4309 /* When below register arrays changed, please update gpr_reg_size, 4310 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4311 to cover all gfx9 ASICs */ 4312 static const struct soc15_reg_entry vgpr_init_regs[] = { 4313 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4314 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4315 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4316 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4317 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4318 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4319 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4320 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4321 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4322 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4323 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4324 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4325 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4326 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4327 }; 4328 4329 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4330 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4331 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4332 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4333 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4334 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4335 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4336 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4337 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4338 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4339 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4340 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4341 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4342 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4343 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4344 }; 4345 4346 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4347 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4348 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4349 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4350 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4351 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4352 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4353 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4354 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4355 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4356 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4357 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4358 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4359 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4360 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4361 }; 4362 4363 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4364 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4365 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4366 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4367 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4368 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4369 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4370 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4371 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4372 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4373 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4374 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4375 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4376 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4377 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4378 }; 4379 4380 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4381 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4382 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4383 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4384 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4385 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4386 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4387 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4388 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4389 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4390 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4391 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4392 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4393 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4394 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4395 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4396 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4397 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4398 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4399 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4400 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4401 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4402 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4403 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4404 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4405 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4406 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4407 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4408 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4409 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4410 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4411 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4412 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4413 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4414 }; 4415 4416 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4417 { 4418 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4419 int i, r; 4420 4421 /* only support when RAS is enabled */ 4422 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4423 return 0; 4424 4425 r = amdgpu_ring_alloc(ring, 7); 4426 if (r) { 4427 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4428 ring->name, r); 4429 return r; 4430 } 4431 4432 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4433 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4434 4435 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4436 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4437 PACKET3_DMA_DATA_DST_SEL(1) | 4438 PACKET3_DMA_DATA_SRC_SEL(2) | 4439 PACKET3_DMA_DATA_ENGINE(0))); 4440 amdgpu_ring_write(ring, 0); 4441 amdgpu_ring_write(ring, 0); 4442 amdgpu_ring_write(ring, 0); 4443 amdgpu_ring_write(ring, 0); 4444 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4445 adev->gds.gds_size); 4446 4447 amdgpu_ring_commit(ring); 4448 4449 for (i = 0; i < adev->usec_timeout; i++) { 4450 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4451 break; 4452 udelay(1); 4453 } 4454 4455 if (i >= adev->usec_timeout) 4456 r = -ETIMEDOUT; 4457 4458 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4459 4460 return r; 4461 } 4462 4463 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4464 { 4465 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4466 struct amdgpu_ib ib; 4467 struct dma_fence *f = NULL; 4468 int r, i; 4469 unsigned total_size, vgpr_offset, sgpr_offset; 4470 u64 gpu_addr; 4471 4472 int compute_dim_x = adev->gfx.config.max_shader_engines * 4473 adev->gfx.config.max_cu_per_sh * 4474 adev->gfx.config.max_sh_per_se; 4475 int sgpr_work_group_size = 5; 4476 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4477 int vgpr_init_shader_size; 4478 const u32 *vgpr_init_shader_ptr; 4479 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4480 4481 /* only support when RAS is enabled */ 4482 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4483 return 0; 4484 4485 /* bail if the compute ring is not ready */ 4486 if (!ring->sched.ready) 4487 return 0; 4488 4489 if (adev->asic_type == CHIP_ARCTURUS) { 4490 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4491 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4492 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4493 } else { 4494 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4495 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4496 vgpr_init_regs_ptr = vgpr_init_regs; 4497 } 4498 4499 total_size = 4500 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4501 total_size += 4502 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4503 total_size += 4504 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4505 total_size = ALIGN(total_size, 256); 4506 vgpr_offset = total_size; 4507 total_size += ALIGN(vgpr_init_shader_size, 256); 4508 sgpr_offset = total_size; 4509 total_size += sizeof(sgpr_init_compute_shader); 4510 4511 /* allocate an indirect buffer to put the commands in */ 4512 memset(&ib, 0, sizeof(ib)); 4513 r = amdgpu_ib_get(adev, NULL, total_size, 4514 AMDGPU_IB_POOL_DIRECT, &ib); 4515 if (r) { 4516 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4517 return r; 4518 } 4519 4520 /* load the compute shaders */ 4521 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4522 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4523 4524 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4525 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4526 4527 /* init the ib length to 0 */ 4528 ib.length_dw = 0; 4529 4530 /* VGPR */ 4531 /* write the register state for the compute dispatch */ 4532 for (i = 0; i < gpr_reg_size; i++) { 4533 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4534 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4535 - PACKET3_SET_SH_REG_START; 4536 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4537 } 4538 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4539 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4540 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4541 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4542 - PACKET3_SET_SH_REG_START; 4543 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4544 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4545 4546 /* write dispatch packet */ 4547 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4548 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4549 ib.ptr[ib.length_dw++] = 1; /* y */ 4550 ib.ptr[ib.length_dw++] = 1; /* z */ 4551 ib.ptr[ib.length_dw++] = 4552 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4553 4554 /* write CS partial flush packet */ 4555 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4556 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4557 4558 /* SGPR1 */ 4559 /* write the register state for the compute dispatch */ 4560 for (i = 0; i < gpr_reg_size; i++) { 4561 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4562 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4563 - PACKET3_SET_SH_REG_START; 4564 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4565 } 4566 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4567 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4568 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4569 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4570 - PACKET3_SET_SH_REG_START; 4571 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4572 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4573 4574 /* write dispatch packet */ 4575 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4576 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4577 ib.ptr[ib.length_dw++] = 1; /* y */ 4578 ib.ptr[ib.length_dw++] = 1; /* z */ 4579 ib.ptr[ib.length_dw++] = 4580 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4581 4582 /* write CS partial flush packet */ 4583 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4584 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4585 4586 /* SGPR2 */ 4587 /* write the register state for the compute dispatch */ 4588 for (i = 0; i < gpr_reg_size; i++) { 4589 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4590 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4591 - PACKET3_SET_SH_REG_START; 4592 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4593 } 4594 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4595 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4596 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4597 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4598 - PACKET3_SET_SH_REG_START; 4599 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4600 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4601 4602 /* write dispatch packet */ 4603 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4604 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4605 ib.ptr[ib.length_dw++] = 1; /* y */ 4606 ib.ptr[ib.length_dw++] = 1; /* z */ 4607 ib.ptr[ib.length_dw++] = 4608 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4609 4610 /* write CS partial flush packet */ 4611 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4612 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4613 4614 /* shedule the ib on the ring */ 4615 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4616 if (r) { 4617 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4618 goto fail; 4619 } 4620 4621 /* wait for the GPU to finish processing the IB */ 4622 r = dma_fence_wait(f, false); 4623 if (r) { 4624 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4625 goto fail; 4626 } 4627 4628 fail: 4629 amdgpu_ib_free(adev, &ib, NULL); 4630 dma_fence_put(f); 4631 4632 return r; 4633 } 4634 4635 static int gfx_v9_0_early_init(void *handle) 4636 { 4637 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4638 4639 if (adev->asic_type == CHIP_ARCTURUS) 4640 adev->gfx.num_gfx_rings = 0; 4641 else 4642 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4643 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4644 AMDGPU_MAX_COMPUTE_RINGS); 4645 gfx_v9_0_set_kiq_pm4_funcs(adev); 4646 gfx_v9_0_set_ring_funcs(adev); 4647 gfx_v9_0_set_irq_funcs(adev); 4648 gfx_v9_0_set_gds_init(adev); 4649 gfx_v9_0_set_rlc_funcs(adev); 4650 4651 return 0; 4652 } 4653 4654 static int gfx_v9_0_ecc_late_init(void *handle) 4655 { 4656 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4657 int r; 4658 4659 /* 4660 * Temp workaround to fix the issue that CP firmware fails to 4661 * update read pointer when CPDMA is writing clearing operation 4662 * to GDS in suspend/resume sequence on several cards. So just 4663 * limit this operation in cold boot sequence. 4664 */ 4665 if (!adev->in_suspend) { 4666 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4667 if (r) 4668 return r; 4669 } 4670 4671 /* requires IBs so do in late init after IB pool is initialized */ 4672 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4673 if (r) 4674 return r; 4675 4676 if (adev->gfx.funcs && 4677 adev->gfx.funcs->reset_ras_error_count) 4678 adev->gfx.funcs->reset_ras_error_count(adev); 4679 4680 r = amdgpu_gfx_ras_late_init(adev); 4681 if (r) 4682 return r; 4683 4684 return 0; 4685 } 4686 4687 static int gfx_v9_0_late_init(void *handle) 4688 { 4689 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4690 int r; 4691 4692 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4693 if (r) 4694 return r; 4695 4696 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4697 if (r) 4698 return r; 4699 4700 r = gfx_v9_0_ecc_late_init(handle); 4701 if (r) 4702 return r; 4703 4704 return 0; 4705 } 4706 4707 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4708 { 4709 uint32_t rlc_setting; 4710 4711 /* if RLC is not enabled, do nothing */ 4712 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4713 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4714 return false; 4715 4716 return true; 4717 } 4718 4719 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4720 { 4721 uint32_t data; 4722 unsigned i; 4723 4724 data = RLC_SAFE_MODE__CMD_MASK; 4725 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4726 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4727 4728 /* wait for RLC_SAFE_MODE */ 4729 for (i = 0; i < adev->usec_timeout; i++) { 4730 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4731 break; 4732 udelay(1); 4733 } 4734 } 4735 4736 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4737 { 4738 uint32_t data; 4739 4740 data = RLC_SAFE_MODE__CMD_MASK; 4741 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4742 } 4743 4744 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4745 bool enable) 4746 { 4747 amdgpu_gfx_rlc_enter_safe_mode(adev); 4748 4749 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4750 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4751 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4752 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4753 } else { 4754 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4755 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4756 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4757 } 4758 4759 amdgpu_gfx_rlc_exit_safe_mode(adev); 4760 } 4761 4762 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4763 bool enable) 4764 { 4765 /* TODO: double check if we need to perform under safe mode */ 4766 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4767 4768 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4769 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4770 else 4771 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4772 4773 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4774 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4775 else 4776 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4777 4778 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4779 } 4780 4781 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4782 bool enable) 4783 { 4784 uint32_t data, def; 4785 4786 amdgpu_gfx_rlc_enter_safe_mode(adev); 4787 4788 /* It is disabled by HW by default */ 4789 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4790 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4791 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4792 4793 if (adev->asic_type != CHIP_VEGA12) 4794 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4795 4796 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4797 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4798 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4799 4800 /* only for Vega10 & Raven1 */ 4801 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4802 4803 if (def != data) 4804 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4805 4806 /* MGLS is a global flag to control all MGLS in GFX */ 4807 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4808 /* 2 - RLC memory Light sleep */ 4809 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4810 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4811 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4812 if (def != data) 4813 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4814 } 4815 /* 3 - CP memory Light sleep */ 4816 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4817 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4818 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4819 if (def != data) 4820 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4821 } 4822 } 4823 } else { 4824 /* 1 - MGCG_OVERRIDE */ 4825 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4826 4827 if (adev->asic_type != CHIP_VEGA12) 4828 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4829 4830 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4831 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4832 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4833 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4834 4835 if (def != data) 4836 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4837 4838 /* 2 - disable MGLS in RLC */ 4839 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4840 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4841 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4842 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4843 } 4844 4845 /* 3 - disable MGLS in CP */ 4846 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4847 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4848 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4849 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4850 } 4851 } 4852 4853 amdgpu_gfx_rlc_exit_safe_mode(adev); 4854 } 4855 4856 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4857 bool enable) 4858 { 4859 uint32_t data, def; 4860 4861 if (adev->asic_type == CHIP_ARCTURUS) 4862 return; 4863 4864 amdgpu_gfx_rlc_enter_safe_mode(adev); 4865 4866 /* Enable 3D CGCG/CGLS */ 4867 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) { 4868 /* write cmd to clear cgcg/cgls ov */ 4869 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4870 /* unset CGCG override */ 4871 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4872 /* update CGCG and CGLS override bits */ 4873 if (def != data) 4874 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4875 4876 /* enable 3Dcgcg FSM(0x0000363f) */ 4877 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4878 4879 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4880 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4881 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4882 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4883 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4884 if (def != data) 4885 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4886 4887 /* set IDLE_POLL_COUNT(0x00900100) */ 4888 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4889 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4890 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4891 if (def != data) 4892 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4893 } else { 4894 /* Disable CGCG/CGLS */ 4895 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4896 /* disable cgcg, cgls should be disabled */ 4897 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4898 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4899 /* disable cgcg and cgls in FSM */ 4900 if (def != data) 4901 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4902 } 4903 4904 amdgpu_gfx_rlc_exit_safe_mode(adev); 4905 } 4906 4907 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4908 bool enable) 4909 { 4910 uint32_t def, data; 4911 4912 amdgpu_gfx_rlc_enter_safe_mode(adev); 4913 4914 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4915 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4916 /* unset CGCG override */ 4917 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4918 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4919 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4920 else 4921 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4922 /* update CGCG and CGLS override bits */ 4923 if (def != data) 4924 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4925 4926 /* enable cgcg FSM(0x0000363F) */ 4927 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4928 4929 if (adev->asic_type == CHIP_ARCTURUS) 4930 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4931 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4932 else 4933 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4934 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4935 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4936 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4937 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4938 if (def != data) 4939 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4940 4941 /* set IDLE_POLL_COUNT(0x00900100) */ 4942 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4943 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4944 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4945 if (def != data) 4946 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4947 } else { 4948 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4949 /* reset CGCG/CGLS bits */ 4950 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4951 /* disable cgcg and cgls in FSM */ 4952 if (def != data) 4953 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4954 } 4955 4956 amdgpu_gfx_rlc_exit_safe_mode(adev); 4957 } 4958 4959 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4960 bool enable) 4961 { 4962 if (enable) { 4963 /* CGCG/CGLS should be enabled after MGCG/MGLS 4964 * === MGCG + MGLS === 4965 */ 4966 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4967 /* === CGCG /CGLS for GFX 3D Only === */ 4968 gfx_v9_0_update_3d_clock_gating(adev, enable); 4969 /* === CGCG + CGLS === */ 4970 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4971 } else { 4972 /* CGCG/CGLS should be disabled before MGCG/MGLS 4973 * === CGCG + CGLS === 4974 */ 4975 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4976 /* === CGCG /CGLS for GFX 3D Only === */ 4977 gfx_v9_0_update_3d_clock_gating(adev, enable); 4978 /* === MGCG + MGLS === */ 4979 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4980 } 4981 return 0; 4982 } 4983 4984 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 4985 { 4986 u32 reg, data; 4987 4988 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 4989 if (amdgpu_sriov_is_pp_one_vf(adev)) 4990 data = RREG32_NO_KIQ(reg); 4991 else 4992 data = RREG32(reg); 4993 4994 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 4995 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 4996 4997 if (amdgpu_sriov_is_pp_one_vf(adev)) 4998 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 4999 else 5000 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5001 } 5002 5003 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5004 uint32_t offset, 5005 struct soc15_reg_rlcg *entries, int arr_size) 5006 { 5007 int i; 5008 uint32_t reg; 5009 5010 if (!entries) 5011 return false; 5012 5013 for (i = 0; i < arr_size; i++) { 5014 const struct soc15_reg_rlcg *entry; 5015 5016 entry = &entries[i]; 5017 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5018 if (offset == reg) 5019 return true; 5020 } 5021 5022 return false; 5023 } 5024 5025 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5026 { 5027 return gfx_v9_0_check_rlcg_range(adev, offset, 5028 (void *)rlcg_access_gc_9_0, 5029 ARRAY_SIZE(rlcg_access_gc_9_0)); 5030 } 5031 5032 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5033 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5034 .set_safe_mode = gfx_v9_0_set_safe_mode, 5035 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5036 .init = gfx_v9_0_rlc_init, 5037 .get_csb_size = gfx_v9_0_get_csb_size, 5038 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5039 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5040 .resume = gfx_v9_0_rlc_resume, 5041 .stop = gfx_v9_0_rlc_stop, 5042 .reset = gfx_v9_0_rlc_reset, 5043 .start = gfx_v9_0_rlc_start, 5044 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5045 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5046 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5047 }; 5048 5049 static int gfx_v9_0_set_powergating_state(void *handle, 5050 enum amd_powergating_state state) 5051 { 5052 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5053 bool enable = (state == AMD_PG_STATE_GATE); 5054 5055 switch (adev->asic_type) { 5056 case CHIP_RAVEN: 5057 case CHIP_RENOIR: 5058 if (!enable) 5059 amdgpu_gfx_off_ctrl(adev, false); 5060 5061 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5062 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5063 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5064 } else { 5065 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5066 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5067 } 5068 5069 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5070 gfx_v9_0_enable_cp_power_gating(adev, true); 5071 else 5072 gfx_v9_0_enable_cp_power_gating(adev, false); 5073 5074 /* update gfx cgpg state */ 5075 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5076 5077 /* update mgcg state */ 5078 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5079 5080 if (enable) 5081 amdgpu_gfx_off_ctrl(adev, true); 5082 break; 5083 case CHIP_VEGA12: 5084 amdgpu_gfx_off_ctrl(adev, enable); 5085 break; 5086 default: 5087 break; 5088 } 5089 5090 return 0; 5091 } 5092 5093 static int gfx_v9_0_set_clockgating_state(void *handle, 5094 enum amd_clockgating_state state) 5095 { 5096 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5097 5098 if (amdgpu_sriov_vf(adev)) 5099 return 0; 5100 5101 switch (adev->asic_type) { 5102 case CHIP_VEGA10: 5103 case CHIP_VEGA12: 5104 case CHIP_VEGA20: 5105 case CHIP_RAVEN: 5106 case CHIP_ARCTURUS: 5107 case CHIP_RENOIR: 5108 gfx_v9_0_update_gfx_clock_gating(adev, 5109 state == AMD_CG_STATE_GATE); 5110 break; 5111 default: 5112 break; 5113 } 5114 return 0; 5115 } 5116 5117 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5118 { 5119 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5120 int data; 5121 5122 if (amdgpu_sriov_vf(adev)) 5123 *flags = 0; 5124 5125 /* AMD_CG_SUPPORT_GFX_MGCG */ 5126 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5127 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5128 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5129 5130 /* AMD_CG_SUPPORT_GFX_CGCG */ 5131 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5132 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5133 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5134 5135 /* AMD_CG_SUPPORT_GFX_CGLS */ 5136 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5137 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5138 5139 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5140 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5141 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5142 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5143 5144 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5145 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5146 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5147 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5148 5149 if (adev->asic_type != CHIP_ARCTURUS) { 5150 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5151 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5152 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5153 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5154 5155 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5156 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5157 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5158 } 5159 } 5160 5161 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5162 { 5163 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5164 } 5165 5166 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5167 { 5168 struct amdgpu_device *adev = ring->adev; 5169 u64 wptr; 5170 5171 /* XXX check if swapping is necessary on BE */ 5172 if (ring->use_doorbell) { 5173 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5174 } else { 5175 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5176 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5177 } 5178 5179 return wptr; 5180 } 5181 5182 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5183 { 5184 struct amdgpu_device *adev = ring->adev; 5185 5186 if (ring->use_doorbell) { 5187 /* XXX check if swapping is necessary on BE */ 5188 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5189 WDOORBELL64(ring->doorbell_index, ring->wptr); 5190 } else { 5191 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5192 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5193 } 5194 } 5195 5196 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5197 { 5198 struct amdgpu_device *adev = ring->adev; 5199 u32 ref_and_mask, reg_mem_engine; 5200 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5201 5202 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5203 switch (ring->me) { 5204 case 1: 5205 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5206 break; 5207 case 2: 5208 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5209 break; 5210 default: 5211 return; 5212 } 5213 reg_mem_engine = 0; 5214 } else { 5215 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5216 reg_mem_engine = 1; /* pfp */ 5217 } 5218 5219 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5220 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5221 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5222 ref_and_mask, ref_and_mask, 0x20); 5223 } 5224 5225 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5226 struct amdgpu_job *job, 5227 struct amdgpu_ib *ib, 5228 uint32_t flags) 5229 { 5230 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5231 u32 header, control = 0; 5232 5233 if (ib->flags & AMDGPU_IB_FLAG_CE) 5234 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5235 else 5236 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5237 5238 control |= ib->length_dw | (vmid << 24); 5239 5240 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5241 control |= INDIRECT_BUFFER_PRE_ENB(1); 5242 5243 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5244 gfx_v9_0_ring_emit_de_meta(ring); 5245 } 5246 5247 amdgpu_ring_write(ring, header); 5248 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5249 amdgpu_ring_write(ring, 5250 #ifdef __BIG_ENDIAN 5251 (2 << 0) | 5252 #endif 5253 lower_32_bits(ib->gpu_addr)); 5254 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5255 amdgpu_ring_write(ring, control); 5256 } 5257 5258 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5259 struct amdgpu_job *job, 5260 struct amdgpu_ib *ib, 5261 uint32_t flags) 5262 { 5263 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5264 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5265 5266 /* Currently, there is a high possibility to get wave ID mismatch 5267 * between ME and GDS, leading to a hw deadlock, because ME generates 5268 * different wave IDs than the GDS expects. This situation happens 5269 * randomly when at least 5 compute pipes use GDS ordered append. 5270 * The wave IDs generated by ME are also wrong after suspend/resume. 5271 * Those are probably bugs somewhere else in the kernel driver. 5272 * 5273 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5274 * GDS to 0 for this ring (me/pipe). 5275 */ 5276 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5277 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5278 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5279 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5280 } 5281 5282 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5283 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5284 amdgpu_ring_write(ring, 5285 #ifdef __BIG_ENDIAN 5286 (2 << 0) | 5287 #endif 5288 lower_32_bits(ib->gpu_addr)); 5289 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5290 amdgpu_ring_write(ring, control); 5291 } 5292 5293 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5294 u64 seq, unsigned flags) 5295 { 5296 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5297 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5298 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5299 5300 /* RELEASE_MEM - flush caches, send int */ 5301 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5302 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5303 EOP_TC_NC_ACTION_EN) : 5304 (EOP_TCL1_ACTION_EN | 5305 EOP_TC_ACTION_EN | 5306 EOP_TC_WB_ACTION_EN | 5307 EOP_TC_MD_ACTION_EN)) | 5308 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5309 EVENT_INDEX(5))); 5310 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5311 5312 /* 5313 * the address should be Qword aligned if 64bit write, Dword 5314 * aligned if only send 32bit data low (discard data high) 5315 */ 5316 if (write64bit) 5317 BUG_ON(addr & 0x7); 5318 else 5319 BUG_ON(addr & 0x3); 5320 amdgpu_ring_write(ring, lower_32_bits(addr)); 5321 amdgpu_ring_write(ring, upper_32_bits(addr)); 5322 amdgpu_ring_write(ring, lower_32_bits(seq)); 5323 amdgpu_ring_write(ring, upper_32_bits(seq)); 5324 amdgpu_ring_write(ring, 0); 5325 } 5326 5327 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5328 { 5329 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5330 uint32_t seq = ring->fence_drv.sync_seq; 5331 uint64_t addr = ring->fence_drv.gpu_addr; 5332 5333 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5334 lower_32_bits(addr), upper_32_bits(addr), 5335 seq, 0xffffffff, 4); 5336 } 5337 5338 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5339 unsigned vmid, uint64_t pd_addr) 5340 { 5341 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5342 5343 /* compute doesn't have PFP */ 5344 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5345 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5346 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5347 amdgpu_ring_write(ring, 0x0); 5348 } 5349 } 5350 5351 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5352 { 5353 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5354 } 5355 5356 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5357 { 5358 u64 wptr; 5359 5360 /* XXX check if swapping is necessary on BE */ 5361 if (ring->use_doorbell) 5362 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5363 else 5364 BUG(); 5365 return wptr; 5366 } 5367 5368 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5369 { 5370 struct amdgpu_device *adev = ring->adev; 5371 5372 /* XXX check if swapping is necessary on BE */ 5373 if (ring->use_doorbell) { 5374 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5375 WDOORBELL64(ring->doorbell_index, ring->wptr); 5376 } else{ 5377 BUG(); /* only DOORBELL method supported on gfx9 now */ 5378 } 5379 } 5380 5381 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5382 u64 seq, unsigned int flags) 5383 { 5384 struct amdgpu_device *adev = ring->adev; 5385 5386 /* we only allocate 32bit for each seq wb address */ 5387 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5388 5389 /* write fence seq to the "addr" */ 5390 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5391 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5392 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5393 amdgpu_ring_write(ring, lower_32_bits(addr)); 5394 amdgpu_ring_write(ring, upper_32_bits(addr)); 5395 amdgpu_ring_write(ring, lower_32_bits(seq)); 5396 5397 if (flags & AMDGPU_FENCE_FLAG_INT) { 5398 /* set register to trigger INT */ 5399 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5400 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5401 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5402 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5403 amdgpu_ring_write(ring, 0); 5404 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5405 } 5406 } 5407 5408 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5409 { 5410 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5411 amdgpu_ring_write(ring, 0); 5412 } 5413 5414 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5415 { 5416 struct v9_ce_ib_state ce_payload = {0}; 5417 uint64_t csa_addr; 5418 int cnt; 5419 5420 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5421 csa_addr = amdgpu_csa_vaddr(ring->adev); 5422 5423 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5424 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5425 WRITE_DATA_DST_SEL(8) | 5426 WR_CONFIRM) | 5427 WRITE_DATA_CACHE_POLICY(0)); 5428 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5429 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5430 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5431 } 5432 5433 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5434 { 5435 struct v9_de_ib_state de_payload = {0}; 5436 uint64_t csa_addr, gds_addr; 5437 int cnt; 5438 5439 csa_addr = amdgpu_csa_vaddr(ring->adev); 5440 gds_addr = csa_addr + 4096; 5441 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5442 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5443 5444 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5445 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5446 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5447 WRITE_DATA_DST_SEL(8) | 5448 WR_CONFIRM) | 5449 WRITE_DATA_CACHE_POLICY(0)); 5450 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5451 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5452 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5453 } 5454 5455 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5456 bool secure) 5457 { 5458 uint32_t v = secure ? FRAME_TMZ : 0; 5459 5460 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5461 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5462 } 5463 5464 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5465 { 5466 uint32_t dw2 = 0; 5467 5468 if (amdgpu_sriov_vf(ring->adev)) 5469 gfx_v9_0_ring_emit_ce_meta(ring); 5470 5471 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5472 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5473 /* set load_global_config & load_global_uconfig */ 5474 dw2 |= 0x8001; 5475 /* set load_cs_sh_regs */ 5476 dw2 |= 0x01000000; 5477 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5478 dw2 |= 0x10002; 5479 5480 /* set load_ce_ram if preamble presented */ 5481 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5482 dw2 |= 0x10000000; 5483 } else { 5484 /* still load_ce_ram if this is the first time preamble presented 5485 * although there is no context switch happens. 5486 */ 5487 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5488 dw2 |= 0x10000000; 5489 } 5490 5491 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5492 amdgpu_ring_write(ring, dw2); 5493 amdgpu_ring_write(ring, 0); 5494 } 5495 5496 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5497 { 5498 unsigned ret; 5499 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5500 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5501 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5502 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5503 ret = ring->wptr & ring->buf_mask; 5504 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5505 return ret; 5506 } 5507 5508 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5509 { 5510 unsigned cur; 5511 BUG_ON(offset > ring->buf_mask); 5512 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5513 5514 cur = (ring->wptr & ring->buf_mask) - 1; 5515 if (likely(cur > offset)) 5516 ring->ring[offset] = cur - offset; 5517 else 5518 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5519 } 5520 5521 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5522 uint32_t reg_val_offs) 5523 { 5524 struct amdgpu_device *adev = ring->adev; 5525 5526 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5527 amdgpu_ring_write(ring, 0 | /* src: register*/ 5528 (5 << 8) | /* dst: memory */ 5529 (1 << 20)); /* write confirm */ 5530 amdgpu_ring_write(ring, reg); 5531 amdgpu_ring_write(ring, 0); 5532 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5533 reg_val_offs * 4)); 5534 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5535 reg_val_offs * 4)); 5536 } 5537 5538 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5539 uint32_t val) 5540 { 5541 uint32_t cmd = 0; 5542 5543 switch (ring->funcs->type) { 5544 case AMDGPU_RING_TYPE_GFX: 5545 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5546 break; 5547 case AMDGPU_RING_TYPE_KIQ: 5548 cmd = (1 << 16); /* no inc addr */ 5549 break; 5550 default: 5551 cmd = WR_CONFIRM; 5552 break; 5553 } 5554 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5555 amdgpu_ring_write(ring, cmd); 5556 amdgpu_ring_write(ring, reg); 5557 amdgpu_ring_write(ring, 0); 5558 amdgpu_ring_write(ring, val); 5559 } 5560 5561 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5562 uint32_t val, uint32_t mask) 5563 { 5564 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5565 } 5566 5567 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5568 uint32_t reg0, uint32_t reg1, 5569 uint32_t ref, uint32_t mask) 5570 { 5571 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5572 struct amdgpu_device *adev = ring->adev; 5573 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5574 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5575 5576 if (fw_version_ok) 5577 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5578 ref, mask, 0x20); 5579 else 5580 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5581 ref, mask); 5582 } 5583 5584 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5585 { 5586 struct amdgpu_device *adev = ring->adev; 5587 uint32_t value = 0; 5588 5589 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5590 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5591 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5592 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5593 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5594 } 5595 5596 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5597 enum amdgpu_interrupt_state state) 5598 { 5599 switch (state) { 5600 case AMDGPU_IRQ_STATE_DISABLE: 5601 case AMDGPU_IRQ_STATE_ENABLE: 5602 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5603 TIME_STAMP_INT_ENABLE, 5604 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5605 break; 5606 default: 5607 break; 5608 } 5609 } 5610 5611 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5612 int me, int pipe, 5613 enum amdgpu_interrupt_state state) 5614 { 5615 u32 mec_int_cntl, mec_int_cntl_reg; 5616 5617 /* 5618 * amdgpu controls only the first MEC. That's why this function only 5619 * handles the setting of interrupts for this specific MEC. All other 5620 * pipes' interrupts are set by amdkfd. 5621 */ 5622 5623 if (me == 1) { 5624 switch (pipe) { 5625 case 0: 5626 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5627 break; 5628 case 1: 5629 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5630 break; 5631 case 2: 5632 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5633 break; 5634 case 3: 5635 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5636 break; 5637 default: 5638 DRM_DEBUG("invalid pipe %d\n", pipe); 5639 return; 5640 } 5641 } else { 5642 DRM_DEBUG("invalid me %d\n", me); 5643 return; 5644 } 5645 5646 switch (state) { 5647 case AMDGPU_IRQ_STATE_DISABLE: 5648 mec_int_cntl = RREG32(mec_int_cntl_reg); 5649 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5650 TIME_STAMP_INT_ENABLE, 0); 5651 WREG32(mec_int_cntl_reg, mec_int_cntl); 5652 break; 5653 case AMDGPU_IRQ_STATE_ENABLE: 5654 mec_int_cntl = RREG32(mec_int_cntl_reg); 5655 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5656 TIME_STAMP_INT_ENABLE, 1); 5657 WREG32(mec_int_cntl_reg, mec_int_cntl); 5658 break; 5659 default: 5660 break; 5661 } 5662 } 5663 5664 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5665 struct amdgpu_irq_src *source, 5666 unsigned type, 5667 enum amdgpu_interrupt_state state) 5668 { 5669 switch (state) { 5670 case AMDGPU_IRQ_STATE_DISABLE: 5671 case AMDGPU_IRQ_STATE_ENABLE: 5672 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5673 PRIV_REG_INT_ENABLE, 5674 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5675 break; 5676 default: 5677 break; 5678 } 5679 5680 return 0; 5681 } 5682 5683 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5684 struct amdgpu_irq_src *source, 5685 unsigned type, 5686 enum amdgpu_interrupt_state state) 5687 { 5688 switch (state) { 5689 case AMDGPU_IRQ_STATE_DISABLE: 5690 case AMDGPU_IRQ_STATE_ENABLE: 5691 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5692 PRIV_INSTR_INT_ENABLE, 5693 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5694 break; 5695 default: 5696 break; 5697 } 5698 5699 return 0; 5700 } 5701 5702 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5703 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5704 CP_ECC_ERROR_INT_ENABLE, 1) 5705 5706 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5707 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5708 CP_ECC_ERROR_INT_ENABLE, 0) 5709 5710 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5711 struct amdgpu_irq_src *source, 5712 unsigned type, 5713 enum amdgpu_interrupt_state state) 5714 { 5715 switch (state) { 5716 case AMDGPU_IRQ_STATE_DISABLE: 5717 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5718 CP_ECC_ERROR_INT_ENABLE, 0); 5719 DISABLE_ECC_ON_ME_PIPE(1, 0); 5720 DISABLE_ECC_ON_ME_PIPE(1, 1); 5721 DISABLE_ECC_ON_ME_PIPE(1, 2); 5722 DISABLE_ECC_ON_ME_PIPE(1, 3); 5723 break; 5724 5725 case AMDGPU_IRQ_STATE_ENABLE: 5726 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5727 CP_ECC_ERROR_INT_ENABLE, 1); 5728 ENABLE_ECC_ON_ME_PIPE(1, 0); 5729 ENABLE_ECC_ON_ME_PIPE(1, 1); 5730 ENABLE_ECC_ON_ME_PIPE(1, 2); 5731 ENABLE_ECC_ON_ME_PIPE(1, 3); 5732 break; 5733 default: 5734 break; 5735 } 5736 5737 return 0; 5738 } 5739 5740 5741 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5742 struct amdgpu_irq_src *src, 5743 unsigned type, 5744 enum amdgpu_interrupt_state state) 5745 { 5746 switch (type) { 5747 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5748 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5749 break; 5750 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5751 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5752 break; 5753 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5754 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5755 break; 5756 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5757 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5758 break; 5759 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5760 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5761 break; 5762 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5763 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5764 break; 5765 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5766 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5767 break; 5768 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5769 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5770 break; 5771 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5772 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5773 break; 5774 default: 5775 break; 5776 } 5777 return 0; 5778 } 5779 5780 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5781 struct amdgpu_irq_src *source, 5782 struct amdgpu_iv_entry *entry) 5783 { 5784 int i; 5785 u8 me_id, pipe_id, queue_id; 5786 struct amdgpu_ring *ring; 5787 5788 DRM_DEBUG("IH: CP EOP\n"); 5789 me_id = (entry->ring_id & 0x0c) >> 2; 5790 pipe_id = (entry->ring_id & 0x03) >> 0; 5791 queue_id = (entry->ring_id & 0x70) >> 4; 5792 5793 switch (me_id) { 5794 case 0: 5795 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5796 break; 5797 case 1: 5798 case 2: 5799 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5800 ring = &adev->gfx.compute_ring[i]; 5801 /* Per-queue interrupt is supported for MEC starting from VI. 5802 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5803 */ 5804 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5805 amdgpu_fence_process(ring); 5806 } 5807 break; 5808 } 5809 return 0; 5810 } 5811 5812 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5813 struct amdgpu_iv_entry *entry) 5814 { 5815 u8 me_id, pipe_id, queue_id; 5816 struct amdgpu_ring *ring; 5817 int i; 5818 5819 me_id = (entry->ring_id & 0x0c) >> 2; 5820 pipe_id = (entry->ring_id & 0x03) >> 0; 5821 queue_id = (entry->ring_id & 0x70) >> 4; 5822 5823 switch (me_id) { 5824 case 0: 5825 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5826 break; 5827 case 1: 5828 case 2: 5829 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5830 ring = &adev->gfx.compute_ring[i]; 5831 if (ring->me == me_id && ring->pipe == pipe_id && 5832 ring->queue == queue_id) 5833 drm_sched_fault(&ring->sched); 5834 } 5835 break; 5836 } 5837 } 5838 5839 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5840 struct amdgpu_irq_src *source, 5841 struct amdgpu_iv_entry *entry) 5842 { 5843 DRM_ERROR("Illegal register access in command stream\n"); 5844 gfx_v9_0_fault(adev, entry); 5845 return 0; 5846 } 5847 5848 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5849 struct amdgpu_irq_src *source, 5850 struct amdgpu_iv_entry *entry) 5851 { 5852 DRM_ERROR("Illegal instruction in command stream\n"); 5853 gfx_v9_0_fault(adev, entry); 5854 return 0; 5855 } 5856 5857 5858 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5859 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5860 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5861 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5862 }, 5863 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5864 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5865 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5866 }, 5867 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5868 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5869 0, 0 5870 }, 5871 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5872 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5873 0, 0 5874 }, 5875 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5876 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5877 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5878 }, 5879 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5880 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5881 0, 0 5882 }, 5883 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5884 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5885 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5886 }, 5887 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5888 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5889 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5890 }, 5891 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5892 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5893 0, 0 5894 }, 5895 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5896 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5897 0, 0 5898 }, 5899 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5900 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5901 0, 0 5902 }, 5903 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5904 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5905 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5906 }, 5907 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5908 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5909 0, 0 5910 }, 5911 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5912 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5913 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5914 }, 5915 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5916 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5917 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5918 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5919 }, 5920 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 5921 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5922 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 5923 0, 0 5924 }, 5925 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 5926 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5927 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 5928 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 5929 }, 5930 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 5931 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5932 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 5933 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 5934 }, 5935 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 5936 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5937 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 5938 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 5939 }, 5940 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 5941 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5942 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 5943 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 5944 }, 5945 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 5946 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 5947 0, 0 5948 }, 5949 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5950 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 5951 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 5952 }, 5953 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5954 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 5955 0, 0 5956 }, 5957 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5958 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 5959 0, 0 5960 }, 5961 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5962 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 5963 0, 0 5964 }, 5965 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5966 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 5967 0, 0 5968 }, 5969 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5970 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 5971 0, 0 5972 }, 5973 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5974 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 5975 0, 0 5976 }, 5977 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5978 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 5979 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 5980 }, 5981 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5982 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 5983 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 5984 }, 5985 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5986 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 5987 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 5988 }, 5989 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5990 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 5991 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 5992 }, 5993 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5994 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 5995 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 5996 }, 5997 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5998 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 5999 0, 0 6000 }, 6001 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6002 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6003 0, 0 6004 }, 6005 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6006 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6007 0, 0 6008 }, 6009 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6010 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6011 0, 0 6012 }, 6013 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6014 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6015 0, 0 6016 }, 6017 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6018 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6019 0, 0 6020 }, 6021 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6022 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6023 0, 0 6024 }, 6025 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6026 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6027 0, 0 6028 }, 6029 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6030 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6031 0, 0 6032 }, 6033 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6034 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6035 0, 0 6036 }, 6037 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6038 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6039 0, 0 6040 }, 6041 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6042 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6043 0, 0 6044 }, 6045 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6046 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6047 0, 0 6048 }, 6049 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6050 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6051 0, 0 6052 }, 6053 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6054 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6055 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6056 }, 6057 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6058 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6059 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6060 }, 6061 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6062 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6063 0, 0 6064 }, 6065 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6066 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6067 0, 0 6068 }, 6069 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6070 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6071 0, 0 6072 }, 6073 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6074 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6075 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6076 }, 6077 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6078 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6079 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6080 }, 6081 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6082 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6083 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6084 }, 6085 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6086 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6087 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6088 }, 6089 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6090 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6091 0, 0 6092 }, 6093 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6094 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6095 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6096 }, 6097 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6098 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6099 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6100 }, 6101 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6102 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6103 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6104 }, 6105 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6106 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6107 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6108 }, 6109 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6110 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6111 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6112 }, 6113 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6114 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6115 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6116 }, 6117 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6118 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6119 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6120 }, 6121 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6122 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6123 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6124 }, 6125 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6126 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6127 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6128 }, 6129 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6130 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6131 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6132 }, 6133 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6134 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6135 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6136 }, 6137 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6138 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6139 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6140 }, 6141 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6142 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6143 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6144 }, 6145 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6146 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6147 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6148 }, 6149 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6150 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6151 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6152 }, 6153 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6154 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6155 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6156 }, 6157 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6158 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6159 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6160 }, 6161 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6162 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6163 0, 0 6164 }, 6165 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6166 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6167 0, 0 6168 }, 6169 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6170 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6171 0, 0 6172 }, 6173 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6174 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6175 0, 0 6176 }, 6177 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6178 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6179 0, 0 6180 }, 6181 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6182 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6183 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6184 }, 6185 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6186 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6187 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6188 }, 6189 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6190 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6191 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6192 }, 6193 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6194 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6195 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6196 }, 6197 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6198 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6199 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6200 }, 6201 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6202 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6203 0, 0 6204 }, 6205 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6206 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6207 0, 0 6208 }, 6209 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6210 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6211 0, 0 6212 }, 6213 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6214 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6215 0, 0 6216 }, 6217 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6218 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6219 0, 0 6220 }, 6221 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6222 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6223 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6224 }, 6225 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6226 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6227 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6228 }, 6229 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6230 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6231 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6232 }, 6233 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6234 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6235 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6236 }, 6237 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6238 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6239 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6240 }, 6241 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6242 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6243 0, 0 6244 }, 6245 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6246 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6247 0, 0 6248 }, 6249 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6250 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6251 0, 0 6252 }, 6253 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6254 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6255 0, 0 6256 }, 6257 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6258 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6259 0, 0 6260 }, 6261 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6262 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6263 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6264 }, 6265 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6266 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6267 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6268 }, 6269 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6270 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6271 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6272 }, 6273 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6274 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6275 0, 0 6276 }, 6277 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6278 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6279 0, 0 6280 }, 6281 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6282 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6283 0, 0 6284 }, 6285 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6286 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6287 0, 0 6288 }, 6289 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6290 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6291 0, 0 6292 }, 6293 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6294 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6295 0, 0 6296 } 6297 }; 6298 6299 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6300 void *inject_if) 6301 { 6302 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6303 int ret; 6304 struct ta_ras_trigger_error_input block_info = { 0 }; 6305 6306 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6307 return -EINVAL; 6308 6309 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6310 return -EINVAL; 6311 6312 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6313 return -EPERM; 6314 6315 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6316 info->head.type)) { 6317 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6318 ras_gfx_subblocks[info->head.sub_block_index].name, 6319 info->head.type); 6320 return -EPERM; 6321 } 6322 6323 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6324 info->head.type)) { 6325 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6326 ras_gfx_subblocks[info->head.sub_block_index].name, 6327 info->head.type); 6328 return -EPERM; 6329 } 6330 6331 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6332 block_info.sub_block_index = 6333 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6334 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6335 block_info.address = info->address; 6336 block_info.value = info->value; 6337 6338 mutex_lock(&adev->grbm_idx_mutex); 6339 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6340 mutex_unlock(&adev->grbm_idx_mutex); 6341 6342 return ret; 6343 } 6344 6345 static const char *vml2_mems[] = { 6346 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6347 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6348 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6349 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6350 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6351 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6352 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6353 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6354 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6355 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6356 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6357 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6358 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6359 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6360 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6361 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6362 }; 6363 6364 static const char *vml2_walker_mems[] = { 6365 "UTC_VML2_CACHE_PDE0_MEM0", 6366 "UTC_VML2_CACHE_PDE0_MEM1", 6367 "UTC_VML2_CACHE_PDE1_MEM0", 6368 "UTC_VML2_CACHE_PDE1_MEM1", 6369 "UTC_VML2_CACHE_PDE2_MEM0", 6370 "UTC_VML2_CACHE_PDE2_MEM1", 6371 "UTC_VML2_RDIF_LOG_FIFO", 6372 }; 6373 6374 static const char *atc_l2_cache_2m_mems[] = { 6375 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6376 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6377 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6378 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6379 }; 6380 6381 static const char *atc_l2_cache_4k_mems[] = { 6382 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6383 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6384 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6385 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6386 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6387 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6388 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6389 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6390 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6391 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6392 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6393 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6394 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6395 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6396 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6397 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6398 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6399 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6400 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6401 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6402 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6403 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6404 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6405 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6406 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6407 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6408 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6409 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6410 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6411 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6412 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6413 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6414 }; 6415 6416 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6417 struct ras_err_data *err_data) 6418 { 6419 uint32_t i, data; 6420 uint32_t sec_count, ded_count; 6421 6422 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6423 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6424 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6425 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6426 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6427 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6428 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6429 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6430 6431 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6432 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6433 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6434 6435 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6436 if (sec_count) { 6437 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6438 "SEC %d\n", i, vml2_mems[i], sec_count); 6439 err_data->ce_count += sec_count; 6440 } 6441 6442 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6443 if (ded_count) { 6444 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6445 "DED %d\n", i, vml2_mems[i], ded_count); 6446 err_data->ue_count += ded_count; 6447 } 6448 } 6449 6450 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6451 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6452 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6453 6454 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6455 SEC_COUNT); 6456 if (sec_count) { 6457 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6458 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6459 err_data->ce_count += sec_count; 6460 } 6461 6462 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6463 DED_COUNT); 6464 if (ded_count) { 6465 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6466 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6467 err_data->ue_count += ded_count; 6468 } 6469 } 6470 6471 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6472 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6473 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6474 6475 sec_count = (data & 0x00006000L) >> 0xd; 6476 if (sec_count) { 6477 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6478 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6479 sec_count); 6480 err_data->ce_count += sec_count; 6481 } 6482 } 6483 6484 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6485 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6486 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6487 6488 sec_count = (data & 0x00006000L) >> 0xd; 6489 if (sec_count) { 6490 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6491 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6492 sec_count); 6493 err_data->ce_count += sec_count; 6494 } 6495 6496 ded_count = (data & 0x00018000L) >> 0xf; 6497 if (ded_count) { 6498 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6499 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6500 ded_count); 6501 err_data->ue_count += ded_count; 6502 } 6503 } 6504 6505 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6506 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6507 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6508 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6509 6510 return 0; 6511 } 6512 6513 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6514 const struct soc15_reg_entry *reg, 6515 uint32_t se_id, uint32_t inst_id, uint32_t value, 6516 uint32_t *sec_count, uint32_t *ded_count) 6517 { 6518 uint32_t i; 6519 uint32_t sec_cnt, ded_cnt; 6520 6521 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6522 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6523 gfx_v9_0_ras_fields[i].seg != reg->seg || 6524 gfx_v9_0_ras_fields[i].inst != reg->inst) 6525 continue; 6526 6527 sec_cnt = (value & 6528 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6529 gfx_v9_0_ras_fields[i].sec_count_shift; 6530 if (sec_cnt) { 6531 dev_info(adev->dev, "GFX SubBlock %s, " 6532 "Instance[%d][%d], SEC %d\n", 6533 gfx_v9_0_ras_fields[i].name, 6534 se_id, inst_id, 6535 sec_cnt); 6536 *sec_count += sec_cnt; 6537 } 6538 6539 ded_cnt = (value & 6540 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6541 gfx_v9_0_ras_fields[i].ded_count_shift; 6542 if (ded_cnt) { 6543 dev_info(adev->dev, "GFX SubBlock %s, " 6544 "Instance[%d][%d], DED %d\n", 6545 gfx_v9_0_ras_fields[i].name, 6546 se_id, inst_id, 6547 ded_cnt); 6548 *ded_count += ded_cnt; 6549 } 6550 } 6551 6552 return 0; 6553 } 6554 6555 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6556 { 6557 int i, j, k; 6558 6559 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6560 return; 6561 6562 /* read back registers to clear the counters */ 6563 mutex_lock(&adev->grbm_idx_mutex); 6564 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6565 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6566 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6567 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6568 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6569 } 6570 } 6571 } 6572 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6573 mutex_unlock(&adev->grbm_idx_mutex); 6574 6575 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6576 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6577 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6578 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6579 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6580 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6581 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6582 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6583 6584 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6585 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6586 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6587 } 6588 6589 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6590 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6591 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6592 } 6593 6594 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6595 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6596 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6597 } 6598 6599 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6600 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6601 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6602 } 6603 6604 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6605 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6606 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6607 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6608 } 6609 6610 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6611 void *ras_error_status) 6612 { 6613 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6614 uint32_t sec_count = 0, ded_count = 0; 6615 uint32_t i, j, k; 6616 uint32_t reg_value; 6617 6618 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6619 return -EINVAL; 6620 6621 err_data->ue_count = 0; 6622 err_data->ce_count = 0; 6623 6624 mutex_lock(&adev->grbm_idx_mutex); 6625 6626 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6627 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6628 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6629 gfx_v9_0_select_se_sh(adev, j, 0, k); 6630 reg_value = 6631 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6632 if (reg_value) 6633 gfx_v9_0_ras_error_count(adev, 6634 &gfx_v9_0_edc_counter_regs[i], 6635 j, k, reg_value, 6636 &sec_count, &ded_count); 6637 } 6638 } 6639 } 6640 6641 err_data->ce_count += sec_count; 6642 err_data->ue_count += ded_count; 6643 6644 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6645 mutex_unlock(&adev->grbm_idx_mutex); 6646 6647 gfx_v9_0_query_utc_edc_status(adev, err_data); 6648 6649 return 0; 6650 } 6651 6652 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6653 { 6654 const unsigned int cp_coher_cntl = 6655 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6656 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6657 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6658 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6659 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6660 6661 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6662 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6663 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6664 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6665 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6666 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6667 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6668 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6669 } 6670 6671 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6672 uint32_t pipe, bool enable) 6673 { 6674 struct amdgpu_device *adev = ring->adev; 6675 uint32_t val; 6676 uint32_t wcl_cs_reg; 6677 6678 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6679 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 6680 6681 switch (pipe) { 6682 case 0: 6683 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 6684 break; 6685 case 1: 6686 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 6687 break; 6688 case 2: 6689 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 6690 break; 6691 case 3: 6692 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 6693 break; 6694 default: 6695 DRM_DEBUG("invalid pipe %d\n", pipe); 6696 return; 6697 } 6698 6699 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6700 6701 } 6702 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6703 { 6704 struct amdgpu_device *adev = ring->adev; 6705 uint32_t val; 6706 int i; 6707 6708 6709 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6710 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6711 * around 25% of gpu resources. 6712 */ 6713 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6714 amdgpu_ring_emit_wreg(ring, 6715 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 6716 val); 6717 6718 /* Restrict waves for normal/low priority compute queues as well 6719 * to get best QoS for high priority compute jobs. 6720 * 6721 * amdgpu controls only 1st ME(0-3 CS pipes). 6722 */ 6723 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6724 if (i != ring->pipe) 6725 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 6726 6727 } 6728 } 6729 6730 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6731 .name = "gfx_v9_0", 6732 .early_init = gfx_v9_0_early_init, 6733 .late_init = gfx_v9_0_late_init, 6734 .sw_init = gfx_v9_0_sw_init, 6735 .sw_fini = gfx_v9_0_sw_fini, 6736 .hw_init = gfx_v9_0_hw_init, 6737 .hw_fini = gfx_v9_0_hw_fini, 6738 .suspend = gfx_v9_0_suspend, 6739 .resume = gfx_v9_0_resume, 6740 .is_idle = gfx_v9_0_is_idle, 6741 .wait_for_idle = gfx_v9_0_wait_for_idle, 6742 .soft_reset = gfx_v9_0_soft_reset, 6743 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6744 .set_powergating_state = gfx_v9_0_set_powergating_state, 6745 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6746 }; 6747 6748 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6749 .type = AMDGPU_RING_TYPE_GFX, 6750 .align_mask = 0xff, 6751 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6752 .support_64bit_ptrs = true, 6753 .vmhub = AMDGPU_GFXHUB_0, 6754 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6755 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6756 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6757 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6758 5 + /* COND_EXEC */ 6759 7 + /* PIPELINE_SYNC */ 6760 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6761 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6762 2 + /* VM_FLUSH */ 6763 8 + /* FENCE for VM_FLUSH */ 6764 20 + /* GDS switch */ 6765 4 + /* double SWITCH_BUFFER, 6766 the first COND_EXEC jump to the place just 6767 prior to this double SWITCH_BUFFER */ 6768 5 + /* COND_EXEC */ 6769 7 + /* HDP_flush */ 6770 4 + /* VGT_flush */ 6771 14 + /* CE_META */ 6772 31 + /* DE_META */ 6773 3 + /* CNTX_CTRL */ 6774 5 + /* HDP_INVL */ 6775 8 + 8 + /* FENCE x2 */ 6776 2 + /* SWITCH_BUFFER */ 6777 7, /* gfx_v9_0_emit_mem_sync */ 6778 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6779 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6780 .emit_fence = gfx_v9_0_ring_emit_fence, 6781 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6782 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6783 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6784 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6785 .test_ring = gfx_v9_0_ring_test_ring, 6786 .test_ib = gfx_v9_0_ring_test_ib, 6787 .insert_nop = amdgpu_ring_insert_nop, 6788 .pad_ib = amdgpu_ring_generic_pad_ib, 6789 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6790 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6791 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6792 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6793 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6794 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6795 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6796 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6797 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6798 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6799 }; 6800 6801 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6802 .type = AMDGPU_RING_TYPE_COMPUTE, 6803 .align_mask = 0xff, 6804 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6805 .support_64bit_ptrs = true, 6806 .vmhub = AMDGPU_GFXHUB_0, 6807 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6808 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6809 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6810 .emit_frame_size = 6811 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6812 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6813 5 + /* hdp invalidate */ 6814 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6815 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6816 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6817 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6818 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6819 7 + /* gfx_v9_0_emit_mem_sync */ 6820 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6821 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6822 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6823 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6824 .emit_fence = gfx_v9_0_ring_emit_fence, 6825 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6826 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6827 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6828 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6829 .test_ring = gfx_v9_0_ring_test_ring, 6830 .test_ib = gfx_v9_0_ring_test_ib, 6831 .insert_nop = amdgpu_ring_insert_nop, 6832 .pad_ib = amdgpu_ring_generic_pad_ib, 6833 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6834 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6835 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6836 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6837 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 6838 }; 6839 6840 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6841 .type = AMDGPU_RING_TYPE_KIQ, 6842 .align_mask = 0xff, 6843 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6844 .support_64bit_ptrs = true, 6845 .vmhub = AMDGPU_GFXHUB_0, 6846 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6847 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6848 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6849 .emit_frame_size = 6850 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6851 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6852 5 + /* hdp invalidate */ 6853 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6854 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6855 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6856 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6857 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6858 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6859 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6860 .test_ring = gfx_v9_0_ring_test_ring, 6861 .insert_nop = amdgpu_ring_insert_nop, 6862 .pad_ib = amdgpu_ring_generic_pad_ib, 6863 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6864 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6865 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6866 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6867 }; 6868 6869 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6870 { 6871 int i; 6872 6873 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6874 6875 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6876 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6877 6878 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6879 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6880 } 6881 6882 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6883 .set = gfx_v9_0_set_eop_interrupt_state, 6884 .process = gfx_v9_0_eop_irq, 6885 }; 6886 6887 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6888 .set = gfx_v9_0_set_priv_reg_fault_state, 6889 .process = gfx_v9_0_priv_reg_irq, 6890 }; 6891 6892 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6893 .set = gfx_v9_0_set_priv_inst_fault_state, 6894 .process = gfx_v9_0_priv_inst_irq, 6895 }; 6896 6897 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6898 .set = gfx_v9_0_set_cp_ecc_error_state, 6899 .process = amdgpu_gfx_cp_ecc_error_irq, 6900 }; 6901 6902 6903 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6904 { 6905 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6906 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6907 6908 adev->gfx.priv_reg_irq.num_types = 1; 6909 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6910 6911 adev->gfx.priv_inst_irq.num_types = 1; 6912 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6913 6914 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6915 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6916 } 6917 6918 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6919 { 6920 switch (adev->asic_type) { 6921 case CHIP_VEGA10: 6922 case CHIP_VEGA12: 6923 case CHIP_VEGA20: 6924 case CHIP_RAVEN: 6925 case CHIP_ARCTURUS: 6926 case CHIP_RENOIR: 6927 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 6928 break; 6929 default: 6930 break; 6931 } 6932 } 6933 6934 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 6935 { 6936 /* init asci gds info */ 6937 switch (adev->asic_type) { 6938 case CHIP_VEGA10: 6939 case CHIP_VEGA12: 6940 case CHIP_VEGA20: 6941 adev->gds.gds_size = 0x10000; 6942 break; 6943 case CHIP_RAVEN: 6944 case CHIP_ARCTURUS: 6945 adev->gds.gds_size = 0x1000; 6946 break; 6947 default: 6948 adev->gds.gds_size = 0x10000; 6949 break; 6950 } 6951 6952 switch (adev->asic_type) { 6953 case CHIP_VEGA10: 6954 case CHIP_VEGA20: 6955 adev->gds.gds_compute_max_wave_id = 0x7ff; 6956 break; 6957 case CHIP_VEGA12: 6958 adev->gds.gds_compute_max_wave_id = 0x27f; 6959 break; 6960 case CHIP_RAVEN: 6961 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 6962 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 6963 else 6964 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 6965 break; 6966 case CHIP_ARCTURUS: 6967 adev->gds.gds_compute_max_wave_id = 0xfff; 6968 break; 6969 default: 6970 /* this really depends on the chip */ 6971 adev->gds.gds_compute_max_wave_id = 0x7ff; 6972 break; 6973 } 6974 6975 adev->gds.gws_size = 64; 6976 adev->gds.oa_size = 16; 6977 } 6978 6979 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6980 u32 bitmap) 6981 { 6982 u32 data; 6983 6984 if (!bitmap) 6985 return; 6986 6987 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6988 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6989 6990 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 6991 } 6992 6993 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6994 { 6995 u32 data, mask; 6996 6997 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 6998 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 6999 7000 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7001 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7002 7003 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7004 7005 return (~data) & mask; 7006 } 7007 7008 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7009 struct amdgpu_cu_info *cu_info) 7010 { 7011 int i, j, k, counter, active_cu_number = 0; 7012 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7013 unsigned disable_masks[4 * 4]; 7014 7015 if (!adev || !cu_info) 7016 return -EINVAL; 7017 7018 /* 7019 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7020 */ 7021 if (adev->gfx.config.max_shader_engines * 7022 adev->gfx.config.max_sh_per_se > 16) 7023 return -EINVAL; 7024 7025 amdgpu_gfx_parse_disable_cu(disable_masks, 7026 adev->gfx.config.max_shader_engines, 7027 adev->gfx.config.max_sh_per_se); 7028 7029 mutex_lock(&adev->grbm_idx_mutex); 7030 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7031 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7032 mask = 1; 7033 ao_bitmap = 0; 7034 counter = 0; 7035 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 7036 gfx_v9_0_set_user_cu_inactive_bitmap( 7037 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7038 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7039 7040 /* 7041 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7042 * 4x4 size array, and it's usually suitable for Vega 7043 * ASICs which has 4*2 SE/SH layout. 7044 * But for Arcturus, SE/SH layout is changed to 8*1. 7045 * To mostly reduce the impact, we make it compatible 7046 * with current bitmap array as below: 7047 * SE4,SH0 --> bitmap[0][1] 7048 * SE5,SH0 --> bitmap[1][1] 7049 * SE6,SH0 --> bitmap[2][1] 7050 * SE7,SH0 --> bitmap[3][1] 7051 */ 7052 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7053 7054 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7055 if (bitmap & mask) { 7056 if (counter < adev->gfx.config.max_cu_per_sh) 7057 ao_bitmap |= mask; 7058 counter ++; 7059 } 7060 mask <<= 1; 7061 } 7062 active_cu_number += counter; 7063 if (i < 2 && j < 2) 7064 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7065 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7066 } 7067 } 7068 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7069 mutex_unlock(&adev->grbm_idx_mutex); 7070 7071 cu_info->number = active_cu_number; 7072 cu_info->ao_cu_mask = ao_cu_mask; 7073 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7074 7075 return 0; 7076 } 7077 7078 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7079 { 7080 .type = AMD_IP_BLOCK_TYPE_GFX, 7081 .major = 9, 7082 .minor = 0, 7083 .rev = 0, 7084 .funcs = &gfx_v9_0_ip_funcs, 7085 }; 7086