1 /* 2 * Copyright 2012 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include "drmP.h" 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include "radeon_audio.h" 31 #include "cikd.h" 32 #include "atom.h" 33 #include "cik_blit_shaders.h" 34 #include "radeon_ucode.h" 35 #include "clearstate_ci.h" 36 #include "radeon_kfd.h" 37 38 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 39 MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 40 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); 41 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); 42 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); 43 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin"); 44 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); 45 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); 46 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); 47 48 MODULE_FIRMWARE("radeon/bonaire_pfp.bin"); 49 MODULE_FIRMWARE("radeon/bonaire_me.bin"); 50 MODULE_FIRMWARE("radeon/bonaire_ce.bin"); 51 MODULE_FIRMWARE("radeon/bonaire_mec.bin"); 52 MODULE_FIRMWARE("radeon/bonaire_mc.bin"); 53 MODULE_FIRMWARE("radeon/bonaire_rlc.bin"); 54 MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); 55 MODULE_FIRMWARE("radeon/bonaire_smc.bin"); 56 57 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); 58 MODULE_FIRMWARE("radeon/HAWAII_me.bin"); 59 MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); 60 MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); 61 MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); 62 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin"); 63 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); 64 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); 65 MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); 66 67 MODULE_FIRMWARE("radeon/hawaii_pfp.bin"); 68 MODULE_FIRMWARE("radeon/hawaii_me.bin"); 69 MODULE_FIRMWARE("radeon/hawaii_ce.bin"); 70 MODULE_FIRMWARE("radeon/hawaii_mec.bin"); 71 MODULE_FIRMWARE("radeon/hawaii_mc.bin"); 72 MODULE_FIRMWARE("radeon/hawaii_rlc.bin"); 73 MODULE_FIRMWARE("radeon/hawaii_sdma.bin"); 74 MODULE_FIRMWARE("radeon/hawaii_smc.bin"); 75 76 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); 77 MODULE_FIRMWARE("radeon/KAVERI_me.bin"); 78 MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); 79 MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); 80 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); 81 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); 82 83 MODULE_FIRMWARE("radeon/kaveri_pfp.bin"); 84 MODULE_FIRMWARE("radeon/kaveri_me.bin"); 85 MODULE_FIRMWARE("radeon/kaveri_ce.bin"); 86 MODULE_FIRMWARE("radeon/kaveri_mec.bin"); 87 MODULE_FIRMWARE("radeon/kaveri_mec2.bin"); 88 MODULE_FIRMWARE("radeon/kaveri_rlc.bin"); 89 MODULE_FIRMWARE("radeon/kaveri_sdma.bin"); 90 91 MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); 92 MODULE_FIRMWARE("radeon/KABINI_me.bin"); 93 MODULE_FIRMWARE("radeon/KABINI_ce.bin"); 94 MODULE_FIRMWARE("radeon/KABINI_mec.bin"); 95 MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); 96 MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); 97 98 MODULE_FIRMWARE("radeon/kabini_pfp.bin"); 99 MODULE_FIRMWARE("radeon/kabini_me.bin"); 100 MODULE_FIRMWARE("radeon/kabini_ce.bin"); 101 MODULE_FIRMWARE("radeon/kabini_mec.bin"); 102 MODULE_FIRMWARE("radeon/kabini_rlc.bin"); 103 MODULE_FIRMWARE("radeon/kabini_sdma.bin"); 104 105 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin"); 106 MODULE_FIRMWARE("radeon/MULLINS_me.bin"); 107 MODULE_FIRMWARE("radeon/MULLINS_ce.bin"); 108 MODULE_FIRMWARE("radeon/MULLINS_mec.bin"); 109 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin"); 110 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin"); 111 112 MODULE_FIRMWARE("radeon/mullins_pfp.bin"); 113 MODULE_FIRMWARE("radeon/mullins_me.bin"); 114 MODULE_FIRMWARE("radeon/mullins_ce.bin"); 115 MODULE_FIRMWARE("radeon/mullins_mec.bin"); 116 MODULE_FIRMWARE("radeon/mullins_rlc.bin"); 117 MODULE_FIRMWARE("radeon/mullins_sdma.bin"); 118 119 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 120 extern void r600_ih_ring_fini(struct radeon_device *rdev); 121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 123 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 124 extern void sumo_rlc_fini(struct radeon_device *rdev); 125 extern int sumo_rlc_init(struct radeon_device *rdev); 126 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); 127 extern void si_rlc_reset(struct radeon_device *rdev); 128 extern void si_init_uvd_internal_cg(struct radeon_device *rdev); 129 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); 130 extern int cik_sdma_resume(struct radeon_device *rdev); 131 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable); 132 extern void cik_sdma_fini(struct radeon_device *rdev); 133 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable); 134 static void cik_rlc_stop(struct radeon_device *rdev); 135 static void cik_pcie_gen3_enable(struct radeon_device *rdev); 136 static void cik_program_aspm(struct radeon_device *rdev); 137 static void cik_init_pg(struct radeon_device *rdev); 138 static void cik_init_cg(struct radeon_device *rdev); 139 static void cik_fini_pg(struct radeon_device *rdev); 140 static void cik_fini_cg(struct radeon_device *rdev); 141 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 142 bool enable); 143 144 /* get temperature in millidegrees */ 145 int ci_get_temp(struct radeon_device *rdev) 146 { 147 u32 temp; 148 int actual_temp = 0; 149 150 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 151 CTF_TEMP_SHIFT; 152 153 if (temp & 0x200) 154 actual_temp = 255; 155 else 156 actual_temp = temp & 0x1ff; 157 158 actual_temp = actual_temp * 1000; 159 160 return actual_temp; 161 } 162 163 /* get temperature in millidegrees */ 164 int kv_get_temp(struct radeon_device *rdev) 165 { 166 u32 temp; 167 int actual_temp = 0; 168 169 temp = RREG32_SMC(0xC0300E0C); 170 171 if (temp) 172 actual_temp = (temp / 8) - 49; 173 else 174 actual_temp = 0; 175 176 actual_temp = actual_temp * 1000; 177 178 return actual_temp; 179 } 180 181 /* 182 * Indirect registers accessor 183 */ 184 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) 185 { 186 unsigned long flags; 187 u32 r; 188 189 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 190 WREG32(PCIE_INDEX, reg); 191 (void)RREG32(PCIE_INDEX); 192 r = RREG32(PCIE_DATA); 193 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 194 return r; 195 } 196 197 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) 198 { 199 unsigned long flags; 200 201 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 202 WREG32(PCIE_INDEX, reg); 203 (void)RREG32(PCIE_INDEX); 204 WREG32(PCIE_DATA, v); 205 (void)RREG32(PCIE_DATA); 206 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 207 } 208 209 static const u32 spectre_rlc_save_restore_register_list[] = 210 { 211 (0x0e00 << 16) | (0xc12c >> 2), 212 0x00000000, 213 (0x0e00 << 16) | (0xc140 >> 2), 214 0x00000000, 215 (0x0e00 << 16) | (0xc150 >> 2), 216 0x00000000, 217 (0x0e00 << 16) | (0xc15c >> 2), 218 0x00000000, 219 (0x0e00 << 16) | (0xc168 >> 2), 220 0x00000000, 221 (0x0e00 << 16) | (0xc170 >> 2), 222 0x00000000, 223 (0x0e00 << 16) | (0xc178 >> 2), 224 0x00000000, 225 (0x0e00 << 16) | (0xc204 >> 2), 226 0x00000000, 227 (0x0e00 << 16) | (0xc2b4 >> 2), 228 0x00000000, 229 (0x0e00 << 16) | (0xc2b8 >> 2), 230 0x00000000, 231 (0x0e00 << 16) | (0xc2bc >> 2), 232 0x00000000, 233 (0x0e00 << 16) | (0xc2c0 >> 2), 234 0x00000000, 235 (0x0e00 << 16) | (0x8228 >> 2), 236 0x00000000, 237 (0x0e00 << 16) | (0x829c >> 2), 238 0x00000000, 239 (0x0e00 << 16) | (0x869c >> 2), 240 0x00000000, 241 (0x0600 << 16) | (0x98f4 >> 2), 242 0x00000000, 243 (0x0e00 << 16) | (0x98f8 >> 2), 244 0x00000000, 245 (0x0e00 << 16) | (0x9900 >> 2), 246 0x00000000, 247 (0x0e00 << 16) | (0xc260 >> 2), 248 0x00000000, 249 (0x0e00 << 16) | (0x90e8 >> 2), 250 0x00000000, 251 (0x0e00 << 16) | (0x3c000 >> 2), 252 0x00000000, 253 (0x0e00 << 16) | (0x3c00c >> 2), 254 0x00000000, 255 (0x0e00 << 16) | (0x8c1c >> 2), 256 0x00000000, 257 (0x0e00 << 16) | (0x9700 >> 2), 258 0x00000000, 259 (0x0e00 << 16) | (0xcd20 >> 2), 260 0x00000000, 261 (0x4e00 << 16) | (0xcd20 >> 2), 262 0x00000000, 263 (0x5e00 << 16) | (0xcd20 >> 2), 264 0x00000000, 265 (0x6e00 << 16) | (0xcd20 >> 2), 266 0x00000000, 267 (0x7e00 << 16) | (0xcd20 >> 2), 268 0x00000000, 269 (0x8e00 << 16) | (0xcd20 >> 2), 270 0x00000000, 271 (0x9e00 << 16) | (0xcd20 >> 2), 272 0x00000000, 273 (0xae00 << 16) | (0xcd20 >> 2), 274 0x00000000, 275 (0xbe00 << 16) | (0xcd20 >> 2), 276 0x00000000, 277 (0x0e00 << 16) | (0x89bc >> 2), 278 0x00000000, 279 (0x0e00 << 16) | (0x8900 >> 2), 280 0x00000000, 281 0x3, 282 (0x0e00 << 16) | (0xc130 >> 2), 283 0x00000000, 284 (0x0e00 << 16) | (0xc134 >> 2), 285 0x00000000, 286 (0x0e00 << 16) | (0xc1fc >> 2), 287 0x00000000, 288 (0x0e00 << 16) | (0xc208 >> 2), 289 0x00000000, 290 (0x0e00 << 16) | (0xc264 >> 2), 291 0x00000000, 292 (0x0e00 << 16) | (0xc268 >> 2), 293 0x00000000, 294 (0x0e00 << 16) | (0xc26c >> 2), 295 0x00000000, 296 (0x0e00 << 16) | (0xc270 >> 2), 297 0x00000000, 298 (0x0e00 << 16) | (0xc274 >> 2), 299 0x00000000, 300 (0x0e00 << 16) | (0xc278 >> 2), 301 0x00000000, 302 (0x0e00 << 16) | (0xc27c >> 2), 303 0x00000000, 304 (0x0e00 << 16) | (0xc280 >> 2), 305 0x00000000, 306 (0x0e00 << 16) | (0xc284 >> 2), 307 0x00000000, 308 (0x0e00 << 16) | (0xc288 >> 2), 309 0x00000000, 310 (0x0e00 << 16) | (0xc28c >> 2), 311 0x00000000, 312 (0x0e00 << 16) | (0xc290 >> 2), 313 0x00000000, 314 (0x0e00 << 16) | (0xc294 >> 2), 315 0x00000000, 316 (0x0e00 << 16) | (0xc298 >> 2), 317 0x00000000, 318 (0x0e00 << 16) | (0xc29c >> 2), 319 0x00000000, 320 (0x0e00 << 16) | (0xc2a0 >> 2), 321 0x00000000, 322 (0x0e00 << 16) | (0xc2a4 >> 2), 323 0x00000000, 324 (0x0e00 << 16) | (0xc2a8 >> 2), 325 0x00000000, 326 (0x0e00 << 16) | (0xc2ac >> 2), 327 0x00000000, 328 (0x0e00 << 16) | (0xc2b0 >> 2), 329 0x00000000, 330 (0x0e00 << 16) | (0x301d0 >> 2), 331 0x00000000, 332 (0x0e00 << 16) | (0x30238 >> 2), 333 0x00000000, 334 (0x0e00 << 16) | (0x30250 >> 2), 335 0x00000000, 336 (0x0e00 << 16) | (0x30254 >> 2), 337 0x00000000, 338 (0x0e00 << 16) | (0x30258 >> 2), 339 0x00000000, 340 (0x0e00 << 16) | (0x3025c >> 2), 341 0x00000000, 342 (0x4e00 << 16) | (0xc900 >> 2), 343 0x00000000, 344 (0x5e00 << 16) | (0xc900 >> 2), 345 0x00000000, 346 (0x6e00 << 16) | (0xc900 >> 2), 347 0x00000000, 348 (0x7e00 << 16) | (0xc900 >> 2), 349 0x00000000, 350 (0x8e00 << 16) | (0xc900 >> 2), 351 0x00000000, 352 (0x9e00 << 16) | (0xc900 >> 2), 353 0x00000000, 354 (0xae00 << 16) | (0xc900 >> 2), 355 0x00000000, 356 (0xbe00 << 16) | (0xc900 >> 2), 357 0x00000000, 358 (0x4e00 << 16) | (0xc904 >> 2), 359 0x00000000, 360 (0x5e00 << 16) | (0xc904 >> 2), 361 0x00000000, 362 (0x6e00 << 16) | (0xc904 >> 2), 363 0x00000000, 364 (0x7e00 << 16) | (0xc904 >> 2), 365 0x00000000, 366 (0x8e00 << 16) | (0xc904 >> 2), 367 0x00000000, 368 (0x9e00 << 16) | (0xc904 >> 2), 369 0x00000000, 370 (0xae00 << 16) | (0xc904 >> 2), 371 0x00000000, 372 (0xbe00 << 16) | (0xc904 >> 2), 373 0x00000000, 374 (0x4e00 << 16) | (0xc908 >> 2), 375 0x00000000, 376 (0x5e00 << 16) | (0xc908 >> 2), 377 0x00000000, 378 (0x6e00 << 16) | (0xc908 >> 2), 379 0x00000000, 380 (0x7e00 << 16) | (0xc908 >> 2), 381 0x00000000, 382 (0x8e00 << 16) | (0xc908 >> 2), 383 0x00000000, 384 (0x9e00 << 16) | (0xc908 >> 2), 385 0x00000000, 386 (0xae00 << 16) | (0xc908 >> 2), 387 0x00000000, 388 (0xbe00 << 16) | (0xc908 >> 2), 389 0x00000000, 390 (0x4e00 << 16) | (0xc90c >> 2), 391 0x00000000, 392 (0x5e00 << 16) | (0xc90c >> 2), 393 0x00000000, 394 (0x6e00 << 16) | (0xc90c >> 2), 395 0x00000000, 396 (0x7e00 << 16) | (0xc90c >> 2), 397 0x00000000, 398 (0x8e00 << 16) | (0xc90c >> 2), 399 0x00000000, 400 (0x9e00 << 16) | (0xc90c >> 2), 401 0x00000000, 402 (0xae00 << 16) | (0xc90c >> 2), 403 0x00000000, 404 (0xbe00 << 16) | (0xc90c >> 2), 405 0x00000000, 406 (0x4e00 << 16) | (0xc910 >> 2), 407 0x00000000, 408 (0x5e00 << 16) | (0xc910 >> 2), 409 0x00000000, 410 (0x6e00 << 16) | (0xc910 >> 2), 411 0x00000000, 412 (0x7e00 << 16) | (0xc910 >> 2), 413 0x00000000, 414 (0x8e00 << 16) | (0xc910 >> 2), 415 0x00000000, 416 (0x9e00 << 16) | (0xc910 >> 2), 417 0x00000000, 418 (0xae00 << 16) | (0xc910 >> 2), 419 0x00000000, 420 (0xbe00 << 16) | (0xc910 >> 2), 421 0x00000000, 422 (0x0e00 << 16) | (0xc99c >> 2), 423 0x00000000, 424 (0x0e00 << 16) | (0x9834 >> 2), 425 0x00000000, 426 (0x0000 << 16) | (0x30f00 >> 2), 427 0x00000000, 428 (0x0001 << 16) | (0x30f00 >> 2), 429 0x00000000, 430 (0x0000 << 16) | (0x30f04 >> 2), 431 0x00000000, 432 (0x0001 << 16) | (0x30f04 >> 2), 433 0x00000000, 434 (0x0000 << 16) | (0x30f08 >> 2), 435 0x00000000, 436 (0x0001 << 16) | (0x30f08 >> 2), 437 0x00000000, 438 (0x0000 << 16) | (0x30f0c >> 2), 439 0x00000000, 440 (0x0001 << 16) | (0x30f0c >> 2), 441 0x00000000, 442 (0x0600 << 16) | (0x9b7c >> 2), 443 0x00000000, 444 (0x0e00 << 16) | (0x8a14 >> 2), 445 0x00000000, 446 (0x0e00 << 16) | (0x8a18 >> 2), 447 0x00000000, 448 (0x0600 << 16) | (0x30a00 >> 2), 449 0x00000000, 450 (0x0e00 << 16) | (0x8bf0 >> 2), 451 0x00000000, 452 (0x0e00 << 16) | (0x8bcc >> 2), 453 0x00000000, 454 (0x0e00 << 16) | (0x8b24 >> 2), 455 0x00000000, 456 (0x0e00 << 16) | (0x30a04 >> 2), 457 0x00000000, 458 (0x0600 << 16) | (0x30a10 >> 2), 459 0x00000000, 460 (0x0600 << 16) | (0x30a14 >> 2), 461 0x00000000, 462 (0x0600 << 16) | (0x30a18 >> 2), 463 0x00000000, 464 (0x0600 << 16) | (0x30a2c >> 2), 465 0x00000000, 466 (0x0e00 << 16) | (0xc700 >> 2), 467 0x00000000, 468 (0x0e00 << 16) | (0xc704 >> 2), 469 0x00000000, 470 (0x0e00 << 16) | (0xc708 >> 2), 471 0x00000000, 472 (0x0e00 << 16) | (0xc768 >> 2), 473 0x00000000, 474 (0x0400 << 16) | (0xc770 >> 2), 475 0x00000000, 476 (0x0400 << 16) | (0xc774 >> 2), 477 0x00000000, 478 (0x0400 << 16) | (0xc778 >> 2), 479 0x00000000, 480 (0x0400 << 16) | (0xc77c >> 2), 481 0x00000000, 482 (0x0400 << 16) | (0xc780 >> 2), 483 0x00000000, 484 (0x0400 << 16) | (0xc784 >> 2), 485 0x00000000, 486 (0x0400 << 16) | (0xc788 >> 2), 487 0x00000000, 488 (0x0400 << 16) | (0xc78c >> 2), 489 0x00000000, 490 (0x0400 << 16) | (0xc798 >> 2), 491 0x00000000, 492 (0x0400 << 16) | (0xc79c >> 2), 493 0x00000000, 494 (0x0400 << 16) | (0xc7a0 >> 2), 495 0x00000000, 496 (0x0400 << 16) | (0xc7a4 >> 2), 497 0x00000000, 498 (0x0400 << 16) | (0xc7a8 >> 2), 499 0x00000000, 500 (0x0400 << 16) | (0xc7ac >> 2), 501 0x00000000, 502 (0x0400 << 16) | (0xc7b0 >> 2), 503 0x00000000, 504 (0x0400 << 16) | (0xc7b4 >> 2), 505 0x00000000, 506 (0x0e00 << 16) | (0x9100 >> 2), 507 0x00000000, 508 (0x0e00 << 16) | (0x3c010 >> 2), 509 0x00000000, 510 (0x0e00 << 16) | (0x92a8 >> 2), 511 0x00000000, 512 (0x0e00 << 16) | (0x92ac >> 2), 513 0x00000000, 514 (0x0e00 << 16) | (0x92b4 >> 2), 515 0x00000000, 516 (0x0e00 << 16) | (0x92b8 >> 2), 517 0x00000000, 518 (0x0e00 << 16) | (0x92bc >> 2), 519 0x00000000, 520 (0x0e00 << 16) | (0x92c0 >> 2), 521 0x00000000, 522 (0x0e00 << 16) | (0x92c4 >> 2), 523 0x00000000, 524 (0x0e00 << 16) | (0x92c8 >> 2), 525 0x00000000, 526 (0x0e00 << 16) | (0x92cc >> 2), 527 0x00000000, 528 (0x0e00 << 16) | (0x92d0 >> 2), 529 0x00000000, 530 (0x0e00 << 16) | (0x8c00 >> 2), 531 0x00000000, 532 (0x0e00 << 16) | (0x8c04 >> 2), 533 0x00000000, 534 (0x0e00 << 16) | (0x8c20 >> 2), 535 0x00000000, 536 (0x0e00 << 16) | (0x8c38 >> 2), 537 0x00000000, 538 (0x0e00 << 16) | (0x8c3c >> 2), 539 0x00000000, 540 (0x0e00 << 16) | (0xae00 >> 2), 541 0x00000000, 542 (0x0e00 << 16) | (0x9604 >> 2), 543 0x00000000, 544 (0x0e00 << 16) | (0xac08 >> 2), 545 0x00000000, 546 (0x0e00 << 16) | (0xac0c >> 2), 547 0x00000000, 548 (0x0e00 << 16) | (0xac10 >> 2), 549 0x00000000, 550 (0x0e00 << 16) | (0xac14 >> 2), 551 0x00000000, 552 (0x0e00 << 16) | (0xac58 >> 2), 553 0x00000000, 554 (0x0e00 << 16) | (0xac68 >> 2), 555 0x00000000, 556 (0x0e00 << 16) | (0xac6c >> 2), 557 0x00000000, 558 (0x0e00 << 16) | (0xac70 >> 2), 559 0x00000000, 560 (0x0e00 << 16) | (0xac74 >> 2), 561 0x00000000, 562 (0x0e00 << 16) | (0xac78 >> 2), 563 0x00000000, 564 (0x0e00 << 16) | (0xac7c >> 2), 565 0x00000000, 566 (0x0e00 << 16) | (0xac80 >> 2), 567 0x00000000, 568 (0x0e00 << 16) | (0xac84 >> 2), 569 0x00000000, 570 (0x0e00 << 16) | (0xac88 >> 2), 571 0x00000000, 572 (0x0e00 << 16) | (0xac8c >> 2), 573 0x00000000, 574 (0x0e00 << 16) | (0x970c >> 2), 575 0x00000000, 576 (0x0e00 << 16) | (0x9714 >> 2), 577 0x00000000, 578 (0x0e00 << 16) | (0x9718 >> 2), 579 0x00000000, 580 (0x0e00 << 16) | (0x971c >> 2), 581 0x00000000, 582 (0x0e00 << 16) | (0x31068 >> 2), 583 0x00000000, 584 (0x4e00 << 16) | (0x31068 >> 2), 585 0x00000000, 586 (0x5e00 << 16) | (0x31068 >> 2), 587 0x00000000, 588 (0x6e00 << 16) | (0x31068 >> 2), 589 0x00000000, 590 (0x7e00 << 16) | (0x31068 >> 2), 591 0x00000000, 592 (0x8e00 << 16) | (0x31068 >> 2), 593 0x00000000, 594 (0x9e00 << 16) | (0x31068 >> 2), 595 0x00000000, 596 (0xae00 << 16) | (0x31068 >> 2), 597 0x00000000, 598 (0xbe00 << 16) | (0x31068 >> 2), 599 0x00000000, 600 (0x0e00 << 16) | (0xcd10 >> 2), 601 0x00000000, 602 (0x0e00 << 16) | (0xcd14 >> 2), 603 0x00000000, 604 (0x0e00 << 16) | (0x88b0 >> 2), 605 0x00000000, 606 (0x0e00 << 16) | (0x88b4 >> 2), 607 0x00000000, 608 (0x0e00 << 16) | (0x88b8 >> 2), 609 0x00000000, 610 (0x0e00 << 16) | (0x88bc >> 2), 611 0x00000000, 612 (0x0400 << 16) | (0x89c0 >> 2), 613 0x00000000, 614 (0x0e00 << 16) | (0x88c4 >> 2), 615 0x00000000, 616 (0x0e00 << 16) | (0x88c8 >> 2), 617 0x00000000, 618 (0x0e00 << 16) | (0x88d0 >> 2), 619 0x00000000, 620 (0x0e00 << 16) | (0x88d4 >> 2), 621 0x00000000, 622 (0x0e00 << 16) | (0x88d8 >> 2), 623 0x00000000, 624 (0x0e00 << 16) | (0x8980 >> 2), 625 0x00000000, 626 (0x0e00 << 16) | (0x30938 >> 2), 627 0x00000000, 628 (0x0e00 << 16) | (0x3093c >> 2), 629 0x00000000, 630 (0x0e00 << 16) | (0x30940 >> 2), 631 0x00000000, 632 (0x0e00 << 16) | (0x89a0 >> 2), 633 0x00000000, 634 (0x0e00 << 16) | (0x30900 >> 2), 635 0x00000000, 636 (0x0e00 << 16) | (0x30904 >> 2), 637 0x00000000, 638 (0x0e00 << 16) | (0x89b4 >> 2), 639 0x00000000, 640 (0x0e00 << 16) | (0x3c210 >> 2), 641 0x00000000, 642 (0x0e00 << 16) | (0x3c214 >> 2), 643 0x00000000, 644 (0x0e00 << 16) | (0x3c218 >> 2), 645 0x00000000, 646 (0x0e00 << 16) | (0x8904 >> 2), 647 0x00000000, 648 0x5, 649 (0x0e00 << 16) | (0x8c28 >> 2), 650 (0x0e00 << 16) | (0x8c2c >> 2), 651 (0x0e00 << 16) | (0x8c30 >> 2), 652 (0x0e00 << 16) | (0x8c34 >> 2), 653 (0x0e00 << 16) | (0x9600 >> 2), 654 }; 655 656 static const u32 kalindi_rlc_save_restore_register_list[] = 657 { 658 (0x0e00 << 16) | (0xc12c >> 2), 659 0x00000000, 660 (0x0e00 << 16) | (0xc140 >> 2), 661 0x00000000, 662 (0x0e00 << 16) | (0xc150 >> 2), 663 0x00000000, 664 (0x0e00 << 16) | (0xc15c >> 2), 665 0x00000000, 666 (0x0e00 << 16) | (0xc168 >> 2), 667 0x00000000, 668 (0x0e00 << 16) | (0xc170 >> 2), 669 0x00000000, 670 (0x0e00 << 16) | (0xc204 >> 2), 671 0x00000000, 672 (0x0e00 << 16) | (0xc2b4 >> 2), 673 0x00000000, 674 (0x0e00 << 16) | (0xc2b8 >> 2), 675 0x00000000, 676 (0x0e00 << 16) | (0xc2bc >> 2), 677 0x00000000, 678 (0x0e00 << 16) | (0xc2c0 >> 2), 679 0x00000000, 680 (0x0e00 << 16) | (0x8228 >> 2), 681 0x00000000, 682 (0x0e00 << 16) | (0x829c >> 2), 683 0x00000000, 684 (0x0e00 << 16) | (0x869c >> 2), 685 0x00000000, 686 (0x0600 << 16) | (0x98f4 >> 2), 687 0x00000000, 688 (0x0e00 << 16) | (0x98f8 >> 2), 689 0x00000000, 690 (0x0e00 << 16) | (0x9900 >> 2), 691 0x00000000, 692 (0x0e00 << 16) | (0xc260 >> 2), 693 0x00000000, 694 (0x0e00 << 16) | (0x90e8 >> 2), 695 0x00000000, 696 (0x0e00 << 16) | (0x3c000 >> 2), 697 0x00000000, 698 (0x0e00 << 16) | (0x3c00c >> 2), 699 0x00000000, 700 (0x0e00 << 16) | (0x8c1c >> 2), 701 0x00000000, 702 (0x0e00 << 16) | (0x9700 >> 2), 703 0x00000000, 704 (0x0e00 << 16) | (0xcd20 >> 2), 705 0x00000000, 706 (0x4e00 << 16) | (0xcd20 >> 2), 707 0x00000000, 708 (0x5e00 << 16) | (0xcd20 >> 2), 709 0x00000000, 710 (0x6e00 << 16) | (0xcd20 >> 2), 711 0x00000000, 712 (0x7e00 << 16) | (0xcd20 >> 2), 713 0x00000000, 714 (0x0e00 << 16) | (0x89bc >> 2), 715 0x00000000, 716 (0x0e00 << 16) | (0x8900 >> 2), 717 0x00000000, 718 0x3, 719 (0x0e00 << 16) | (0xc130 >> 2), 720 0x00000000, 721 (0x0e00 << 16) | (0xc134 >> 2), 722 0x00000000, 723 (0x0e00 << 16) | (0xc1fc >> 2), 724 0x00000000, 725 (0x0e00 << 16) | (0xc208 >> 2), 726 0x00000000, 727 (0x0e00 << 16) | (0xc264 >> 2), 728 0x00000000, 729 (0x0e00 << 16) | (0xc268 >> 2), 730 0x00000000, 731 (0x0e00 << 16) | (0xc26c >> 2), 732 0x00000000, 733 (0x0e00 << 16) | (0xc270 >> 2), 734 0x00000000, 735 (0x0e00 << 16) | (0xc274 >> 2), 736 0x00000000, 737 (0x0e00 << 16) | (0xc28c >> 2), 738 0x00000000, 739 (0x0e00 << 16) | (0xc290 >> 2), 740 0x00000000, 741 (0x0e00 << 16) | (0xc294 >> 2), 742 0x00000000, 743 (0x0e00 << 16) | (0xc298 >> 2), 744 0x00000000, 745 (0x0e00 << 16) | (0xc2a0 >> 2), 746 0x00000000, 747 (0x0e00 << 16) | (0xc2a4 >> 2), 748 0x00000000, 749 (0x0e00 << 16) | (0xc2a8 >> 2), 750 0x00000000, 751 (0x0e00 << 16) | (0xc2ac >> 2), 752 0x00000000, 753 (0x0e00 << 16) | (0x301d0 >> 2), 754 0x00000000, 755 (0x0e00 << 16) | (0x30238 >> 2), 756 0x00000000, 757 (0x0e00 << 16) | (0x30250 >> 2), 758 0x00000000, 759 (0x0e00 << 16) | (0x30254 >> 2), 760 0x00000000, 761 (0x0e00 << 16) | (0x30258 >> 2), 762 0x00000000, 763 (0x0e00 << 16) | (0x3025c >> 2), 764 0x00000000, 765 (0x4e00 << 16) | (0xc900 >> 2), 766 0x00000000, 767 (0x5e00 << 16) | (0xc900 >> 2), 768 0x00000000, 769 (0x6e00 << 16) | (0xc900 >> 2), 770 0x00000000, 771 (0x7e00 << 16) | (0xc900 >> 2), 772 0x00000000, 773 (0x4e00 << 16) | (0xc904 >> 2), 774 0x00000000, 775 (0x5e00 << 16) | (0xc904 >> 2), 776 0x00000000, 777 (0x6e00 << 16) | (0xc904 >> 2), 778 0x00000000, 779 (0x7e00 << 16) | (0xc904 >> 2), 780 0x00000000, 781 (0x4e00 << 16) | (0xc908 >> 2), 782 0x00000000, 783 (0x5e00 << 16) | (0xc908 >> 2), 784 0x00000000, 785 (0x6e00 << 16) | (0xc908 >> 2), 786 0x00000000, 787 (0x7e00 << 16) | (0xc908 >> 2), 788 0x00000000, 789 (0x4e00 << 16) | (0xc90c >> 2), 790 0x00000000, 791 (0x5e00 << 16) | (0xc90c >> 2), 792 0x00000000, 793 (0x6e00 << 16) | (0xc90c >> 2), 794 0x00000000, 795 (0x7e00 << 16) | (0xc90c >> 2), 796 0x00000000, 797 (0x4e00 << 16) | (0xc910 >> 2), 798 0x00000000, 799 (0x5e00 << 16) | (0xc910 >> 2), 800 0x00000000, 801 (0x6e00 << 16) | (0xc910 >> 2), 802 0x00000000, 803 (0x7e00 << 16) | (0xc910 >> 2), 804 0x00000000, 805 (0x0e00 << 16) | (0xc99c >> 2), 806 0x00000000, 807 (0x0e00 << 16) | (0x9834 >> 2), 808 0x00000000, 809 (0x0000 << 16) | (0x30f00 >> 2), 810 0x00000000, 811 (0x0000 << 16) | (0x30f04 >> 2), 812 0x00000000, 813 (0x0000 << 16) | (0x30f08 >> 2), 814 0x00000000, 815 (0x0000 << 16) | (0x30f0c >> 2), 816 0x00000000, 817 (0x0600 << 16) | (0x9b7c >> 2), 818 0x00000000, 819 (0x0e00 << 16) | (0x8a14 >> 2), 820 0x00000000, 821 (0x0e00 << 16) | (0x8a18 >> 2), 822 0x00000000, 823 (0x0600 << 16) | (0x30a00 >> 2), 824 0x00000000, 825 (0x0e00 << 16) | (0x8bf0 >> 2), 826 0x00000000, 827 (0x0e00 << 16) | (0x8bcc >> 2), 828 0x00000000, 829 (0x0e00 << 16) | (0x8b24 >> 2), 830 0x00000000, 831 (0x0e00 << 16) | (0x30a04 >> 2), 832 0x00000000, 833 (0x0600 << 16) | (0x30a10 >> 2), 834 0x00000000, 835 (0x0600 << 16) | (0x30a14 >> 2), 836 0x00000000, 837 (0x0600 << 16) | (0x30a18 >> 2), 838 0x00000000, 839 (0x0600 << 16) | (0x30a2c >> 2), 840 0x00000000, 841 (0x0e00 << 16) | (0xc700 >> 2), 842 0x00000000, 843 (0x0e00 << 16) | (0xc704 >> 2), 844 0x00000000, 845 (0x0e00 << 16) | (0xc708 >> 2), 846 0x00000000, 847 (0x0e00 << 16) | (0xc768 >> 2), 848 0x00000000, 849 (0x0400 << 16) | (0xc770 >> 2), 850 0x00000000, 851 (0x0400 << 16) | (0xc774 >> 2), 852 0x00000000, 853 (0x0400 << 16) | (0xc798 >> 2), 854 0x00000000, 855 (0x0400 << 16) | (0xc79c >> 2), 856 0x00000000, 857 (0x0e00 << 16) | (0x9100 >> 2), 858 0x00000000, 859 (0x0e00 << 16) | (0x3c010 >> 2), 860 0x00000000, 861 (0x0e00 << 16) | (0x8c00 >> 2), 862 0x00000000, 863 (0x0e00 << 16) | (0x8c04 >> 2), 864 0x00000000, 865 (0x0e00 << 16) | (0x8c20 >> 2), 866 0x00000000, 867 (0x0e00 << 16) | (0x8c38 >> 2), 868 0x00000000, 869 (0x0e00 << 16) | (0x8c3c >> 2), 870 0x00000000, 871 (0x0e00 << 16) | (0xae00 >> 2), 872 0x00000000, 873 (0x0e00 << 16) | (0x9604 >> 2), 874 0x00000000, 875 (0x0e00 << 16) | (0xac08 >> 2), 876 0x00000000, 877 (0x0e00 << 16) | (0xac0c >> 2), 878 0x00000000, 879 (0x0e00 << 16) | (0xac10 >> 2), 880 0x00000000, 881 (0x0e00 << 16) | (0xac14 >> 2), 882 0x00000000, 883 (0x0e00 << 16) | (0xac58 >> 2), 884 0x00000000, 885 (0x0e00 << 16) | (0xac68 >> 2), 886 0x00000000, 887 (0x0e00 << 16) | (0xac6c >> 2), 888 0x00000000, 889 (0x0e00 << 16) | (0xac70 >> 2), 890 0x00000000, 891 (0x0e00 << 16) | (0xac74 >> 2), 892 0x00000000, 893 (0x0e00 << 16) | (0xac78 >> 2), 894 0x00000000, 895 (0x0e00 << 16) | (0xac7c >> 2), 896 0x00000000, 897 (0x0e00 << 16) | (0xac80 >> 2), 898 0x00000000, 899 (0x0e00 << 16) | (0xac84 >> 2), 900 0x00000000, 901 (0x0e00 << 16) | (0xac88 >> 2), 902 0x00000000, 903 (0x0e00 << 16) | (0xac8c >> 2), 904 0x00000000, 905 (0x0e00 << 16) | (0x970c >> 2), 906 0x00000000, 907 (0x0e00 << 16) | (0x9714 >> 2), 908 0x00000000, 909 (0x0e00 << 16) | (0x9718 >> 2), 910 0x00000000, 911 (0x0e00 << 16) | (0x971c >> 2), 912 0x00000000, 913 (0x0e00 << 16) | (0x31068 >> 2), 914 0x00000000, 915 (0x4e00 << 16) | (0x31068 >> 2), 916 0x00000000, 917 (0x5e00 << 16) | (0x31068 >> 2), 918 0x00000000, 919 (0x6e00 << 16) | (0x31068 >> 2), 920 0x00000000, 921 (0x7e00 << 16) | (0x31068 >> 2), 922 0x00000000, 923 (0x0e00 << 16) | (0xcd10 >> 2), 924 0x00000000, 925 (0x0e00 << 16) | (0xcd14 >> 2), 926 0x00000000, 927 (0x0e00 << 16) | (0x88b0 >> 2), 928 0x00000000, 929 (0x0e00 << 16) | (0x88b4 >> 2), 930 0x00000000, 931 (0x0e00 << 16) | (0x88b8 >> 2), 932 0x00000000, 933 (0x0e00 << 16) | (0x88bc >> 2), 934 0x00000000, 935 (0x0400 << 16) | (0x89c0 >> 2), 936 0x00000000, 937 (0x0e00 << 16) | (0x88c4 >> 2), 938 0x00000000, 939 (0x0e00 << 16) | (0x88c8 >> 2), 940 0x00000000, 941 (0x0e00 << 16) | (0x88d0 >> 2), 942 0x00000000, 943 (0x0e00 << 16) | (0x88d4 >> 2), 944 0x00000000, 945 (0x0e00 << 16) | (0x88d8 >> 2), 946 0x00000000, 947 (0x0e00 << 16) | (0x8980 >> 2), 948 0x00000000, 949 (0x0e00 << 16) | (0x30938 >> 2), 950 0x00000000, 951 (0x0e00 << 16) | (0x3093c >> 2), 952 0x00000000, 953 (0x0e00 << 16) | (0x30940 >> 2), 954 0x00000000, 955 (0x0e00 << 16) | (0x89a0 >> 2), 956 0x00000000, 957 (0x0e00 << 16) | (0x30900 >> 2), 958 0x00000000, 959 (0x0e00 << 16) | (0x30904 >> 2), 960 0x00000000, 961 (0x0e00 << 16) | (0x89b4 >> 2), 962 0x00000000, 963 (0x0e00 << 16) | (0x3e1fc >> 2), 964 0x00000000, 965 (0x0e00 << 16) | (0x3c210 >> 2), 966 0x00000000, 967 (0x0e00 << 16) | (0x3c214 >> 2), 968 0x00000000, 969 (0x0e00 << 16) | (0x3c218 >> 2), 970 0x00000000, 971 (0x0e00 << 16) | (0x8904 >> 2), 972 0x00000000, 973 0x5, 974 (0x0e00 << 16) | (0x8c28 >> 2), 975 (0x0e00 << 16) | (0x8c2c >> 2), 976 (0x0e00 << 16) | (0x8c30 >> 2), 977 (0x0e00 << 16) | (0x8c34 >> 2), 978 (0x0e00 << 16) | (0x9600 >> 2), 979 }; 980 981 static const u32 bonaire_golden_spm_registers[] = 982 { 983 0x30800, 0xe0ffffff, 0xe0000000 984 }; 985 986 static const u32 bonaire_golden_common_registers[] = 987 { 988 0xc770, 0xffffffff, 0x00000800, 989 0xc774, 0xffffffff, 0x00000800, 990 0xc798, 0xffffffff, 0x00007fbf, 991 0xc79c, 0xffffffff, 0x00007faf 992 }; 993 994 static const u32 bonaire_golden_registers[] = 995 { 996 0x3354, 0x00000333, 0x00000333, 997 0x3350, 0x000c0fc0, 0x00040200, 998 0x9a10, 0x00010000, 0x00058208, 999 0x3c000, 0xffff1fff, 0x00140000, 1000 0x3c200, 0xfdfc0fff, 0x00000100, 1001 0x3c234, 0x40000000, 0x40000200, 1002 0x9830, 0xffffffff, 0x00000000, 1003 0x9834, 0xf00fffff, 0x00000400, 1004 0x9838, 0x0002021c, 0x00020200, 1005 0xc78, 0x00000080, 0x00000000, 1006 0x5bb0, 0x000000f0, 0x00000070, 1007 0x5bc0, 0xf0311fff, 0x80300000, 1008 0x98f8, 0x73773777, 0x12010001, 1009 0x350c, 0x00810000, 0x408af000, 1010 0x7030, 0x31000111, 0x00000011, 1011 0x2f48, 0x73773777, 0x12010001, 1012 0x220c, 0x00007fb6, 0x0021a1b1, 1013 0x2210, 0x00007fb6, 0x002021b1, 1014 0x2180, 0x00007fb6, 0x00002191, 1015 0x2218, 0x00007fb6, 0x002121b1, 1016 0x221c, 0x00007fb6, 0x002021b1, 1017 0x21dc, 0x00007fb6, 0x00002191, 1018 0x21e0, 0x00007fb6, 0x00002191, 1019 0x3628, 0x0000003f, 0x0000000a, 1020 0x362c, 0x0000003f, 0x0000000a, 1021 0x2ae4, 0x00073ffe, 0x000022a2, 1022 0x240c, 0x000007ff, 0x00000000, 1023 0x8a14, 0xf000003f, 0x00000007, 1024 0x8bf0, 0x00002001, 0x00000001, 1025 0x8b24, 0xffffffff, 0x00ffffff, 1026 0x30a04, 0x0000ff0f, 0x00000000, 1027 0x28a4c, 0x07ffffff, 0x06000000, 1028 0x4d8, 0x00000fff, 0x00000100, 1029 0x3e78, 0x00000001, 0x00000002, 1030 0x9100, 0x03000000, 0x0362c688, 1031 0x8c00, 0x000000ff, 0x00000001, 1032 0xe40, 0x00001fff, 0x00001fff, 1033 0x9060, 0x0000007f, 0x00000020, 1034 0x9508, 0x00010000, 0x00010000, 1035 0xac14, 0x000003ff, 0x000000f3, 1036 0xac0c, 0xffffffff, 0x00001032 1037 }; 1038 1039 static const u32 bonaire_mgcg_cgcg_init[] = 1040 { 1041 0xc420, 0xffffffff, 0xfffffffc, 1042 0x30800, 0xffffffff, 0xe0000000, 1043 0x3c2a0, 0xffffffff, 0x00000100, 1044 0x3c208, 0xffffffff, 0x00000100, 1045 0x3c2c0, 0xffffffff, 0xc0000100, 1046 0x3c2c8, 0xffffffff, 0xc0000100, 1047 0x3c2c4, 0xffffffff, 0xc0000100, 1048 0x55e4, 0xffffffff, 0x00600100, 1049 0x3c280, 0xffffffff, 0x00000100, 1050 0x3c214, 0xffffffff, 0x06000100, 1051 0x3c220, 0xffffffff, 0x00000100, 1052 0x3c218, 0xffffffff, 0x06000100, 1053 0x3c204, 0xffffffff, 0x00000100, 1054 0x3c2e0, 0xffffffff, 0x00000100, 1055 0x3c224, 0xffffffff, 0x00000100, 1056 0x3c200, 0xffffffff, 0x00000100, 1057 0x3c230, 0xffffffff, 0x00000100, 1058 0x3c234, 0xffffffff, 0x00000100, 1059 0x3c250, 0xffffffff, 0x00000100, 1060 0x3c254, 0xffffffff, 0x00000100, 1061 0x3c258, 0xffffffff, 0x00000100, 1062 0x3c25c, 0xffffffff, 0x00000100, 1063 0x3c260, 0xffffffff, 0x00000100, 1064 0x3c27c, 0xffffffff, 0x00000100, 1065 0x3c278, 0xffffffff, 0x00000100, 1066 0x3c210, 0xffffffff, 0x06000100, 1067 0x3c290, 0xffffffff, 0x00000100, 1068 0x3c274, 0xffffffff, 0x00000100, 1069 0x3c2b4, 0xffffffff, 0x00000100, 1070 0x3c2b0, 0xffffffff, 0x00000100, 1071 0x3c270, 0xffffffff, 0x00000100, 1072 0x30800, 0xffffffff, 0xe0000000, 1073 0x3c020, 0xffffffff, 0x00010000, 1074 0x3c024, 0xffffffff, 0x00030002, 1075 0x3c028, 0xffffffff, 0x00040007, 1076 0x3c02c, 0xffffffff, 0x00060005, 1077 0x3c030, 0xffffffff, 0x00090008, 1078 0x3c034, 0xffffffff, 0x00010000, 1079 0x3c038, 0xffffffff, 0x00030002, 1080 0x3c03c, 0xffffffff, 0x00040007, 1081 0x3c040, 0xffffffff, 0x00060005, 1082 0x3c044, 0xffffffff, 0x00090008, 1083 0x3c048, 0xffffffff, 0x00010000, 1084 0x3c04c, 0xffffffff, 0x00030002, 1085 0x3c050, 0xffffffff, 0x00040007, 1086 0x3c054, 0xffffffff, 0x00060005, 1087 0x3c058, 0xffffffff, 0x00090008, 1088 0x3c05c, 0xffffffff, 0x00010000, 1089 0x3c060, 0xffffffff, 0x00030002, 1090 0x3c064, 0xffffffff, 0x00040007, 1091 0x3c068, 0xffffffff, 0x00060005, 1092 0x3c06c, 0xffffffff, 0x00090008, 1093 0x3c070, 0xffffffff, 0x00010000, 1094 0x3c074, 0xffffffff, 0x00030002, 1095 0x3c078, 0xffffffff, 0x00040007, 1096 0x3c07c, 0xffffffff, 0x00060005, 1097 0x3c080, 0xffffffff, 0x00090008, 1098 0x3c084, 0xffffffff, 0x00010000, 1099 0x3c088, 0xffffffff, 0x00030002, 1100 0x3c08c, 0xffffffff, 0x00040007, 1101 0x3c090, 0xffffffff, 0x00060005, 1102 0x3c094, 0xffffffff, 0x00090008, 1103 0x3c098, 0xffffffff, 0x00010000, 1104 0x3c09c, 0xffffffff, 0x00030002, 1105 0x3c0a0, 0xffffffff, 0x00040007, 1106 0x3c0a4, 0xffffffff, 0x00060005, 1107 0x3c0a8, 0xffffffff, 0x00090008, 1108 0x3c000, 0xffffffff, 0x96e00200, 1109 0x8708, 0xffffffff, 0x00900100, 1110 0xc424, 0xffffffff, 0x0020003f, 1111 0x38, 0xffffffff, 0x0140001c, 1112 0x3c, 0x000f0000, 0x000f0000, 1113 0x220, 0xffffffff, 0xC060000C, 1114 0x224, 0xc0000fff, 0x00000100, 1115 0xf90, 0xffffffff, 0x00000100, 1116 0xf98, 0x00000101, 0x00000000, 1117 0x20a8, 0xffffffff, 0x00000104, 1118 0x55e4, 0xff000fff, 0x00000100, 1119 0x30cc, 0xc0000fff, 0x00000104, 1120 0xc1e4, 0x00000001, 0x00000001, 1121 0xd00c, 0xff000ff0, 0x00000100, 1122 0xd80c, 0xff000ff0, 0x00000100 1123 }; 1124 1125 static const u32 spectre_golden_spm_registers[] = 1126 { 1127 0x30800, 0xe0ffffff, 0xe0000000 1128 }; 1129 1130 static const u32 spectre_golden_common_registers[] = 1131 { 1132 0xc770, 0xffffffff, 0x00000800, 1133 0xc774, 0xffffffff, 0x00000800, 1134 0xc798, 0xffffffff, 0x00007fbf, 1135 0xc79c, 0xffffffff, 0x00007faf 1136 }; 1137 1138 static const u32 spectre_golden_registers[] = 1139 { 1140 0x3c000, 0xffff1fff, 0x96940200, 1141 0x3c00c, 0xffff0001, 0xff000000, 1142 0x3c200, 0xfffc0fff, 0x00000100, 1143 0x6ed8, 0x00010101, 0x00010000, 1144 0x9834, 0xf00fffff, 0x00000400, 1145 0x9838, 0xfffffffc, 0x00020200, 1146 0x5bb0, 0x000000f0, 0x00000070, 1147 0x5bc0, 0xf0311fff, 0x80300000, 1148 0x98f8, 0x73773777, 0x12010001, 1149 0x9b7c, 0x00ff0000, 0x00fc0000, 1150 0x2f48, 0x73773777, 0x12010001, 1151 0x8a14, 0xf000003f, 0x00000007, 1152 0x8b24, 0xffffffff, 0x00ffffff, 1153 0x28350, 0x3f3f3fff, 0x00000082, 1154 0x28354, 0x0000003f, 0x00000000, 1155 0x3e78, 0x00000001, 0x00000002, 1156 0x913c, 0xffff03df, 0x00000004, 1157 0xc768, 0x00000008, 0x00000008, 1158 0x8c00, 0x000008ff, 0x00000800, 1159 0x9508, 0x00010000, 0x00010000, 1160 0xac0c, 0xffffffff, 0x54763210, 1161 0x214f8, 0x01ff01ff, 0x00000002, 1162 0x21498, 0x007ff800, 0x00200000, 1163 0x2015c, 0xffffffff, 0x00000f40, 1164 0x30934, 0xffffffff, 0x00000001 1165 }; 1166 1167 static const u32 spectre_mgcg_cgcg_init[] = 1168 { 1169 0xc420, 0xffffffff, 0xfffffffc, 1170 0x30800, 0xffffffff, 0xe0000000, 1171 0x3c2a0, 0xffffffff, 0x00000100, 1172 0x3c208, 0xffffffff, 0x00000100, 1173 0x3c2c0, 0xffffffff, 0x00000100, 1174 0x3c2c8, 0xffffffff, 0x00000100, 1175 0x3c2c4, 0xffffffff, 0x00000100, 1176 0x55e4, 0xffffffff, 0x00600100, 1177 0x3c280, 0xffffffff, 0x00000100, 1178 0x3c214, 0xffffffff, 0x06000100, 1179 0x3c220, 0xffffffff, 0x00000100, 1180 0x3c218, 0xffffffff, 0x06000100, 1181 0x3c204, 0xffffffff, 0x00000100, 1182 0x3c2e0, 0xffffffff, 0x00000100, 1183 0x3c224, 0xffffffff, 0x00000100, 1184 0x3c200, 0xffffffff, 0x00000100, 1185 0x3c230, 0xffffffff, 0x00000100, 1186 0x3c234, 0xffffffff, 0x00000100, 1187 0x3c250, 0xffffffff, 0x00000100, 1188 0x3c254, 0xffffffff, 0x00000100, 1189 0x3c258, 0xffffffff, 0x00000100, 1190 0x3c25c, 0xffffffff, 0x00000100, 1191 0x3c260, 0xffffffff, 0x00000100, 1192 0x3c27c, 0xffffffff, 0x00000100, 1193 0x3c278, 0xffffffff, 0x00000100, 1194 0x3c210, 0xffffffff, 0x06000100, 1195 0x3c290, 0xffffffff, 0x00000100, 1196 0x3c274, 0xffffffff, 0x00000100, 1197 0x3c2b4, 0xffffffff, 0x00000100, 1198 0x3c2b0, 0xffffffff, 0x00000100, 1199 0x3c270, 0xffffffff, 0x00000100, 1200 0x30800, 0xffffffff, 0xe0000000, 1201 0x3c020, 0xffffffff, 0x00010000, 1202 0x3c024, 0xffffffff, 0x00030002, 1203 0x3c028, 0xffffffff, 0x00040007, 1204 0x3c02c, 0xffffffff, 0x00060005, 1205 0x3c030, 0xffffffff, 0x00090008, 1206 0x3c034, 0xffffffff, 0x00010000, 1207 0x3c038, 0xffffffff, 0x00030002, 1208 0x3c03c, 0xffffffff, 0x00040007, 1209 0x3c040, 0xffffffff, 0x00060005, 1210 0x3c044, 0xffffffff, 0x00090008, 1211 0x3c048, 0xffffffff, 0x00010000, 1212 0x3c04c, 0xffffffff, 0x00030002, 1213 0x3c050, 0xffffffff, 0x00040007, 1214 0x3c054, 0xffffffff, 0x00060005, 1215 0x3c058, 0xffffffff, 0x00090008, 1216 0x3c05c, 0xffffffff, 0x00010000, 1217 0x3c060, 0xffffffff, 0x00030002, 1218 0x3c064, 0xffffffff, 0x00040007, 1219 0x3c068, 0xffffffff, 0x00060005, 1220 0x3c06c, 0xffffffff, 0x00090008, 1221 0x3c070, 0xffffffff, 0x00010000, 1222 0x3c074, 0xffffffff, 0x00030002, 1223 0x3c078, 0xffffffff, 0x00040007, 1224 0x3c07c, 0xffffffff, 0x00060005, 1225 0x3c080, 0xffffffff, 0x00090008, 1226 0x3c084, 0xffffffff, 0x00010000, 1227 0x3c088, 0xffffffff, 0x00030002, 1228 0x3c08c, 0xffffffff, 0x00040007, 1229 0x3c090, 0xffffffff, 0x00060005, 1230 0x3c094, 0xffffffff, 0x00090008, 1231 0x3c098, 0xffffffff, 0x00010000, 1232 0x3c09c, 0xffffffff, 0x00030002, 1233 0x3c0a0, 0xffffffff, 0x00040007, 1234 0x3c0a4, 0xffffffff, 0x00060005, 1235 0x3c0a8, 0xffffffff, 0x00090008, 1236 0x3c0ac, 0xffffffff, 0x00010000, 1237 0x3c0b0, 0xffffffff, 0x00030002, 1238 0x3c0b4, 0xffffffff, 0x00040007, 1239 0x3c0b8, 0xffffffff, 0x00060005, 1240 0x3c0bc, 0xffffffff, 0x00090008, 1241 0x3c000, 0xffffffff, 0x96e00200, 1242 0x8708, 0xffffffff, 0x00900100, 1243 0xc424, 0xffffffff, 0x0020003f, 1244 0x38, 0xffffffff, 0x0140001c, 1245 0x3c, 0x000f0000, 0x000f0000, 1246 0x220, 0xffffffff, 0xC060000C, 1247 0x224, 0xc0000fff, 0x00000100, 1248 0xf90, 0xffffffff, 0x00000100, 1249 0xf98, 0x00000101, 0x00000000, 1250 0x20a8, 0xffffffff, 0x00000104, 1251 0x55e4, 0xff000fff, 0x00000100, 1252 0x30cc, 0xc0000fff, 0x00000104, 1253 0xc1e4, 0x00000001, 0x00000001, 1254 0xd00c, 0xff000ff0, 0x00000100, 1255 0xd80c, 0xff000ff0, 0x00000100 1256 }; 1257 1258 static const u32 kalindi_golden_spm_registers[] = 1259 { 1260 0x30800, 0xe0ffffff, 0xe0000000 1261 }; 1262 1263 static const u32 kalindi_golden_common_registers[] = 1264 { 1265 0xc770, 0xffffffff, 0x00000800, 1266 0xc774, 0xffffffff, 0x00000800, 1267 0xc798, 0xffffffff, 0x00007fbf, 1268 0xc79c, 0xffffffff, 0x00007faf 1269 }; 1270 1271 static const u32 kalindi_golden_registers[] = 1272 { 1273 0x3c000, 0xffffdfff, 0x6e944040, 1274 0x55e4, 0xff607fff, 0xfc000100, 1275 0x3c220, 0xff000fff, 0x00000100, 1276 0x3c224, 0xff000fff, 0x00000100, 1277 0x3c200, 0xfffc0fff, 0x00000100, 1278 0x6ed8, 0x00010101, 0x00010000, 1279 0x9830, 0xffffffff, 0x00000000, 1280 0x9834, 0xf00fffff, 0x00000400, 1281 0x5bb0, 0x000000f0, 0x00000070, 1282 0x5bc0, 0xf0311fff, 0x80300000, 1283 0x98f8, 0x73773777, 0x12010001, 1284 0x98fc, 0xffffffff, 0x00000010, 1285 0x9b7c, 0x00ff0000, 0x00fc0000, 1286 0x8030, 0x00001f0f, 0x0000100a, 1287 0x2f48, 0x73773777, 0x12010001, 1288 0x2408, 0x000fffff, 0x000c007f, 1289 0x8a14, 0xf000003f, 0x00000007, 1290 0x8b24, 0x3fff3fff, 0x00ffcfff, 1291 0x30a04, 0x0000ff0f, 0x00000000, 1292 0x28a4c, 0x07ffffff, 0x06000000, 1293 0x4d8, 0x00000fff, 0x00000100, 1294 0x3e78, 0x00000001, 0x00000002, 1295 0xc768, 0x00000008, 0x00000008, 1296 0x8c00, 0x000000ff, 0x00000003, 1297 0x214f8, 0x01ff01ff, 0x00000002, 1298 0x21498, 0x007ff800, 0x00200000, 1299 0x2015c, 0xffffffff, 0x00000f40, 1300 0x88c4, 0x001f3ae3, 0x00000082, 1301 0x88d4, 0x0000001f, 0x00000010, 1302 0x30934, 0xffffffff, 0x00000000 1303 }; 1304 1305 static const u32 kalindi_mgcg_cgcg_init[] = 1306 { 1307 0xc420, 0xffffffff, 0xfffffffc, 1308 0x30800, 0xffffffff, 0xe0000000, 1309 0x3c2a0, 0xffffffff, 0x00000100, 1310 0x3c208, 0xffffffff, 0x00000100, 1311 0x3c2c0, 0xffffffff, 0x00000100, 1312 0x3c2c8, 0xffffffff, 0x00000100, 1313 0x3c2c4, 0xffffffff, 0x00000100, 1314 0x55e4, 0xffffffff, 0x00600100, 1315 0x3c280, 0xffffffff, 0x00000100, 1316 0x3c214, 0xffffffff, 0x06000100, 1317 0x3c220, 0xffffffff, 0x00000100, 1318 0x3c218, 0xffffffff, 0x06000100, 1319 0x3c204, 0xffffffff, 0x00000100, 1320 0x3c2e0, 0xffffffff, 0x00000100, 1321 0x3c224, 0xffffffff, 0x00000100, 1322 0x3c200, 0xffffffff, 0x00000100, 1323 0x3c230, 0xffffffff, 0x00000100, 1324 0x3c234, 0xffffffff, 0x00000100, 1325 0x3c250, 0xffffffff, 0x00000100, 1326 0x3c254, 0xffffffff, 0x00000100, 1327 0x3c258, 0xffffffff, 0x00000100, 1328 0x3c25c, 0xffffffff, 0x00000100, 1329 0x3c260, 0xffffffff, 0x00000100, 1330 0x3c27c, 0xffffffff, 0x00000100, 1331 0x3c278, 0xffffffff, 0x00000100, 1332 0x3c210, 0xffffffff, 0x06000100, 1333 0x3c290, 0xffffffff, 0x00000100, 1334 0x3c274, 0xffffffff, 0x00000100, 1335 0x3c2b4, 0xffffffff, 0x00000100, 1336 0x3c2b0, 0xffffffff, 0x00000100, 1337 0x3c270, 0xffffffff, 0x00000100, 1338 0x30800, 0xffffffff, 0xe0000000, 1339 0x3c020, 0xffffffff, 0x00010000, 1340 0x3c024, 0xffffffff, 0x00030002, 1341 0x3c028, 0xffffffff, 0x00040007, 1342 0x3c02c, 0xffffffff, 0x00060005, 1343 0x3c030, 0xffffffff, 0x00090008, 1344 0x3c034, 0xffffffff, 0x00010000, 1345 0x3c038, 0xffffffff, 0x00030002, 1346 0x3c03c, 0xffffffff, 0x00040007, 1347 0x3c040, 0xffffffff, 0x00060005, 1348 0x3c044, 0xffffffff, 0x00090008, 1349 0x3c000, 0xffffffff, 0x96e00200, 1350 0x8708, 0xffffffff, 0x00900100, 1351 0xc424, 0xffffffff, 0x0020003f, 1352 0x38, 0xffffffff, 0x0140001c, 1353 0x3c, 0x000f0000, 0x000f0000, 1354 0x220, 0xffffffff, 0xC060000C, 1355 0x224, 0xc0000fff, 0x00000100, 1356 0x20a8, 0xffffffff, 0x00000104, 1357 0x55e4, 0xff000fff, 0x00000100, 1358 0x30cc, 0xc0000fff, 0x00000104, 1359 0xc1e4, 0x00000001, 0x00000001, 1360 0xd00c, 0xff000ff0, 0x00000100, 1361 0xd80c, 0xff000ff0, 0x00000100 1362 }; 1363 1364 static const u32 hawaii_golden_spm_registers[] = 1365 { 1366 0x30800, 0xe0ffffff, 0xe0000000 1367 }; 1368 1369 static const u32 hawaii_golden_common_registers[] = 1370 { 1371 0x30800, 0xffffffff, 0xe0000000, 1372 0x28350, 0xffffffff, 0x3a00161a, 1373 0x28354, 0xffffffff, 0x0000002e, 1374 0x9a10, 0xffffffff, 0x00018208, 1375 0x98f8, 0xffffffff, 0x12011003 1376 }; 1377 1378 static const u32 hawaii_golden_registers[] = 1379 { 1380 0x3354, 0x00000333, 0x00000333, 1381 0x9a10, 0x00010000, 0x00058208, 1382 0x9830, 0xffffffff, 0x00000000, 1383 0x9834, 0xf00fffff, 0x00000400, 1384 0x9838, 0x0002021c, 0x00020200, 1385 0xc78, 0x00000080, 0x00000000, 1386 0x5bb0, 0x000000f0, 0x00000070, 1387 0x5bc0, 0xf0311fff, 0x80300000, 1388 0x350c, 0x00810000, 0x408af000, 1389 0x7030, 0x31000111, 0x00000011, 1390 0x2f48, 0x73773777, 0x12010001, 1391 0x2120, 0x0000007f, 0x0000001b, 1392 0x21dc, 0x00007fb6, 0x00002191, 1393 0x3628, 0x0000003f, 0x0000000a, 1394 0x362c, 0x0000003f, 0x0000000a, 1395 0x2ae4, 0x00073ffe, 0x000022a2, 1396 0x240c, 0x000007ff, 0x00000000, 1397 0x8bf0, 0x00002001, 0x00000001, 1398 0x8b24, 0xffffffff, 0x00ffffff, 1399 0x30a04, 0x0000ff0f, 0x00000000, 1400 0x28a4c, 0x07ffffff, 0x06000000, 1401 0x3e78, 0x00000001, 0x00000002, 1402 0xc768, 0x00000008, 0x00000008, 1403 0xc770, 0x00000f00, 0x00000800, 1404 0xc774, 0x00000f00, 0x00000800, 1405 0xc798, 0x00ffffff, 0x00ff7fbf, 1406 0xc79c, 0x00ffffff, 0x00ff7faf, 1407 0x8c00, 0x000000ff, 0x00000800, 1408 0xe40, 0x00001fff, 0x00001fff, 1409 0x9060, 0x0000007f, 0x00000020, 1410 0x9508, 0x00010000, 0x00010000, 1411 0xae00, 0x00100000, 0x000ff07c, 1412 0xac14, 0x000003ff, 0x0000000f, 1413 0xac10, 0xffffffff, 0x7564fdec, 1414 0xac0c, 0xffffffff, 0x3120b9a8, 1415 0xac08, 0x20000000, 0x0f9c0000 1416 }; 1417 1418 static const u32 hawaii_mgcg_cgcg_init[] = 1419 { 1420 0xc420, 0xffffffff, 0xfffffffd, 1421 0x30800, 0xffffffff, 0xe0000000, 1422 0x3c2a0, 0xffffffff, 0x00000100, 1423 0x3c208, 0xffffffff, 0x00000100, 1424 0x3c2c0, 0xffffffff, 0x00000100, 1425 0x3c2c8, 0xffffffff, 0x00000100, 1426 0x3c2c4, 0xffffffff, 0x00000100, 1427 0x55e4, 0xffffffff, 0x00200100, 1428 0x3c280, 0xffffffff, 0x00000100, 1429 0x3c214, 0xffffffff, 0x06000100, 1430 0x3c220, 0xffffffff, 0x00000100, 1431 0x3c218, 0xffffffff, 0x06000100, 1432 0x3c204, 0xffffffff, 0x00000100, 1433 0x3c2e0, 0xffffffff, 0x00000100, 1434 0x3c224, 0xffffffff, 0x00000100, 1435 0x3c200, 0xffffffff, 0x00000100, 1436 0x3c230, 0xffffffff, 0x00000100, 1437 0x3c234, 0xffffffff, 0x00000100, 1438 0x3c250, 0xffffffff, 0x00000100, 1439 0x3c254, 0xffffffff, 0x00000100, 1440 0x3c258, 0xffffffff, 0x00000100, 1441 0x3c25c, 0xffffffff, 0x00000100, 1442 0x3c260, 0xffffffff, 0x00000100, 1443 0x3c27c, 0xffffffff, 0x00000100, 1444 0x3c278, 0xffffffff, 0x00000100, 1445 0x3c210, 0xffffffff, 0x06000100, 1446 0x3c290, 0xffffffff, 0x00000100, 1447 0x3c274, 0xffffffff, 0x00000100, 1448 0x3c2b4, 0xffffffff, 0x00000100, 1449 0x3c2b0, 0xffffffff, 0x00000100, 1450 0x3c270, 0xffffffff, 0x00000100, 1451 0x30800, 0xffffffff, 0xe0000000, 1452 0x3c020, 0xffffffff, 0x00010000, 1453 0x3c024, 0xffffffff, 0x00030002, 1454 0x3c028, 0xffffffff, 0x00040007, 1455 0x3c02c, 0xffffffff, 0x00060005, 1456 0x3c030, 0xffffffff, 0x00090008, 1457 0x3c034, 0xffffffff, 0x00010000, 1458 0x3c038, 0xffffffff, 0x00030002, 1459 0x3c03c, 0xffffffff, 0x00040007, 1460 0x3c040, 0xffffffff, 0x00060005, 1461 0x3c044, 0xffffffff, 0x00090008, 1462 0x3c048, 0xffffffff, 0x00010000, 1463 0x3c04c, 0xffffffff, 0x00030002, 1464 0x3c050, 0xffffffff, 0x00040007, 1465 0x3c054, 0xffffffff, 0x00060005, 1466 0x3c058, 0xffffffff, 0x00090008, 1467 0x3c05c, 0xffffffff, 0x00010000, 1468 0x3c060, 0xffffffff, 0x00030002, 1469 0x3c064, 0xffffffff, 0x00040007, 1470 0x3c068, 0xffffffff, 0x00060005, 1471 0x3c06c, 0xffffffff, 0x00090008, 1472 0x3c070, 0xffffffff, 0x00010000, 1473 0x3c074, 0xffffffff, 0x00030002, 1474 0x3c078, 0xffffffff, 0x00040007, 1475 0x3c07c, 0xffffffff, 0x00060005, 1476 0x3c080, 0xffffffff, 0x00090008, 1477 0x3c084, 0xffffffff, 0x00010000, 1478 0x3c088, 0xffffffff, 0x00030002, 1479 0x3c08c, 0xffffffff, 0x00040007, 1480 0x3c090, 0xffffffff, 0x00060005, 1481 0x3c094, 0xffffffff, 0x00090008, 1482 0x3c098, 0xffffffff, 0x00010000, 1483 0x3c09c, 0xffffffff, 0x00030002, 1484 0x3c0a0, 0xffffffff, 0x00040007, 1485 0x3c0a4, 0xffffffff, 0x00060005, 1486 0x3c0a8, 0xffffffff, 0x00090008, 1487 0x3c0ac, 0xffffffff, 0x00010000, 1488 0x3c0b0, 0xffffffff, 0x00030002, 1489 0x3c0b4, 0xffffffff, 0x00040007, 1490 0x3c0b8, 0xffffffff, 0x00060005, 1491 0x3c0bc, 0xffffffff, 0x00090008, 1492 0x3c0c0, 0xffffffff, 0x00010000, 1493 0x3c0c4, 0xffffffff, 0x00030002, 1494 0x3c0c8, 0xffffffff, 0x00040007, 1495 0x3c0cc, 0xffffffff, 0x00060005, 1496 0x3c0d0, 0xffffffff, 0x00090008, 1497 0x3c0d4, 0xffffffff, 0x00010000, 1498 0x3c0d8, 0xffffffff, 0x00030002, 1499 0x3c0dc, 0xffffffff, 0x00040007, 1500 0x3c0e0, 0xffffffff, 0x00060005, 1501 0x3c0e4, 0xffffffff, 0x00090008, 1502 0x3c0e8, 0xffffffff, 0x00010000, 1503 0x3c0ec, 0xffffffff, 0x00030002, 1504 0x3c0f0, 0xffffffff, 0x00040007, 1505 0x3c0f4, 0xffffffff, 0x00060005, 1506 0x3c0f8, 0xffffffff, 0x00090008, 1507 0xc318, 0xffffffff, 0x00020200, 1508 0x3350, 0xffffffff, 0x00000200, 1509 0x15c0, 0xffffffff, 0x00000400, 1510 0x55e8, 0xffffffff, 0x00000000, 1511 0x2f50, 0xffffffff, 0x00000902, 1512 0x3c000, 0xffffffff, 0x96940200, 1513 0x8708, 0xffffffff, 0x00900100, 1514 0xc424, 0xffffffff, 0x0020003f, 1515 0x38, 0xffffffff, 0x0140001c, 1516 0x3c, 0x000f0000, 0x000f0000, 1517 0x220, 0xffffffff, 0xc060000c, 1518 0x224, 0xc0000fff, 0x00000100, 1519 0xf90, 0xffffffff, 0x00000100, 1520 0xf98, 0x00000101, 0x00000000, 1521 0x20a8, 0xffffffff, 0x00000104, 1522 0x55e4, 0xff000fff, 0x00000100, 1523 0x30cc, 0xc0000fff, 0x00000104, 1524 0xc1e4, 0x00000001, 0x00000001, 1525 0xd00c, 0xff000ff0, 0x00000100, 1526 0xd80c, 0xff000ff0, 0x00000100 1527 }; 1528 1529 static const u32 godavari_golden_registers[] = 1530 { 1531 0x55e4, 0xff607fff, 0xfc000100, 1532 0x6ed8, 0x00010101, 0x00010000, 1533 0x9830, 0xffffffff, 0x00000000, 1534 0x98302, 0xf00fffff, 0x00000400, 1535 0x6130, 0xffffffff, 0x00010000, 1536 0x5bb0, 0x000000f0, 0x00000070, 1537 0x5bc0, 0xf0311fff, 0x80300000, 1538 0x98f8, 0x73773777, 0x12010001, 1539 0x98fc, 0xffffffff, 0x00000010, 1540 0x8030, 0x00001f0f, 0x0000100a, 1541 0x2f48, 0x73773777, 0x12010001, 1542 0x2408, 0x000fffff, 0x000c007f, 1543 0x8a14, 0xf000003f, 0x00000007, 1544 0x8b24, 0xffffffff, 0x00ff0fff, 1545 0x30a04, 0x0000ff0f, 0x00000000, 1546 0x28a4c, 0x07ffffff, 0x06000000, 1547 0x4d8, 0x00000fff, 0x00000100, 1548 0xd014, 0x00010000, 0x00810001, 1549 0xd814, 0x00010000, 0x00810001, 1550 0x3e78, 0x00000001, 0x00000002, 1551 0xc768, 0x00000008, 0x00000008, 1552 0xc770, 0x00000f00, 0x00000800, 1553 0xc774, 0x00000f00, 0x00000800, 1554 0xc798, 0x00ffffff, 0x00ff7fbf, 1555 0xc79c, 0x00ffffff, 0x00ff7faf, 1556 0x8c00, 0x000000ff, 0x00000001, 1557 0x214f8, 0x01ff01ff, 0x00000002, 1558 0x21498, 0x007ff800, 0x00200000, 1559 0x2015c, 0xffffffff, 0x00000f40, 1560 0x88c4, 0x001f3ae3, 0x00000082, 1561 0x88d4, 0x0000001f, 0x00000010, 1562 0x30934, 0xffffffff, 0x00000000 1563 }; 1564 1565 1566 static void cik_init_golden_registers(struct radeon_device *rdev) 1567 { 1568 /* Some of the registers might be dependent on GRBM_GFX_INDEX */ 1569 mutex_lock(&rdev->grbm_idx_mutex); 1570 switch (rdev->family) { 1571 case CHIP_BONAIRE: 1572 radeon_program_register_sequence(rdev, 1573 bonaire_mgcg_cgcg_init, 1574 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 1575 radeon_program_register_sequence(rdev, 1576 bonaire_golden_registers, 1577 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 1578 radeon_program_register_sequence(rdev, 1579 bonaire_golden_common_registers, 1580 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 1581 radeon_program_register_sequence(rdev, 1582 bonaire_golden_spm_registers, 1583 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 1584 break; 1585 case CHIP_KABINI: 1586 radeon_program_register_sequence(rdev, 1587 kalindi_mgcg_cgcg_init, 1588 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1589 radeon_program_register_sequence(rdev, 1590 kalindi_golden_registers, 1591 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 1592 radeon_program_register_sequence(rdev, 1593 kalindi_golden_common_registers, 1594 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1595 radeon_program_register_sequence(rdev, 1596 kalindi_golden_spm_registers, 1597 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1598 break; 1599 case CHIP_MULLINS: 1600 radeon_program_register_sequence(rdev, 1601 kalindi_mgcg_cgcg_init, 1602 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1603 radeon_program_register_sequence(rdev, 1604 godavari_golden_registers, 1605 (const u32)ARRAY_SIZE(godavari_golden_registers)); 1606 radeon_program_register_sequence(rdev, 1607 kalindi_golden_common_registers, 1608 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1609 radeon_program_register_sequence(rdev, 1610 kalindi_golden_spm_registers, 1611 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1612 break; 1613 case CHIP_KAVERI: 1614 radeon_program_register_sequence(rdev, 1615 spectre_mgcg_cgcg_init, 1616 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 1617 radeon_program_register_sequence(rdev, 1618 spectre_golden_registers, 1619 (const u32)ARRAY_SIZE(spectre_golden_registers)); 1620 radeon_program_register_sequence(rdev, 1621 spectre_golden_common_registers, 1622 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 1623 radeon_program_register_sequence(rdev, 1624 spectre_golden_spm_registers, 1625 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 1626 break; 1627 case CHIP_HAWAII: 1628 radeon_program_register_sequence(rdev, 1629 hawaii_mgcg_cgcg_init, 1630 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 1631 radeon_program_register_sequence(rdev, 1632 hawaii_golden_registers, 1633 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 1634 radeon_program_register_sequence(rdev, 1635 hawaii_golden_common_registers, 1636 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 1637 radeon_program_register_sequence(rdev, 1638 hawaii_golden_spm_registers, 1639 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 1640 break; 1641 default: 1642 break; 1643 } 1644 mutex_unlock(&rdev->grbm_idx_mutex); 1645 } 1646 1647 /** 1648 * cik_get_xclk - get the xclk 1649 * 1650 * @rdev: radeon_device pointer 1651 * 1652 * Returns the reference clock used by the gfx engine 1653 * (CIK). 1654 */ 1655 u32 cik_get_xclk(struct radeon_device *rdev) 1656 { 1657 u32 reference_clock = rdev->clock.spll.reference_freq; 1658 1659 if (rdev->flags & RADEON_IS_IGP) { 1660 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) 1661 return reference_clock / 2; 1662 } else { 1663 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) 1664 return reference_clock / 4; 1665 } 1666 return reference_clock; 1667 } 1668 1669 /** 1670 * cik_mm_rdoorbell - read a doorbell dword 1671 * 1672 * @rdev: radeon_device pointer 1673 * @index: doorbell index 1674 * 1675 * Returns the value in the doorbell aperture at the 1676 * requested doorbell index (CIK). 1677 */ 1678 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) 1679 { 1680 if (index < rdev->doorbell.num_doorbells) { 1681 return readl(rdev->doorbell.ptr + index); 1682 } else { 1683 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 1684 return 0; 1685 } 1686 } 1687 1688 /** 1689 * cik_mm_wdoorbell - write a doorbell dword 1690 * 1691 * @rdev: radeon_device pointer 1692 * @index: doorbell index 1693 * @v: value to write 1694 * 1695 * Writes @v to the doorbell aperture at the 1696 * requested doorbell index (CIK). 1697 */ 1698 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) 1699 { 1700 if (index < rdev->doorbell.num_doorbells) { 1701 writel(v, rdev->doorbell.ptr + index); 1702 } else { 1703 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 1704 } 1705 } 1706 1707 #define BONAIRE_IO_MC_REGS_SIZE 36 1708 1709 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = 1710 { 1711 {0x00000070, 0x04400000}, 1712 {0x00000071, 0x80c01803}, 1713 {0x00000072, 0x00004004}, 1714 {0x00000073, 0x00000100}, 1715 {0x00000074, 0x00ff0000}, 1716 {0x00000075, 0x34000000}, 1717 {0x00000076, 0x08000014}, 1718 {0x00000077, 0x00cc08ec}, 1719 {0x00000078, 0x00000400}, 1720 {0x00000079, 0x00000000}, 1721 {0x0000007a, 0x04090000}, 1722 {0x0000007c, 0x00000000}, 1723 {0x0000007e, 0x4408a8e8}, 1724 {0x0000007f, 0x00000304}, 1725 {0x00000080, 0x00000000}, 1726 {0x00000082, 0x00000001}, 1727 {0x00000083, 0x00000002}, 1728 {0x00000084, 0xf3e4f400}, 1729 {0x00000085, 0x052024e3}, 1730 {0x00000087, 0x00000000}, 1731 {0x00000088, 0x01000000}, 1732 {0x0000008a, 0x1c0a0000}, 1733 {0x0000008b, 0xff010000}, 1734 {0x0000008d, 0xffffefff}, 1735 {0x0000008e, 0xfff3efff}, 1736 {0x0000008f, 0xfff3efbf}, 1737 {0x00000092, 0xf7ffffff}, 1738 {0x00000093, 0xffffff7f}, 1739 {0x00000095, 0x00101101}, 1740 {0x00000096, 0x00000fff}, 1741 {0x00000097, 0x00116fff}, 1742 {0x00000098, 0x60010000}, 1743 {0x00000099, 0x10010000}, 1744 {0x0000009a, 0x00006000}, 1745 {0x0000009b, 0x00001000}, 1746 {0x0000009f, 0x00b48000} 1747 }; 1748 1749 #define HAWAII_IO_MC_REGS_SIZE 22 1750 1751 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = 1752 { 1753 {0x0000007d, 0x40000000}, 1754 {0x0000007e, 0x40180304}, 1755 {0x0000007f, 0x0000ff00}, 1756 {0x00000081, 0x00000000}, 1757 {0x00000083, 0x00000800}, 1758 {0x00000086, 0x00000000}, 1759 {0x00000087, 0x00000100}, 1760 {0x00000088, 0x00020100}, 1761 {0x00000089, 0x00000000}, 1762 {0x0000008b, 0x00040000}, 1763 {0x0000008c, 0x00000100}, 1764 {0x0000008e, 0xff010000}, 1765 {0x00000090, 0xffffefff}, 1766 {0x00000091, 0xfff3efff}, 1767 {0x00000092, 0xfff3efbf}, 1768 {0x00000093, 0xf7ffffff}, 1769 {0x00000094, 0xffffff7f}, 1770 {0x00000095, 0x00000fff}, 1771 {0x00000096, 0x00116fff}, 1772 {0x00000097, 0x60010000}, 1773 {0x00000098, 0x10010000}, 1774 {0x0000009f, 0x00c79000} 1775 }; 1776 1777 1778 /** 1779 * cik_srbm_select - select specific register instances 1780 * 1781 * @rdev: radeon_device pointer 1782 * @me: selected ME (micro engine) 1783 * @pipe: pipe 1784 * @queue: queue 1785 * @vmid: VMID 1786 * 1787 * Switches the currently active registers instances. Some 1788 * registers are instanced per VMID, others are instanced per 1789 * me/pipe/queue combination. 1790 */ 1791 static void cik_srbm_select(struct radeon_device *rdev, 1792 u32 me, u32 pipe, u32 queue, u32 vmid) 1793 { 1794 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | 1795 MEID(me & 0x3) | 1796 VMID(vmid & 0xf) | 1797 QUEUEID(queue & 0x7)); 1798 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); 1799 } 1800 1801 /* ucode loading */ 1802 /** 1803 * ci_mc_load_microcode - load MC ucode into the hw 1804 * 1805 * @rdev: radeon_device pointer 1806 * 1807 * Load the GDDR MC ucode into the hw (CIK). 1808 * Returns 0 on success, error on failure. 1809 */ 1810 int ci_mc_load_microcode(struct radeon_device *rdev) 1811 { 1812 const __be32 *fw_data = NULL; 1813 const __le32 *new_fw_data = NULL; 1814 u32 running, blackout = 0, tmp; 1815 u32 *io_mc_regs = NULL; 1816 const __le32 *new_io_mc_regs = NULL; 1817 int i, regs_size, ucode_size; 1818 1819 if (!rdev->mc_fw) 1820 return -EINVAL; 1821 1822 if (rdev->new_fw) { 1823 const struct mc_firmware_header_v1_0 *hdr = 1824 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; 1825 1826 radeon_ucode_print_mc_hdr(&hdr->header); 1827 1828 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); 1829 new_io_mc_regs = (const __le32 *) 1830 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); 1831 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1832 new_fw_data = (const __le32 *) 1833 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1834 } else { 1835 ucode_size = rdev->mc_fw->size / 4; 1836 1837 switch (rdev->family) { 1838 case CHIP_BONAIRE: 1839 io_mc_regs = (u32 *)&bonaire_io_mc_regs; 1840 regs_size = BONAIRE_IO_MC_REGS_SIZE; 1841 break; 1842 case CHIP_HAWAII: 1843 io_mc_regs = (u32 *)&hawaii_io_mc_regs; 1844 regs_size = HAWAII_IO_MC_REGS_SIZE; 1845 break; 1846 default: 1847 return -EINVAL; 1848 } 1849 fw_data = (const __be32 *)rdev->mc_fw->data; 1850 } 1851 1852 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1853 1854 if (running == 0) { 1855 if (running) { 1856 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1857 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1858 } 1859 1860 /* reset the engine and set to writable */ 1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1863 1864 /* load mc io regs */ 1865 for (i = 0; i < regs_size; i++) { 1866 if (rdev->new_fw) { 1867 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); 1868 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); 1869 } else { 1870 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1871 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1872 } 1873 } 1874 1875 tmp = RREG32(MC_SEQ_MISC0); 1876 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { 1877 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5); 1878 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023); 1879 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9); 1880 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0); 1881 } 1882 1883 /* load the MC ucode */ 1884 for (i = 0; i < ucode_size; i++) { 1885 if (rdev->new_fw) 1886 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); 1887 else 1888 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1889 } 1890 1891 /* put the engine back into the active state */ 1892 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1893 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1894 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1895 1896 /* wait for training to complete */ 1897 for (i = 0; i < rdev->usec_timeout; i++) { 1898 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1899 break; 1900 udelay(1); 1901 } 1902 for (i = 0; i < rdev->usec_timeout; i++) { 1903 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1904 break; 1905 udelay(1); 1906 } 1907 1908 if (running) 1909 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1910 } 1911 1912 return 0; 1913 } 1914 1915 /** 1916 * cik_init_microcode - load ucode images from disk 1917 * 1918 * @rdev: radeon_device pointer 1919 * 1920 * Use the firmware interface to load the ucode images into 1921 * the driver (not loaded into hw). 1922 * Returns 0 on success, error on failure. 1923 */ 1924 static int cik_init_microcode(struct radeon_device *rdev) 1925 { 1926 const char *chip_name; 1927 const char *new_chip_name; 1928 size_t pfp_req_size, me_req_size, ce_req_size, 1929 mec_req_size, rlc_req_size, mc_req_size = 0, 1930 sdma_req_size, smc_req_size = 0, mc2_req_size = 0; 1931 char fw_name[30]; 1932 int new_fw = 0; 1933 int err; 1934 int num_fw; 1935 1936 DRM_DEBUG("\n"); 1937 1938 switch (rdev->family) { 1939 case CHIP_BONAIRE: 1940 chip_name = "BONAIRE"; 1941 new_chip_name = "bonaire"; 1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1943 me_req_size = CIK_ME_UCODE_SIZE * 4; 1944 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1947 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4; 1948 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4; 1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1950 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4); 1951 num_fw = 8; 1952 break; 1953 case CHIP_HAWAII: 1954 chip_name = "HAWAII"; 1955 new_chip_name = "hawaii"; 1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1957 me_req_size = CIK_ME_UCODE_SIZE * 4; 1958 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1960 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1961 mc_req_size = HAWAII_MC_UCODE_SIZE * 4; 1962 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4; 1963 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1964 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4); 1965 num_fw = 8; 1966 break; 1967 case CHIP_KAVERI: 1968 chip_name = "KAVERI"; 1969 new_chip_name = "kaveri"; 1970 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1971 me_req_size = CIK_ME_UCODE_SIZE * 4; 1972 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1973 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1974 rlc_req_size = KV_RLC_UCODE_SIZE * 4; 1975 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1976 num_fw = 7; 1977 break; 1978 case CHIP_KABINI: 1979 chip_name = "KABINI"; 1980 new_chip_name = "kabini"; 1981 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1982 me_req_size = CIK_ME_UCODE_SIZE * 4; 1983 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1984 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1985 rlc_req_size = KB_RLC_UCODE_SIZE * 4; 1986 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1987 num_fw = 6; 1988 break; 1989 case CHIP_MULLINS: 1990 chip_name = "MULLINS"; 1991 new_chip_name = "mullins"; 1992 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1993 me_req_size = CIK_ME_UCODE_SIZE * 4; 1994 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1995 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1996 rlc_req_size = ML_RLC_UCODE_SIZE * 4; 1997 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1998 num_fw = 6; 1999 break; 2000 default: BUG(); 2001 } 2002 2003 DRM_INFO("Loading %s Microcode\n", new_chip_name); 2004 2005 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name); 2006 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2007 if (err) { 2008 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 2009 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2010 if (err) 2011 goto out; 2012 if (rdev->pfp_fw->size != pfp_req_size) { 2013 printk(KERN_ERR 2014 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2015 rdev->pfp_fw->size, fw_name); 2016 err = -EINVAL; 2017 goto out; 2018 } 2019 } else { 2020 err = radeon_ucode_validate(rdev->pfp_fw); 2021 if (err) { 2022 printk(KERN_ERR 2023 "cik_fw: validation failed for firmware \"%s\"\n", 2024 fw_name); 2025 goto out; 2026 } else { 2027 new_fw++; 2028 } 2029 } 2030 2031 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name); 2032 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2033 if (err) { 2034 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 2035 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2036 if (err) 2037 goto out; 2038 if (rdev->me_fw->size != me_req_size) { 2039 printk(KERN_ERR 2040 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2041 rdev->me_fw->size, fw_name); 2042 err = -EINVAL; 2043 } 2044 } else { 2045 err = radeon_ucode_validate(rdev->me_fw); 2046 if (err) { 2047 printk(KERN_ERR 2048 "cik_fw: validation failed for firmware \"%s\"\n", 2049 fw_name); 2050 goto out; 2051 } else { 2052 new_fw++; 2053 } 2054 } 2055 2056 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name); 2057 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2058 if (err) { 2059 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 2060 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2061 if (err) 2062 goto out; 2063 if (rdev->ce_fw->size != ce_req_size) { 2064 printk(KERN_ERR 2065 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2066 rdev->ce_fw->size, fw_name); 2067 err = -EINVAL; 2068 } 2069 } else { 2070 err = radeon_ucode_validate(rdev->ce_fw); 2071 if (err) { 2072 printk(KERN_ERR 2073 "cik_fw: validation failed for firmware \"%s\"\n", 2074 fw_name); 2075 goto out; 2076 } else { 2077 new_fw++; 2078 } 2079 } 2080 2081 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name); 2082 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2083 if (err) { 2084 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name); 2085 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2086 if (err) 2087 goto out; 2088 if (rdev->mec_fw->size != mec_req_size) { 2089 printk(KERN_ERR 2090 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2091 rdev->mec_fw->size, fw_name); 2092 err = -EINVAL; 2093 } 2094 } else { 2095 err = radeon_ucode_validate(rdev->mec_fw); 2096 if (err) { 2097 printk(KERN_ERR 2098 "cik_fw: validation failed for firmware \"%s\"\n", 2099 fw_name); 2100 goto out; 2101 } else { 2102 new_fw++; 2103 } 2104 } 2105 2106 if (rdev->family == CHIP_KAVERI) { 2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name); 2108 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); 2109 if (err) { 2110 goto out; 2111 } else { 2112 err = radeon_ucode_validate(rdev->mec2_fw); 2113 if (err) { 2114 goto out; 2115 } else { 2116 new_fw++; 2117 } 2118 } 2119 } 2120 2121 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name); 2122 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2123 if (err) { 2124 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); 2125 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2126 if (err) 2127 goto out; 2128 if (rdev->rlc_fw->size != rlc_req_size) { 2129 printk(KERN_ERR 2130 "cik_rlc: Bogus length %zu in firmware \"%s\"\n", 2131 rdev->rlc_fw->size, fw_name); 2132 err = -EINVAL; 2133 } 2134 } else { 2135 err = radeon_ucode_validate(rdev->rlc_fw); 2136 if (err) { 2137 printk(KERN_ERR 2138 "cik_fw: validation failed for firmware \"%s\"\n", 2139 fw_name); 2140 goto out; 2141 } else { 2142 new_fw++; 2143 } 2144 } 2145 2146 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name); 2147 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2148 if (err) { 2149 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name); 2150 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2151 if (err) 2152 goto out; 2153 if (rdev->sdma_fw->size != sdma_req_size) { 2154 printk(KERN_ERR 2155 "cik_sdma: Bogus length %zu in firmware \"%s\"\n", 2156 rdev->sdma_fw->size, fw_name); 2157 err = -EINVAL; 2158 } 2159 } else { 2160 err = radeon_ucode_validate(rdev->sdma_fw); 2161 if (err) { 2162 printk(KERN_ERR 2163 "cik_fw: validation failed for firmware \"%s\"\n", 2164 fw_name); 2165 goto out; 2166 } else { 2167 new_fw++; 2168 } 2169 } 2170 2171 /* No SMC, MC ucode on APUs */ 2172 if (!(rdev->flags & RADEON_IS_IGP)) { 2173 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name); 2174 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2175 if (err) { 2176 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name); 2177 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2178 if (err) { 2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 2180 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2181 if (err) 2182 goto out; 2183 } 2184 if ((rdev->mc_fw->size != mc_req_size) && 2185 (rdev->mc_fw->size != mc2_req_size)){ 2186 printk(KERN_ERR 2187 "cik_mc: Bogus length %zu in firmware \"%s\"\n", 2188 rdev->mc_fw->size, fw_name); 2189 err = -EINVAL; 2190 } 2191 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); 2192 } else { 2193 err = radeon_ucode_validate(rdev->mc_fw); 2194 if (err) { 2195 printk(KERN_ERR 2196 "cik_fw: validation failed for firmware \"%s\"\n", 2197 fw_name); 2198 goto out; 2199 } else { 2200 new_fw++; 2201 } 2202 } 2203 2204 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name); 2205 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2206 if (err) { 2207 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 2208 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2209 if (err) { 2210 printk(KERN_ERR 2211 "smc: error loading firmware \"%s\"\n", 2212 fw_name); 2213 release_firmware(rdev->smc_fw); 2214 rdev->smc_fw = NULL; 2215 err = 0; 2216 } else if (rdev->smc_fw->size != smc_req_size) { 2217 printk(KERN_ERR 2218 "cik_smc: Bogus length %zu in firmware \"%s\"\n", 2219 rdev->smc_fw->size, fw_name); 2220 err = -EINVAL; 2221 } 2222 } else { 2223 err = radeon_ucode_validate(rdev->smc_fw); 2224 if (err) { 2225 printk(KERN_ERR 2226 "cik_fw: validation failed for firmware \"%s\"\n", 2227 fw_name); 2228 goto out; 2229 } else { 2230 new_fw++; 2231 } 2232 } 2233 } 2234 2235 if (new_fw == 0) { 2236 rdev->new_fw = false; 2237 } else if (new_fw < num_fw) { 2238 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n"); 2239 err = -EINVAL; 2240 } else { 2241 rdev->new_fw = true; 2242 } 2243 2244 out: 2245 if (err) { 2246 if (err != -EINVAL) 2247 printk(KERN_ERR 2248 "cik_cp: Failed to load firmware \"%s\"\n", 2249 fw_name); 2250 release_firmware(rdev->pfp_fw); 2251 rdev->pfp_fw = NULL; 2252 release_firmware(rdev->me_fw); 2253 rdev->me_fw = NULL; 2254 release_firmware(rdev->ce_fw); 2255 rdev->ce_fw = NULL; 2256 release_firmware(rdev->mec_fw); 2257 rdev->mec_fw = NULL; 2258 release_firmware(rdev->mec2_fw); 2259 rdev->mec2_fw = NULL; 2260 release_firmware(rdev->rlc_fw); 2261 rdev->rlc_fw = NULL; 2262 release_firmware(rdev->sdma_fw); 2263 rdev->sdma_fw = NULL; 2264 release_firmware(rdev->mc_fw); 2265 rdev->mc_fw = NULL; 2266 release_firmware(rdev->smc_fw); 2267 rdev->smc_fw = NULL; 2268 } 2269 return err; 2270 } 2271 2272 /* 2273 * Core functions 2274 */ 2275 /** 2276 * cik_tiling_mode_table_init - init the hw tiling table 2277 * 2278 * @rdev: radeon_device pointer 2279 * 2280 * Starting with SI, the tiling setup is done globally in a 2281 * set of 32 tiling modes. Rather than selecting each set of 2282 * parameters per surface as on older asics, we just select 2283 * which index in the tiling table we want to use, and the 2284 * surface uses those parameters (CIK). 2285 */ 2286 static void cik_tiling_mode_table_init(struct radeon_device *rdev) 2287 { 2288 const u32 num_tile_mode_states = 32; 2289 const u32 num_secondary_tile_mode_states = 16; 2290 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2291 u32 num_pipe_configs; 2292 u32 num_rbs = rdev->config.cik.max_backends_per_se * 2293 rdev->config.cik.max_shader_engines; 2294 2295 switch (rdev->config.cik.mem_row_size_in_kb) { 2296 case 1: 2297 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2298 break; 2299 case 2: 2300 default: 2301 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2302 break; 2303 case 4: 2304 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2305 break; 2306 } 2307 2308 num_pipe_configs = rdev->config.cik.max_tile_pipes; 2309 if (num_pipe_configs > 8) 2310 num_pipe_configs = 16; 2311 2312 if (num_pipe_configs == 16) { 2313 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2314 switch (reg_offset) { 2315 case 0: 2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2317 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2320 break; 2321 case 1: 2322 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2323 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2324 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2325 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2326 break; 2327 case 2: 2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2329 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2332 break; 2333 case 3: 2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2335 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2336 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2338 break; 2339 case 4: 2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2343 TILE_SPLIT(split_equal_to_row_size)); 2344 break; 2345 case 5: 2346 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2347 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2348 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2349 break; 2350 case 6: 2351 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2355 break; 2356 case 7: 2357 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2360 TILE_SPLIT(split_equal_to_row_size)); 2361 break; 2362 case 8: 2363 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2365 break; 2366 case 9: 2367 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2370 break; 2371 case 10: 2372 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2373 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2374 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2376 break; 2377 case 11: 2378 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2381 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2382 break; 2383 case 12: 2384 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2385 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2387 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2388 break; 2389 case 13: 2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2392 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2393 break; 2394 case 14: 2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2396 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2397 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2398 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2399 break; 2400 case 16: 2401 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2402 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2404 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2405 break; 2406 case 17: 2407 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2408 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2410 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2411 break; 2412 case 27: 2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2416 break; 2417 case 28: 2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2419 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2421 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2422 break; 2423 case 29: 2424 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2425 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2428 break; 2429 case 30: 2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2434 break; 2435 default: 2436 gb_tile_moden = 0; 2437 break; 2438 } 2439 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2440 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2441 } 2442 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2443 switch (reg_offset) { 2444 case 0: 2445 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2448 NUM_BANKS(ADDR_SURF_16_BANK)); 2449 break; 2450 case 1: 2451 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2454 NUM_BANKS(ADDR_SURF_16_BANK)); 2455 break; 2456 case 2: 2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2460 NUM_BANKS(ADDR_SURF_16_BANK)); 2461 break; 2462 case 3: 2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2466 NUM_BANKS(ADDR_SURF_16_BANK)); 2467 break; 2468 case 4: 2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2472 NUM_BANKS(ADDR_SURF_8_BANK)); 2473 break; 2474 case 5: 2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2478 NUM_BANKS(ADDR_SURF_4_BANK)); 2479 break; 2480 case 6: 2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2484 NUM_BANKS(ADDR_SURF_2_BANK)); 2485 break; 2486 case 8: 2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2490 NUM_BANKS(ADDR_SURF_16_BANK)); 2491 break; 2492 case 9: 2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2496 NUM_BANKS(ADDR_SURF_16_BANK)); 2497 break; 2498 case 10: 2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2502 NUM_BANKS(ADDR_SURF_16_BANK)); 2503 break; 2504 case 11: 2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2508 NUM_BANKS(ADDR_SURF_8_BANK)); 2509 break; 2510 case 12: 2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2514 NUM_BANKS(ADDR_SURF_4_BANK)); 2515 break; 2516 case 13: 2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2520 NUM_BANKS(ADDR_SURF_2_BANK)); 2521 break; 2522 case 14: 2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2526 NUM_BANKS(ADDR_SURF_2_BANK)); 2527 break; 2528 default: 2529 gb_tile_moden = 0; 2530 break; 2531 } 2532 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2533 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2534 } 2535 } else if (num_pipe_configs == 8) { 2536 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2537 switch (reg_offset) { 2538 case 0: 2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2543 break; 2544 case 1: 2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2549 break; 2550 case 2: 2551 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2552 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2555 break; 2556 case 3: 2557 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2560 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2561 break; 2562 case 4: 2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2566 TILE_SPLIT(split_equal_to_row_size)); 2567 break; 2568 case 5: 2569 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2572 break; 2573 case 6: 2574 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2575 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2577 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2578 break; 2579 case 7: 2580 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2583 TILE_SPLIT(split_equal_to_row_size)); 2584 break; 2585 case 8: 2586 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2587 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2588 break; 2589 case 9: 2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2592 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2593 break; 2594 case 10: 2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2599 break; 2600 case 11: 2601 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2604 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2605 break; 2606 case 12: 2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2611 break; 2612 case 13: 2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2615 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2616 break; 2617 case 14: 2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2619 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2620 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2622 break; 2623 case 16: 2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2625 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2626 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2628 break; 2629 case 17: 2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2631 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2632 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2634 break; 2635 case 27: 2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2637 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2638 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2639 break; 2640 case 28: 2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2642 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2643 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2645 break; 2646 case 29: 2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2648 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2649 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2651 break; 2652 case 30: 2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2654 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2655 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2657 break; 2658 default: 2659 gb_tile_moden = 0; 2660 break; 2661 } 2662 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2664 } 2665 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2666 switch (reg_offset) { 2667 case 0: 2668 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2671 NUM_BANKS(ADDR_SURF_16_BANK)); 2672 break; 2673 case 1: 2674 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2677 NUM_BANKS(ADDR_SURF_16_BANK)); 2678 break; 2679 case 2: 2680 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2683 NUM_BANKS(ADDR_SURF_16_BANK)); 2684 break; 2685 case 3: 2686 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2689 NUM_BANKS(ADDR_SURF_16_BANK)); 2690 break; 2691 case 4: 2692 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2695 NUM_BANKS(ADDR_SURF_8_BANK)); 2696 break; 2697 case 5: 2698 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2701 NUM_BANKS(ADDR_SURF_4_BANK)); 2702 break; 2703 case 6: 2704 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2705 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2706 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2707 NUM_BANKS(ADDR_SURF_2_BANK)); 2708 break; 2709 case 8: 2710 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2713 NUM_BANKS(ADDR_SURF_16_BANK)); 2714 break; 2715 case 9: 2716 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2719 NUM_BANKS(ADDR_SURF_16_BANK)); 2720 break; 2721 case 10: 2722 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2723 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2724 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2725 NUM_BANKS(ADDR_SURF_16_BANK)); 2726 break; 2727 case 11: 2728 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2731 NUM_BANKS(ADDR_SURF_16_BANK)); 2732 break; 2733 case 12: 2734 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2737 NUM_BANKS(ADDR_SURF_8_BANK)); 2738 break; 2739 case 13: 2740 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2741 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2742 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2743 NUM_BANKS(ADDR_SURF_4_BANK)); 2744 break; 2745 case 14: 2746 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2749 NUM_BANKS(ADDR_SURF_2_BANK)); 2750 break; 2751 default: 2752 gb_tile_moden = 0; 2753 break; 2754 } 2755 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2756 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2757 } 2758 } else if (num_pipe_configs == 4) { 2759 if (num_rbs == 4) { 2760 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2761 switch (reg_offset) { 2762 case 0: 2763 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2765 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2766 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2767 break; 2768 case 1: 2769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2771 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2773 break; 2774 case 2: 2775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2777 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2779 break; 2780 case 3: 2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2785 break; 2786 case 4: 2787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2790 TILE_SPLIT(split_equal_to_row_size)); 2791 break; 2792 case 5: 2793 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2794 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2796 break; 2797 case 6: 2798 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2799 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2800 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2802 break; 2803 case 7: 2804 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2806 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2807 TILE_SPLIT(split_equal_to_row_size)); 2808 break; 2809 case 8: 2810 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2811 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2812 break; 2813 case 9: 2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2815 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2817 break; 2818 case 10: 2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2821 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2823 break; 2824 case 11: 2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2827 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2829 break; 2830 case 12: 2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2835 break; 2836 case 13: 2837 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2838 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2839 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2840 break; 2841 case 14: 2842 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2843 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2844 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2845 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2846 break; 2847 case 16: 2848 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2849 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2850 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2852 break; 2853 case 17: 2854 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2855 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2856 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2857 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2858 break; 2859 case 27: 2860 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2861 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2862 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2863 break; 2864 case 28: 2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2866 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2867 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2869 break; 2870 case 29: 2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2872 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2873 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2875 break; 2876 case 30: 2877 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2881 break; 2882 default: 2883 gb_tile_moden = 0; 2884 break; 2885 } 2886 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2887 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2888 } 2889 } else if (num_rbs < 4) { 2890 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2891 switch (reg_offset) { 2892 case 0: 2893 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2895 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2896 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2897 break; 2898 case 1: 2899 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2900 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2901 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2903 break; 2904 case 2: 2905 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2906 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2907 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2909 break; 2910 case 3: 2911 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2912 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2913 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2914 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2915 break; 2916 case 4: 2917 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2920 TILE_SPLIT(split_equal_to_row_size)); 2921 break; 2922 case 5: 2923 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2924 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2926 break; 2927 case 6: 2928 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2929 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2930 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2931 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2932 break; 2933 case 7: 2934 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2935 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2936 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2937 TILE_SPLIT(split_equal_to_row_size)); 2938 break; 2939 case 8: 2940 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2941 PIPE_CONFIG(ADDR_SURF_P4_8x16)); 2942 break; 2943 case 9: 2944 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2945 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2946 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2947 break; 2948 case 10: 2949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2950 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2951 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2953 break; 2954 case 11: 2955 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2956 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2957 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2959 break; 2960 case 12: 2961 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2962 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2964 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2965 break; 2966 case 13: 2967 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2968 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2969 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2970 break; 2971 case 14: 2972 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2973 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2974 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2975 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2976 break; 2977 case 16: 2978 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2979 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2980 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2982 break; 2983 case 17: 2984 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2985 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2986 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2987 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2988 break; 2989 case 27: 2990 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2991 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2992 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2993 break; 2994 case 28: 2995 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2996 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2997 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2998 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2999 break; 3000 case 29: 3001 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3002 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3003 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 3004 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3005 break; 3006 case 30: 3007 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3008 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3009 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 3010 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3011 break; 3012 default: 3013 gb_tile_moden = 0; 3014 break; 3015 } 3016 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3018 } 3019 } 3020 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 3021 switch (reg_offset) { 3022 case 0: 3023 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3026 NUM_BANKS(ADDR_SURF_16_BANK)); 3027 break; 3028 case 1: 3029 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3030 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3031 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3032 NUM_BANKS(ADDR_SURF_16_BANK)); 3033 break; 3034 case 2: 3035 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3038 NUM_BANKS(ADDR_SURF_16_BANK)); 3039 break; 3040 case 3: 3041 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3042 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3043 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3044 NUM_BANKS(ADDR_SURF_16_BANK)); 3045 break; 3046 case 4: 3047 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3048 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3049 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3050 NUM_BANKS(ADDR_SURF_16_BANK)); 3051 break; 3052 case 5: 3053 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3054 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3055 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3056 NUM_BANKS(ADDR_SURF_8_BANK)); 3057 break; 3058 case 6: 3059 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3060 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3061 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3062 NUM_BANKS(ADDR_SURF_4_BANK)); 3063 break; 3064 case 8: 3065 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3066 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3067 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3068 NUM_BANKS(ADDR_SURF_16_BANK)); 3069 break; 3070 case 9: 3071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3074 NUM_BANKS(ADDR_SURF_16_BANK)); 3075 break; 3076 case 10: 3077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3080 NUM_BANKS(ADDR_SURF_16_BANK)); 3081 break; 3082 case 11: 3083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3086 NUM_BANKS(ADDR_SURF_16_BANK)); 3087 break; 3088 case 12: 3089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3092 NUM_BANKS(ADDR_SURF_16_BANK)); 3093 break; 3094 case 13: 3095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3098 NUM_BANKS(ADDR_SURF_8_BANK)); 3099 break; 3100 case 14: 3101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3104 NUM_BANKS(ADDR_SURF_4_BANK)); 3105 break; 3106 default: 3107 gb_tile_moden = 0; 3108 break; 3109 } 3110 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3111 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3112 } 3113 } else if (num_pipe_configs == 2) { 3114 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 3115 switch (reg_offset) { 3116 case 0: 3117 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3119 PIPE_CONFIG(ADDR_SURF_P2) | 3120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 3121 break; 3122 case 1: 3123 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3125 PIPE_CONFIG(ADDR_SURF_P2) | 3126 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 3127 break; 3128 case 2: 3129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3130 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3131 PIPE_CONFIG(ADDR_SURF_P2) | 3132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3133 break; 3134 case 3: 3135 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3137 PIPE_CONFIG(ADDR_SURF_P2) | 3138 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 3139 break; 3140 case 4: 3141 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3142 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3143 PIPE_CONFIG(ADDR_SURF_P2) | 3144 TILE_SPLIT(split_equal_to_row_size)); 3145 break; 3146 case 5: 3147 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3148 PIPE_CONFIG(ADDR_SURF_P2) | 3149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3150 break; 3151 case 6: 3152 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3153 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3154 PIPE_CONFIG(ADDR_SURF_P2) | 3155 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3156 break; 3157 case 7: 3158 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3159 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3160 PIPE_CONFIG(ADDR_SURF_P2) | 3161 TILE_SPLIT(split_equal_to_row_size)); 3162 break; 3163 case 8: 3164 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3165 PIPE_CONFIG(ADDR_SURF_P2); 3166 break; 3167 case 9: 3168 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3170 PIPE_CONFIG(ADDR_SURF_P2)); 3171 break; 3172 case 10: 3173 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3174 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3175 PIPE_CONFIG(ADDR_SURF_P2) | 3176 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3177 break; 3178 case 11: 3179 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3180 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3181 PIPE_CONFIG(ADDR_SURF_P2) | 3182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3183 break; 3184 case 12: 3185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3186 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3187 PIPE_CONFIG(ADDR_SURF_P2) | 3188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3189 break; 3190 case 13: 3191 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3192 PIPE_CONFIG(ADDR_SURF_P2) | 3193 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 3194 break; 3195 case 14: 3196 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3198 PIPE_CONFIG(ADDR_SURF_P2) | 3199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3200 break; 3201 case 16: 3202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3204 PIPE_CONFIG(ADDR_SURF_P2) | 3205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3206 break; 3207 case 17: 3208 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3209 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3210 PIPE_CONFIG(ADDR_SURF_P2) | 3211 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3212 break; 3213 case 27: 3214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3215 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3216 PIPE_CONFIG(ADDR_SURF_P2)); 3217 break; 3218 case 28: 3219 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3220 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3221 PIPE_CONFIG(ADDR_SURF_P2) | 3222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3223 break; 3224 case 29: 3225 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3226 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3227 PIPE_CONFIG(ADDR_SURF_P2) | 3228 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3229 break; 3230 case 30: 3231 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3232 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3233 PIPE_CONFIG(ADDR_SURF_P2) | 3234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3235 break; 3236 default: 3237 gb_tile_moden = 0; 3238 break; 3239 } 3240 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 3241 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3242 } 3243 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 3244 switch (reg_offset) { 3245 case 0: 3246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3249 NUM_BANKS(ADDR_SURF_16_BANK)); 3250 break; 3251 case 1: 3252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3255 NUM_BANKS(ADDR_SURF_16_BANK)); 3256 break; 3257 case 2: 3258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3261 NUM_BANKS(ADDR_SURF_16_BANK)); 3262 break; 3263 case 3: 3264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3267 NUM_BANKS(ADDR_SURF_16_BANK)); 3268 break; 3269 case 4: 3270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3273 NUM_BANKS(ADDR_SURF_16_BANK)); 3274 break; 3275 case 5: 3276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3279 NUM_BANKS(ADDR_SURF_16_BANK)); 3280 break; 3281 case 6: 3282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3285 NUM_BANKS(ADDR_SURF_8_BANK)); 3286 break; 3287 case 8: 3288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3291 NUM_BANKS(ADDR_SURF_16_BANK)); 3292 break; 3293 case 9: 3294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3297 NUM_BANKS(ADDR_SURF_16_BANK)); 3298 break; 3299 case 10: 3300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3303 NUM_BANKS(ADDR_SURF_16_BANK)); 3304 break; 3305 case 11: 3306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3309 NUM_BANKS(ADDR_SURF_16_BANK)); 3310 break; 3311 case 12: 3312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3315 NUM_BANKS(ADDR_SURF_16_BANK)); 3316 break; 3317 case 13: 3318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3321 NUM_BANKS(ADDR_SURF_16_BANK)); 3322 break; 3323 case 14: 3324 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3327 NUM_BANKS(ADDR_SURF_8_BANK)); 3328 break; 3329 default: 3330 gb_tile_moden = 0; 3331 break; 3332 } 3333 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3334 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3335 } 3336 } else 3337 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); 3338 } 3339 3340 /** 3341 * cik_select_se_sh - select which SE, SH to address 3342 * 3343 * @rdev: radeon_device pointer 3344 * @se_num: shader engine to address 3345 * @sh_num: sh block to address 3346 * 3347 * Select which SE, SH combinations to address. Certain 3348 * registers are instanced per SE or SH. 0xffffffff means 3349 * broadcast to all SEs or SHs (CIK). 3350 */ 3351 static void cik_select_se_sh(struct radeon_device *rdev, 3352 u32 se_num, u32 sh_num) 3353 { 3354 u32 data = INSTANCE_BROADCAST_WRITES; 3355 3356 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 3357 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 3358 else if (se_num == 0xffffffff) 3359 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 3360 else if (sh_num == 0xffffffff) 3361 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 3362 else 3363 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 3364 WREG32(GRBM_GFX_INDEX, data); 3365 } 3366 3367 /** 3368 * cik_create_bitmask - create a bitmask 3369 * 3370 * @bit_width: length of the mask 3371 * 3372 * create a variable length bit mask (CIK). 3373 * Returns the bitmask. 3374 */ 3375 static u32 cik_create_bitmask(u32 bit_width) 3376 { 3377 u32 i, mask = 0; 3378 3379 for (i = 0; i < bit_width; i++) { 3380 mask <<= 1; 3381 mask |= 1; 3382 } 3383 return mask; 3384 } 3385 3386 /** 3387 * cik_get_rb_disabled - computes the mask of disabled RBs 3388 * 3389 * @rdev: radeon_device pointer 3390 * @max_rb_num: max RBs (render backends) for the asic 3391 * @se_num: number of SEs (shader engines) for the asic 3392 * @sh_per_se: number of SH blocks per SE for the asic 3393 * 3394 * Calculates the bitmask of disabled RBs (CIK). 3395 * Returns the disabled RB bitmask. 3396 */ 3397 static u32 cik_get_rb_disabled(struct radeon_device *rdev, 3398 u32 max_rb_num_per_se, 3399 u32 sh_per_se) 3400 { 3401 u32 data, mask; 3402 3403 data = RREG32(CC_RB_BACKEND_DISABLE); 3404 if (data & 1) 3405 data &= BACKEND_DISABLE_MASK; 3406 else 3407 data = 0; 3408 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 3409 3410 data >>= BACKEND_DISABLE_SHIFT; 3411 3412 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); 3413 3414 return data & mask; 3415 } 3416 3417 /** 3418 * cik_setup_rb - setup the RBs on the asic 3419 * 3420 * @rdev: radeon_device pointer 3421 * @se_num: number of SEs (shader engines) for the asic 3422 * @sh_per_se: number of SH blocks per SE for the asic 3423 * @max_rb_num: max RBs (render backends) for the asic 3424 * 3425 * Configures per-SE/SH RB registers (CIK). 3426 */ 3427 static void cik_setup_rb(struct radeon_device *rdev, 3428 u32 se_num, u32 sh_per_se, 3429 u32 max_rb_num_per_se) 3430 { 3431 int i, j; 3432 u32 data, mask; 3433 u32 disabled_rbs = 0; 3434 u32 enabled_rbs = 0; 3435 3436 mutex_lock(&rdev->grbm_idx_mutex); 3437 for (i = 0; i < se_num; i++) { 3438 for (j = 0; j < sh_per_se; j++) { 3439 cik_select_se_sh(rdev, i, j); 3440 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3441 if (rdev->family == CHIP_HAWAII) 3442 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); 3443 else 3444 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); 3445 } 3446 } 3447 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3448 mutex_unlock(&rdev->grbm_idx_mutex); 3449 3450 mask = 1; 3451 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3452 if (!(disabled_rbs & mask)) 3453 enabled_rbs |= mask; 3454 mask <<= 1; 3455 } 3456 3457 rdev->config.cik.backend_enable_mask = enabled_rbs; 3458 3459 mutex_lock(&rdev->grbm_idx_mutex); 3460 for (i = 0; i < se_num; i++) { 3461 cik_select_se_sh(rdev, i, 0xffffffff); 3462 data = 0; 3463 for (j = 0; j < sh_per_se; j++) { 3464 switch (enabled_rbs & 3) { 3465 case 0: 3466 if (j == 0) 3467 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); 3468 else 3469 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); 3470 break; 3471 case 1: 3472 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3473 break; 3474 case 2: 3475 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3476 break; 3477 case 3: 3478 default: 3479 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3480 break; 3481 } 3482 enabled_rbs >>= 2; 3483 } 3484 WREG32(PA_SC_RASTER_CONFIG, data); 3485 } 3486 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3487 mutex_unlock(&rdev->grbm_idx_mutex); 3488 } 3489 3490 /** 3491 * cik_gpu_init - setup the 3D engine 3492 * 3493 * @rdev: radeon_device pointer 3494 * 3495 * Configures the 3D engine and tiling configuration 3496 * registers so that the 3D engine is usable. 3497 */ 3498 static void cik_gpu_init(struct radeon_device *rdev) 3499 { 3500 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); 3501 u32 mc_shared_chmap, mc_arb_ramcfg; 3502 u32 hdp_host_path_cntl; 3503 u32 tmp; 3504 int i, j; 3505 3506 switch (rdev->family) { 3507 case CHIP_BONAIRE: 3508 rdev->config.cik.max_shader_engines = 2; 3509 rdev->config.cik.max_tile_pipes = 4; 3510 rdev->config.cik.max_cu_per_sh = 7; 3511 rdev->config.cik.max_sh_per_se = 1; 3512 rdev->config.cik.max_backends_per_se = 2; 3513 rdev->config.cik.max_texture_channel_caches = 4; 3514 rdev->config.cik.max_gprs = 256; 3515 rdev->config.cik.max_gs_threads = 32; 3516 rdev->config.cik.max_hw_contexts = 8; 3517 3518 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3519 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3520 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3521 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3522 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3523 break; 3524 case CHIP_HAWAII: 3525 rdev->config.cik.max_shader_engines = 4; 3526 rdev->config.cik.max_tile_pipes = 16; 3527 rdev->config.cik.max_cu_per_sh = 11; 3528 rdev->config.cik.max_sh_per_se = 1; 3529 rdev->config.cik.max_backends_per_se = 4; 3530 rdev->config.cik.max_texture_channel_caches = 16; 3531 rdev->config.cik.max_gprs = 256; 3532 rdev->config.cik.max_gs_threads = 32; 3533 rdev->config.cik.max_hw_contexts = 8; 3534 3535 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3536 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3537 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3538 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3539 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; 3540 break; 3541 case CHIP_KAVERI: 3542 rdev->config.cik.max_shader_engines = 1; 3543 rdev->config.cik.max_tile_pipes = 4; 3544 if ((rdev->pdev->device == 0x1304) || 3545 (rdev->pdev->device == 0x1305) || 3546 (rdev->pdev->device == 0x130C) || 3547 (rdev->pdev->device == 0x130F) || 3548 (rdev->pdev->device == 0x1310) || 3549 (rdev->pdev->device == 0x1311) || 3550 (rdev->pdev->device == 0x131C)) { 3551 rdev->config.cik.max_cu_per_sh = 8; 3552 rdev->config.cik.max_backends_per_se = 2; 3553 } else if ((rdev->pdev->device == 0x1309) || 3554 (rdev->pdev->device == 0x130A) || 3555 (rdev->pdev->device == 0x130D) || 3556 (rdev->pdev->device == 0x1313) || 3557 (rdev->pdev->device == 0x131D)) { 3558 rdev->config.cik.max_cu_per_sh = 6; 3559 rdev->config.cik.max_backends_per_se = 2; 3560 } else if ((rdev->pdev->device == 0x1306) || 3561 (rdev->pdev->device == 0x1307) || 3562 (rdev->pdev->device == 0x130B) || 3563 (rdev->pdev->device == 0x130E) || 3564 (rdev->pdev->device == 0x1315) || 3565 (rdev->pdev->device == 0x1318) || 3566 (rdev->pdev->device == 0x131B)) { 3567 rdev->config.cik.max_cu_per_sh = 4; 3568 rdev->config.cik.max_backends_per_se = 1; 3569 } else { 3570 rdev->config.cik.max_cu_per_sh = 3; 3571 rdev->config.cik.max_backends_per_se = 1; 3572 } 3573 rdev->config.cik.max_sh_per_se = 1; 3574 rdev->config.cik.max_texture_channel_caches = 4; 3575 rdev->config.cik.max_gprs = 256; 3576 rdev->config.cik.max_gs_threads = 16; 3577 rdev->config.cik.max_hw_contexts = 8; 3578 3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3584 break; 3585 case CHIP_KABINI: 3586 case CHIP_MULLINS: 3587 default: 3588 rdev->config.cik.max_shader_engines = 1; 3589 rdev->config.cik.max_tile_pipes = 2; 3590 rdev->config.cik.max_cu_per_sh = 2; 3591 rdev->config.cik.max_sh_per_se = 1; 3592 rdev->config.cik.max_backends_per_se = 1; 3593 rdev->config.cik.max_texture_channel_caches = 2; 3594 rdev->config.cik.max_gprs = 256; 3595 rdev->config.cik.max_gs_threads = 16; 3596 rdev->config.cik.max_hw_contexts = 8; 3597 3598 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3599 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3600 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3601 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3602 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3603 break; 3604 } 3605 3606 /* Initialize HDP */ 3607 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3608 WREG32((0x2c14 + j), 0x00000000); 3609 WREG32((0x2c18 + j), 0x00000000); 3610 WREG32((0x2c1c + j), 0x00000000); 3611 WREG32((0x2c20 + j), 0x00000000); 3612 WREG32((0x2c24 + j), 0x00000000); 3613 } 3614 3615 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3616 3617 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3618 3619 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3620 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3621 3622 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; 3623 rdev->config.cik.mem_max_burst_length_bytes = 256; 3624 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3625 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3626 if (rdev->config.cik.mem_row_size_in_kb > 4) 3627 rdev->config.cik.mem_row_size_in_kb = 4; 3628 /* XXX use MC settings? */ 3629 rdev->config.cik.shader_engine_tile_size = 32; 3630 rdev->config.cik.num_gpus = 1; 3631 rdev->config.cik.multi_gpu_tile_size = 64; 3632 3633 /* fix up row size */ 3634 gb_addr_config &= ~ROW_SIZE_MASK; 3635 switch (rdev->config.cik.mem_row_size_in_kb) { 3636 case 1: 3637 default: 3638 gb_addr_config |= ROW_SIZE(0); 3639 break; 3640 case 2: 3641 gb_addr_config |= ROW_SIZE(1); 3642 break; 3643 case 4: 3644 gb_addr_config |= ROW_SIZE(2); 3645 break; 3646 } 3647 3648 /* setup tiling info dword. gb_addr_config is not adequate since it does 3649 * not have bank info, so create a custom tiling dword. 3650 * bits 3:0 num_pipes 3651 * bits 7:4 num_banks 3652 * bits 11:8 group_size 3653 * bits 15:12 row_size 3654 */ 3655 rdev->config.cik.tile_config = 0; 3656 switch (rdev->config.cik.num_tile_pipes) { 3657 case 1: 3658 rdev->config.cik.tile_config |= (0 << 0); 3659 break; 3660 case 2: 3661 rdev->config.cik.tile_config |= (1 << 0); 3662 break; 3663 case 4: 3664 rdev->config.cik.tile_config |= (2 << 0); 3665 break; 3666 case 8: 3667 default: 3668 /* XXX what about 12? */ 3669 rdev->config.cik.tile_config |= (3 << 0); 3670 break; 3671 } 3672 rdev->config.cik.tile_config |= 3673 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 3674 rdev->config.cik.tile_config |= 3675 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3676 rdev->config.cik.tile_config |= 3677 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3678 3679 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3680 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3681 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3682 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); 3683 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); 3684 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3685 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3686 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3687 3688 cik_tiling_mode_table_init(rdev); 3689 3690 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, 3691 rdev->config.cik.max_sh_per_se, 3692 rdev->config.cik.max_backends_per_se); 3693 3694 rdev->config.cik.active_cus = 0; 3695 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 3696 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 3697 rdev->config.cik.active_cus += 3698 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); 3699 } 3700 } 3701 3702 /* set HW defaults for 3D engine */ 3703 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3704 3705 mutex_lock(&rdev->grbm_idx_mutex); 3706 /* 3707 * making sure that the following register writes will be broadcasted 3708 * to all the shaders 3709 */ 3710 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3711 WREG32(SX_DEBUG_1, 0x20); 3712 3713 WREG32(TA_CNTL_AUX, 0x00010000); 3714 3715 tmp = RREG32(SPI_CONFIG_CNTL); 3716 tmp |= 0x03000000; 3717 WREG32(SPI_CONFIG_CNTL, tmp); 3718 3719 WREG32(SQ_CONFIG, 1); 3720 3721 WREG32(DB_DEBUG, 0); 3722 3723 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; 3724 tmp |= 0x00000400; 3725 WREG32(DB_DEBUG2, tmp); 3726 3727 tmp = RREG32(DB_DEBUG3) & ~0x0002021c; 3728 tmp |= 0x00020200; 3729 WREG32(DB_DEBUG3, tmp); 3730 3731 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; 3732 tmp |= 0x00018208; 3733 WREG32(CB_HW_CONTROL, tmp); 3734 3735 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3736 3737 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | 3738 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | 3739 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | 3740 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); 3741 3742 WREG32(VGT_NUM_INSTANCES, 1); 3743 3744 WREG32(CP_PERFMON_CNTL, 0); 3745 3746 WREG32(SQ_CONFIG, 0); 3747 3748 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3749 FORCE_EOV_MAX_REZ_CNT(255))); 3750 3751 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3752 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3753 3754 WREG32(VGT_GS_VERTEX_REUSE, 16); 3755 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3756 3757 tmp = RREG32(HDP_MISC_CNTL); 3758 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3759 WREG32(HDP_MISC_CNTL, tmp); 3760 3761 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3762 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3763 3764 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3765 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); 3766 mutex_unlock(&rdev->grbm_idx_mutex); 3767 3768 udelay(50); 3769 } 3770 3771 /* 3772 * GPU scratch registers helpers function. 3773 */ 3774 /** 3775 * cik_scratch_init - setup driver info for CP scratch regs 3776 * 3777 * @rdev: radeon_device pointer 3778 * 3779 * Set up the number and offset of the CP scratch registers. 3780 * NOTE: use of CP scratch registers is a legacy inferface and 3781 * is not used by default on newer asics (r6xx+). On newer asics, 3782 * memory buffers are used for fences rather than scratch regs. 3783 */ 3784 static void cik_scratch_init(struct radeon_device *rdev) 3785 { 3786 int i; 3787 3788 rdev->scratch.num_reg = 7; 3789 rdev->scratch.reg_base = SCRATCH_REG0; 3790 for (i = 0; i < rdev->scratch.num_reg; i++) { 3791 rdev->scratch.free[i] = true; 3792 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3793 } 3794 } 3795 3796 /** 3797 * cik_ring_test - basic gfx ring test 3798 * 3799 * @rdev: radeon_device pointer 3800 * @ring: radeon_ring structure holding ring information 3801 * 3802 * Allocate a scratch register and write to it using the gfx ring (CIK). 3803 * Provides a basic gfx ring test to verify that the ring is working. 3804 * Used by cik_cp_gfx_resume(); 3805 * Returns 0 on success, error on failure. 3806 */ 3807 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) 3808 { 3809 uint32_t scratch; 3810 uint32_t tmp = 0; 3811 unsigned i; 3812 int r; 3813 3814 r = radeon_scratch_get(rdev, &scratch); 3815 if (r) { 3816 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); 3817 return r; 3818 } 3819 WREG32(scratch, 0xCAFEDEAD); 3820 r = radeon_ring_lock(rdev, ring, 3); 3821 if (r) { 3822 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); 3823 radeon_scratch_free(rdev, scratch); 3824 return r; 3825 } 3826 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3827 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); 3828 radeon_ring_write(ring, 0xDEADBEEF); 3829 radeon_ring_unlock_commit(rdev, ring, false); 3830 3831 for (i = 0; i < rdev->usec_timeout; i++) { 3832 tmp = RREG32(scratch); 3833 if (tmp == 0xDEADBEEF) 3834 break; 3835 DRM_UDELAY(1); 3836 } 3837 if (i < rdev->usec_timeout) { 3838 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 3839 } else { 3840 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 3841 ring->idx, scratch, tmp); 3842 r = -EINVAL; 3843 } 3844 radeon_scratch_free(rdev, scratch); 3845 return r; 3846 } 3847 3848 /** 3849 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp 3850 * 3851 * @rdev: radeon_device pointer 3852 * @ridx: radeon ring index 3853 * 3854 * Emits an hdp flush on the cp. 3855 */ 3856 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, 3857 int ridx) 3858 { 3859 struct radeon_ring *ring = &rdev->ring[ridx]; 3860 u32 ref_and_mask; 3861 3862 switch (ring->idx) { 3863 case CAYMAN_RING_TYPE_CP1_INDEX: 3864 case CAYMAN_RING_TYPE_CP2_INDEX: 3865 default: 3866 switch (ring->me) { 3867 case 0: 3868 ref_and_mask = CP2 << ring->pipe; 3869 break; 3870 case 1: 3871 ref_and_mask = CP6 << ring->pipe; 3872 break; 3873 default: 3874 return; 3875 } 3876 break; 3877 case RADEON_RING_TYPE_GFX_INDEX: 3878 ref_and_mask = CP0; 3879 break; 3880 } 3881 3882 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 3883 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 3884 WAIT_REG_MEM_FUNCTION(3) | /* == */ 3885 WAIT_REG_MEM_ENGINE(1))); /* pfp */ 3886 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); 3887 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); 3888 radeon_ring_write(ring, ref_and_mask); 3889 radeon_ring_write(ring, ref_and_mask); 3890 radeon_ring_write(ring, 0x20); /* poll interval */ 3891 } 3892 3893 /** 3894 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring 3895 * 3896 * @rdev: radeon_device pointer 3897 * @fence: radeon fence object 3898 * 3899 * Emits a fence sequnce number on the gfx ring and flushes 3900 * GPU caches. 3901 */ 3902 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, 3903 struct radeon_fence *fence) 3904 { 3905 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3906 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3907 3908 /* Workaround for cache flush problems. First send a dummy EOP 3909 * event down the pipe with seq one below. 3910 */ 3911 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3912 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3913 EOP_TC_ACTION_EN | 3914 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3915 EVENT_INDEX(5))); 3916 radeon_ring_write(ring, addr & 0xfffffffc); 3917 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 3918 DATA_SEL(1) | INT_SEL(0)); 3919 radeon_ring_write(ring, fence->seq - 1); 3920 radeon_ring_write(ring, 0); 3921 3922 /* Then send the real EOP event down the pipe. */ 3923 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3924 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3925 EOP_TC_ACTION_EN | 3926 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3927 EVENT_INDEX(5))); 3928 radeon_ring_write(ring, addr & 0xfffffffc); 3929 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); 3930 radeon_ring_write(ring, fence->seq); 3931 radeon_ring_write(ring, 0); 3932 } 3933 3934 /** 3935 * cik_fence_compute_ring_emit - emit a fence on the compute ring 3936 * 3937 * @rdev: radeon_device pointer 3938 * @fence: radeon fence object 3939 * 3940 * Emits a fence sequnce number on the compute ring and flushes 3941 * GPU caches. 3942 */ 3943 void cik_fence_compute_ring_emit(struct radeon_device *rdev, 3944 struct radeon_fence *fence) 3945 { 3946 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3947 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3948 3949 /* RELEASE_MEM - flush caches, send int */ 3950 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 3951 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3952 EOP_TC_ACTION_EN | 3953 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3954 EVENT_INDEX(5))); 3955 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); 3956 radeon_ring_write(ring, addr & 0xfffffffc); 3957 radeon_ring_write(ring, upper_32_bits(addr)); 3958 radeon_ring_write(ring, fence->seq); 3959 radeon_ring_write(ring, 0); 3960 } 3961 3962 /** 3963 * cik_semaphore_ring_emit - emit a semaphore on the CP ring 3964 * 3965 * @rdev: radeon_device pointer 3966 * @ring: radeon ring buffer object 3967 * @semaphore: radeon semaphore object 3968 * @emit_wait: Is this a sempahore wait? 3969 * 3970 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP 3971 * from running ahead of semaphore waits. 3972 */ 3973 bool cik_semaphore_ring_emit(struct radeon_device *rdev, 3974 struct radeon_ring *ring, 3975 struct radeon_semaphore *semaphore, 3976 bool emit_wait) 3977 { 3978 uint64_t addr = semaphore->gpu_addr; 3979 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; 3980 3981 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); 3982 radeon_ring_write(ring, lower_32_bits(addr)); 3983 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); 3984 3985 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) { 3986 /* Prevent the PFP from running ahead of the semaphore wait */ 3987 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3988 radeon_ring_write(ring, 0x0); 3989 } 3990 3991 return true; 3992 } 3993 3994 /** 3995 * cik_copy_cpdma - copy pages using the CP DMA engine 3996 * 3997 * @rdev: radeon_device pointer 3998 * @src_offset: src GPU address 3999 * @dst_offset: dst GPU address 4000 * @num_gpu_pages: number of GPU pages to xfer 4001 * @resv: reservation object to sync to 4002 * 4003 * Copy GPU paging using the CP DMA engine (CIK+). 4004 * Used by the radeon ttm implementation to move pages if 4005 * registered as the asic copy callback. 4006 */ 4007 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, 4008 uint64_t src_offset, uint64_t dst_offset, 4009 unsigned num_gpu_pages, 4010 struct reservation_object *resv) 4011 { 4012 struct radeon_fence *fence; 4013 struct radeon_sync sync; 4014 int ring_index = rdev->asic->copy.blit_ring_index; 4015 struct radeon_ring *ring = &rdev->ring[ring_index]; 4016 u32 size_in_bytes, cur_size_in_bytes, control; 4017 int i, num_loops; 4018 int r = 0; 4019 4020 radeon_sync_create(&sync); 4021 4022 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 4023 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); 4024 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); 4025 if (r) { 4026 DRM_ERROR("radeon: moving bo (%d).\n", r); 4027 radeon_sync_free(rdev, &sync, NULL); 4028 return ERR_PTR(r); 4029 } 4030 4031 radeon_sync_resv(rdev, &sync, resv, false); 4032 radeon_sync_rings(rdev, &sync, ring->idx); 4033 4034 for (i = 0; i < num_loops; i++) { 4035 cur_size_in_bytes = size_in_bytes; 4036 if (cur_size_in_bytes > 0x1fffff) 4037 cur_size_in_bytes = 0x1fffff; 4038 size_in_bytes -= cur_size_in_bytes; 4039 control = 0; 4040 if (size_in_bytes == 0) 4041 control |= PACKET3_DMA_DATA_CP_SYNC; 4042 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4043 radeon_ring_write(ring, control); 4044 radeon_ring_write(ring, lower_32_bits(src_offset)); 4045 radeon_ring_write(ring, upper_32_bits(src_offset)); 4046 radeon_ring_write(ring, lower_32_bits(dst_offset)); 4047 radeon_ring_write(ring, upper_32_bits(dst_offset)); 4048 radeon_ring_write(ring, cur_size_in_bytes); 4049 src_offset += cur_size_in_bytes; 4050 dst_offset += cur_size_in_bytes; 4051 } 4052 4053 r = radeon_fence_emit(rdev, &fence, ring->idx); 4054 if (r) { 4055 radeon_ring_unlock_undo(rdev, ring); 4056 radeon_sync_free(rdev, &sync, NULL); 4057 return ERR_PTR(r); 4058 } 4059 4060 radeon_ring_unlock_commit(rdev, ring, false); 4061 radeon_sync_free(rdev, &sync, fence); 4062 4063 return fence; 4064 } 4065 4066 /* 4067 * IB stuff 4068 */ 4069 /** 4070 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring 4071 * 4072 * @rdev: radeon_device pointer 4073 * @ib: radeon indirect buffer object 4074 * 4075 * Emits an DE (drawing engine) or CE (constant engine) IB 4076 * on the gfx ring. IBs are usually generated by userspace 4077 * acceleration drivers and submitted to the kernel for 4078 * sheduling on the ring. This function schedules the IB 4079 * on the gfx ring for execution by the GPU. 4080 */ 4081 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 4082 { 4083 struct radeon_ring *ring = &rdev->ring[ib->ring]; 4084 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; 4085 u32 header, control = INDIRECT_BUFFER_VALID; 4086 4087 if (ib->is_const_ib) { 4088 /* set switch buffer packet before const IB */ 4089 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 4090 radeon_ring_write(ring, 0); 4091 4092 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 4093 } else { 4094 u32 next_rptr; 4095 if (ring->rptr_save_reg) { 4096 next_rptr = ring->wptr + 3 + 4; 4097 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 4098 radeon_ring_write(ring, ((ring->rptr_save_reg - 4099 PACKET3_SET_UCONFIG_REG_START) >> 2)); 4100 radeon_ring_write(ring, next_rptr); 4101 } else if (rdev->wb.enabled) { 4102 next_rptr = ring->wptr + 5 + 4; 4103 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4104 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); 4105 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 4106 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); 4107 radeon_ring_write(ring, next_rptr); 4108 } 4109 4110 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 4111 } 4112 4113 control |= ib->length_dw | (vm_id << 24); 4114 4115 radeon_ring_write(ring, header); 4116 radeon_ring_write(ring, 4117 #ifdef __BIG_ENDIAN 4118 (2 << 0) | 4119 #endif 4120 (ib->gpu_addr & 0xFFFFFFFC)); 4121 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 4122 radeon_ring_write(ring, control); 4123 } 4124 4125 /** 4126 * cik_ib_test - basic gfx ring IB test 4127 * 4128 * @rdev: radeon_device pointer 4129 * @ring: radeon_ring structure holding ring information 4130 * 4131 * Allocate an IB and execute it on the gfx ring (CIK). 4132 * Provides a basic gfx ring test to verify that IBs are working. 4133 * Returns 0 on success, error on failure. 4134 */ 4135 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) 4136 { 4137 struct radeon_ib ib; 4138 uint32_t scratch; 4139 uint32_t tmp = 0; 4140 unsigned i; 4141 int r; 4142 4143 r = radeon_scratch_get(rdev, &scratch); 4144 if (r) { 4145 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); 4146 return r; 4147 } 4148 WREG32(scratch, 0xCAFEDEAD); 4149 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); 4150 if (r) { 4151 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 4152 radeon_scratch_free(rdev, scratch); 4153 return r; 4154 } 4155 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); 4156 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); 4157 ib.ptr[2] = 0xDEADBEEF; 4158 ib.length_dw = 3; 4159 r = radeon_ib_schedule(rdev, &ib, NULL, false); 4160 if (r) { 4161 radeon_scratch_free(rdev, scratch); 4162 radeon_ib_free(rdev, &ib); 4163 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); 4164 return r; 4165 } 4166 r = radeon_fence_wait(ib.fence, false); 4167 if (r) { 4168 DRM_ERROR("radeon: fence wait failed (%d).\n", r); 4169 radeon_scratch_free(rdev, scratch); 4170 radeon_ib_free(rdev, &ib); 4171 return r; 4172 } 4173 for (i = 0; i < rdev->usec_timeout; i++) { 4174 tmp = RREG32(scratch); 4175 if (tmp == 0xDEADBEEF) 4176 break; 4177 DRM_UDELAY(1); 4178 } 4179 if (i < rdev->usec_timeout) { 4180 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); 4181 } else { 4182 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 4183 scratch, tmp); 4184 r = -EINVAL; 4185 } 4186 radeon_scratch_free(rdev, scratch); 4187 radeon_ib_free(rdev, &ib); 4188 return r; 4189 } 4190 4191 /* 4192 * CP. 4193 * On CIK, gfx and compute now have independant command processors. 4194 * 4195 * GFX 4196 * Gfx consists of a single ring and can process both gfx jobs and 4197 * compute jobs. The gfx CP consists of three microengines (ME): 4198 * PFP - Pre-Fetch Parser 4199 * ME - Micro Engine 4200 * CE - Constant Engine 4201 * The PFP and ME make up what is considered the Drawing Engine (DE). 4202 * The CE is an asynchronous engine used for updating buffer desciptors 4203 * used by the DE so that they can be loaded into cache in parallel 4204 * while the DE is processing state update packets. 4205 * 4206 * Compute 4207 * The compute CP consists of two microengines (ME): 4208 * MEC1 - Compute MicroEngine 1 4209 * MEC2 - Compute MicroEngine 2 4210 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. 4211 * The queues are exposed to userspace and are programmed directly 4212 * by the compute runtime. 4213 */ 4214 /** 4215 * cik_cp_gfx_enable - enable/disable the gfx CP MEs 4216 * 4217 * @rdev: radeon_device pointer 4218 * @enable: enable or disable the MEs 4219 * 4220 * Halts or unhalts the gfx MEs. 4221 */ 4222 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) 4223 { 4224 if (enable) 4225 WREG32(CP_ME_CNTL, 0); 4226 else { 4227 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4228 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 4229 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 4230 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4231 } 4232 udelay(50); 4233 } 4234 4235 /** 4236 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode 4237 * 4238 * @rdev: radeon_device pointer 4239 * 4240 * Loads the gfx PFP, ME, and CE ucode. 4241 * Returns 0 for success, -EINVAL if the ucode is not available. 4242 */ 4243 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) 4244 { 4245 int i; 4246 4247 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 4248 return -EINVAL; 4249 4250 cik_cp_gfx_enable(rdev, false); 4251 4252 if (rdev->new_fw) { 4253 const struct gfx_firmware_header_v1_0 *pfp_hdr = 4254 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 4255 const struct gfx_firmware_header_v1_0 *ce_hdr = 4256 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 4257 const struct gfx_firmware_header_v1_0 *me_hdr = 4258 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 4259 const __le32 *fw_data; 4260 u32 fw_size; 4261 4262 radeon_ucode_print_gfx_hdr(&pfp_hdr->header); 4263 radeon_ucode_print_gfx_hdr(&ce_hdr->header); 4264 radeon_ucode_print_gfx_hdr(&me_hdr->header); 4265 4266 /* PFP */ 4267 fw_data = (const __le32 *) 4268 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 4269 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 4270 WREG32(CP_PFP_UCODE_ADDR, 0); 4271 for (i = 0; i < fw_size; i++) 4272 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 4273 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version)); 4274 4275 /* CE */ 4276 fw_data = (const __le32 *) 4277 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 4278 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 4279 WREG32(CP_CE_UCODE_ADDR, 0); 4280 for (i = 0; i < fw_size; i++) 4281 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 4282 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version)); 4283 4284 /* ME */ 4285 fw_data = (const __be32 *) 4286 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 4287 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 4288 WREG32(CP_ME_RAM_WADDR, 0); 4289 for (i = 0; i < fw_size; i++) 4290 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 4291 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4292 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4293 } else { 4294 const __be32 *fw_data; 4295 4296 /* PFP */ 4297 fw_data = (const __be32 *)rdev->pfp_fw->data; 4298 WREG32(CP_PFP_UCODE_ADDR, 0); 4299 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) 4300 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 4301 WREG32(CP_PFP_UCODE_ADDR, 0); 4302 4303 /* CE */ 4304 fw_data = (const __be32 *)rdev->ce_fw->data; 4305 WREG32(CP_CE_UCODE_ADDR, 0); 4306 for (i = 0; i < CIK_CE_UCODE_SIZE; i++) 4307 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 4308 WREG32(CP_CE_UCODE_ADDR, 0); 4309 4310 /* ME */ 4311 fw_data = (const __be32 *)rdev->me_fw->data; 4312 WREG32(CP_ME_RAM_WADDR, 0); 4313 for (i = 0; i < CIK_ME_UCODE_SIZE; i++) 4314 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 4315 WREG32(CP_ME_RAM_WADDR, 0); 4316 } 4317 4318 return 0; 4319 } 4320 4321 /** 4322 * cik_cp_gfx_start - start the gfx ring 4323 * 4324 * @rdev: radeon_device pointer 4325 * 4326 * Enables the ring and loads the clear state context and other 4327 * packets required to init the ring. 4328 * Returns 0 for success, error for failure. 4329 */ 4330 static int cik_cp_gfx_start(struct radeon_device *rdev) 4331 { 4332 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4333 int r, i; 4334 4335 /* init the CP */ 4336 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); 4337 WREG32(CP_ENDIAN_SWAP, 0); 4338 WREG32(CP_DEVICE_ID, 1); 4339 4340 cik_cp_gfx_enable(rdev, true); 4341 4342 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); 4343 if (r) { 4344 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 4345 return r; 4346 } 4347 4348 /* init the CE partitions. CE only used for gfx on CIK */ 4349 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4350 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4351 radeon_ring_write(ring, 0x8000); 4352 radeon_ring_write(ring, 0x8000); 4353 4354 /* setup clear context state */ 4355 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4356 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4357 4358 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4359 radeon_ring_write(ring, 0x80000000); 4360 radeon_ring_write(ring, 0x80000000); 4361 4362 for (i = 0; i < cik_default_size; i++) 4363 radeon_ring_write(ring, cik_default_state[i]); 4364 4365 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4366 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4367 4368 /* set clear context state */ 4369 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4370 radeon_ring_write(ring, 0); 4371 4372 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4373 radeon_ring_write(ring, 0x00000316); 4374 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 4375 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 4376 4377 radeon_ring_unlock_commit(rdev, ring, false); 4378 4379 return 0; 4380 } 4381 4382 /** 4383 * cik_cp_gfx_fini - stop the gfx ring 4384 * 4385 * @rdev: radeon_device pointer 4386 * 4387 * Stop the gfx ring and tear down the driver ring 4388 * info. 4389 */ 4390 static void cik_cp_gfx_fini(struct radeon_device *rdev) 4391 { 4392 cik_cp_gfx_enable(rdev, false); 4393 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4394 } 4395 4396 /** 4397 * cik_cp_gfx_resume - setup the gfx ring buffer registers 4398 * 4399 * @rdev: radeon_device pointer 4400 * 4401 * Program the location and size of the gfx ring buffer 4402 * and test it to make sure it's working. 4403 * Returns 0 for success, error for failure. 4404 */ 4405 static int cik_cp_gfx_resume(struct radeon_device *rdev) 4406 { 4407 struct radeon_ring *ring; 4408 u32 tmp; 4409 u32 rb_bufsz; 4410 u64 rb_addr; 4411 int r; 4412 4413 WREG32(CP_SEM_WAIT_TIMER, 0x0); 4414 if (rdev->family != CHIP_HAWAII) 4415 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 4416 4417 /* Set the write pointer delay */ 4418 WREG32(CP_RB_WPTR_DELAY, 0); 4419 4420 /* set the RB to use vmid 0 */ 4421 WREG32(CP_RB_VMID, 0); 4422 4423 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 4424 4425 /* ring 0 - compute and gfx */ 4426 /* Set ring buffer size */ 4427 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4428 rb_bufsz = order_base_2(ring->ring_size / 8); 4429 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 4430 #ifdef __BIG_ENDIAN 4431 tmp |= BUF_SWAP_32BIT; 4432 #endif 4433 WREG32(CP_RB0_CNTL, tmp); 4434 4435 /* Initialize the ring buffer's read and write pointers */ 4436 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 4437 ring->wptr = 0; 4438 WREG32(CP_RB0_WPTR, ring->wptr); 4439 4440 /* set the wb address wether it's enabled or not */ 4441 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 4442 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 4443 4444 /* scratch register shadowing is no longer supported */ 4445 WREG32(SCRATCH_UMSK, 0); 4446 4447 if (!rdev->wb.enabled) 4448 tmp |= RB_NO_UPDATE; 4449 4450 mdelay(1); 4451 WREG32(CP_RB0_CNTL, tmp); 4452 4453 rb_addr = ring->gpu_addr >> 8; 4454 WREG32(CP_RB0_BASE, rb_addr); 4455 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4456 4457 /* start the ring */ 4458 cik_cp_gfx_start(rdev); 4459 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 4460 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4461 if (r) { 4462 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4463 return r; 4464 } 4465 4466 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4467 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 4468 4469 return 0; 4470 } 4471 4472 u32 cik_gfx_get_rptr(struct radeon_device *rdev, 4473 struct radeon_ring *ring) 4474 { 4475 u32 rptr; 4476 4477 if (rdev->wb.enabled) 4478 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4479 else 4480 rptr = RREG32(CP_RB0_RPTR); 4481 4482 return rptr; 4483 } 4484 4485 u32 cik_gfx_get_wptr(struct radeon_device *rdev, 4486 struct radeon_ring *ring) 4487 { 4488 u32 wptr; 4489 4490 wptr = RREG32(CP_RB0_WPTR); 4491 4492 return wptr; 4493 } 4494 4495 void cik_gfx_set_wptr(struct radeon_device *rdev, 4496 struct radeon_ring *ring) 4497 { 4498 WREG32(CP_RB0_WPTR, ring->wptr); 4499 (void)RREG32(CP_RB0_WPTR); 4500 } 4501 4502 u32 cik_compute_get_rptr(struct radeon_device *rdev, 4503 struct radeon_ring *ring) 4504 { 4505 u32 rptr; 4506 4507 if (rdev->wb.enabled) { 4508 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4509 } else { 4510 mutex_lock(&rdev->srbm_mutex); 4511 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4512 rptr = RREG32(CP_HQD_PQ_RPTR); 4513 cik_srbm_select(rdev, 0, 0, 0, 0); 4514 mutex_unlock(&rdev->srbm_mutex); 4515 } 4516 4517 return rptr; 4518 } 4519 4520 u32 cik_compute_get_wptr(struct radeon_device *rdev, 4521 struct radeon_ring *ring) 4522 { 4523 u32 wptr; 4524 4525 if (rdev->wb.enabled) { 4526 /* XXX check if swapping is necessary on BE */ 4527 wptr = rdev->wb.wb[ring->wptr_offs/4]; 4528 } else { 4529 mutex_lock(&rdev->srbm_mutex); 4530 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4531 wptr = RREG32(CP_HQD_PQ_WPTR); 4532 cik_srbm_select(rdev, 0, 0, 0, 0); 4533 mutex_unlock(&rdev->srbm_mutex); 4534 } 4535 4536 return wptr; 4537 } 4538 4539 void cik_compute_set_wptr(struct radeon_device *rdev, 4540 struct radeon_ring *ring) 4541 { 4542 /* XXX check if swapping is necessary on BE */ 4543 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; 4544 WDOORBELL32(ring->doorbell_index, ring->wptr); 4545 } 4546 4547 /** 4548 * cik_cp_compute_enable - enable/disable the compute CP MEs 4549 * 4550 * @rdev: radeon_device pointer 4551 * @enable: enable or disable the MEs 4552 * 4553 * Halts or unhalts the compute MEs. 4554 */ 4555 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) 4556 { 4557 if (enable) 4558 WREG32(CP_MEC_CNTL, 0); 4559 else { 4560 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); 4561 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 4562 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 4563 } 4564 udelay(50); 4565 } 4566 4567 /** 4568 * cik_cp_compute_load_microcode - load the compute CP ME ucode 4569 * 4570 * @rdev: radeon_device pointer 4571 * 4572 * Loads the compute MEC1&2 ucode. 4573 * Returns 0 for success, -EINVAL if the ucode is not available. 4574 */ 4575 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) 4576 { 4577 int i; 4578 4579 if (!rdev->mec_fw) 4580 return -EINVAL; 4581 4582 cik_cp_compute_enable(rdev, false); 4583 4584 if (rdev->new_fw) { 4585 const struct gfx_firmware_header_v1_0 *mec_hdr = 4586 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 4587 const __le32 *fw_data; 4588 u32 fw_size; 4589 4590 radeon_ucode_print_gfx_hdr(&mec_hdr->header); 4591 4592 /* MEC1 */ 4593 fw_data = (const __le32 *) 4594 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 4595 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 4596 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4597 for (i = 0; i < fw_size; i++) 4598 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++)); 4599 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version)); 4600 4601 /* MEC2 */ 4602 if (rdev->family == CHIP_KAVERI) { 4603 const struct gfx_firmware_header_v1_0 *mec2_hdr = 4604 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 4605 4606 fw_data = (const __le32 *) 4607 (rdev->mec2_fw->data + 4608 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); 4609 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; 4610 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4611 for (i = 0; i < fw_size; i++) 4612 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++)); 4613 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version)); 4614 } 4615 } else { 4616 const __be32 *fw_data; 4617 4618 /* MEC1 */ 4619 fw_data = (const __be32 *)rdev->mec_fw->data; 4620 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4621 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4622 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); 4623 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4624 4625 if (rdev->family == CHIP_KAVERI) { 4626 /* MEC2 */ 4627 fw_data = (const __be32 *)rdev->mec_fw->data; 4628 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4629 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4630 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); 4631 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4632 } 4633 } 4634 4635 return 0; 4636 } 4637 4638 /** 4639 * cik_cp_compute_start - start the compute queues 4640 * 4641 * @rdev: radeon_device pointer 4642 * 4643 * Enable the compute queues. 4644 * Returns 0 for success, error for failure. 4645 */ 4646 static int cik_cp_compute_start(struct radeon_device *rdev) 4647 { 4648 cik_cp_compute_enable(rdev, true); 4649 4650 return 0; 4651 } 4652 4653 /** 4654 * cik_cp_compute_fini - stop the compute queues 4655 * 4656 * @rdev: radeon_device pointer 4657 * 4658 * Stop the compute queues and tear down the driver queue 4659 * info. 4660 */ 4661 static void cik_cp_compute_fini(struct radeon_device *rdev) 4662 { 4663 int i, idx, r; 4664 4665 cik_cp_compute_enable(rdev, false); 4666 4667 for (i = 0; i < 2; i++) { 4668 if (i == 0) 4669 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4670 else 4671 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4672 4673 if (rdev->ring[idx].mqd_obj) { 4674 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4675 if (unlikely(r != 0)) 4676 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); 4677 4678 radeon_bo_unpin(rdev->ring[idx].mqd_obj); 4679 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4680 4681 radeon_bo_unref(&rdev->ring[idx].mqd_obj); 4682 rdev->ring[idx].mqd_obj = NULL; 4683 } 4684 } 4685 } 4686 4687 static void cik_mec_fini(struct radeon_device *rdev) 4688 { 4689 int r; 4690 4691 if (rdev->mec.hpd_eop_obj) { 4692 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4693 if (unlikely(r != 0)) 4694 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); 4695 radeon_bo_unpin(rdev->mec.hpd_eop_obj); 4696 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4697 4698 radeon_bo_unref(&rdev->mec.hpd_eop_obj); 4699 rdev->mec.hpd_eop_obj = NULL; 4700 } 4701 } 4702 4703 #define MEC_HPD_SIZE 2048 4704 4705 static int cik_mec_init(struct radeon_device *rdev) 4706 { 4707 int r; 4708 u32 *hpd; 4709 4710 /* 4711 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total 4712 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total 4713 * Nonetheless, we assign only 1 pipe because all other pipes will 4714 * be handled by KFD 4715 */ 4716 rdev->mec.num_mec = 1; 4717 rdev->mec.num_pipe = 1; 4718 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; 4719 4720 if (rdev->mec.hpd_eop_obj == NULL) { 4721 r = radeon_bo_create(rdev, 4722 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, 4723 PAGE_SIZE, true, 4724 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, 4725 &rdev->mec.hpd_eop_obj); 4726 if (r) { 4727 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); 4728 return r; 4729 } 4730 } 4731 4732 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4733 if (unlikely(r != 0)) { 4734 cik_mec_fini(rdev); 4735 return r; 4736 } 4737 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, 4738 &rdev->mec.hpd_eop_gpu_addr); 4739 if (r) { 4740 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); 4741 cik_mec_fini(rdev); 4742 return r; 4743 } 4744 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); 4745 if (r) { 4746 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); 4747 cik_mec_fini(rdev); 4748 return r; 4749 } 4750 4751 /* clear memory. Not sure if this is required or not */ 4752 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); 4753 4754 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); 4755 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4756 4757 return 0; 4758 } 4759 4760 struct hqd_registers 4761 { 4762 u32 cp_mqd_base_addr; 4763 u32 cp_mqd_base_addr_hi; 4764 u32 cp_hqd_active; 4765 u32 cp_hqd_vmid; 4766 u32 cp_hqd_persistent_state; 4767 u32 cp_hqd_pipe_priority; 4768 u32 cp_hqd_queue_priority; 4769 u32 cp_hqd_quantum; 4770 u32 cp_hqd_pq_base; 4771 u32 cp_hqd_pq_base_hi; 4772 u32 cp_hqd_pq_rptr; 4773 u32 cp_hqd_pq_rptr_report_addr; 4774 u32 cp_hqd_pq_rptr_report_addr_hi; 4775 u32 cp_hqd_pq_wptr_poll_addr; 4776 u32 cp_hqd_pq_wptr_poll_addr_hi; 4777 u32 cp_hqd_pq_doorbell_control; 4778 u32 cp_hqd_pq_wptr; 4779 u32 cp_hqd_pq_control; 4780 u32 cp_hqd_ib_base_addr; 4781 u32 cp_hqd_ib_base_addr_hi; 4782 u32 cp_hqd_ib_rptr; 4783 u32 cp_hqd_ib_control; 4784 u32 cp_hqd_iq_timer; 4785 u32 cp_hqd_iq_rptr; 4786 u32 cp_hqd_dequeue_request; 4787 u32 cp_hqd_dma_offload; 4788 u32 cp_hqd_sema_cmd; 4789 u32 cp_hqd_msg_type; 4790 u32 cp_hqd_atomic0_preop_lo; 4791 u32 cp_hqd_atomic0_preop_hi; 4792 u32 cp_hqd_atomic1_preop_lo; 4793 u32 cp_hqd_atomic1_preop_hi; 4794 u32 cp_hqd_hq_scheduler0; 4795 u32 cp_hqd_hq_scheduler1; 4796 u32 cp_mqd_control; 4797 }; 4798 4799 struct bonaire_mqd 4800 { 4801 u32 header; 4802 u32 dispatch_initiator; 4803 u32 dimensions[3]; 4804 u32 start_idx[3]; 4805 u32 num_threads[3]; 4806 u32 pipeline_stat_enable; 4807 u32 perf_counter_enable; 4808 u32 pgm[2]; 4809 u32 tba[2]; 4810 u32 tma[2]; 4811 u32 pgm_rsrc[2]; 4812 u32 vmid; 4813 u32 resource_limits; 4814 u32 static_thread_mgmt01[2]; 4815 u32 tmp_ring_size; 4816 u32 static_thread_mgmt23[2]; 4817 u32 restart[3]; 4818 u32 thread_trace_enable; 4819 u32 reserved1; 4820 u32 user_data[16]; 4821 u32 vgtcs_invoke_count[2]; 4822 struct hqd_registers queue_state; 4823 u32 dequeue_cntr; 4824 u32 interrupt_queue[64]; 4825 }; 4826 4827 /** 4828 * cik_cp_compute_resume - setup the compute queue registers 4829 * 4830 * @rdev: radeon_device pointer 4831 * 4832 * Program the compute queues and test them to make sure they 4833 * are working. 4834 * Returns 0 for success, error for failure. 4835 */ 4836 static int cik_cp_compute_resume(struct radeon_device *rdev) 4837 { 4838 int r, i, j, idx; 4839 u32 tmp; 4840 bool use_doorbell = true; 4841 u64 hqd_gpu_addr; 4842 u64 mqd_gpu_addr; 4843 u64 eop_gpu_addr; 4844 u64 wb_gpu_addr; 4845 u32 *buf; 4846 struct bonaire_mqd *mqd; 4847 4848 r = cik_cp_compute_start(rdev); 4849 if (r) 4850 return r; 4851 4852 /* fix up chicken bits */ 4853 tmp = RREG32(CP_CPF_DEBUG); 4854 tmp |= (1 << 23); 4855 WREG32(CP_CPF_DEBUG, tmp); 4856 4857 /* init the pipes */ 4858 mutex_lock(&rdev->srbm_mutex); 4859 4860 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr; 4861 4862 cik_srbm_select(rdev, 0, 0, 0, 0); 4863 4864 /* write the EOP addr */ 4865 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); 4866 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); 4867 4868 /* set the VMID assigned */ 4869 WREG32(CP_HPD_EOP_VMID, 0); 4870 4871 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4872 tmp = RREG32(CP_HPD_EOP_CONTROL); 4873 tmp &= ~EOP_SIZE_MASK; 4874 tmp |= order_base_2(MEC_HPD_SIZE / 8); 4875 WREG32(CP_HPD_EOP_CONTROL, tmp); 4876 4877 mutex_unlock(&rdev->srbm_mutex); 4878 4879 /* init the queues. Just two for now. */ 4880 for (i = 0; i < 2; i++) { 4881 if (i == 0) 4882 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4883 else 4884 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4885 4886 if (rdev->ring[idx].mqd_obj == NULL) { 4887 r = radeon_bo_create(rdev, 4888 sizeof(struct bonaire_mqd), 4889 PAGE_SIZE, true, 4890 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4891 NULL, &rdev->ring[idx].mqd_obj); 4892 if (r) { 4893 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); 4894 return r; 4895 } 4896 } 4897 4898 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4899 if (unlikely(r != 0)) { 4900 cik_cp_compute_fini(rdev); 4901 return r; 4902 } 4903 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, 4904 &mqd_gpu_addr); 4905 if (r) { 4906 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); 4907 cik_cp_compute_fini(rdev); 4908 return r; 4909 } 4910 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); 4911 if (r) { 4912 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); 4913 cik_cp_compute_fini(rdev); 4914 return r; 4915 } 4916 4917 /* init the mqd struct */ 4918 memset(buf, 0, sizeof(struct bonaire_mqd)); 4919 4920 mqd = (struct bonaire_mqd *)buf; 4921 mqd->header = 0xC0310800; 4922 mqd->static_thread_mgmt01[0] = 0xffffffff; 4923 mqd->static_thread_mgmt01[1] = 0xffffffff; 4924 mqd->static_thread_mgmt23[0] = 0xffffffff; 4925 mqd->static_thread_mgmt23[1] = 0xffffffff; 4926 4927 mutex_lock(&rdev->srbm_mutex); 4928 cik_srbm_select(rdev, rdev->ring[idx].me, 4929 rdev->ring[idx].pipe, 4930 rdev->ring[idx].queue, 0); 4931 4932 /* disable wptr polling */ 4933 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4934 tmp &= ~WPTR_POLL_EN; 4935 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4936 4937 /* enable doorbell? */ 4938 mqd->queue_state.cp_hqd_pq_doorbell_control = 4939 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4940 if (use_doorbell) 4941 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4942 else 4943 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; 4944 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4945 mqd->queue_state.cp_hqd_pq_doorbell_control); 4946 4947 /* disable the queue if it's active */ 4948 mqd->queue_state.cp_hqd_dequeue_request = 0; 4949 mqd->queue_state.cp_hqd_pq_rptr = 0; 4950 mqd->queue_state.cp_hqd_pq_wptr= 0; 4951 if (RREG32(CP_HQD_ACTIVE) & 1) { 4952 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4953 for (j = 0; j < rdev->usec_timeout; j++) { 4954 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4955 break; 4956 udelay(1); 4957 } 4958 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); 4959 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); 4960 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4961 } 4962 4963 /* set the pointer to the MQD */ 4964 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; 4965 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 4966 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); 4967 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); 4968 /* set MQD vmid to 0 */ 4969 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); 4970 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; 4971 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); 4972 4973 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4974 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; 4975 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; 4976 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4977 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); 4978 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); 4979 4980 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4981 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); 4982 mqd->queue_state.cp_hqd_pq_control &= 4983 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); 4984 4985 mqd->queue_state.cp_hqd_pq_control |= 4986 order_base_2(rdev->ring[idx].ring_size / 8); 4987 mqd->queue_state.cp_hqd_pq_control |= 4988 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); 4989 #ifdef __BIG_ENDIAN 4990 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; 4991 #endif 4992 mqd->queue_state.cp_hqd_pq_control &= 4993 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); 4994 mqd->queue_state.cp_hqd_pq_control |= 4995 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ 4996 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); 4997 4998 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ 4999 if (i == 0) 5000 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; 5001 else 5002 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; 5003 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; 5004 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 5005 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); 5006 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, 5007 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); 5008 5009 /* set the wb address wether it's enabled or not */ 5010 if (i == 0) 5011 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; 5012 else 5013 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; 5014 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; 5015 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = 5016 upper_32_bits(wb_gpu_addr) & 0xffff; 5017 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, 5018 mqd->queue_state.cp_hqd_pq_rptr_report_addr); 5019 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, 5020 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); 5021 5022 /* enable the doorbell if requested */ 5023 if (use_doorbell) { 5024 mqd->queue_state.cp_hqd_pq_doorbell_control = 5025 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 5026 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; 5027 mqd->queue_state.cp_hqd_pq_doorbell_control |= 5028 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); 5029 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 5030 mqd->queue_state.cp_hqd_pq_doorbell_control &= 5031 ~(DOORBELL_SOURCE | DOORBELL_HIT); 5032 5033 } else { 5034 mqd->queue_state.cp_hqd_pq_doorbell_control = 0; 5035 } 5036 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 5037 mqd->queue_state.cp_hqd_pq_doorbell_control); 5038 5039 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 5040 rdev->ring[idx].wptr = 0; 5041 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; 5042 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 5043 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); 5044 5045 /* set the vmid for the queue */ 5046 mqd->queue_state.cp_hqd_vmid = 0; 5047 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); 5048 5049 /* activate the queue */ 5050 mqd->queue_state.cp_hqd_active = 1; 5051 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); 5052 5053 cik_srbm_select(rdev, 0, 0, 0, 0); 5054 mutex_unlock(&rdev->srbm_mutex); 5055 5056 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); 5057 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 5058 5059 rdev->ring[idx].ready = true; 5060 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); 5061 if (r) 5062 rdev->ring[idx].ready = false; 5063 } 5064 5065 return 0; 5066 } 5067 5068 static void cik_cp_enable(struct radeon_device *rdev, bool enable) 5069 { 5070 cik_cp_gfx_enable(rdev, enable); 5071 cik_cp_compute_enable(rdev, enable); 5072 } 5073 5074 static int cik_cp_load_microcode(struct radeon_device *rdev) 5075 { 5076 int r; 5077 5078 r = cik_cp_gfx_load_microcode(rdev); 5079 if (r) 5080 return r; 5081 r = cik_cp_compute_load_microcode(rdev); 5082 if (r) 5083 return r; 5084 5085 return 0; 5086 } 5087 5088 static void cik_cp_fini(struct radeon_device *rdev) 5089 { 5090 cik_cp_gfx_fini(rdev); 5091 cik_cp_compute_fini(rdev); 5092 } 5093 5094 static int cik_cp_resume(struct radeon_device *rdev) 5095 { 5096 int r; 5097 5098 cik_enable_gui_idle_interrupt(rdev, false); 5099 5100 r = cik_cp_load_microcode(rdev); 5101 if (r) 5102 return r; 5103 5104 r = cik_cp_gfx_resume(rdev); 5105 if (r) 5106 return r; 5107 r = cik_cp_compute_resume(rdev); 5108 if (r) 5109 return r; 5110 5111 cik_enable_gui_idle_interrupt(rdev, true); 5112 5113 return 0; 5114 } 5115 5116 static void cik_print_gpu_status_regs(struct radeon_device *rdev) 5117 { 5118 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 5119 RREG32(GRBM_STATUS)); 5120 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 5121 RREG32(GRBM_STATUS2)); 5122 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 5123 RREG32(GRBM_STATUS_SE0)); 5124 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 5125 RREG32(GRBM_STATUS_SE1)); 5126 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", 5127 RREG32(GRBM_STATUS_SE2)); 5128 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", 5129 RREG32(GRBM_STATUS_SE3)); 5130 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 5131 RREG32(SRBM_STATUS)); 5132 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", 5133 RREG32(SRBM_STATUS2)); 5134 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", 5135 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); 5136 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", 5137 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); 5138 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); 5139 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", 5140 RREG32(CP_STALLED_STAT1)); 5141 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", 5142 RREG32(CP_STALLED_STAT2)); 5143 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", 5144 RREG32(CP_STALLED_STAT3)); 5145 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", 5146 RREG32(CP_CPF_BUSY_STAT)); 5147 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", 5148 RREG32(CP_CPF_STALLED_STAT1)); 5149 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); 5150 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); 5151 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", 5152 RREG32(CP_CPC_STALLED_STAT1)); 5153 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); 5154 } 5155 5156 /** 5157 * cik_gpu_check_soft_reset - check which blocks are busy 5158 * 5159 * @rdev: radeon_device pointer 5160 * 5161 * Check which blocks are busy and return the relevant reset 5162 * mask to be used by cik_gpu_soft_reset(). 5163 * Returns a mask of the blocks to be reset. 5164 */ 5165 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) 5166 { 5167 u32 reset_mask = 0; 5168 u32 tmp; 5169 5170 /* GRBM_STATUS */ 5171 tmp = RREG32(GRBM_STATUS); 5172 if (tmp & (PA_BUSY | SC_BUSY | 5173 BCI_BUSY | SX_BUSY | 5174 TA_BUSY | VGT_BUSY | 5175 DB_BUSY | CB_BUSY | 5176 GDS_BUSY | SPI_BUSY | 5177 IA_BUSY | IA_BUSY_NO_DMA)) 5178 reset_mask |= RADEON_RESET_GFX; 5179 5180 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) 5181 reset_mask |= RADEON_RESET_CP; 5182 5183 /* GRBM_STATUS2 */ 5184 tmp = RREG32(GRBM_STATUS2); 5185 if (tmp & RLC_BUSY) 5186 reset_mask |= RADEON_RESET_RLC; 5187 5188 /* SDMA0_STATUS_REG */ 5189 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); 5190 if (!(tmp & SDMA_IDLE)) 5191 reset_mask |= RADEON_RESET_DMA; 5192 5193 /* SDMA1_STATUS_REG */ 5194 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); 5195 if (!(tmp & SDMA_IDLE)) 5196 reset_mask |= RADEON_RESET_DMA1; 5197 5198 /* SRBM_STATUS2 */ 5199 tmp = RREG32(SRBM_STATUS2); 5200 if (tmp & SDMA_BUSY) 5201 reset_mask |= RADEON_RESET_DMA; 5202 5203 if (tmp & SDMA1_BUSY) 5204 reset_mask |= RADEON_RESET_DMA1; 5205 5206 /* SRBM_STATUS */ 5207 tmp = RREG32(SRBM_STATUS); 5208 5209 if (tmp & IH_BUSY) 5210 reset_mask |= RADEON_RESET_IH; 5211 5212 if (tmp & SEM_BUSY) 5213 reset_mask |= RADEON_RESET_SEM; 5214 5215 if (tmp & GRBM_RQ_PENDING) 5216 reset_mask |= RADEON_RESET_GRBM; 5217 5218 if (tmp & VMC_BUSY) 5219 reset_mask |= RADEON_RESET_VMC; 5220 5221 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 5222 MCC_BUSY | MCD_BUSY)) 5223 reset_mask |= RADEON_RESET_MC; 5224 5225 if (evergreen_is_display_hung(rdev)) 5226 reset_mask |= RADEON_RESET_DISPLAY; 5227 5228 /* Skip MC reset as it's mostly likely not hung, just busy */ 5229 if (reset_mask & RADEON_RESET_MC) { 5230 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 5231 reset_mask &= ~RADEON_RESET_MC; 5232 } 5233 5234 return reset_mask; 5235 } 5236 5237 /** 5238 * cik_gpu_soft_reset - soft reset GPU 5239 * 5240 * @rdev: radeon_device pointer 5241 * @reset_mask: mask of which blocks to reset 5242 * 5243 * Soft reset the blocks specified in @reset_mask. 5244 */ 5245 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 5246 { 5247 struct evergreen_mc_save save; 5248 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 5249 u32 tmp; 5250 5251 if (reset_mask == 0) 5252 return; 5253 5254 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 5255 5256 cik_print_gpu_status_regs(rdev); 5257 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5258 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 5259 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5260 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 5261 5262 /* disable CG/PG */ 5263 cik_fini_pg(rdev); 5264 cik_fini_cg(rdev); 5265 5266 /* stop the rlc */ 5267 cik_rlc_stop(rdev); 5268 5269 /* Disable GFX parsing/prefetching */ 5270 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5271 5272 /* Disable MEC parsing/prefetching */ 5273 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5274 5275 if (reset_mask & RADEON_RESET_DMA) { 5276 /* sdma0 */ 5277 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5278 tmp |= SDMA_HALT; 5279 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5280 } 5281 if (reset_mask & RADEON_RESET_DMA1) { 5282 /* sdma1 */ 5283 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5284 tmp |= SDMA_HALT; 5285 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5286 } 5287 5288 evergreen_mc_stop(rdev, &save); 5289 if (evergreen_mc_wait_for_idle(rdev)) { 5290 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5291 } 5292 5293 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) 5294 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; 5295 5296 if (reset_mask & RADEON_RESET_CP) { 5297 grbm_soft_reset |= SOFT_RESET_CP; 5298 5299 srbm_soft_reset |= SOFT_RESET_GRBM; 5300 } 5301 5302 if (reset_mask & RADEON_RESET_DMA) 5303 srbm_soft_reset |= SOFT_RESET_SDMA; 5304 5305 if (reset_mask & RADEON_RESET_DMA1) 5306 srbm_soft_reset |= SOFT_RESET_SDMA1; 5307 5308 if (reset_mask & RADEON_RESET_DISPLAY) 5309 srbm_soft_reset |= SOFT_RESET_DC; 5310 5311 if (reset_mask & RADEON_RESET_RLC) 5312 grbm_soft_reset |= SOFT_RESET_RLC; 5313 5314 if (reset_mask & RADEON_RESET_SEM) 5315 srbm_soft_reset |= SOFT_RESET_SEM; 5316 5317 if (reset_mask & RADEON_RESET_IH) 5318 srbm_soft_reset |= SOFT_RESET_IH; 5319 5320 if (reset_mask & RADEON_RESET_GRBM) 5321 srbm_soft_reset |= SOFT_RESET_GRBM; 5322 5323 if (reset_mask & RADEON_RESET_VMC) 5324 srbm_soft_reset |= SOFT_RESET_VMC; 5325 5326 if (!(rdev->flags & RADEON_IS_IGP)) { 5327 if (reset_mask & RADEON_RESET_MC) 5328 srbm_soft_reset |= SOFT_RESET_MC; 5329 } 5330 5331 if (grbm_soft_reset) { 5332 tmp = RREG32(GRBM_SOFT_RESET); 5333 tmp |= grbm_soft_reset; 5334 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 5335 WREG32(GRBM_SOFT_RESET, tmp); 5336 tmp = RREG32(GRBM_SOFT_RESET); 5337 5338 udelay(50); 5339 5340 tmp &= ~grbm_soft_reset; 5341 WREG32(GRBM_SOFT_RESET, tmp); 5342 tmp = RREG32(GRBM_SOFT_RESET); 5343 } 5344 5345 if (srbm_soft_reset) { 5346 tmp = RREG32(SRBM_SOFT_RESET); 5347 tmp |= srbm_soft_reset; 5348 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 5349 WREG32(SRBM_SOFT_RESET, tmp); 5350 tmp = RREG32(SRBM_SOFT_RESET); 5351 5352 udelay(50); 5353 5354 tmp &= ~srbm_soft_reset; 5355 WREG32(SRBM_SOFT_RESET, tmp); 5356 tmp = RREG32(SRBM_SOFT_RESET); 5357 } 5358 5359 /* Wait a little for things to settle down */ 5360 udelay(50); 5361 5362 evergreen_mc_resume(rdev, &save); 5363 udelay(50); 5364 5365 cik_print_gpu_status_regs(rdev); 5366 } 5367 5368 struct kv_reset_save_regs { 5369 u32 gmcon_reng_execute; 5370 u32 gmcon_misc; 5371 u32 gmcon_misc3; 5372 }; 5373 5374 static void kv_save_regs_for_reset(struct radeon_device *rdev, 5375 struct kv_reset_save_regs *save) 5376 { 5377 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); 5378 save->gmcon_misc = RREG32(GMCON_MISC); 5379 save->gmcon_misc3 = RREG32(GMCON_MISC3); 5380 5381 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); 5382 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | 5383 STCTRL_STUTTER_EN)); 5384 } 5385 5386 static void kv_restore_regs_for_reset(struct radeon_device *rdev, 5387 struct kv_reset_save_regs *save) 5388 { 5389 int i; 5390 5391 WREG32(GMCON_PGFSM_WRITE, 0); 5392 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); 5393 5394 for (i = 0; i < 5; i++) 5395 WREG32(GMCON_PGFSM_WRITE, 0); 5396 5397 WREG32(GMCON_PGFSM_WRITE, 0); 5398 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); 5399 5400 for (i = 0; i < 5; i++) 5401 WREG32(GMCON_PGFSM_WRITE, 0); 5402 5403 WREG32(GMCON_PGFSM_WRITE, 0x210000); 5404 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); 5405 5406 for (i = 0; i < 5; i++) 5407 WREG32(GMCON_PGFSM_WRITE, 0); 5408 5409 WREG32(GMCON_PGFSM_WRITE, 0x21003); 5410 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); 5411 5412 for (i = 0; i < 5; i++) 5413 WREG32(GMCON_PGFSM_WRITE, 0); 5414 5415 WREG32(GMCON_PGFSM_WRITE, 0x2b00); 5416 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); 5417 5418 for (i = 0; i < 5; i++) 5419 WREG32(GMCON_PGFSM_WRITE, 0); 5420 5421 WREG32(GMCON_PGFSM_WRITE, 0); 5422 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); 5423 5424 for (i = 0; i < 5; i++) 5425 WREG32(GMCON_PGFSM_WRITE, 0); 5426 5427 WREG32(GMCON_PGFSM_WRITE, 0x420000); 5428 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); 5429 5430 for (i = 0; i < 5; i++) 5431 WREG32(GMCON_PGFSM_WRITE, 0); 5432 5433 WREG32(GMCON_PGFSM_WRITE, 0x120202); 5434 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); 5435 5436 for (i = 0; i < 5; i++) 5437 WREG32(GMCON_PGFSM_WRITE, 0); 5438 5439 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); 5440 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); 5441 5442 for (i = 0; i < 5; i++) 5443 WREG32(GMCON_PGFSM_WRITE, 0); 5444 5445 WREG32(GMCON_PGFSM_WRITE, 0x373f3e); 5446 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); 5447 5448 for (i = 0; i < 5; i++) 5449 WREG32(GMCON_PGFSM_WRITE, 0); 5450 5451 WREG32(GMCON_PGFSM_WRITE, 0x3e1332); 5452 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); 5453 5454 WREG32(GMCON_MISC3, save->gmcon_misc3); 5455 WREG32(GMCON_MISC, save->gmcon_misc); 5456 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); 5457 } 5458 5459 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) 5460 { 5461 struct evergreen_mc_save save; 5462 struct kv_reset_save_regs kv_save = { 0 }; 5463 u32 tmp, i; 5464 5465 dev_info(rdev->dev, "GPU pci config reset\n"); 5466 5467 /* disable dpm? */ 5468 5469 /* disable cg/pg */ 5470 cik_fini_pg(rdev); 5471 cik_fini_cg(rdev); 5472 5473 /* Disable GFX parsing/prefetching */ 5474 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5475 5476 /* Disable MEC parsing/prefetching */ 5477 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5478 5479 /* sdma0 */ 5480 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5481 tmp |= SDMA_HALT; 5482 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5483 /* sdma1 */ 5484 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5485 tmp |= SDMA_HALT; 5486 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5487 /* XXX other engines? */ 5488 5489 /* halt the rlc, disable cp internal ints */ 5490 cik_rlc_stop(rdev); 5491 5492 udelay(50); 5493 5494 /* disable mem access */ 5495 evergreen_mc_stop(rdev, &save); 5496 if (evergreen_mc_wait_for_idle(rdev)) { 5497 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 5498 } 5499 5500 if (rdev->flags & RADEON_IS_IGP) 5501 kv_save_regs_for_reset(rdev, &kv_save); 5502 5503 /* disable BM */ 5504 pci_clear_master(rdev->pdev); 5505 /* reset */ 5506 radeon_pci_config_reset(rdev); 5507 5508 udelay(100); 5509 5510 /* wait for asic to come out of reset */ 5511 for (i = 0; i < rdev->usec_timeout; i++) { 5512 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 5513 break; 5514 udelay(1); 5515 } 5516 5517 /* does asic init need to be run first??? */ 5518 if (rdev->flags & RADEON_IS_IGP) 5519 kv_restore_regs_for_reset(rdev, &kv_save); 5520 } 5521 5522 /** 5523 * cik_asic_reset - soft reset GPU 5524 * 5525 * @rdev: radeon_device pointer 5526 * 5527 * Look up which blocks are hung and attempt 5528 * to reset them. 5529 * Returns 0 for success. 5530 */ 5531 int cik_asic_reset(struct radeon_device *rdev) 5532 { 5533 u32 reset_mask; 5534 5535 reset_mask = cik_gpu_check_soft_reset(rdev); 5536 5537 if (reset_mask) 5538 r600_set_bios_scratch_engine_hung(rdev, true); 5539 5540 /* try soft reset */ 5541 cik_gpu_soft_reset(rdev, reset_mask); 5542 5543 reset_mask = cik_gpu_check_soft_reset(rdev); 5544 5545 /* try pci config reset */ 5546 if (reset_mask && radeon_hard_reset) 5547 cik_gpu_pci_config_reset(rdev); 5548 5549 reset_mask = cik_gpu_check_soft_reset(rdev); 5550 5551 if (!reset_mask) 5552 r600_set_bios_scratch_engine_hung(rdev, false); 5553 5554 return 0; 5555 } 5556 5557 /** 5558 * cik_gfx_is_lockup - check if the 3D engine is locked up 5559 * 5560 * @rdev: radeon_device pointer 5561 * @ring: radeon_ring structure holding ring information 5562 * 5563 * Check if the 3D engine is locked up (CIK). 5564 * Returns true if the engine is locked, false if not. 5565 */ 5566 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 5567 { 5568 u32 reset_mask = cik_gpu_check_soft_reset(rdev); 5569 5570 if (!(reset_mask & (RADEON_RESET_GFX | 5571 RADEON_RESET_COMPUTE | 5572 RADEON_RESET_CP))) { 5573 radeon_ring_lockup_update(rdev, ring); 5574 return false; 5575 } 5576 return radeon_ring_test_lockup(rdev, ring); 5577 } 5578 5579 /* MC */ 5580 /** 5581 * cik_mc_program - program the GPU memory controller 5582 * 5583 * @rdev: radeon_device pointer 5584 * 5585 * Set the location of vram, gart, and AGP in the GPU's 5586 * physical address space (CIK). 5587 */ 5588 static void cik_mc_program(struct radeon_device *rdev) 5589 { 5590 struct evergreen_mc_save save; 5591 u32 tmp; 5592 int i, j; 5593 5594 /* Initialize HDP */ 5595 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 5596 WREG32((0x2c14 + j), 0x00000000); 5597 WREG32((0x2c18 + j), 0x00000000); 5598 WREG32((0x2c1c + j), 0x00000000); 5599 WREG32((0x2c20 + j), 0x00000000); 5600 WREG32((0x2c24 + j), 0x00000000); 5601 } 5602 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 5603 5604 evergreen_mc_stop(rdev, &save); 5605 if (radeon_mc_wait_for_idle(rdev)) { 5606 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5607 } 5608 /* Lockout access through VGA aperture*/ 5609 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 5610 /* Update configuration */ 5611 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 5612 rdev->mc.vram_start >> 12); 5613 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 5614 rdev->mc.vram_end >> 12); 5615 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 5616 rdev->vram_scratch.gpu_addr >> 12); 5617 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 5618 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 5619 WREG32(MC_VM_FB_LOCATION, tmp); 5620 /* XXX double check these! */ 5621 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 5622 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 5623 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 5624 WREG32(MC_VM_AGP_BASE, 0); 5625 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 5626 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 5627 if (radeon_mc_wait_for_idle(rdev)) { 5628 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5629 } 5630 evergreen_mc_resume(rdev, &save); 5631 /* we need to own VRAM, so turn off the VGA renderer here 5632 * to stop it overwriting our objects */ 5633 rv515_vga_render_disable(rdev); 5634 } 5635 5636 /** 5637 * cik_mc_init - initialize the memory controller driver params 5638 * 5639 * @rdev: radeon_device pointer 5640 * 5641 * Look up the amount of vram, vram width, and decide how to place 5642 * vram and gart within the GPU's physical address space (CIK). 5643 * Returns 0 for success. 5644 */ 5645 static int cik_mc_init(struct radeon_device *rdev) 5646 { 5647 u32 tmp; 5648 int chansize, numchan; 5649 5650 /* Get VRAM informations */ 5651 rdev->mc.vram_is_ddr = true; 5652 tmp = RREG32(MC_ARB_RAMCFG); 5653 if (tmp & CHANSIZE_MASK) { 5654 chansize = 64; 5655 } else { 5656 chansize = 32; 5657 } 5658 tmp = RREG32(MC_SHARED_CHMAP); 5659 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 5660 case 0: 5661 default: 5662 numchan = 1; 5663 break; 5664 case 1: 5665 numchan = 2; 5666 break; 5667 case 2: 5668 numchan = 4; 5669 break; 5670 case 3: 5671 numchan = 8; 5672 break; 5673 case 4: 5674 numchan = 3; 5675 break; 5676 case 5: 5677 numchan = 6; 5678 break; 5679 case 6: 5680 numchan = 10; 5681 break; 5682 case 7: 5683 numchan = 12; 5684 break; 5685 case 8: 5686 numchan = 16; 5687 break; 5688 } 5689 rdev->mc.vram_width = numchan * chansize; 5690 /* Could aper size report 0 ? */ 5691 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 5692 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 5693 /* size in MB on si */ 5694 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5695 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5696 rdev->mc.visible_vram_size = rdev->mc.aper_size; 5697 si_vram_gtt_location(rdev, &rdev->mc); 5698 radeon_update_bandwidth_info(rdev); 5699 5700 return 0; 5701 } 5702 5703 /* 5704 * GART 5705 * VMID 0 is the physical GPU addresses as used by the kernel. 5706 * VMIDs 1-15 are used for userspace clients and are handled 5707 * by the radeon vm/hsa code. 5708 */ 5709 /** 5710 * cik_pcie_gart_tlb_flush - gart tlb flush callback 5711 * 5712 * @rdev: radeon_device pointer 5713 * 5714 * Flush the TLB for the VMID 0 page table (CIK). 5715 */ 5716 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) 5717 { 5718 /* flush hdp cache */ 5719 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); 5720 5721 /* bits 0-15 are the VM contexts0-15 */ 5722 WREG32(VM_INVALIDATE_REQUEST, 0x1); 5723 } 5724 5725 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev) 5726 { 5727 int i; 5728 uint32_t sh_mem_bases, sh_mem_config; 5729 5730 sh_mem_bases = 0x6000 | 0x6000 << 16; 5731 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED); 5732 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED); 5733 5734 mutex_lock(&rdev->srbm_mutex); 5735 for (i = 8; i < 16; i++) { 5736 cik_srbm_select(rdev, 0, 0, 0, i); 5737 /* CP and shaders */ 5738 WREG32(SH_MEM_CONFIG, sh_mem_config); 5739 WREG32(SH_MEM_APE1_BASE, 1); 5740 WREG32(SH_MEM_APE1_LIMIT, 0); 5741 WREG32(SH_MEM_BASES, sh_mem_bases); 5742 } 5743 cik_srbm_select(rdev, 0, 0, 0, 0); 5744 mutex_unlock(&rdev->srbm_mutex); 5745 } 5746 5747 /** 5748 * cik_pcie_gart_enable - gart enable 5749 * 5750 * @rdev: radeon_device pointer 5751 * 5752 * This sets up the TLBs, programs the page tables for VMID0, 5753 * sets up the hw for VMIDs 1-15 which are allocated on 5754 * demand, and sets up the global locations for the LDS, GDS, 5755 * and GPUVM for FSA64 clients (CIK). 5756 * Returns 0 for success, errors for failure. 5757 */ 5758 static int cik_pcie_gart_enable(struct radeon_device *rdev) 5759 { 5760 int r, i; 5761 5762 if (rdev->gart.robj == NULL) { 5763 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 5764 return -EINVAL; 5765 } 5766 r = radeon_gart_table_vram_pin(rdev); 5767 if (r) 5768 return r; 5769 /* Setup TLB control */ 5770 WREG32(MC_VM_MX_L1_TLB_CNTL, 5771 (0xA << 7) | 5772 ENABLE_L1_TLB | 5773 ENABLE_L1_FRAGMENT_PROCESSING | 5774 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5775 ENABLE_ADVANCED_DRIVER_MODEL | 5776 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5777 /* Setup L2 cache */ 5778 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 5779 ENABLE_L2_FRAGMENT_PROCESSING | 5780 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5781 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5782 EFFECTIVE_L2_QUEUE_SIZE(7) | 5783 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5784 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 5785 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5786 BANK_SELECT(4) | 5787 L2_CACHE_BIGK_FRAGMENT_SIZE(4)); 5788 /* setup context0 */ 5789 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 5790 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 5791 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 5792 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 5793 (u32)(rdev->dummy_page.addr >> 12)); 5794 WREG32(VM_CONTEXT0_CNTL2, 0); 5795 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 5796 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 5797 5798 WREG32(0x15D4, 0); 5799 WREG32(0x15D8, 0); 5800 WREG32(0x15DC, 0); 5801 5802 /* restore context1-15 */ 5803 /* set vm size, must be a multiple of 4 */ 5804 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 5805 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 5806 for (i = 1; i < 16; i++) { 5807 if (i < 8) 5808 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 5809 rdev->vm_manager.saved_table_addr[i]); 5810 else 5811 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 5812 rdev->vm_manager.saved_table_addr[i]); 5813 } 5814 5815 /* enable context1-15 */ 5816 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 5817 (u32)(rdev->dummy_page.addr >> 12)); 5818 WREG32(VM_CONTEXT1_CNTL2, 4); 5819 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 5820 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | 5821 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5822 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5823 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5824 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5825 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 5826 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 5827 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 5828 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 5829 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 5830 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 5831 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5832 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 5833 5834 if (rdev->family == CHIP_KAVERI) { 5835 u32 tmp = RREG32(CHUB_CONTROL); 5836 tmp &= ~BYPASS_VM; 5837 WREG32(CHUB_CONTROL, tmp); 5838 } 5839 5840 /* XXX SH_MEM regs */ 5841 /* where to put LDS, scratch, GPUVM in FSA64 space */ 5842 mutex_lock(&rdev->srbm_mutex); 5843 for (i = 0; i < 16; i++) { 5844 cik_srbm_select(rdev, 0, 0, 0, i); 5845 /* CP and shaders */ 5846 WREG32(SH_MEM_CONFIG, 0); 5847 WREG32(SH_MEM_APE1_BASE, 1); 5848 WREG32(SH_MEM_APE1_LIMIT, 0); 5849 WREG32(SH_MEM_BASES, 0); 5850 /* SDMA GFX */ 5851 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); 5852 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); 5853 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); 5854 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); 5855 /* XXX SDMA RLC - todo */ 5856 } 5857 cik_srbm_select(rdev, 0, 0, 0, 0); 5858 mutex_unlock(&rdev->srbm_mutex); 5859 5860 cik_pcie_init_compute_vmid(rdev); 5861 5862 cik_pcie_gart_tlb_flush(rdev); 5863 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 5864 (unsigned)(rdev->mc.gtt_size >> 20), 5865 (unsigned long long)rdev->gart.table_addr); 5866 rdev->gart.ready = true; 5867 return 0; 5868 } 5869 5870 /** 5871 * cik_pcie_gart_disable - gart disable 5872 * 5873 * @rdev: radeon_device pointer 5874 * 5875 * This disables all VM page table (CIK). 5876 */ 5877 static void cik_pcie_gart_disable(struct radeon_device *rdev) 5878 { 5879 unsigned i; 5880 5881 for (i = 1; i < 16; ++i) { 5882 uint32_t reg; 5883 if (i < 8) 5884 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); 5885 else 5886 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); 5887 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); 5888 } 5889 5890 /* Disable all tables */ 5891 WREG32(VM_CONTEXT0_CNTL, 0); 5892 WREG32(VM_CONTEXT1_CNTL, 0); 5893 /* Setup TLB control */ 5894 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5895 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5896 /* Setup L2 cache */ 5897 WREG32(VM_L2_CNTL, 5898 ENABLE_L2_FRAGMENT_PROCESSING | 5899 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5900 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5901 EFFECTIVE_L2_QUEUE_SIZE(7) | 5902 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5903 WREG32(VM_L2_CNTL2, 0); 5904 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5905 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5906 radeon_gart_table_vram_unpin(rdev); 5907 } 5908 5909 /** 5910 * cik_pcie_gart_fini - vm fini callback 5911 * 5912 * @rdev: radeon_device pointer 5913 * 5914 * Tears down the driver GART/VM setup (CIK). 5915 */ 5916 static void cik_pcie_gart_fini(struct radeon_device *rdev) 5917 { 5918 cik_pcie_gart_disable(rdev); 5919 radeon_gart_table_vram_free(rdev); 5920 radeon_gart_fini(rdev); 5921 } 5922 5923 /* vm parser */ 5924 /** 5925 * cik_ib_parse - vm ib_parse callback 5926 * 5927 * @rdev: radeon_device pointer 5928 * @ib: indirect buffer pointer 5929 * 5930 * CIK uses hw IB checking so this is a nop (CIK). 5931 */ 5932 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 5933 { 5934 return 0; 5935 } 5936 5937 /* 5938 * vm 5939 * VMID 0 is the physical GPU addresses as used by the kernel. 5940 * VMIDs 1-15 are used for userspace clients and are handled 5941 * by the radeon vm/hsa code. 5942 */ 5943 /** 5944 * cik_vm_init - cik vm init callback 5945 * 5946 * @rdev: radeon_device pointer 5947 * 5948 * Inits cik specific vm parameters (number of VMs, base of vram for 5949 * VMIDs 1-15) (CIK). 5950 * Returns 0 for success. 5951 */ 5952 int cik_vm_init(struct radeon_device *rdev) 5953 { 5954 /* 5955 * number of VMs 5956 * VMID 0 is reserved for System 5957 * radeon graphics/compute will use VMIDs 1-7 5958 * amdkfd will use VMIDs 8-15 5959 */ 5960 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS; 5961 /* base offset of vram pages */ 5962 if (rdev->flags & RADEON_IS_IGP) { 5963 u64 tmp = RREG32(MC_VM_FB_OFFSET); 5964 tmp <<= 22; 5965 rdev->vm_manager.vram_base_offset = tmp; 5966 } else 5967 rdev->vm_manager.vram_base_offset = 0; 5968 5969 return 0; 5970 } 5971 5972 /** 5973 * cik_vm_fini - cik vm fini callback 5974 * 5975 * @rdev: radeon_device pointer 5976 * 5977 * Tear down any asic specific VM setup (CIK). 5978 */ 5979 void cik_vm_fini(struct radeon_device *rdev) 5980 { 5981 } 5982 5983 /** 5984 * cik_vm_decode_fault - print human readable fault info 5985 * 5986 * @rdev: radeon_device pointer 5987 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 5988 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 5989 * 5990 * Print human readable fault information (CIK). 5991 */ 5992 static void cik_vm_decode_fault(struct radeon_device *rdev, 5993 u32 status, u32 addr, u32 mc_client) 5994 { 5995 u32 mc_id; 5996 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 5997 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 5998 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, 5999 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; 6000 6001 if (rdev->family == CHIP_HAWAII) 6002 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 6003 else 6004 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 6005 6006 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", 6007 protections, vmid, addr, 6008 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 6009 block, mc_client, mc_id); 6010 } 6011 6012 /** 6013 * cik_vm_flush - cik vm flush using the CP 6014 * 6015 * @rdev: radeon_device pointer 6016 * 6017 * Update the page table base and flush the VM TLB 6018 * using the CP (CIK). 6019 */ 6020 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, 6021 unsigned vm_id, uint64_t pd_addr) 6022 { 6023 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX); 6024 6025 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6026 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6027 WRITE_DATA_DST_SEL(0))); 6028 if (vm_id < 8) { 6029 radeon_ring_write(ring, 6030 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); 6031 } else { 6032 radeon_ring_write(ring, 6033 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); 6034 } 6035 radeon_ring_write(ring, 0); 6036 radeon_ring_write(ring, pd_addr >> 12); 6037 6038 /* update SH_MEM_* regs */ 6039 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6040 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6041 WRITE_DATA_DST_SEL(0))); 6042 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 6043 radeon_ring_write(ring, 0); 6044 radeon_ring_write(ring, VMID(vm_id)); 6045 6046 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); 6047 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6048 WRITE_DATA_DST_SEL(0))); 6049 radeon_ring_write(ring, SH_MEM_BASES >> 2); 6050 radeon_ring_write(ring, 0); 6051 6052 radeon_ring_write(ring, 0); /* SH_MEM_BASES */ 6053 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */ 6054 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ 6055 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ 6056 6057 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6058 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6059 WRITE_DATA_DST_SEL(0))); 6060 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 6061 radeon_ring_write(ring, 0); 6062 radeon_ring_write(ring, VMID(0)); 6063 6064 /* HDP flush */ 6065 cik_hdp_flush_cp_ring_emit(rdev, ring->idx); 6066 6067 /* bits 0-15 are the VM contexts0-15 */ 6068 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6069 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6070 WRITE_DATA_DST_SEL(0))); 6071 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 6072 radeon_ring_write(ring, 0); 6073 radeon_ring_write(ring, 1 << vm_id); 6074 6075 /* wait for the invalidate to complete */ 6076 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6077 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 6078 WAIT_REG_MEM_FUNCTION(0) | /* always */ 6079 WAIT_REG_MEM_ENGINE(0))); /* me */ 6080 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 6081 radeon_ring_write(ring, 0); 6082 radeon_ring_write(ring, 0); /* ref */ 6083 radeon_ring_write(ring, 0); /* mask */ 6084 radeon_ring_write(ring, 0x20); /* poll interval */ 6085 6086 /* compute doesn't have PFP */ 6087 if (usepfp) { 6088 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6089 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6090 radeon_ring_write(ring, 0x0); 6091 } 6092 } 6093 6094 /* 6095 * RLC 6096 * The RLC is a multi-purpose microengine that handles a 6097 * variety of functions, the most important of which is 6098 * the interrupt controller. 6099 */ 6100 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 6101 bool enable) 6102 { 6103 u32 tmp = RREG32(CP_INT_CNTL_RING0); 6104 6105 if (enable) 6106 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6107 else 6108 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6109 WREG32(CP_INT_CNTL_RING0, tmp); 6110 } 6111 6112 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) 6113 { 6114 u32 tmp; 6115 6116 tmp = RREG32(RLC_LB_CNTL); 6117 if (enable) 6118 tmp |= LOAD_BALANCE_ENABLE; 6119 else 6120 tmp &= ~LOAD_BALANCE_ENABLE; 6121 WREG32(RLC_LB_CNTL, tmp); 6122 } 6123 6124 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) 6125 { 6126 u32 i, j, k; 6127 u32 mask; 6128 6129 mutex_lock(&rdev->grbm_idx_mutex); 6130 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6131 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6132 cik_select_se_sh(rdev, i, j); 6133 for (k = 0; k < rdev->usec_timeout; k++) { 6134 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) 6135 break; 6136 udelay(1); 6137 } 6138 } 6139 } 6140 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6141 mutex_unlock(&rdev->grbm_idx_mutex); 6142 6143 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; 6144 for (k = 0; k < rdev->usec_timeout; k++) { 6145 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 6146 break; 6147 udelay(1); 6148 } 6149 } 6150 6151 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) 6152 { 6153 u32 tmp; 6154 6155 tmp = RREG32(RLC_CNTL); 6156 if (tmp != rlc) 6157 WREG32(RLC_CNTL, rlc); 6158 } 6159 6160 static u32 cik_halt_rlc(struct radeon_device *rdev) 6161 { 6162 u32 data, orig; 6163 6164 orig = data = RREG32(RLC_CNTL); 6165 6166 if (data & RLC_ENABLE) { 6167 u32 i; 6168 6169 data &= ~RLC_ENABLE; 6170 WREG32(RLC_CNTL, data); 6171 6172 for (i = 0; i < rdev->usec_timeout; i++) { 6173 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) 6174 break; 6175 udelay(1); 6176 } 6177 6178 cik_wait_for_rlc_serdes(rdev); 6179 } 6180 6181 return orig; 6182 } 6183 6184 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) 6185 { 6186 u32 tmp, i, mask; 6187 6188 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); 6189 WREG32(RLC_GPR_REG2, tmp); 6190 6191 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; 6192 for (i = 0; i < rdev->usec_timeout; i++) { 6193 if ((RREG32(RLC_GPM_STAT) & mask) == mask) 6194 break; 6195 udelay(1); 6196 } 6197 6198 for (i = 0; i < rdev->usec_timeout; i++) { 6199 if ((RREG32(RLC_GPR_REG2) & REQ) == 0) 6200 break; 6201 udelay(1); 6202 } 6203 } 6204 6205 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) 6206 { 6207 u32 tmp; 6208 6209 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); 6210 WREG32(RLC_GPR_REG2, tmp); 6211 } 6212 6213 /** 6214 * cik_rlc_stop - stop the RLC ME 6215 * 6216 * @rdev: radeon_device pointer 6217 * 6218 * Halt the RLC ME (MicroEngine) (CIK). 6219 */ 6220 static void cik_rlc_stop(struct radeon_device *rdev) 6221 { 6222 WREG32(RLC_CNTL, 0); 6223 6224 cik_enable_gui_idle_interrupt(rdev, false); 6225 6226 cik_wait_for_rlc_serdes(rdev); 6227 } 6228 6229 /** 6230 * cik_rlc_start - start the RLC ME 6231 * 6232 * @rdev: radeon_device pointer 6233 * 6234 * Unhalt the RLC ME (MicroEngine) (CIK). 6235 */ 6236 static void cik_rlc_start(struct radeon_device *rdev) 6237 { 6238 WREG32(RLC_CNTL, RLC_ENABLE); 6239 6240 cik_enable_gui_idle_interrupt(rdev, true); 6241 6242 udelay(50); 6243 } 6244 6245 /** 6246 * cik_rlc_resume - setup the RLC hw 6247 * 6248 * @rdev: radeon_device pointer 6249 * 6250 * Initialize the RLC registers, load the ucode, 6251 * and start the RLC (CIK). 6252 * Returns 0 for success, -EINVAL if the ucode is not available. 6253 */ 6254 static int cik_rlc_resume(struct radeon_device *rdev) 6255 { 6256 u32 i, size, tmp; 6257 6258 if (!rdev->rlc_fw) 6259 return -EINVAL; 6260 6261 cik_rlc_stop(rdev); 6262 6263 /* disable CG */ 6264 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; 6265 WREG32(RLC_CGCG_CGLS_CTRL, tmp); 6266 6267 si_rlc_reset(rdev); 6268 6269 cik_init_pg(rdev); 6270 6271 cik_init_cg(rdev); 6272 6273 WREG32(RLC_LB_CNTR_INIT, 0); 6274 WREG32(RLC_LB_CNTR_MAX, 0x00008000); 6275 6276 mutex_lock(&rdev->grbm_idx_mutex); 6277 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6278 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 6279 WREG32(RLC_LB_PARAMS, 0x00600408); 6280 WREG32(RLC_LB_CNTL, 0x80000004); 6281 mutex_unlock(&rdev->grbm_idx_mutex); 6282 6283 WREG32(RLC_MC_CNTL, 0); 6284 WREG32(RLC_UCODE_CNTL, 0); 6285 6286 if (rdev->new_fw) { 6287 const struct rlc_firmware_header_v1_0 *hdr = 6288 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; 6289 const __le32 *fw_data = (const __le32 *) 6290 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6291 6292 radeon_ucode_print_rlc_hdr(&hdr->header); 6293 6294 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 6295 WREG32(RLC_GPM_UCODE_ADDR, 0); 6296 for (i = 0; i < size; i++) 6297 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 6298 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version)); 6299 } else { 6300 const __be32 *fw_data; 6301 6302 switch (rdev->family) { 6303 case CHIP_BONAIRE: 6304 case CHIP_HAWAII: 6305 default: 6306 size = BONAIRE_RLC_UCODE_SIZE; 6307 break; 6308 case CHIP_KAVERI: 6309 size = KV_RLC_UCODE_SIZE; 6310 break; 6311 case CHIP_KABINI: 6312 size = KB_RLC_UCODE_SIZE; 6313 break; 6314 case CHIP_MULLINS: 6315 size = ML_RLC_UCODE_SIZE; 6316 break; 6317 } 6318 6319 fw_data = (const __be32 *)rdev->rlc_fw->data; 6320 WREG32(RLC_GPM_UCODE_ADDR, 0); 6321 for (i = 0; i < size; i++) 6322 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); 6323 WREG32(RLC_GPM_UCODE_ADDR, 0); 6324 } 6325 6326 /* XXX - find out what chips support lbpw */ 6327 cik_enable_lbpw(rdev, false); 6328 6329 if (rdev->family == CHIP_BONAIRE) 6330 WREG32(RLC_DRIVER_DMA_STATUS, 0); 6331 6332 cik_rlc_start(rdev); 6333 6334 return 0; 6335 } 6336 6337 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) 6338 { 6339 u32 data, orig, tmp, tmp2; 6340 6341 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 6342 6343 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 6344 cik_enable_gui_idle_interrupt(rdev, true); 6345 6346 tmp = cik_halt_rlc(rdev); 6347 6348 mutex_lock(&rdev->grbm_idx_mutex); 6349 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6350 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6351 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6352 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; 6353 WREG32(RLC_SERDES_WR_CTRL, tmp2); 6354 mutex_unlock(&rdev->grbm_idx_mutex); 6355 6356 cik_update_rlc(rdev, tmp); 6357 6358 data |= CGCG_EN | CGLS_EN; 6359 } else { 6360 cik_enable_gui_idle_interrupt(rdev, false); 6361 6362 RREG32(CB_CGTT_SCLK_CTRL); 6363 RREG32(CB_CGTT_SCLK_CTRL); 6364 RREG32(CB_CGTT_SCLK_CTRL); 6365 RREG32(CB_CGTT_SCLK_CTRL); 6366 6367 data &= ~(CGCG_EN | CGLS_EN); 6368 } 6369 6370 if (orig != data) 6371 WREG32(RLC_CGCG_CGLS_CTRL, data); 6372 6373 } 6374 6375 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) 6376 { 6377 u32 data, orig, tmp = 0; 6378 6379 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 6380 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { 6381 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 6382 orig = data = RREG32(CP_MEM_SLP_CNTL); 6383 data |= CP_MEM_LS_EN; 6384 if (orig != data) 6385 WREG32(CP_MEM_SLP_CNTL, data); 6386 } 6387 } 6388 6389 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6390 data |= 0x00000001; 6391 data &= 0xfffffffd; 6392 if (orig != data) 6393 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6394 6395 tmp = cik_halt_rlc(rdev); 6396 6397 mutex_lock(&rdev->grbm_idx_mutex); 6398 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6399 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6400 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6401 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; 6402 WREG32(RLC_SERDES_WR_CTRL, data); 6403 mutex_unlock(&rdev->grbm_idx_mutex); 6404 6405 cik_update_rlc(rdev, tmp); 6406 6407 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { 6408 orig = data = RREG32(CGTS_SM_CTRL_REG); 6409 data &= ~SM_MODE_MASK; 6410 data |= SM_MODE(0x2); 6411 data |= SM_MODE_ENABLE; 6412 data &= ~CGTS_OVERRIDE; 6413 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && 6414 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) 6415 data &= ~CGTS_LS_OVERRIDE; 6416 data &= ~ON_MONITOR_ADD_MASK; 6417 data |= ON_MONITOR_ADD_EN; 6418 data |= ON_MONITOR_ADD(0x96); 6419 if (orig != data) 6420 WREG32(CGTS_SM_CTRL_REG, data); 6421 } 6422 } else { 6423 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6424 data |= 0x00000003; 6425 if (orig != data) 6426 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6427 6428 data = RREG32(RLC_MEM_SLP_CNTL); 6429 if (data & RLC_MEM_LS_EN) { 6430 data &= ~RLC_MEM_LS_EN; 6431 WREG32(RLC_MEM_SLP_CNTL, data); 6432 } 6433 6434 data = RREG32(CP_MEM_SLP_CNTL); 6435 if (data & CP_MEM_LS_EN) { 6436 data &= ~CP_MEM_LS_EN; 6437 WREG32(CP_MEM_SLP_CNTL, data); 6438 } 6439 6440 orig = data = RREG32(CGTS_SM_CTRL_REG); 6441 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; 6442 if (orig != data) 6443 WREG32(CGTS_SM_CTRL_REG, data); 6444 6445 tmp = cik_halt_rlc(rdev); 6446 6447 mutex_lock(&rdev->grbm_idx_mutex); 6448 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6449 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6450 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6451 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; 6452 WREG32(RLC_SERDES_WR_CTRL, data); 6453 mutex_unlock(&rdev->grbm_idx_mutex); 6454 6455 cik_update_rlc(rdev, tmp); 6456 } 6457 } 6458 6459 static const u32 mc_cg_registers[] = 6460 { 6461 MC_HUB_MISC_HUB_CG, 6462 MC_HUB_MISC_SIP_CG, 6463 MC_HUB_MISC_VM_CG, 6464 MC_XPB_CLK_GAT, 6465 ATC_MISC_CG, 6466 MC_CITF_MISC_WR_CG, 6467 MC_CITF_MISC_RD_CG, 6468 MC_CITF_MISC_VM_CG, 6469 VM_L2_CG, 6470 }; 6471 6472 static void cik_enable_mc_ls(struct radeon_device *rdev, 6473 bool enable) 6474 { 6475 int i; 6476 u32 orig, data; 6477 6478 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6479 orig = data = RREG32(mc_cg_registers[i]); 6480 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 6481 data |= MC_LS_ENABLE; 6482 else 6483 data &= ~MC_LS_ENABLE; 6484 if (data != orig) 6485 WREG32(mc_cg_registers[i], data); 6486 } 6487 } 6488 6489 static void cik_enable_mc_mgcg(struct radeon_device *rdev, 6490 bool enable) 6491 { 6492 int i; 6493 u32 orig, data; 6494 6495 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6496 orig = data = RREG32(mc_cg_registers[i]); 6497 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 6498 data |= MC_CG_ENABLE; 6499 else 6500 data &= ~MC_CG_ENABLE; 6501 if (data != orig) 6502 WREG32(mc_cg_registers[i], data); 6503 } 6504 } 6505 6506 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, 6507 bool enable) 6508 { 6509 u32 orig, data; 6510 6511 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 6512 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); 6513 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); 6514 } else { 6515 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); 6516 data |= 0xff000000; 6517 if (data != orig) 6518 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); 6519 6520 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); 6521 data |= 0xff000000; 6522 if (data != orig) 6523 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); 6524 } 6525 } 6526 6527 static void cik_enable_sdma_mgls(struct radeon_device *rdev, 6528 bool enable) 6529 { 6530 u32 orig, data; 6531 6532 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { 6533 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6534 data |= 0x100; 6535 if (orig != data) 6536 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6537 6538 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6539 data |= 0x100; 6540 if (orig != data) 6541 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6542 } else { 6543 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6544 data &= ~0x100; 6545 if (orig != data) 6546 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6547 6548 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6549 data &= ~0x100; 6550 if (orig != data) 6551 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6552 } 6553 } 6554 6555 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, 6556 bool enable) 6557 { 6558 u32 orig, data; 6559 6560 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 6561 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6562 data = 0xfff; 6563 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6564 6565 orig = data = RREG32(UVD_CGC_CTRL); 6566 data |= DCM; 6567 if (orig != data) 6568 WREG32(UVD_CGC_CTRL, data); 6569 } else { 6570 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6571 data &= ~0xfff; 6572 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6573 6574 orig = data = RREG32(UVD_CGC_CTRL); 6575 data &= ~DCM; 6576 if (orig != data) 6577 WREG32(UVD_CGC_CTRL, data); 6578 } 6579 } 6580 6581 static void cik_enable_bif_mgls(struct radeon_device *rdev, 6582 bool enable) 6583 { 6584 u32 orig, data; 6585 6586 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 6587 6588 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 6589 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 6590 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 6591 else 6592 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 6593 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 6594 6595 if (orig != data) 6596 WREG32_PCIE_PORT(PCIE_CNTL2, data); 6597 } 6598 6599 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, 6600 bool enable) 6601 { 6602 u32 orig, data; 6603 6604 orig = data = RREG32(HDP_HOST_PATH_CNTL); 6605 6606 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 6607 data &= ~CLOCK_GATING_DIS; 6608 else 6609 data |= CLOCK_GATING_DIS; 6610 6611 if (orig != data) 6612 WREG32(HDP_HOST_PATH_CNTL, data); 6613 } 6614 6615 static void cik_enable_hdp_ls(struct radeon_device *rdev, 6616 bool enable) 6617 { 6618 u32 orig, data; 6619 6620 orig = data = RREG32(HDP_MEM_POWER_LS); 6621 6622 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 6623 data |= HDP_LS_ENABLE; 6624 else 6625 data &= ~HDP_LS_ENABLE; 6626 6627 if (orig != data) 6628 WREG32(HDP_MEM_POWER_LS, data); 6629 } 6630 6631 void cik_update_cg(struct radeon_device *rdev, 6632 u32 block, bool enable) 6633 { 6634 6635 if (block & RADEON_CG_BLOCK_GFX) { 6636 cik_enable_gui_idle_interrupt(rdev, false); 6637 /* order matters! */ 6638 if (enable) { 6639 cik_enable_mgcg(rdev, true); 6640 cik_enable_cgcg(rdev, true); 6641 } else { 6642 cik_enable_cgcg(rdev, false); 6643 cik_enable_mgcg(rdev, false); 6644 } 6645 cik_enable_gui_idle_interrupt(rdev, true); 6646 } 6647 6648 if (block & RADEON_CG_BLOCK_MC) { 6649 if (!(rdev->flags & RADEON_IS_IGP)) { 6650 cik_enable_mc_mgcg(rdev, enable); 6651 cik_enable_mc_ls(rdev, enable); 6652 } 6653 } 6654 6655 if (block & RADEON_CG_BLOCK_SDMA) { 6656 cik_enable_sdma_mgcg(rdev, enable); 6657 cik_enable_sdma_mgls(rdev, enable); 6658 } 6659 6660 if (block & RADEON_CG_BLOCK_BIF) { 6661 cik_enable_bif_mgls(rdev, enable); 6662 } 6663 6664 if (block & RADEON_CG_BLOCK_UVD) { 6665 if (rdev->has_uvd) 6666 cik_enable_uvd_mgcg(rdev, enable); 6667 } 6668 6669 if (block & RADEON_CG_BLOCK_HDP) { 6670 cik_enable_hdp_mgcg(rdev, enable); 6671 cik_enable_hdp_ls(rdev, enable); 6672 } 6673 6674 if (block & RADEON_CG_BLOCK_VCE) { 6675 vce_v2_0_enable_mgcg(rdev, enable); 6676 } 6677 } 6678 6679 static void cik_init_cg(struct radeon_device *rdev) 6680 { 6681 6682 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); 6683 6684 if (rdev->has_uvd) 6685 si_init_uvd_internal_cg(rdev); 6686 6687 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6688 RADEON_CG_BLOCK_SDMA | 6689 RADEON_CG_BLOCK_BIF | 6690 RADEON_CG_BLOCK_UVD | 6691 RADEON_CG_BLOCK_HDP), true); 6692 } 6693 6694 static void cik_fini_cg(struct radeon_device *rdev) 6695 { 6696 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6697 RADEON_CG_BLOCK_SDMA | 6698 RADEON_CG_BLOCK_BIF | 6699 RADEON_CG_BLOCK_UVD | 6700 RADEON_CG_BLOCK_HDP), false); 6701 6702 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); 6703 } 6704 6705 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, 6706 bool enable) 6707 { 6708 u32 data, orig; 6709 6710 orig = data = RREG32(RLC_PG_CNTL); 6711 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6712 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6713 else 6714 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6715 if (orig != data) 6716 WREG32(RLC_PG_CNTL, data); 6717 } 6718 6719 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, 6720 bool enable) 6721 { 6722 u32 data, orig; 6723 6724 orig = data = RREG32(RLC_PG_CNTL); 6725 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6726 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6727 else 6728 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6729 if (orig != data) 6730 WREG32(RLC_PG_CNTL, data); 6731 } 6732 6733 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) 6734 { 6735 u32 data, orig; 6736 6737 orig = data = RREG32(RLC_PG_CNTL); 6738 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) 6739 data &= ~DISABLE_CP_PG; 6740 else 6741 data |= DISABLE_CP_PG; 6742 if (orig != data) 6743 WREG32(RLC_PG_CNTL, data); 6744 } 6745 6746 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) 6747 { 6748 u32 data, orig; 6749 6750 orig = data = RREG32(RLC_PG_CNTL); 6751 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) 6752 data &= ~DISABLE_GDS_PG; 6753 else 6754 data |= DISABLE_GDS_PG; 6755 if (orig != data) 6756 WREG32(RLC_PG_CNTL, data); 6757 } 6758 6759 #define CP_ME_TABLE_SIZE 96 6760 #define CP_ME_TABLE_OFFSET 2048 6761 #define CP_MEC_TABLE_OFFSET 4096 6762 6763 void cik_init_cp_pg_table(struct radeon_device *rdev) 6764 { 6765 volatile u32 *dst_ptr; 6766 int me, i, max_me = 4; 6767 u32 bo_offset = 0; 6768 u32 table_offset, table_size; 6769 6770 if (rdev->family == CHIP_KAVERI) 6771 max_me = 5; 6772 6773 if (rdev->rlc.cp_table_ptr == NULL) 6774 return; 6775 6776 /* write the cp table buffer */ 6777 dst_ptr = rdev->rlc.cp_table_ptr; 6778 for (me = 0; me < max_me; me++) { 6779 if (rdev->new_fw) { 6780 const __le32 *fw_data; 6781 const struct gfx_firmware_header_v1_0 *hdr; 6782 6783 if (me == 0) { 6784 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 6785 fw_data = (const __le32 *) 6786 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6787 table_offset = le32_to_cpu(hdr->jt_offset); 6788 table_size = le32_to_cpu(hdr->jt_size); 6789 } else if (me == 1) { 6790 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 6791 fw_data = (const __le32 *) 6792 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6793 table_offset = le32_to_cpu(hdr->jt_offset); 6794 table_size = le32_to_cpu(hdr->jt_size); 6795 } else if (me == 2) { 6796 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 6797 fw_data = (const __le32 *) 6798 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6799 table_offset = le32_to_cpu(hdr->jt_offset); 6800 table_size = le32_to_cpu(hdr->jt_size); 6801 } else if (me == 3) { 6802 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 6803 fw_data = (const __le32 *) 6804 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6805 table_offset = le32_to_cpu(hdr->jt_offset); 6806 table_size = le32_to_cpu(hdr->jt_size); 6807 } else { 6808 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 6809 fw_data = (const __le32 *) 6810 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6811 table_offset = le32_to_cpu(hdr->jt_offset); 6812 table_size = le32_to_cpu(hdr->jt_size); 6813 } 6814 6815 for (i = 0; i < table_size; i ++) { 6816 dst_ptr[bo_offset + i] = 6817 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); 6818 } 6819 bo_offset += table_size; 6820 } else { 6821 const __be32 *fw_data; 6822 table_size = CP_ME_TABLE_SIZE; 6823 6824 if (me == 0) { 6825 fw_data = (const __be32 *)rdev->ce_fw->data; 6826 table_offset = CP_ME_TABLE_OFFSET; 6827 } else if (me == 1) { 6828 fw_data = (const __be32 *)rdev->pfp_fw->data; 6829 table_offset = CP_ME_TABLE_OFFSET; 6830 } else if (me == 2) { 6831 fw_data = (const __be32 *)rdev->me_fw->data; 6832 table_offset = CP_ME_TABLE_OFFSET; 6833 } else { 6834 fw_data = (const __be32 *)rdev->mec_fw->data; 6835 table_offset = CP_MEC_TABLE_OFFSET; 6836 } 6837 6838 for (i = 0; i < table_size; i ++) { 6839 dst_ptr[bo_offset + i] = 6840 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); 6841 } 6842 bo_offset += table_size; 6843 } 6844 } 6845 } 6846 6847 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, 6848 bool enable) 6849 { 6850 u32 data, orig; 6851 6852 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 6853 orig = data = RREG32(RLC_PG_CNTL); 6854 data |= GFX_PG_ENABLE; 6855 if (orig != data) 6856 WREG32(RLC_PG_CNTL, data); 6857 6858 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6859 data |= AUTO_PG_EN; 6860 if (orig != data) 6861 WREG32(RLC_AUTO_PG_CTRL, data); 6862 } else { 6863 orig = data = RREG32(RLC_PG_CNTL); 6864 data &= ~GFX_PG_ENABLE; 6865 if (orig != data) 6866 WREG32(RLC_PG_CNTL, data); 6867 6868 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6869 data &= ~AUTO_PG_EN; 6870 if (orig != data) 6871 WREG32(RLC_AUTO_PG_CTRL, data); 6872 6873 data = RREG32(DB_RENDER_CONTROL); 6874 } 6875 } 6876 6877 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 6878 { 6879 u32 mask = 0, tmp, tmp1; 6880 int i; 6881 6882 mutex_lock(&rdev->grbm_idx_mutex); 6883 cik_select_se_sh(rdev, se, sh); 6884 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 6885 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 6886 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6887 mutex_unlock(&rdev->grbm_idx_mutex); 6888 6889 tmp &= 0xffff0000; 6890 6891 tmp |= tmp1; 6892 tmp >>= 16; 6893 6894 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { 6895 mask <<= 1; 6896 mask |= 1; 6897 } 6898 6899 return (~tmp) & mask; 6900 } 6901 6902 static void cik_init_ao_cu_mask(struct radeon_device *rdev) 6903 { 6904 u32 i, j, k, active_cu_number = 0; 6905 u32 mask, counter, cu_bitmap; 6906 u32 tmp = 0; 6907 6908 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6909 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6910 mask = 1; 6911 cu_bitmap = 0; 6912 counter = 0; 6913 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { 6914 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { 6915 if (counter < 2) 6916 cu_bitmap |= mask; 6917 counter ++; 6918 } 6919 mask <<= 1; 6920 } 6921 6922 active_cu_number += counter; 6923 tmp |= (cu_bitmap << (i * 16 + j * 8)); 6924 } 6925 } 6926 6927 WREG32(RLC_PG_AO_CU_MASK, tmp); 6928 6929 tmp = RREG32(RLC_MAX_PG_CU); 6930 tmp &= ~MAX_PU_CU_MASK; 6931 tmp |= MAX_PU_CU(active_cu_number); 6932 WREG32(RLC_MAX_PG_CU, tmp); 6933 } 6934 6935 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, 6936 bool enable) 6937 { 6938 u32 data, orig; 6939 6940 orig = data = RREG32(RLC_PG_CNTL); 6941 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) 6942 data |= STATIC_PER_CU_PG_ENABLE; 6943 else 6944 data &= ~STATIC_PER_CU_PG_ENABLE; 6945 if (orig != data) 6946 WREG32(RLC_PG_CNTL, data); 6947 } 6948 6949 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, 6950 bool enable) 6951 { 6952 u32 data, orig; 6953 6954 orig = data = RREG32(RLC_PG_CNTL); 6955 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) 6956 data |= DYN_PER_CU_PG_ENABLE; 6957 else 6958 data &= ~DYN_PER_CU_PG_ENABLE; 6959 if (orig != data) 6960 WREG32(RLC_PG_CNTL, data); 6961 } 6962 6963 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 6964 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D 6965 6966 static void cik_init_gfx_cgpg(struct radeon_device *rdev) 6967 { 6968 u32 data, orig; 6969 u32 i; 6970 6971 if (rdev->rlc.cs_data) { 6972 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6973 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); 6974 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); 6975 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); 6976 } else { 6977 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6978 for (i = 0; i < 3; i++) 6979 WREG32(RLC_GPM_SCRATCH_DATA, 0); 6980 } 6981 if (rdev->rlc.reg_list) { 6982 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); 6983 for (i = 0; i < rdev->rlc.reg_list_size; i++) 6984 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); 6985 } 6986 6987 orig = data = RREG32(RLC_PG_CNTL); 6988 data |= GFX_PG_SRC; 6989 if (orig != data) 6990 WREG32(RLC_PG_CNTL, data); 6991 6992 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 6993 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); 6994 6995 data = RREG32(CP_RB_WPTR_POLL_CNTL); 6996 data &= ~IDLE_POLL_COUNT_MASK; 6997 data |= IDLE_POLL_COUNT(0x60); 6998 WREG32(CP_RB_WPTR_POLL_CNTL, data); 6999 7000 data = 0x10101010; 7001 WREG32(RLC_PG_DELAY, data); 7002 7003 data = RREG32(RLC_PG_DELAY_2); 7004 data &= ~0xff; 7005 data |= 0x3; 7006 WREG32(RLC_PG_DELAY_2, data); 7007 7008 data = RREG32(RLC_AUTO_PG_CTRL); 7009 data &= ~GRBM_REG_SGIT_MASK; 7010 data |= GRBM_REG_SGIT(0x700); 7011 WREG32(RLC_AUTO_PG_CTRL, data); 7012 7013 } 7014 7015 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) 7016 { 7017 cik_enable_gfx_cgpg(rdev, enable); 7018 cik_enable_gfx_static_mgpg(rdev, enable); 7019 cik_enable_gfx_dynamic_mgpg(rdev, enable); 7020 } 7021 7022 u32 cik_get_csb_size(struct radeon_device *rdev) 7023 { 7024 u32 count = 0; 7025 const struct cs_section_def *sect = NULL; 7026 const struct cs_extent_def *ext = NULL; 7027 7028 if (rdev->rlc.cs_data == NULL) 7029 return 0; 7030 7031 /* begin clear state */ 7032 count += 2; 7033 /* context control state */ 7034 count += 3; 7035 7036 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 7037 for (ext = sect->section; ext->extent != NULL; ++ext) { 7038 if (sect->id == SECT_CONTEXT) 7039 count += 2 + ext->reg_count; 7040 else 7041 return 0; 7042 } 7043 } 7044 /* pa_sc_raster_config/pa_sc_raster_config1 */ 7045 count += 4; 7046 /* end clear state */ 7047 count += 2; 7048 /* clear state */ 7049 count += 2; 7050 7051 return count; 7052 } 7053 7054 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 7055 { 7056 u32 count = 0, i; 7057 const struct cs_section_def *sect = NULL; 7058 const struct cs_extent_def *ext = NULL; 7059 7060 if (rdev->rlc.cs_data == NULL) 7061 return; 7062 if (buffer == NULL) 7063 return; 7064 7065 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 7066 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 7067 7068 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 7069 buffer[count++] = cpu_to_le32(0x80000000); 7070 buffer[count++] = cpu_to_le32(0x80000000); 7071 7072 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 7073 for (ext = sect->section; ext->extent != NULL; ++ext) { 7074 if (sect->id == SECT_CONTEXT) { 7075 buffer[count++] = 7076 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 7077 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 7078 for (i = 0; i < ext->reg_count; i++) 7079 buffer[count++] = cpu_to_le32(ext->extent[i]); 7080 } else { 7081 return; 7082 } 7083 } 7084 } 7085 7086 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 7087 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 7088 switch (rdev->family) { 7089 case CHIP_BONAIRE: 7090 buffer[count++] = cpu_to_le32(0x16000012); 7091 buffer[count++] = cpu_to_le32(0x00000000); 7092 break; 7093 case CHIP_KAVERI: 7094 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 7095 buffer[count++] = cpu_to_le32(0x00000000); 7096 break; 7097 case CHIP_KABINI: 7098 case CHIP_MULLINS: 7099 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 7100 buffer[count++] = cpu_to_le32(0x00000000); 7101 break; 7102 case CHIP_HAWAII: 7103 buffer[count++] = cpu_to_le32(0x3a00161a); 7104 buffer[count++] = cpu_to_le32(0x0000002e); 7105 break; 7106 default: 7107 buffer[count++] = cpu_to_le32(0x00000000); 7108 buffer[count++] = cpu_to_le32(0x00000000); 7109 break; 7110 } 7111 7112 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 7113 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 7114 7115 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 7116 buffer[count++] = cpu_to_le32(0); 7117 } 7118 7119 static void cik_init_pg(struct radeon_device *rdev) 7120 { 7121 if (rdev->pg_flags) { 7122 cik_enable_sck_slowdown_on_pu(rdev, true); 7123 cik_enable_sck_slowdown_on_pd(rdev, true); 7124 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7125 cik_init_gfx_cgpg(rdev); 7126 cik_enable_cp_pg(rdev, true); 7127 cik_enable_gds_pg(rdev, true); 7128 } 7129 cik_init_ao_cu_mask(rdev); 7130 cik_update_gfx_pg(rdev, true); 7131 } 7132 } 7133 7134 static void cik_fini_pg(struct radeon_device *rdev) 7135 { 7136 if (rdev->pg_flags) { 7137 cik_update_gfx_pg(rdev, false); 7138 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7139 cik_enable_cp_pg(rdev, false); 7140 cik_enable_gds_pg(rdev, false); 7141 } 7142 } 7143 } 7144 7145 /* 7146 * Interrupts 7147 * Starting with r6xx, interrupts are handled via a ring buffer. 7148 * Ring buffers are areas of GPU accessible memory that the GPU 7149 * writes interrupt vectors into and the host reads vectors out of. 7150 * There is a rptr (read pointer) that determines where the 7151 * host is currently reading, and a wptr (write pointer) 7152 * which determines where the GPU has written. When the 7153 * pointers are equal, the ring is idle. When the GPU 7154 * writes vectors to the ring buffer, it increments the 7155 * wptr. When there is an interrupt, the host then starts 7156 * fetching commands and processing them until the pointers are 7157 * equal again at which point it updates the rptr. 7158 */ 7159 7160 /** 7161 * cik_enable_interrupts - Enable the interrupt ring buffer 7162 * 7163 * @rdev: radeon_device pointer 7164 * 7165 * Enable the interrupt ring buffer (CIK). 7166 */ 7167 static void cik_enable_interrupts(struct radeon_device *rdev) 7168 { 7169 u32 ih_cntl = RREG32(IH_CNTL); 7170 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7171 7172 ih_cntl |= ENABLE_INTR; 7173 ih_rb_cntl |= IH_RB_ENABLE; 7174 WREG32(IH_CNTL, ih_cntl); 7175 WREG32(IH_RB_CNTL, ih_rb_cntl); 7176 rdev->ih.enabled = true; 7177 } 7178 7179 /** 7180 * cik_disable_interrupts - Disable the interrupt ring buffer 7181 * 7182 * @rdev: radeon_device pointer 7183 * 7184 * Disable the interrupt ring buffer (CIK). 7185 */ 7186 static void cik_disable_interrupts(struct radeon_device *rdev) 7187 { 7188 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7189 u32 ih_cntl = RREG32(IH_CNTL); 7190 7191 ih_rb_cntl &= ~IH_RB_ENABLE; 7192 ih_cntl &= ~ENABLE_INTR; 7193 WREG32(IH_RB_CNTL, ih_rb_cntl); 7194 WREG32(IH_CNTL, ih_cntl); 7195 /* set rptr, wptr to 0 */ 7196 WREG32(IH_RB_RPTR, 0); 7197 WREG32(IH_RB_WPTR, 0); 7198 rdev->ih.enabled = false; 7199 rdev->ih.rptr = 0; 7200 } 7201 7202 /** 7203 * cik_disable_interrupt_state - Disable all interrupt sources 7204 * 7205 * @rdev: radeon_device pointer 7206 * 7207 * Clear all interrupt enable bits used by the driver (CIK). 7208 */ 7209 static void cik_disable_interrupt_state(struct radeon_device *rdev) 7210 { 7211 u32 tmp; 7212 7213 /* gfx ring */ 7214 tmp = RREG32(CP_INT_CNTL_RING0) & 7215 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7216 WREG32(CP_INT_CNTL_RING0, tmp); 7217 /* sdma */ 7218 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7219 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); 7220 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7221 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); 7222 /* compute queues */ 7223 WREG32(CP_ME1_PIPE0_INT_CNTL, 0); 7224 WREG32(CP_ME1_PIPE1_INT_CNTL, 0); 7225 WREG32(CP_ME1_PIPE2_INT_CNTL, 0); 7226 WREG32(CP_ME1_PIPE3_INT_CNTL, 0); 7227 WREG32(CP_ME2_PIPE0_INT_CNTL, 0); 7228 WREG32(CP_ME2_PIPE1_INT_CNTL, 0); 7229 WREG32(CP_ME2_PIPE2_INT_CNTL, 0); 7230 WREG32(CP_ME2_PIPE3_INT_CNTL, 0); 7231 /* grbm */ 7232 WREG32(GRBM_INT_CNTL, 0); 7233 /* vline/vblank, etc. */ 7234 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7235 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7236 if (rdev->num_crtc >= 4) { 7237 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7238 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7239 } 7240 if (rdev->num_crtc >= 6) { 7241 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7242 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7243 } 7244 /* pflip */ 7245 if (rdev->num_crtc >= 2) { 7246 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7247 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7248 } 7249 if (rdev->num_crtc >= 4) { 7250 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7251 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7252 } 7253 if (rdev->num_crtc >= 6) { 7254 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7255 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7256 } 7257 7258 /* dac hotplug */ 7259 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 7260 7261 /* digital hotplug */ 7262 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7263 WREG32(DC_HPD1_INT_CONTROL, tmp); 7264 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7265 WREG32(DC_HPD2_INT_CONTROL, tmp); 7266 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7267 WREG32(DC_HPD3_INT_CONTROL, tmp); 7268 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7269 WREG32(DC_HPD4_INT_CONTROL, tmp); 7270 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7271 WREG32(DC_HPD5_INT_CONTROL, tmp); 7272 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7273 WREG32(DC_HPD6_INT_CONTROL, tmp); 7274 7275 } 7276 7277 /** 7278 * cik_irq_init - init and enable the interrupt ring 7279 * 7280 * @rdev: radeon_device pointer 7281 * 7282 * Allocate a ring buffer for the interrupt controller, 7283 * enable the RLC, disable interrupts, enable the IH 7284 * ring buffer and enable it (CIK). 7285 * Called at device load and reume. 7286 * Returns 0 for success, errors for failure. 7287 */ 7288 static int cik_irq_init(struct radeon_device *rdev) 7289 { 7290 int ret = 0; 7291 int rb_bufsz; 7292 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 7293 7294 /* allocate ring */ 7295 ret = r600_ih_ring_alloc(rdev); 7296 if (ret) 7297 return ret; 7298 7299 /* disable irqs */ 7300 cik_disable_interrupts(rdev); 7301 7302 /* init rlc */ 7303 ret = cik_rlc_resume(rdev); 7304 if (ret) { 7305 r600_ih_ring_fini(rdev); 7306 return ret; 7307 } 7308 7309 /* setup interrupt control */ 7310 /* XXX this should actually be a bus address, not an MC address. same on older asics */ 7311 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 7312 interrupt_cntl = RREG32(INTERRUPT_CNTL); 7313 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 7314 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 7315 */ 7316 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 7317 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 7318 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 7319 WREG32(INTERRUPT_CNTL, interrupt_cntl); 7320 7321 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 7322 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 7323 7324 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 7325 IH_WPTR_OVERFLOW_CLEAR | 7326 (rb_bufsz << 1)); 7327 7328 if (rdev->wb.enabled) 7329 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 7330 7331 /* set the writeback address whether it's enabled or not */ 7332 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 7333 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 7334 7335 WREG32(IH_RB_CNTL, ih_rb_cntl); 7336 7337 /* set rptr, wptr to 0 */ 7338 WREG32(IH_RB_RPTR, 0); 7339 WREG32(IH_RB_WPTR, 0); 7340 7341 /* Default settings for IH_CNTL (disabled at first) */ 7342 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 7343 /* RPTR_REARM only works if msi's are enabled */ 7344 if (rdev->msi_enabled) 7345 ih_cntl |= RPTR_REARM; 7346 WREG32(IH_CNTL, ih_cntl); 7347 7348 /* force the active interrupt state to all disabled */ 7349 cik_disable_interrupt_state(rdev); 7350 7351 pci_set_master(rdev->pdev); 7352 7353 /* enable irqs */ 7354 cik_enable_interrupts(rdev); 7355 7356 return ret; 7357 } 7358 7359 /** 7360 * cik_irq_set - enable/disable interrupt sources 7361 * 7362 * @rdev: radeon_device pointer 7363 * 7364 * Enable interrupt sources on the GPU (vblanks, hpd, 7365 * etc.) (CIK). 7366 * Returns 0 for success, errors for failure. 7367 */ 7368 int cik_irq_set(struct radeon_device *rdev) 7369 { 7370 u32 cp_int_cntl; 7371 u32 cp_m1p0; 7372 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 7373 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 7374 u32 grbm_int_cntl = 0; 7375 u32 dma_cntl, dma_cntl1; 7376 7377 if (!rdev->irq.installed) { 7378 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 7379 return -EINVAL; 7380 } 7381 /* don't enable anything if the ih is disabled */ 7382 if (!rdev->ih.enabled) { 7383 cik_disable_interrupts(rdev); 7384 /* force the active interrupt state to all disabled */ 7385 cik_disable_interrupt_state(rdev); 7386 return 0; 7387 } 7388 7389 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 7390 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7391 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; 7392 7393 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 7394 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 7395 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 7396 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 7397 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 7398 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 7399 7400 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7401 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7402 7403 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7404 7405 /* enable CP interrupts on all rings */ 7406 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 7407 DRM_DEBUG("cik_irq_set: sw int gfx\n"); 7408 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 7409 } 7410 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 7411 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7412 DRM_DEBUG("si_irq_set: sw int cp1\n"); 7413 if (ring->me == 1) { 7414 switch (ring->pipe) { 7415 case 0: 7416 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7417 break; 7418 default: 7419 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7420 break; 7421 } 7422 } else { 7423 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); 7424 } 7425 } 7426 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 7427 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7428 DRM_DEBUG("si_irq_set: sw int cp2\n"); 7429 if (ring->me == 1) { 7430 switch (ring->pipe) { 7431 case 0: 7432 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7433 break; 7434 default: 7435 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7436 break; 7437 } 7438 } else { 7439 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); 7440 } 7441 } 7442 7443 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 7444 DRM_DEBUG("cik_irq_set: sw int dma\n"); 7445 dma_cntl |= TRAP_ENABLE; 7446 } 7447 7448 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 7449 DRM_DEBUG("cik_irq_set: sw int dma1\n"); 7450 dma_cntl1 |= TRAP_ENABLE; 7451 } 7452 7453 if (rdev->irq.crtc_vblank_int[0] || 7454 atomic_read(&rdev->irq.pflip[0])) { 7455 DRM_DEBUG("cik_irq_set: vblank 0\n"); 7456 crtc1 |= VBLANK_INTERRUPT_MASK; 7457 } 7458 if (rdev->irq.crtc_vblank_int[1] || 7459 atomic_read(&rdev->irq.pflip[1])) { 7460 DRM_DEBUG("cik_irq_set: vblank 1\n"); 7461 crtc2 |= VBLANK_INTERRUPT_MASK; 7462 } 7463 if (rdev->irq.crtc_vblank_int[2] || 7464 atomic_read(&rdev->irq.pflip[2])) { 7465 DRM_DEBUG("cik_irq_set: vblank 2\n"); 7466 crtc3 |= VBLANK_INTERRUPT_MASK; 7467 } 7468 if (rdev->irq.crtc_vblank_int[3] || 7469 atomic_read(&rdev->irq.pflip[3])) { 7470 DRM_DEBUG("cik_irq_set: vblank 3\n"); 7471 crtc4 |= VBLANK_INTERRUPT_MASK; 7472 } 7473 if (rdev->irq.crtc_vblank_int[4] || 7474 atomic_read(&rdev->irq.pflip[4])) { 7475 DRM_DEBUG("cik_irq_set: vblank 4\n"); 7476 crtc5 |= VBLANK_INTERRUPT_MASK; 7477 } 7478 if (rdev->irq.crtc_vblank_int[5] || 7479 atomic_read(&rdev->irq.pflip[5])) { 7480 DRM_DEBUG("cik_irq_set: vblank 5\n"); 7481 crtc6 |= VBLANK_INTERRUPT_MASK; 7482 } 7483 if (rdev->irq.hpd[0]) { 7484 DRM_DEBUG("cik_irq_set: hpd 1\n"); 7485 hpd1 |= DC_HPDx_INT_EN; 7486 } 7487 if (rdev->irq.hpd[1]) { 7488 DRM_DEBUG("cik_irq_set: hpd 2\n"); 7489 hpd2 |= DC_HPDx_INT_EN; 7490 } 7491 if (rdev->irq.hpd[2]) { 7492 DRM_DEBUG("cik_irq_set: hpd 3\n"); 7493 hpd3 |= DC_HPDx_INT_EN; 7494 } 7495 if (rdev->irq.hpd[3]) { 7496 DRM_DEBUG("cik_irq_set: hpd 4\n"); 7497 hpd4 |= DC_HPDx_INT_EN; 7498 } 7499 if (rdev->irq.hpd[4]) { 7500 DRM_DEBUG("cik_irq_set: hpd 5\n"); 7501 hpd5 |= DC_HPDx_INT_EN; 7502 } 7503 if (rdev->irq.hpd[5]) { 7504 DRM_DEBUG("cik_irq_set: hpd 6\n"); 7505 hpd6 |= DC_HPDx_INT_EN; 7506 } 7507 7508 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 7509 7510 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); 7511 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); 7512 7513 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); 7514 7515 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 7516 7517 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 7518 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 7519 if (rdev->num_crtc >= 4) { 7520 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 7521 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 7522 } 7523 if (rdev->num_crtc >= 6) { 7524 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 7525 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 7526 } 7527 7528 if (rdev->num_crtc >= 2) { 7529 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 7530 GRPH_PFLIP_INT_MASK); 7531 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 7532 GRPH_PFLIP_INT_MASK); 7533 } 7534 if (rdev->num_crtc >= 4) { 7535 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 7536 GRPH_PFLIP_INT_MASK); 7537 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 7538 GRPH_PFLIP_INT_MASK); 7539 } 7540 if (rdev->num_crtc >= 6) { 7541 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 7542 GRPH_PFLIP_INT_MASK); 7543 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 7544 GRPH_PFLIP_INT_MASK); 7545 } 7546 7547 WREG32(DC_HPD1_INT_CONTROL, hpd1); 7548 WREG32(DC_HPD2_INT_CONTROL, hpd2); 7549 WREG32(DC_HPD3_INT_CONTROL, hpd3); 7550 WREG32(DC_HPD4_INT_CONTROL, hpd4); 7551 WREG32(DC_HPD5_INT_CONTROL, hpd5); 7552 WREG32(DC_HPD6_INT_CONTROL, hpd6); 7553 7554 return 0; 7555 } 7556 7557 /** 7558 * cik_irq_ack - ack interrupt sources 7559 * 7560 * @rdev: radeon_device pointer 7561 * 7562 * Ack interrupt sources on the GPU (vblanks, hpd, 7563 * etc.) (CIK). Certain interrupts sources are sw 7564 * generated and do not require an explicit ack. 7565 */ 7566 static inline void cik_irq_ack(struct radeon_device *rdev) 7567 { 7568 u32 tmp; 7569 7570 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); 7571 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 7572 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 7573 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 7574 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 7575 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 7576 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); 7577 7578 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + 7579 EVERGREEN_CRTC0_REGISTER_OFFSET); 7580 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + 7581 EVERGREEN_CRTC1_REGISTER_OFFSET); 7582 if (rdev->num_crtc >= 4) { 7583 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + 7584 EVERGREEN_CRTC2_REGISTER_OFFSET); 7585 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + 7586 EVERGREEN_CRTC3_REGISTER_OFFSET); 7587 } 7588 if (rdev->num_crtc >= 6) { 7589 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + 7590 EVERGREEN_CRTC4_REGISTER_OFFSET); 7591 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + 7592 EVERGREEN_CRTC5_REGISTER_OFFSET); 7593 } 7594 7595 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 7596 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, 7597 GRPH_PFLIP_INT_CLEAR); 7598 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 7599 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, 7600 GRPH_PFLIP_INT_CLEAR); 7601 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) 7602 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 7603 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) 7604 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 7605 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 7606 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 7607 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) 7608 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 7609 7610 if (rdev->num_crtc >= 4) { 7611 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 7612 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, 7613 GRPH_PFLIP_INT_CLEAR); 7614 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 7615 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, 7616 GRPH_PFLIP_INT_CLEAR); 7617 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 7618 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 7619 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 7620 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 7621 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 7622 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 7623 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 7624 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 7625 } 7626 7627 if (rdev->num_crtc >= 6) { 7628 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 7629 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, 7630 GRPH_PFLIP_INT_CLEAR); 7631 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 7632 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, 7633 GRPH_PFLIP_INT_CLEAR); 7634 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 7635 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 7636 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 7637 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 7638 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 7639 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 7640 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 7641 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 7642 } 7643 7644 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7645 tmp = RREG32(DC_HPD1_INT_CONTROL); 7646 tmp |= DC_HPDx_INT_ACK; 7647 WREG32(DC_HPD1_INT_CONTROL, tmp); 7648 } 7649 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7650 tmp = RREG32(DC_HPD2_INT_CONTROL); 7651 tmp |= DC_HPDx_INT_ACK; 7652 WREG32(DC_HPD2_INT_CONTROL, tmp); 7653 } 7654 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7655 tmp = RREG32(DC_HPD3_INT_CONTROL); 7656 tmp |= DC_HPDx_INT_ACK; 7657 WREG32(DC_HPD3_INT_CONTROL, tmp); 7658 } 7659 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7660 tmp = RREG32(DC_HPD4_INT_CONTROL); 7661 tmp |= DC_HPDx_INT_ACK; 7662 WREG32(DC_HPD4_INT_CONTROL, tmp); 7663 } 7664 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7665 tmp = RREG32(DC_HPD5_INT_CONTROL); 7666 tmp |= DC_HPDx_INT_ACK; 7667 WREG32(DC_HPD5_INT_CONTROL, tmp); 7668 } 7669 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7670 tmp = RREG32(DC_HPD5_INT_CONTROL); 7671 tmp |= DC_HPDx_INT_ACK; 7672 WREG32(DC_HPD6_INT_CONTROL, tmp); 7673 } 7674 } 7675 7676 /** 7677 * cik_irq_disable - disable interrupts 7678 * 7679 * @rdev: radeon_device pointer 7680 * 7681 * Disable interrupts on the hw (CIK). 7682 */ 7683 static void cik_irq_disable(struct radeon_device *rdev) 7684 { 7685 cik_disable_interrupts(rdev); 7686 /* Wait and acknowledge irq */ 7687 mdelay(1); 7688 cik_irq_ack(rdev); 7689 cik_disable_interrupt_state(rdev); 7690 } 7691 7692 /** 7693 * cik_irq_disable - disable interrupts for suspend 7694 * 7695 * @rdev: radeon_device pointer 7696 * 7697 * Disable interrupts and stop the RLC (CIK). 7698 * Used for suspend. 7699 */ 7700 static void cik_irq_suspend(struct radeon_device *rdev) 7701 { 7702 cik_irq_disable(rdev); 7703 cik_rlc_stop(rdev); 7704 } 7705 7706 /** 7707 * cik_irq_fini - tear down interrupt support 7708 * 7709 * @rdev: radeon_device pointer 7710 * 7711 * Disable interrupts on the hw and free the IH ring 7712 * buffer (CIK). 7713 * Used for driver unload. 7714 */ 7715 static void cik_irq_fini(struct radeon_device *rdev) 7716 { 7717 cik_irq_suspend(rdev); 7718 r600_ih_ring_fini(rdev); 7719 } 7720 7721 /** 7722 * cik_get_ih_wptr - get the IH ring buffer wptr 7723 * 7724 * @rdev: radeon_device pointer 7725 * 7726 * Get the IH ring buffer wptr from either the register 7727 * or the writeback memory buffer (CIK). Also check for 7728 * ring buffer overflow and deal with it. 7729 * Used by cik_irq_process(). 7730 * Returns the value of the wptr. 7731 */ 7732 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) 7733 { 7734 u32 wptr, tmp; 7735 7736 if (rdev->wb.enabled) 7737 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 7738 else 7739 wptr = RREG32(IH_RB_WPTR); 7740 7741 if (wptr & RB_OVERFLOW) { 7742 wptr &= ~RB_OVERFLOW; 7743 /* When a ring buffer overflow happen start parsing interrupt 7744 * from the last not overwritten vector (wptr + 16). Hopefully 7745 * this should allow us to catchup. 7746 */ 7747 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 7748 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 7749 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 7750 tmp = RREG32(IH_RB_CNTL); 7751 tmp |= IH_WPTR_OVERFLOW_CLEAR; 7752 WREG32(IH_RB_CNTL, tmp); 7753 } 7754 return (wptr & rdev->ih.ptr_mask); 7755 } 7756 7757 /* CIK IV Ring 7758 * Each IV ring entry is 128 bits: 7759 * [7:0] - interrupt source id 7760 * [31:8] - reserved 7761 * [59:32] - interrupt source data 7762 * [63:60] - reserved 7763 * [71:64] - RINGID 7764 * CP: 7765 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] 7766 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher 7767 * - for gfx, hw shader state (0=PS...5=LS, 6=CS) 7768 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes 7769 * PIPE_ID - ME0 0=3D 7770 * - ME1&2 compute dispatcher (4 pipes each) 7771 * SDMA: 7772 * INSTANCE_ID [1:0], QUEUE_ID[1:0] 7773 * INSTANCE_ID - 0 = sdma0, 1 = sdma1 7774 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 7775 * [79:72] - VMID 7776 * [95:80] - PASID 7777 * [127:96] - reserved 7778 */ 7779 /** 7780 * cik_irq_process - interrupt handler 7781 * 7782 * @rdev: radeon_device pointer 7783 * 7784 * Interrupt hander (CIK). Walk the IH ring, 7785 * ack interrupts and schedule work to handle 7786 * interrupt events. 7787 * Returns irq process return code. 7788 */ 7789 int cik_irq_process(struct radeon_device *rdev) 7790 { 7791 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7792 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7793 u32 wptr; 7794 u32 rptr; 7795 u32 src_id, src_data, ring_id; 7796 u8 me_id, pipe_id, queue_id; 7797 u32 ring_index; 7798 bool queue_hotplug = false; 7799 bool queue_reset = false; 7800 u32 addr, status, mc_client; 7801 bool queue_thermal = false; 7802 7803 if (!rdev->ih.enabled || rdev->shutdown) 7804 return IRQ_NONE; 7805 7806 wptr = cik_get_ih_wptr(rdev); 7807 7808 restart_ih: 7809 /* is somebody else already processing irqs? */ 7810 if (atomic_xchg(&rdev->ih.lock, 1)) 7811 return IRQ_NONE; 7812 7813 rptr = rdev->ih.rptr; 7814 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 7815 7816 /* Order reading of wptr vs. reading of IH ring data */ 7817 rmb(); 7818 7819 /* display interrupts */ 7820 cik_irq_ack(rdev); 7821 7822 while (rptr != wptr) { 7823 /* wptr/rptr are in bytes! */ 7824 ring_index = rptr / 4; 7825 7826 radeon_kfd_interrupt(rdev, 7827 (const void *) &rdev->ih.ring[ring_index]); 7828 7829 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7830 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7831 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7832 7833 switch (src_id) { 7834 case 1: /* D1 vblank/vline */ 7835 switch (src_data) { 7836 case 0: /* D1 vblank */ 7837 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) { 7838 if (rdev->irq.crtc_vblank_int[0]) { 7839 drm_handle_vblank(rdev->ddev, 0); 7840 rdev->pm.vblank_sync = true; 7841 wake_up(&rdev->irq.vblank_queue); 7842 } 7843 if (atomic_read(&rdev->irq.pflip[0])) 7844 radeon_crtc_handle_vblank(rdev, 0); 7845 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 7846 DRM_DEBUG("IH: D1 vblank\n"); 7847 } 7848 break; 7849 case 1: /* D1 vline */ 7850 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) { 7851 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; 7852 DRM_DEBUG("IH: D1 vline\n"); 7853 } 7854 break; 7855 default: 7856 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7857 break; 7858 } 7859 break; 7860 case 2: /* D2 vblank/vline */ 7861 switch (src_data) { 7862 case 0: /* D2 vblank */ 7863 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 7864 if (rdev->irq.crtc_vblank_int[1]) { 7865 drm_handle_vblank(rdev->ddev, 1); 7866 rdev->pm.vblank_sync = true; 7867 wake_up(&rdev->irq.vblank_queue); 7868 } 7869 if (atomic_read(&rdev->irq.pflip[1])) 7870 radeon_crtc_handle_vblank(rdev, 1); 7871 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 7872 DRM_DEBUG("IH: D2 vblank\n"); 7873 } 7874 break; 7875 case 1: /* D2 vline */ 7876 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 7877 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 7878 DRM_DEBUG("IH: D2 vline\n"); 7879 } 7880 break; 7881 default: 7882 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7883 break; 7884 } 7885 break; 7886 case 3: /* D3 vblank/vline */ 7887 switch (src_data) { 7888 case 0: /* D3 vblank */ 7889 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 7890 if (rdev->irq.crtc_vblank_int[2]) { 7891 drm_handle_vblank(rdev->ddev, 2); 7892 rdev->pm.vblank_sync = true; 7893 wake_up(&rdev->irq.vblank_queue); 7894 } 7895 if (atomic_read(&rdev->irq.pflip[2])) 7896 radeon_crtc_handle_vblank(rdev, 2); 7897 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 7898 DRM_DEBUG("IH: D3 vblank\n"); 7899 } 7900 break; 7901 case 1: /* D3 vline */ 7902 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 7903 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 7904 DRM_DEBUG("IH: D3 vline\n"); 7905 } 7906 break; 7907 default: 7908 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7909 break; 7910 } 7911 break; 7912 case 4: /* D4 vblank/vline */ 7913 switch (src_data) { 7914 case 0: /* D4 vblank */ 7915 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 7916 if (rdev->irq.crtc_vblank_int[3]) { 7917 drm_handle_vblank(rdev->ddev, 3); 7918 rdev->pm.vblank_sync = true; 7919 wake_up(&rdev->irq.vblank_queue); 7920 } 7921 if (atomic_read(&rdev->irq.pflip[3])) 7922 radeon_crtc_handle_vblank(rdev, 3); 7923 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 7924 DRM_DEBUG("IH: D4 vblank\n"); 7925 } 7926 break; 7927 case 1: /* D4 vline */ 7928 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 7929 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 7930 DRM_DEBUG("IH: D4 vline\n"); 7931 } 7932 break; 7933 default: 7934 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7935 break; 7936 } 7937 break; 7938 case 5: /* D5 vblank/vline */ 7939 switch (src_data) { 7940 case 0: /* D5 vblank */ 7941 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 7942 if (rdev->irq.crtc_vblank_int[4]) { 7943 drm_handle_vblank(rdev->ddev, 4); 7944 rdev->pm.vblank_sync = true; 7945 wake_up(&rdev->irq.vblank_queue); 7946 } 7947 if (atomic_read(&rdev->irq.pflip[4])) 7948 radeon_crtc_handle_vblank(rdev, 4); 7949 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 7950 DRM_DEBUG("IH: D5 vblank\n"); 7951 } 7952 break; 7953 case 1: /* D5 vline */ 7954 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 7955 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 7956 DRM_DEBUG("IH: D5 vline\n"); 7957 } 7958 break; 7959 default: 7960 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7961 break; 7962 } 7963 break; 7964 case 6: /* D6 vblank/vline */ 7965 switch (src_data) { 7966 case 0: /* D6 vblank */ 7967 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 7968 if (rdev->irq.crtc_vblank_int[5]) { 7969 drm_handle_vblank(rdev->ddev, 5); 7970 rdev->pm.vblank_sync = true; 7971 wake_up(&rdev->irq.vblank_queue); 7972 } 7973 if (atomic_read(&rdev->irq.pflip[5])) 7974 radeon_crtc_handle_vblank(rdev, 5); 7975 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 7976 DRM_DEBUG("IH: D6 vblank\n"); 7977 } 7978 break; 7979 case 1: /* D6 vline */ 7980 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 7981 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 7982 DRM_DEBUG("IH: D6 vline\n"); 7983 } 7984 break; 7985 default: 7986 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7987 break; 7988 } 7989 break; 7990 case 8: /* D1 page flip */ 7991 case 10: /* D2 page flip */ 7992 case 12: /* D3 page flip */ 7993 case 14: /* D4 page flip */ 7994 case 16: /* D5 page flip */ 7995 case 18: /* D6 page flip */ 7996 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 7997 if (radeon_use_pflipirq > 0) 7998 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 7999 break; 8000 case 42: /* HPD hotplug */ 8001 switch (src_data) { 8002 case 0: 8003 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 8004 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; 8005 queue_hotplug = true; 8006 DRM_DEBUG("IH: HPD1\n"); 8007 } 8008 break; 8009 case 1: 8010 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 8011 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; 8012 queue_hotplug = true; 8013 DRM_DEBUG("IH: HPD2\n"); 8014 } 8015 break; 8016 case 2: 8017 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 8018 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 8019 queue_hotplug = true; 8020 DRM_DEBUG("IH: HPD3\n"); 8021 } 8022 break; 8023 case 3: 8024 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 8025 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 8026 queue_hotplug = true; 8027 DRM_DEBUG("IH: HPD4\n"); 8028 } 8029 break; 8030 case 4: 8031 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 8032 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 8033 queue_hotplug = true; 8034 DRM_DEBUG("IH: HPD5\n"); 8035 } 8036 break; 8037 case 5: 8038 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 8039 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 8040 queue_hotplug = true; 8041 DRM_DEBUG("IH: HPD6\n"); 8042 } 8043 break; 8044 default: 8045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8046 break; 8047 } 8048 break; 8049 case 124: /* UVD */ 8050 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 8051 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 8052 break; 8053 case 146: 8054 case 147: 8055 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 8056 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 8057 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); 8058 /* reset addr and status */ 8059 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 8060 if (addr == 0x0 && status == 0x0) 8061 break; 8062 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 8063 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 8064 addr); 8065 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 8066 status); 8067 cik_vm_decode_fault(rdev, status, addr, mc_client); 8068 break; 8069 case 167: /* VCE */ 8070 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); 8071 switch (src_data) { 8072 case 0: 8073 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); 8074 break; 8075 case 1: 8076 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); 8077 break; 8078 default: 8079 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 8080 break; 8081 } 8082 break; 8083 case 176: /* GFX RB CP_INT */ 8084 case 177: /* GFX IB CP_INT */ 8085 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8086 break; 8087 case 181: /* CP EOP event */ 8088 DRM_DEBUG("IH: CP EOP\n"); 8089 /* XXX check the bitfield order! */ 8090 me_id = (ring_id & 0x60) >> 5; 8091 pipe_id = (ring_id & 0x18) >> 3; 8092 queue_id = (ring_id & 0x7) >> 0; 8093 switch (me_id) { 8094 case 0: 8095 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8096 break; 8097 case 1: 8098 case 2: 8099 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) 8100 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8101 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) 8102 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8103 break; 8104 } 8105 break; 8106 case 184: /* CP Privileged reg access */ 8107 DRM_ERROR("Illegal register access in command stream\n"); 8108 /* XXX check the bitfield order! */ 8109 me_id = (ring_id & 0x60) >> 5; 8110 pipe_id = (ring_id & 0x18) >> 3; 8111 queue_id = (ring_id & 0x7) >> 0; 8112 switch (me_id) { 8113 case 0: 8114 /* This results in a full GPU reset, but all we need to do is soft 8115 * reset the CP for gfx 8116 */ 8117 queue_reset = true; 8118 break; 8119 case 1: 8120 /* XXX compute */ 8121 queue_reset = true; 8122 break; 8123 case 2: 8124 /* XXX compute */ 8125 queue_reset = true; 8126 break; 8127 } 8128 break; 8129 case 185: /* CP Privileged inst */ 8130 DRM_ERROR("Illegal instruction in command stream\n"); 8131 /* XXX check the bitfield order! */ 8132 me_id = (ring_id & 0x60) >> 5; 8133 pipe_id = (ring_id & 0x18) >> 3; 8134 queue_id = (ring_id & 0x7) >> 0; 8135 switch (me_id) { 8136 case 0: 8137 /* This results in a full GPU reset, but all we need to do is soft 8138 * reset the CP for gfx 8139 */ 8140 queue_reset = true; 8141 break; 8142 case 1: 8143 /* XXX compute */ 8144 queue_reset = true; 8145 break; 8146 case 2: 8147 /* XXX compute */ 8148 queue_reset = true; 8149 break; 8150 } 8151 break; 8152 case 224: /* SDMA trap event */ 8153 /* XXX check the bitfield order! */ 8154 me_id = (ring_id & 0x3) >> 0; 8155 queue_id = (ring_id & 0xc) >> 2; 8156 DRM_DEBUG("IH: SDMA trap\n"); 8157 switch (me_id) { 8158 case 0: 8159 switch (queue_id) { 8160 case 0: 8161 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 8162 break; 8163 case 1: 8164 /* XXX compute */ 8165 break; 8166 case 2: 8167 /* XXX compute */ 8168 break; 8169 } 8170 break; 8171 case 1: 8172 switch (queue_id) { 8173 case 0: 8174 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8175 break; 8176 case 1: 8177 /* XXX compute */ 8178 break; 8179 case 2: 8180 /* XXX compute */ 8181 break; 8182 } 8183 break; 8184 } 8185 break; 8186 case 230: /* thermal low to high */ 8187 DRM_DEBUG("IH: thermal low to high\n"); 8188 rdev->pm.dpm.thermal.high_to_low = false; 8189 queue_thermal = true; 8190 break; 8191 case 231: /* thermal high to low */ 8192 DRM_DEBUG("IH: thermal high to low\n"); 8193 rdev->pm.dpm.thermal.high_to_low = true; 8194 queue_thermal = true; 8195 break; 8196 case 233: /* GUI IDLE */ 8197 DRM_DEBUG("IH: GUI idle\n"); 8198 break; 8199 case 241: /* SDMA Privileged inst */ 8200 case 247: /* SDMA Privileged inst */ 8201 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 8202 /* XXX check the bitfield order! */ 8203 me_id = (ring_id & 0x3) >> 0; 8204 queue_id = (ring_id & 0xc) >> 2; 8205 switch (me_id) { 8206 case 0: 8207 switch (queue_id) { 8208 case 0: 8209 queue_reset = true; 8210 break; 8211 case 1: 8212 /* XXX compute */ 8213 queue_reset = true; 8214 break; 8215 case 2: 8216 /* XXX compute */ 8217 queue_reset = true; 8218 break; 8219 } 8220 break; 8221 case 1: 8222 switch (queue_id) { 8223 case 0: 8224 queue_reset = true; 8225 break; 8226 case 1: 8227 /* XXX compute */ 8228 queue_reset = true; 8229 break; 8230 case 2: 8231 /* XXX compute */ 8232 queue_reset = true; 8233 break; 8234 } 8235 break; 8236 } 8237 break; 8238 default: 8239 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8240 break; 8241 } 8242 8243 /* wptr/rptr are in bytes! */ 8244 rptr += 16; 8245 rptr &= rdev->ih.ptr_mask; 8246 WREG32(IH_RB_RPTR, rptr); 8247 } 8248 if (queue_hotplug) 8249 schedule_work(&rdev->hotplug_work); 8250 if (queue_reset) { 8251 rdev->needs_reset = true; 8252 wake_up_all(&rdev->fence_queue); 8253 } 8254 if (queue_thermal) 8255 schedule_work(&rdev->pm.dpm.thermal.work); 8256 rdev->ih.rptr = rptr; 8257 atomic_set(&rdev->ih.lock, 0); 8258 8259 /* make sure wptr hasn't changed while processing */ 8260 wptr = cik_get_ih_wptr(rdev); 8261 if (wptr != rptr) 8262 goto restart_ih; 8263 8264 return IRQ_HANDLED; 8265 } 8266 8267 /* 8268 * startup/shutdown callbacks 8269 */ 8270 /** 8271 * cik_startup - program the asic to a functional state 8272 * 8273 * @rdev: radeon_device pointer 8274 * 8275 * Programs the asic to a functional state (CIK). 8276 * Called by cik_init() and cik_resume(). 8277 * Returns 0 for success, error for failure. 8278 */ 8279 static int cik_startup(struct radeon_device *rdev) 8280 { 8281 struct radeon_ring *ring; 8282 u32 nop; 8283 int r; 8284 8285 /* enable pcie gen2/3 link */ 8286 cik_pcie_gen3_enable(rdev); 8287 /* enable aspm */ 8288 cik_program_aspm(rdev); 8289 8290 /* scratch needs to be initialized before MC */ 8291 r = r600_vram_scratch_init(rdev); 8292 if (r) 8293 return r; 8294 8295 cik_mc_program(rdev); 8296 8297 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { 8298 r = ci_mc_load_microcode(rdev); 8299 if (r) { 8300 DRM_ERROR("Failed to load MC firmware!\n"); 8301 return r; 8302 } 8303 } 8304 8305 r = cik_pcie_gart_enable(rdev); 8306 if (r) 8307 return r; 8308 cik_gpu_init(rdev); 8309 8310 /* allocate rlc buffers */ 8311 if (rdev->flags & RADEON_IS_IGP) { 8312 if (rdev->family == CHIP_KAVERI) { 8313 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; 8314 rdev->rlc.reg_list_size = 8315 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); 8316 } else { 8317 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; 8318 rdev->rlc.reg_list_size = 8319 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); 8320 } 8321 } 8322 rdev->rlc.cs_data = ci_cs_data; 8323 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; 8324 r = sumo_rlc_init(rdev); 8325 if (r) { 8326 DRM_ERROR("Failed to init rlc BOs!\n"); 8327 return r; 8328 } 8329 8330 /* allocate wb buffer */ 8331 r = radeon_wb_init(rdev); 8332 if (r) 8333 return r; 8334 8335 /* allocate mec buffers */ 8336 r = cik_mec_init(rdev); 8337 if (r) { 8338 DRM_ERROR("Failed to init MEC BOs!\n"); 8339 return r; 8340 } 8341 8342 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 8343 if (r) { 8344 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8345 return r; 8346 } 8347 8348 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8349 if (r) { 8350 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8351 return r; 8352 } 8353 8354 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8355 if (r) { 8356 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8357 return r; 8358 } 8359 8360 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 8361 if (r) { 8362 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8363 return r; 8364 } 8365 8366 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8367 if (r) { 8368 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8369 return r; 8370 } 8371 8372 r = radeon_uvd_resume(rdev); 8373 if (!r) { 8374 r = uvd_v4_2_resume(rdev); 8375 if (!r) { 8376 r = radeon_fence_driver_start_ring(rdev, 8377 R600_RING_TYPE_UVD_INDEX); 8378 if (r) 8379 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 8380 } 8381 } 8382 if (r) 8383 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 8384 8385 r = radeon_vce_resume(rdev); 8386 if (!r) { 8387 r = vce_v2_0_resume(rdev); 8388 if (!r) 8389 r = radeon_fence_driver_start_ring(rdev, 8390 TN_RING_TYPE_VCE1_INDEX); 8391 if (!r) 8392 r = radeon_fence_driver_start_ring(rdev, 8393 TN_RING_TYPE_VCE2_INDEX); 8394 } 8395 if (r) { 8396 dev_err(rdev->dev, "VCE init error (%d).\n", r); 8397 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; 8398 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; 8399 } 8400 8401 /* Enable IRQ */ 8402 if (!rdev->irq.installed) { 8403 r = radeon_irq_kms_init(rdev); 8404 if (r) 8405 return r; 8406 } 8407 8408 r = cik_irq_init(rdev); 8409 if (r) { 8410 DRM_ERROR("radeon: IH init failed (%d).\n", r); 8411 radeon_irq_kms_fini(rdev); 8412 return r; 8413 } 8414 cik_irq_set(rdev); 8415 8416 if (rdev->family == CHIP_HAWAII) { 8417 if (rdev->new_fw) 8418 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8419 else 8420 nop = RADEON_CP_PACKET2; 8421 } else { 8422 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8423 } 8424 8425 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8426 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 8427 nop); 8428 if (r) 8429 return r; 8430 8431 /* set up the compute queues */ 8432 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8433 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8434 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 8435 nop); 8436 if (r) 8437 return r; 8438 ring->me = 1; /* first MEC */ 8439 ring->pipe = 0; /* first pipe */ 8440 ring->queue = 0; /* first queue */ 8441 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; 8442 8443 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8444 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8445 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 8446 nop); 8447 if (r) 8448 return r; 8449 /* dGPU only have 1 MEC */ 8450 ring->me = 1; /* first MEC */ 8451 ring->pipe = 0; /* first pipe */ 8452 ring->queue = 1; /* second queue */ 8453 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; 8454 8455 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8456 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 8457 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8458 if (r) 8459 return r; 8460 8461 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8462 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 8463 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8464 if (r) 8465 return r; 8466 8467 r = cik_cp_resume(rdev); 8468 if (r) 8469 return r; 8470 8471 r = cik_sdma_resume(rdev); 8472 if (r) 8473 return r; 8474 8475 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8476 if (ring->ring_size) { 8477 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8478 RADEON_CP_PACKET2); 8479 if (!r) 8480 r = uvd_v1_0_init(rdev); 8481 if (r) 8482 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 8483 } 8484 8485 r = -ENOENT; 8486 8487 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8488 if (ring->ring_size) 8489 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8490 VCE_CMD_NO_OP); 8491 8492 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8493 if (ring->ring_size) 8494 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8495 VCE_CMD_NO_OP); 8496 8497 if (!r) 8498 r = vce_v1_0_init(rdev); 8499 else if (r != -ENOENT) 8500 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); 8501 8502 r = radeon_ib_pool_init(rdev); 8503 if (r) { 8504 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 8505 return r; 8506 } 8507 8508 r = radeon_vm_manager_init(rdev); 8509 if (r) { 8510 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 8511 return r; 8512 } 8513 8514 r = radeon_audio_init(rdev); 8515 if (r) 8516 return r; 8517 8518 r = radeon_kfd_resume(rdev); 8519 if (r) 8520 return r; 8521 8522 return 0; 8523 } 8524 8525 /** 8526 * cik_resume - resume the asic to a functional state 8527 * 8528 * @rdev: radeon_device pointer 8529 * 8530 * Programs the asic to a functional state (CIK). 8531 * Called at resume. 8532 * Returns 0 for success, error for failure. 8533 */ 8534 int cik_resume(struct radeon_device *rdev) 8535 { 8536 int r; 8537 8538 /* post card */ 8539 atom_asic_init(rdev->mode_info.atom_context); 8540 8541 /* init golden registers */ 8542 cik_init_golden_registers(rdev); 8543 8544 if (rdev->pm.pm_method == PM_METHOD_DPM) 8545 radeon_pm_resume(rdev); 8546 8547 rdev->accel_working = true; 8548 r = cik_startup(rdev); 8549 if (r) { 8550 DRM_ERROR("cik startup failed on resume\n"); 8551 rdev->accel_working = false; 8552 return r; 8553 } 8554 8555 return r; 8556 8557 } 8558 8559 /** 8560 * cik_suspend - suspend the asic 8561 * 8562 * @rdev: radeon_device pointer 8563 * 8564 * Bring the chip into a state suitable for suspend (CIK). 8565 * Called at suspend. 8566 * Returns 0 for success. 8567 */ 8568 int cik_suspend(struct radeon_device *rdev) 8569 { 8570 radeon_kfd_suspend(rdev); 8571 radeon_pm_suspend(rdev); 8572 radeon_audio_fini(rdev); 8573 radeon_vm_manager_fini(rdev); 8574 cik_cp_enable(rdev, false); 8575 cik_sdma_enable(rdev, false); 8576 uvd_v1_0_fini(rdev); 8577 radeon_uvd_suspend(rdev); 8578 radeon_vce_suspend(rdev); 8579 cik_fini_pg(rdev); 8580 cik_fini_cg(rdev); 8581 cik_irq_suspend(rdev); 8582 radeon_wb_disable(rdev); 8583 cik_pcie_gart_disable(rdev); 8584 return 0; 8585 } 8586 8587 /* Plan is to move initialization in that function and use 8588 * helper function so that radeon_device_init pretty much 8589 * do nothing more than calling asic specific function. This 8590 * should also allow to remove a bunch of callback function 8591 * like vram_info. 8592 */ 8593 /** 8594 * cik_init - asic specific driver and hw init 8595 * 8596 * @rdev: radeon_device pointer 8597 * 8598 * Setup asic specific driver variables and program the hw 8599 * to a functional state (CIK). 8600 * Called at driver startup. 8601 * Returns 0 for success, errors for failure. 8602 */ 8603 int cik_init(struct radeon_device *rdev) 8604 { 8605 struct radeon_ring *ring; 8606 int r; 8607 8608 /* Read BIOS */ 8609 if (!radeon_get_bios(rdev)) { 8610 if (ASIC_IS_AVIVO(rdev)) 8611 return -EINVAL; 8612 } 8613 /* Must be an ATOMBIOS */ 8614 if (!rdev->is_atom_bios) { 8615 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 8616 return -EINVAL; 8617 } 8618 r = radeon_atombios_init(rdev); 8619 if (r) 8620 return r; 8621 8622 /* Post card if necessary */ 8623 if (!radeon_card_posted(rdev)) { 8624 if (!rdev->bios) { 8625 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 8626 return -EINVAL; 8627 } 8628 DRM_INFO("GPU not posted. posting now...\n"); 8629 atom_asic_init(rdev->mode_info.atom_context); 8630 } 8631 /* init golden registers */ 8632 cik_init_golden_registers(rdev); 8633 /* Initialize scratch registers */ 8634 cik_scratch_init(rdev); 8635 /* Initialize surface registers */ 8636 radeon_surface_init(rdev); 8637 /* Initialize clocks */ 8638 radeon_get_clock_info(rdev->ddev); 8639 8640 /* Fence driver */ 8641 r = radeon_fence_driver_init(rdev); 8642 if (r) 8643 return r; 8644 8645 /* initialize memory controller */ 8646 r = cik_mc_init(rdev); 8647 if (r) 8648 return r; 8649 /* Memory manager */ 8650 r = radeon_bo_init(rdev); 8651 if (r) 8652 return r; 8653 8654 if (rdev->flags & RADEON_IS_IGP) { 8655 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8656 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { 8657 r = cik_init_microcode(rdev); 8658 if (r) { 8659 DRM_ERROR("Failed to load firmware!\n"); 8660 return r; 8661 } 8662 } 8663 } else { 8664 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8665 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || 8666 !rdev->mc_fw) { 8667 r = cik_init_microcode(rdev); 8668 if (r) { 8669 DRM_ERROR("Failed to load firmware!\n"); 8670 return r; 8671 } 8672 } 8673 } 8674 8675 /* Initialize power management */ 8676 radeon_pm_init(rdev); 8677 8678 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8679 ring->ring_obj = NULL; 8680 r600_ring_init(rdev, ring, 1024 * 1024); 8681 8682 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8683 ring->ring_obj = NULL; 8684 r600_ring_init(rdev, ring, 1024 * 1024); 8685 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8686 if (r) 8687 return r; 8688 8689 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8690 ring->ring_obj = NULL; 8691 r600_ring_init(rdev, ring, 1024 * 1024); 8692 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8693 if (r) 8694 return r; 8695 8696 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8697 ring->ring_obj = NULL; 8698 r600_ring_init(rdev, ring, 256 * 1024); 8699 8700 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8701 ring->ring_obj = NULL; 8702 r600_ring_init(rdev, ring, 256 * 1024); 8703 8704 r = radeon_uvd_init(rdev); 8705 if (!r) { 8706 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8707 ring->ring_obj = NULL; 8708 r600_ring_init(rdev, ring, 4096); 8709 } 8710 8711 r = radeon_vce_init(rdev); 8712 if (!r) { 8713 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8714 ring->ring_obj = NULL; 8715 r600_ring_init(rdev, ring, 4096); 8716 8717 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8718 ring->ring_obj = NULL; 8719 r600_ring_init(rdev, ring, 4096); 8720 } 8721 8722 rdev->ih.ring_obj = NULL; 8723 r600_ih_ring_init(rdev, 64 * 1024); 8724 8725 r = r600_pcie_gart_init(rdev); 8726 if (r) 8727 return r; 8728 8729 rdev->accel_working = true; 8730 r = cik_startup(rdev); 8731 if (r) { 8732 dev_err(rdev->dev, "disabling GPU acceleration\n"); 8733 cik_cp_fini(rdev); 8734 cik_sdma_fini(rdev); 8735 cik_irq_fini(rdev); 8736 sumo_rlc_fini(rdev); 8737 cik_mec_fini(rdev); 8738 radeon_wb_fini(rdev); 8739 radeon_ib_pool_fini(rdev); 8740 radeon_vm_manager_fini(rdev); 8741 radeon_irq_kms_fini(rdev); 8742 cik_pcie_gart_fini(rdev); 8743 rdev->accel_working = false; 8744 } 8745 8746 /* Don't start up if the MC ucode is missing. 8747 * The default clocks and voltages before the MC ucode 8748 * is loaded are not suffient for advanced operations. 8749 */ 8750 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 8751 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 8752 return -EINVAL; 8753 } 8754 8755 return 0; 8756 } 8757 8758 /** 8759 * cik_fini - asic specific driver and hw fini 8760 * 8761 * @rdev: radeon_device pointer 8762 * 8763 * Tear down the asic specific driver variables and program the hw 8764 * to an idle state (CIK). 8765 * Called at driver unload. 8766 */ 8767 void cik_fini(struct radeon_device *rdev) 8768 { 8769 radeon_pm_fini(rdev); 8770 cik_cp_fini(rdev); 8771 cik_sdma_fini(rdev); 8772 cik_fini_pg(rdev); 8773 cik_fini_cg(rdev); 8774 cik_irq_fini(rdev); 8775 sumo_rlc_fini(rdev); 8776 cik_mec_fini(rdev); 8777 radeon_wb_fini(rdev); 8778 radeon_vm_manager_fini(rdev); 8779 radeon_ib_pool_fini(rdev); 8780 radeon_irq_kms_fini(rdev); 8781 uvd_v1_0_fini(rdev); 8782 radeon_uvd_fini(rdev); 8783 radeon_vce_fini(rdev); 8784 cik_pcie_gart_fini(rdev); 8785 r600_vram_scratch_fini(rdev); 8786 radeon_gem_fini(rdev); 8787 radeon_fence_driver_fini(rdev); 8788 radeon_bo_fini(rdev); 8789 radeon_atombios_fini(rdev); 8790 kfree(rdev->bios); 8791 rdev->bios = NULL; 8792 } 8793 8794 void dce8_program_fmt(struct drm_encoder *encoder) 8795 { 8796 struct drm_device *dev = encoder->dev; 8797 struct radeon_device *rdev = dev->dev_private; 8798 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 8799 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 8800 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 8801 int bpc = 0; 8802 u32 tmp = 0; 8803 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 8804 8805 if (connector) { 8806 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 8807 bpc = radeon_get_monitor_bpc(connector); 8808 dither = radeon_connector->dither; 8809 } 8810 8811 /* LVDS/eDP FMT is set up by atom */ 8812 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 8813 return; 8814 8815 /* not needed for analog */ 8816 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 8817 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 8818 return; 8819 8820 if (bpc == 0) 8821 return; 8822 8823 switch (bpc) { 8824 case 6: 8825 if (dither == RADEON_FMT_DITHER_ENABLE) 8826 /* XXX sort out optimal dither settings */ 8827 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8828 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); 8829 else 8830 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); 8831 break; 8832 case 8: 8833 if (dither == RADEON_FMT_DITHER_ENABLE) 8834 /* XXX sort out optimal dither settings */ 8835 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8836 FMT_RGB_RANDOM_ENABLE | 8837 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); 8838 else 8839 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); 8840 break; 8841 case 10: 8842 if (dither == RADEON_FMT_DITHER_ENABLE) 8843 /* XXX sort out optimal dither settings */ 8844 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8845 FMT_RGB_RANDOM_ENABLE | 8846 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); 8847 else 8848 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); 8849 break; 8850 default: 8851 /* not needed */ 8852 break; 8853 } 8854 8855 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 8856 } 8857 8858 /* display watermark setup */ 8859 /** 8860 * dce8_line_buffer_adjust - Set up the line buffer 8861 * 8862 * @rdev: radeon_device pointer 8863 * @radeon_crtc: the selected display controller 8864 * @mode: the current display mode on the selected display 8865 * controller 8866 * 8867 * Setup up the line buffer allocation for 8868 * the selected display controller (CIK). 8869 * Returns the line buffer size in pixels. 8870 */ 8871 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, 8872 struct radeon_crtc *radeon_crtc, 8873 struct drm_display_mode *mode) 8874 { 8875 u32 tmp, buffer_alloc, i; 8876 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 8877 /* 8878 * Line Buffer Setup 8879 * There are 6 line buffers, one for each display controllers. 8880 * There are 3 partitions per LB. Select the number of partitions 8881 * to enable based on the display width. For display widths larger 8882 * than 4096, you need use to use 2 display controllers and combine 8883 * them using the stereo blender. 8884 */ 8885 if (radeon_crtc->base.enabled && mode) { 8886 if (mode->crtc_hdisplay < 1920) { 8887 tmp = 1; 8888 buffer_alloc = 2; 8889 } else if (mode->crtc_hdisplay < 2560) { 8890 tmp = 2; 8891 buffer_alloc = 2; 8892 } else if (mode->crtc_hdisplay < 4096) { 8893 tmp = 0; 8894 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8895 } else { 8896 DRM_DEBUG_KMS("Mode too big for LB!\n"); 8897 tmp = 0; 8898 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8899 } 8900 } else { 8901 tmp = 1; 8902 buffer_alloc = 0; 8903 } 8904 8905 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, 8906 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); 8907 8908 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 8909 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 8910 for (i = 0; i < rdev->usec_timeout; i++) { 8911 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 8912 DMIF_BUFFERS_ALLOCATED_COMPLETED) 8913 break; 8914 udelay(1); 8915 } 8916 8917 if (radeon_crtc->base.enabled && mode) { 8918 switch (tmp) { 8919 case 0: 8920 default: 8921 return 4096 * 2; 8922 case 1: 8923 return 1920 * 2; 8924 case 2: 8925 return 2560 * 2; 8926 } 8927 } 8928 8929 /* controller not enabled, so no lb used */ 8930 return 0; 8931 } 8932 8933 /** 8934 * cik_get_number_of_dram_channels - get the number of dram channels 8935 * 8936 * @rdev: radeon_device pointer 8937 * 8938 * Look up the number of video ram channels (CIK). 8939 * Used for display watermark bandwidth calculations 8940 * Returns the number of dram channels 8941 */ 8942 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) 8943 { 8944 u32 tmp = RREG32(MC_SHARED_CHMAP); 8945 8946 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 8947 case 0: 8948 default: 8949 return 1; 8950 case 1: 8951 return 2; 8952 case 2: 8953 return 4; 8954 case 3: 8955 return 8; 8956 case 4: 8957 return 3; 8958 case 5: 8959 return 6; 8960 case 6: 8961 return 10; 8962 case 7: 8963 return 12; 8964 case 8: 8965 return 16; 8966 } 8967 } 8968 8969 struct dce8_wm_params { 8970 u32 dram_channels; /* number of dram channels */ 8971 u32 yclk; /* bandwidth per dram data pin in kHz */ 8972 u32 sclk; /* engine clock in kHz */ 8973 u32 disp_clk; /* display clock in kHz */ 8974 u32 src_width; /* viewport width */ 8975 u32 active_time; /* active display time in ns */ 8976 u32 blank_time; /* blank time in ns */ 8977 bool interlaced; /* mode is interlaced */ 8978 fixed20_12 vsc; /* vertical scale ratio */ 8979 u32 num_heads; /* number of active crtcs */ 8980 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 8981 u32 lb_size; /* line buffer allocated to pipe */ 8982 u32 vtaps; /* vertical scaler taps */ 8983 }; 8984 8985 /** 8986 * dce8_dram_bandwidth - get the dram bandwidth 8987 * 8988 * @wm: watermark calculation data 8989 * 8990 * Calculate the raw dram bandwidth (CIK). 8991 * Used for display watermark bandwidth calculations 8992 * Returns the dram bandwidth in MBytes/s 8993 */ 8994 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) 8995 { 8996 /* Calculate raw DRAM Bandwidth */ 8997 fixed20_12 dram_efficiency; /* 0.7 */ 8998 fixed20_12 yclk, dram_channels, bandwidth; 8999 fixed20_12 a; 9000 9001 a.full = dfixed_const(1000); 9002 yclk.full = dfixed_const(wm->yclk); 9003 yclk.full = dfixed_div(yclk, a); 9004 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9005 a.full = dfixed_const(10); 9006 dram_efficiency.full = dfixed_const(7); 9007 dram_efficiency.full = dfixed_div(dram_efficiency, a); 9008 bandwidth.full = dfixed_mul(dram_channels, yclk); 9009 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 9010 9011 return dfixed_trunc(bandwidth); 9012 } 9013 9014 /** 9015 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display 9016 * 9017 * @wm: watermark calculation data 9018 * 9019 * Calculate the dram bandwidth used for display (CIK). 9020 * Used for display watermark bandwidth calculations 9021 * Returns the dram bandwidth for display in MBytes/s 9022 */ 9023 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9024 { 9025 /* Calculate DRAM Bandwidth and the part allocated to display. */ 9026 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 9027 fixed20_12 yclk, dram_channels, bandwidth; 9028 fixed20_12 a; 9029 9030 a.full = dfixed_const(1000); 9031 yclk.full = dfixed_const(wm->yclk); 9032 yclk.full = dfixed_div(yclk, a); 9033 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9034 a.full = dfixed_const(10); 9035 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 9036 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 9037 bandwidth.full = dfixed_mul(dram_channels, yclk); 9038 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 9039 9040 return dfixed_trunc(bandwidth); 9041 } 9042 9043 /** 9044 * dce8_data_return_bandwidth - get the data return bandwidth 9045 * 9046 * @wm: watermark calculation data 9047 * 9048 * Calculate the data return bandwidth used for display (CIK). 9049 * Used for display watermark bandwidth calculations 9050 * Returns the data return bandwidth in MBytes/s 9051 */ 9052 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) 9053 { 9054 /* Calculate the display Data return Bandwidth */ 9055 fixed20_12 return_efficiency; /* 0.8 */ 9056 fixed20_12 sclk, bandwidth; 9057 fixed20_12 a; 9058 9059 a.full = dfixed_const(1000); 9060 sclk.full = dfixed_const(wm->sclk); 9061 sclk.full = dfixed_div(sclk, a); 9062 a.full = dfixed_const(10); 9063 return_efficiency.full = dfixed_const(8); 9064 return_efficiency.full = dfixed_div(return_efficiency, a); 9065 a.full = dfixed_const(32); 9066 bandwidth.full = dfixed_mul(a, sclk); 9067 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 9068 9069 return dfixed_trunc(bandwidth); 9070 } 9071 9072 /** 9073 * dce8_dmif_request_bandwidth - get the dmif bandwidth 9074 * 9075 * @wm: watermark calculation data 9076 * 9077 * Calculate the dmif bandwidth used for display (CIK). 9078 * Used for display watermark bandwidth calculations 9079 * Returns the dmif bandwidth in MBytes/s 9080 */ 9081 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) 9082 { 9083 /* Calculate the DMIF Request Bandwidth */ 9084 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 9085 fixed20_12 disp_clk, bandwidth; 9086 fixed20_12 a, b; 9087 9088 a.full = dfixed_const(1000); 9089 disp_clk.full = dfixed_const(wm->disp_clk); 9090 disp_clk.full = dfixed_div(disp_clk, a); 9091 a.full = dfixed_const(32); 9092 b.full = dfixed_mul(a, disp_clk); 9093 9094 a.full = dfixed_const(10); 9095 disp_clk_request_efficiency.full = dfixed_const(8); 9096 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 9097 9098 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); 9099 9100 return dfixed_trunc(bandwidth); 9101 } 9102 9103 /** 9104 * dce8_available_bandwidth - get the min available bandwidth 9105 * 9106 * @wm: watermark calculation data 9107 * 9108 * Calculate the min available bandwidth used for display (CIK). 9109 * Used for display watermark bandwidth calculations 9110 * Returns the min available bandwidth in MBytes/s 9111 */ 9112 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) 9113 { 9114 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 9115 u32 dram_bandwidth = dce8_dram_bandwidth(wm); 9116 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); 9117 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); 9118 9119 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 9120 } 9121 9122 /** 9123 * dce8_average_bandwidth - get the average available bandwidth 9124 * 9125 * @wm: watermark calculation data 9126 * 9127 * Calculate the average available bandwidth used for display (CIK). 9128 * Used for display watermark bandwidth calculations 9129 * Returns the average available bandwidth in MBytes/s 9130 */ 9131 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) 9132 { 9133 /* Calculate the display mode Average Bandwidth 9134 * DisplayMode should contain the source and destination dimensions, 9135 * timing, etc. 9136 */ 9137 fixed20_12 bpp; 9138 fixed20_12 line_time; 9139 fixed20_12 src_width; 9140 fixed20_12 bandwidth; 9141 fixed20_12 a; 9142 9143 a.full = dfixed_const(1000); 9144 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 9145 line_time.full = dfixed_div(line_time, a); 9146 bpp.full = dfixed_const(wm->bytes_per_pixel); 9147 src_width.full = dfixed_const(wm->src_width); 9148 bandwidth.full = dfixed_mul(src_width, bpp); 9149 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 9150 bandwidth.full = dfixed_div(bandwidth, line_time); 9151 9152 return dfixed_trunc(bandwidth); 9153 } 9154 9155 /** 9156 * dce8_latency_watermark - get the latency watermark 9157 * 9158 * @wm: watermark calculation data 9159 * 9160 * Calculate the latency watermark (CIK). 9161 * Used for display watermark bandwidth calculations 9162 * Returns the latency watermark in ns 9163 */ 9164 static u32 dce8_latency_watermark(struct dce8_wm_params *wm) 9165 { 9166 /* First calculate the latency in ns */ 9167 u32 mc_latency = 2000; /* 2000 ns. */ 9168 u32 available_bandwidth = dce8_available_bandwidth(wm); 9169 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 9170 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 9171 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 9172 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 9173 (wm->num_heads * cursor_line_pair_return_time); 9174 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 9175 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 9176 u32 tmp, dmif_size = 12288; 9177 fixed20_12 a, b, c; 9178 9179 if (wm->num_heads == 0) 9180 return 0; 9181 9182 a.full = dfixed_const(2); 9183 b.full = dfixed_const(1); 9184 if ((wm->vsc.full > a.full) || 9185 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 9186 (wm->vtaps >= 5) || 9187 ((wm->vsc.full >= a.full) && wm->interlaced)) 9188 max_src_lines_per_dst_line = 4; 9189 else 9190 max_src_lines_per_dst_line = 2; 9191 9192 a.full = dfixed_const(available_bandwidth); 9193 b.full = dfixed_const(wm->num_heads); 9194 a.full = dfixed_div(a, b); 9195 9196 b.full = dfixed_const(mc_latency + 512); 9197 c.full = dfixed_const(wm->disp_clk); 9198 b.full = dfixed_div(b, c); 9199 9200 c.full = dfixed_const(dmif_size); 9201 b.full = dfixed_div(c, b); 9202 9203 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 9204 9205 b.full = dfixed_const(1000); 9206 c.full = dfixed_const(wm->disp_clk); 9207 b.full = dfixed_div(c, b); 9208 c.full = dfixed_const(wm->bytes_per_pixel); 9209 b.full = dfixed_mul(b, c); 9210 9211 lb_fill_bw = min(tmp, dfixed_trunc(b)); 9212 9213 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 9214 b.full = dfixed_const(1000); 9215 c.full = dfixed_const(lb_fill_bw); 9216 b.full = dfixed_div(c, b); 9217 a.full = dfixed_div(a, b); 9218 line_fill_time = dfixed_trunc(a); 9219 9220 if (line_fill_time < wm->active_time) 9221 return latency; 9222 else 9223 return latency + (line_fill_time - wm->active_time); 9224 9225 } 9226 9227 /** 9228 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check 9229 * average and available dram bandwidth 9230 * 9231 * @wm: watermark calculation data 9232 * 9233 * Check if the display average bandwidth fits in the display 9234 * dram bandwidth (CIK). 9235 * Used for display watermark bandwidth calculations 9236 * Returns true if the display fits, false if not. 9237 */ 9238 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9239 { 9240 if (dce8_average_bandwidth(wm) <= 9241 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) 9242 return true; 9243 else 9244 return false; 9245 } 9246 9247 /** 9248 * dce8_average_bandwidth_vs_available_bandwidth - check 9249 * average and available bandwidth 9250 * 9251 * @wm: watermark calculation data 9252 * 9253 * Check if the display average bandwidth fits in the display 9254 * available bandwidth (CIK). 9255 * Used for display watermark bandwidth calculations 9256 * Returns true if the display fits, false if not. 9257 */ 9258 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) 9259 { 9260 if (dce8_average_bandwidth(wm) <= 9261 (dce8_available_bandwidth(wm) / wm->num_heads)) 9262 return true; 9263 else 9264 return false; 9265 } 9266 9267 /** 9268 * dce8_check_latency_hiding - check latency hiding 9269 * 9270 * @wm: watermark calculation data 9271 * 9272 * Check latency hiding (CIK). 9273 * Used for display watermark bandwidth calculations 9274 * Returns true if the display fits, false if not. 9275 */ 9276 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) 9277 { 9278 u32 lb_partitions = wm->lb_size / wm->src_width; 9279 u32 line_time = wm->active_time + wm->blank_time; 9280 u32 latency_tolerant_lines; 9281 u32 latency_hiding; 9282 fixed20_12 a; 9283 9284 a.full = dfixed_const(1); 9285 if (wm->vsc.full > a.full) 9286 latency_tolerant_lines = 1; 9287 else { 9288 if (lb_partitions <= (wm->vtaps + 1)) 9289 latency_tolerant_lines = 1; 9290 else 9291 latency_tolerant_lines = 2; 9292 } 9293 9294 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 9295 9296 if (dce8_latency_watermark(wm) <= latency_hiding) 9297 return true; 9298 else 9299 return false; 9300 } 9301 9302 /** 9303 * dce8_program_watermarks - program display watermarks 9304 * 9305 * @rdev: radeon_device pointer 9306 * @radeon_crtc: the selected display controller 9307 * @lb_size: line buffer size 9308 * @num_heads: number of display controllers in use 9309 * 9310 * Calculate and program the display watermarks for the 9311 * selected display controller (CIK). 9312 */ 9313 static void dce8_program_watermarks(struct radeon_device *rdev, 9314 struct radeon_crtc *radeon_crtc, 9315 u32 lb_size, u32 num_heads) 9316 { 9317 struct drm_display_mode *mode = &radeon_crtc->base.mode; 9318 struct dce8_wm_params wm_low, wm_high; 9319 u32 pixel_period; 9320 u32 line_time = 0; 9321 u32 latency_watermark_a = 0, latency_watermark_b = 0; 9322 u32 tmp, wm_mask; 9323 9324 if (radeon_crtc->base.enabled && num_heads && mode) { 9325 pixel_period = 1000000 / (u32)mode->clock; 9326 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 9327 9328 /* watermark for high clocks */ 9329 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9330 rdev->pm.dpm_enabled) { 9331 wm_high.yclk = 9332 radeon_dpm_get_mclk(rdev, false) * 10; 9333 wm_high.sclk = 9334 radeon_dpm_get_sclk(rdev, false) * 10; 9335 } else { 9336 wm_high.yclk = rdev->pm.current_mclk * 10; 9337 wm_high.sclk = rdev->pm.current_sclk * 10; 9338 } 9339 9340 wm_high.disp_clk = mode->clock; 9341 wm_high.src_width = mode->crtc_hdisplay; 9342 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 9343 wm_high.blank_time = line_time - wm_high.active_time; 9344 wm_high.interlaced = false; 9345 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9346 wm_high.interlaced = true; 9347 wm_high.vsc = radeon_crtc->vsc; 9348 wm_high.vtaps = 1; 9349 if (radeon_crtc->rmx_type != RMX_OFF) 9350 wm_high.vtaps = 2; 9351 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9352 wm_high.lb_size = lb_size; 9353 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); 9354 wm_high.num_heads = num_heads; 9355 9356 /* set for high clocks */ 9357 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); 9358 9359 /* possibly force display priority to high */ 9360 /* should really do this at mode validation time... */ 9361 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 9362 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || 9363 !dce8_check_latency_hiding(&wm_high) || 9364 (rdev->disp_priority == 2)) { 9365 DRM_DEBUG_KMS("force priority to high\n"); 9366 } 9367 9368 /* watermark for low clocks */ 9369 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9370 rdev->pm.dpm_enabled) { 9371 wm_low.yclk = 9372 radeon_dpm_get_mclk(rdev, true) * 10; 9373 wm_low.sclk = 9374 radeon_dpm_get_sclk(rdev, true) * 10; 9375 } else { 9376 wm_low.yclk = rdev->pm.current_mclk * 10; 9377 wm_low.sclk = rdev->pm.current_sclk * 10; 9378 } 9379 9380 wm_low.disp_clk = mode->clock; 9381 wm_low.src_width = mode->crtc_hdisplay; 9382 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 9383 wm_low.blank_time = line_time - wm_low.active_time; 9384 wm_low.interlaced = false; 9385 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9386 wm_low.interlaced = true; 9387 wm_low.vsc = radeon_crtc->vsc; 9388 wm_low.vtaps = 1; 9389 if (radeon_crtc->rmx_type != RMX_OFF) 9390 wm_low.vtaps = 2; 9391 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9392 wm_low.lb_size = lb_size; 9393 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); 9394 wm_low.num_heads = num_heads; 9395 9396 /* set for low clocks */ 9397 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); 9398 9399 /* possibly force display priority to high */ 9400 /* should really do this at mode validation time... */ 9401 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 9402 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || 9403 !dce8_check_latency_hiding(&wm_low) || 9404 (rdev->disp_priority == 2)) { 9405 DRM_DEBUG_KMS("force priority to high\n"); 9406 } 9407 } 9408 9409 /* select wm A */ 9410 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9411 tmp = wm_mask; 9412 tmp &= ~LATENCY_WATERMARK_MASK(3); 9413 tmp |= LATENCY_WATERMARK_MASK(1); 9414 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9415 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9416 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 9417 LATENCY_HIGH_WATERMARK(line_time))); 9418 /* select wm B */ 9419 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9420 tmp &= ~LATENCY_WATERMARK_MASK(3); 9421 tmp |= LATENCY_WATERMARK_MASK(2); 9422 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9423 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9424 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 9425 LATENCY_HIGH_WATERMARK(line_time))); 9426 /* restore original selection */ 9427 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); 9428 9429 /* save values for DPM */ 9430 radeon_crtc->line_time = line_time; 9431 radeon_crtc->wm_high = latency_watermark_a; 9432 radeon_crtc->wm_low = latency_watermark_b; 9433 } 9434 9435 /** 9436 * dce8_bandwidth_update - program display watermarks 9437 * 9438 * @rdev: radeon_device pointer 9439 * 9440 * Calculate and program the display watermarks and line 9441 * buffer allocation (CIK). 9442 */ 9443 void dce8_bandwidth_update(struct radeon_device *rdev) 9444 { 9445 struct drm_display_mode *mode = NULL; 9446 u32 num_heads = 0, lb_size; 9447 int i; 9448 9449 if (!rdev->mode_info.mode_config_initialized) 9450 return; 9451 9452 radeon_update_display_priority(rdev); 9453 9454 for (i = 0; i < rdev->num_crtc; i++) { 9455 if (rdev->mode_info.crtcs[i]->base.enabled) 9456 num_heads++; 9457 } 9458 for (i = 0; i < rdev->num_crtc; i++) { 9459 mode = &rdev->mode_info.crtcs[i]->base.mode; 9460 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); 9461 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 9462 } 9463 } 9464 9465 /** 9466 * cik_get_gpu_clock_counter - return GPU clock counter snapshot 9467 * 9468 * @rdev: radeon_device pointer 9469 * 9470 * Fetches a GPU clock counter snapshot (SI). 9471 * Returns the 64 bit clock counter snapshot. 9472 */ 9473 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) 9474 { 9475 uint64_t clock; 9476 9477 mutex_lock(&rdev->gpu_clock_mutex); 9478 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 9479 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 9480 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 9481 mutex_unlock(&rdev->gpu_clock_mutex); 9482 return clock; 9483 } 9484 9485 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, 9486 u32 cntl_reg, u32 status_reg) 9487 { 9488 int r, i; 9489 struct atom_clock_dividers dividers; 9490 uint32_t tmp; 9491 9492 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9493 clock, false, ÷rs); 9494 if (r) 9495 return r; 9496 9497 tmp = RREG32_SMC(cntl_reg); 9498 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); 9499 tmp |= dividers.post_divider; 9500 WREG32_SMC(cntl_reg, tmp); 9501 9502 for (i = 0; i < 100; i++) { 9503 if (RREG32_SMC(status_reg) & DCLK_STATUS) 9504 break; 9505 mdelay(10); 9506 } 9507 if (i == 100) 9508 return -ETIMEDOUT; 9509 9510 return 0; 9511 } 9512 9513 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 9514 { 9515 int r = 0; 9516 9517 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 9518 if (r) 9519 return r; 9520 9521 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 9522 return r; 9523 } 9524 9525 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) 9526 { 9527 int r, i; 9528 struct atom_clock_dividers dividers; 9529 u32 tmp; 9530 9531 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9532 ecclk, false, ÷rs); 9533 if (r) 9534 return r; 9535 9536 for (i = 0; i < 100; i++) { 9537 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9538 break; 9539 mdelay(10); 9540 } 9541 if (i == 100) 9542 return -ETIMEDOUT; 9543 9544 tmp = RREG32_SMC(CG_ECLK_CNTL); 9545 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); 9546 tmp |= dividers.post_divider; 9547 WREG32_SMC(CG_ECLK_CNTL, tmp); 9548 9549 for (i = 0; i < 100; i++) { 9550 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9551 break; 9552 mdelay(10); 9553 } 9554 if (i == 100) 9555 return -ETIMEDOUT; 9556 9557 return 0; 9558 } 9559 9560 static void cik_pcie_gen3_enable(struct radeon_device *rdev) 9561 { 9562 struct pci_dev *root = rdev->pdev->bus->self; 9563 int bridge_pos, gpu_pos; 9564 u32 speed_cntl, mask, current_data_rate; 9565 int ret, i; 9566 u16 tmp16; 9567 9568 if (pci_is_root_bus(rdev->pdev->bus)) 9569 return; 9570 9571 if (radeon_pcie_gen2 == 0) 9572 return; 9573 9574 if (rdev->flags & RADEON_IS_IGP) 9575 return; 9576 9577 if (!(rdev->flags & RADEON_IS_PCIE)) 9578 return; 9579 9580 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 9581 if (ret != 0) 9582 return; 9583 9584 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 9585 return; 9586 9587 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9588 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 9589 LC_CURRENT_DATA_RATE_SHIFT; 9590 if (mask & DRM_PCIE_SPEED_80) { 9591 if (current_data_rate == 2) { 9592 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 9593 return; 9594 } 9595 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 9596 } else if (mask & DRM_PCIE_SPEED_50) { 9597 if (current_data_rate == 1) { 9598 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 9599 return; 9600 } 9601 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 9602 } 9603 9604 bridge_pos = pci_pcie_cap(root); 9605 if (!bridge_pos) 9606 return; 9607 9608 gpu_pos = pci_pcie_cap(rdev->pdev); 9609 if (!gpu_pos) 9610 return; 9611 9612 if (mask & DRM_PCIE_SPEED_80) { 9613 /* re-try equalization if gen3 is not already enabled */ 9614 if (current_data_rate != 2) { 9615 u16 bridge_cfg, gpu_cfg; 9616 u16 bridge_cfg2, gpu_cfg2; 9617 u32 max_lw, current_lw, tmp; 9618 9619 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9620 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9621 9622 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 9623 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9624 9625 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 9626 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9627 9628 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9629 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 9630 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 9631 9632 if (current_lw < max_lw) { 9633 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9634 if (tmp & LC_RENEGOTIATION_SUPPORT) { 9635 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 9636 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 9637 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 9638 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 9639 } 9640 } 9641 9642 for (i = 0; i < 10; i++) { 9643 /* check status */ 9644 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 9645 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 9646 break; 9647 9648 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9649 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9650 9651 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 9652 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 9653 9654 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9655 tmp |= LC_SET_QUIESCE; 9656 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9657 9658 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9659 tmp |= LC_REDO_EQ; 9660 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9661 9662 mdelay(100); 9663 9664 /* linkctl */ 9665 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 9666 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9667 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 9668 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9669 9670 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 9671 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9672 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 9673 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9674 9675 /* linkctl2 */ 9676 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 9677 tmp16 &= ~((1 << 4) | (7 << 9)); 9678 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 9679 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 9680 9681 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9682 tmp16 &= ~((1 << 4) | (7 << 9)); 9683 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 9684 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9685 9686 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9687 tmp &= ~LC_SET_QUIESCE; 9688 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9689 } 9690 } 9691 } 9692 9693 /* set the link speed */ 9694 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 9695 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 9696 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9697 9698 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9699 tmp16 &= ~0xf; 9700 if (mask & DRM_PCIE_SPEED_80) 9701 tmp16 |= 3; /* gen3 */ 9702 else if (mask & DRM_PCIE_SPEED_50) 9703 tmp16 |= 2; /* gen2 */ 9704 else 9705 tmp16 |= 1; /* gen1 */ 9706 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9707 9708 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9709 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 9710 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9711 9712 for (i = 0; i < rdev->usec_timeout; i++) { 9713 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9714 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 9715 break; 9716 udelay(1); 9717 } 9718 } 9719 9720 static void cik_program_aspm(struct radeon_device *rdev) 9721 { 9722 u32 data, orig; 9723 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 9724 bool disable_clkreq = false; 9725 9726 if (radeon_aspm == 0) 9727 return; 9728 9729 /* XXX double check IGPs */ 9730 if (rdev->flags & RADEON_IS_IGP) 9731 return; 9732 9733 if (!(rdev->flags & RADEON_IS_PCIE)) 9734 return; 9735 9736 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9737 data &= ~LC_XMIT_N_FTS_MASK; 9738 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 9739 if (orig != data) 9740 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 9741 9742 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 9743 data |= LC_GO_TO_RECOVERY; 9744 if (orig != data) 9745 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 9746 9747 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); 9748 data |= P_IGNORE_EDB_ERR; 9749 if (orig != data) 9750 WREG32_PCIE_PORT(PCIE_P_CNTL, data); 9751 9752 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9753 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 9754 data |= LC_PMI_TO_L1_DIS; 9755 if (!disable_l0s) 9756 data |= LC_L0S_INACTIVITY(7); 9757 9758 if (!disable_l1) { 9759 data |= LC_L1_INACTIVITY(7); 9760 data &= ~LC_PMI_TO_L1_DIS; 9761 if (orig != data) 9762 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9763 9764 if (!disable_plloff_in_l1) { 9765 bool clk_req_support; 9766 9767 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); 9768 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9769 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9770 if (orig != data) 9771 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); 9772 9773 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); 9774 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9775 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9776 if (orig != data) 9777 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); 9778 9779 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); 9780 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9781 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9782 if (orig != data) 9783 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); 9784 9785 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); 9786 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9787 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9788 if (orig != data) 9789 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); 9790 9791 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9792 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 9793 data |= LC_DYN_LANES_PWR_STATE(3); 9794 if (orig != data) 9795 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 9796 9797 if (!disable_clkreq && 9798 !pci_is_root_bus(rdev->pdev->bus)) { 9799 struct pci_dev *root = rdev->pdev->bus->self; 9800 u32 lnkcap; 9801 9802 clk_req_support = false; 9803 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 9804 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 9805 clk_req_support = true; 9806 } else { 9807 clk_req_support = false; 9808 } 9809 9810 if (clk_req_support) { 9811 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 9812 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 9813 if (orig != data) 9814 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 9815 9816 orig = data = RREG32_SMC(THM_CLK_CNTL); 9817 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 9818 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 9819 if (orig != data) 9820 WREG32_SMC(THM_CLK_CNTL, data); 9821 9822 orig = data = RREG32_SMC(MISC_CLK_CTRL); 9823 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 9824 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 9825 if (orig != data) 9826 WREG32_SMC(MISC_CLK_CTRL, data); 9827 9828 orig = data = RREG32_SMC(CG_CLKPIN_CNTL); 9829 data &= ~BCLK_AS_XCLK; 9830 if (orig != data) 9831 WREG32_SMC(CG_CLKPIN_CNTL, data); 9832 9833 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); 9834 data &= ~FORCE_BIF_REFCLK_EN; 9835 if (orig != data) 9836 WREG32_SMC(CG_CLKPIN_CNTL_2, data); 9837 9838 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); 9839 data &= ~MPLL_CLKOUT_SEL_MASK; 9840 data |= MPLL_CLKOUT_SEL(4); 9841 if (orig != data) 9842 WREG32_SMC(MPLL_BYPASSCLK_SEL, data); 9843 } 9844 } 9845 } else { 9846 if (orig != data) 9847 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9848 } 9849 9850 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 9851 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 9852 if (orig != data) 9853 WREG32_PCIE_PORT(PCIE_CNTL2, data); 9854 9855 if (!disable_l0s) { 9856 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9857 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 9858 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9859 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 9860 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9861 data &= ~LC_L0S_INACTIVITY_MASK; 9862 if (orig != data) 9863 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9864 } 9865 } 9866 } 9867 } 9868