1 /* 2 * Copyright 2012 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include "drmP.h" 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include "radeon_audio.h" 31 #include "cikd.h" 32 #include "atom.h" 33 #include "cik_blit_shaders.h" 34 #include "radeon_ucode.h" 35 #include "clearstate_ci.h" 36 #include "radeon_kfd.h" 37 38 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 39 MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 40 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); 41 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); 42 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); 43 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin"); 44 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); 45 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); 46 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); 47 48 MODULE_FIRMWARE("radeon/bonaire_pfp.bin"); 49 MODULE_FIRMWARE("radeon/bonaire_me.bin"); 50 MODULE_FIRMWARE("radeon/bonaire_ce.bin"); 51 MODULE_FIRMWARE("radeon/bonaire_mec.bin"); 52 MODULE_FIRMWARE("radeon/bonaire_mc.bin"); 53 MODULE_FIRMWARE("radeon/bonaire_rlc.bin"); 54 MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); 55 MODULE_FIRMWARE("radeon/bonaire_smc.bin"); 56 57 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); 58 MODULE_FIRMWARE("radeon/HAWAII_me.bin"); 59 MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); 60 MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); 61 MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); 62 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin"); 63 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); 64 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); 65 MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); 66 67 MODULE_FIRMWARE("radeon/hawaii_pfp.bin"); 68 MODULE_FIRMWARE("radeon/hawaii_me.bin"); 69 MODULE_FIRMWARE("radeon/hawaii_ce.bin"); 70 MODULE_FIRMWARE("radeon/hawaii_mec.bin"); 71 MODULE_FIRMWARE("radeon/hawaii_mc.bin"); 72 MODULE_FIRMWARE("radeon/hawaii_rlc.bin"); 73 MODULE_FIRMWARE("radeon/hawaii_sdma.bin"); 74 MODULE_FIRMWARE("radeon/hawaii_smc.bin"); 75 76 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); 77 MODULE_FIRMWARE("radeon/KAVERI_me.bin"); 78 MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); 79 MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); 80 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); 81 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); 82 83 MODULE_FIRMWARE("radeon/kaveri_pfp.bin"); 84 MODULE_FIRMWARE("radeon/kaveri_me.bin"); 85 MODULE_FIRMWARE("radeon/kaveri_ce.bin"); 86 MODULE_FIRMWARE("radeon/kaveri_mec.bin"); 87 MODULE_FIRMWARE("radeon/kaveri_mec2.bin"); 88 MODULE_FIRMWARE("radeon/kaveri_rlc.bin"); 89 MODULE_FIRMWARE("radeon/kaveri_sdma.bin"); 90 91 MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); 92 MODULE_FIRMWARE("radeon/KABINI_me.bin"); 93 MODULE_FIRMWARE("radeon/KABINI_ce.bin"); 94 MODULE_FIRMWARE("radeon/KABINI_mec.bin"); 95 MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); 96 MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); 97 98 MODULE_FIRMWARE("radeon/kabini_pfp.bin"); 99 MODULE_FIRMWARE("radeon/kabini_me.bin"); 100 MODULE_FIRMWARE("radeon/kabini_ce.bin"); 101 MODULE_FIRMWARE("radeon/kabini_mec.bin"); 102 MODULE_FIRMWARE("radeon/kabini_rlc.bin"); 103 MODULE_FIRMWARE("radeon/kabini_sdma.bin"); 104 105 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin"); 106 MODULE_FIRMWARE("radeon/MULLINS_me.bin"); 107 MODULE_FIRMWARE("radeon/MULLINS_ce.bin"); 108 MODULE_FIRMWARE("radeon/MULLINS_mec.bin"); 109 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin"); 110 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin"); 111 112 MODULE_FIRMWARE("radeon/mullins_pfp.bin"); 113 MODULE_FIRMWARE("radeon/mullins_me.bin"); 114 MODULE_FIRMWARE("radeon/mullins_ce.bin"); 115 MODULE_FIRMWARE("radeon/mullins_mec.bin"); 116 MODULE_FIRMWARE("radeon/mullins_rlc.bin"); 117 MODULE_FIRMWARE("radeon/mullins_sdma.bin"); 118 119 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 120 extern void r600_ih_ring_fini(struct radeon_device *rdev); 121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 123 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 124 extern void sumo_rlc_fini(struct radeon_device *rdev); 125 extern int sumo_rlc_init(struct radeon_device *rdev); 126 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); 127 extern void si_rlc_reset(struct radeon_device *rdev); 128 extern void si_init_uvd_internal_cg(struct radeon_device *rdev); 129 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); 130 extern int cik_sdma_resume(struct radeon_device *rdev); 131 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable); 132 extern void cik_sdma_fini(struct radeon_device *rdev); 133 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable); 134 static void cik_rlc_stop(struct radeon_device *rdev); 135 static void cik_pcie_gen3_enable(struct radeon_device *rdev); 136 static void cik_program_aspm(struct radeon_device *rdev); 137 static void cik_init_pg(struct radeon_device *rdev); 138 static void cik_init_cg(struct radeon_device *rdev); 139 static void cik_fini_pg(struct radeon_device *rdev); 140 static void cik_fini_cg(struct radeon_device *rdev); 141 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 142 bool enable); 143 144 /* get temperature in millidegrees */ 145 int ci_get_temp(struct radeon_device *rdev) 146 { 147 u32 temp; 148 int actual_temp = 0; 149 150 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 151 CTF_TEMP_SHIFT; 152 153 if (temp & 0x200) 154 actual_temp = 255; 155 else 156 actual_temp = temp & 0x1ff; 157 158 actual_temp = actual_temp * 1000; 159 160 return actual_temp; 161 } 162 163 /* get temperature in millidegrees */ 164 int kv_get_temp(struct radeon_device *rdev) 165 { 166 u32 temp; 167 int actual_temp = 0; 168 169 temp = RREG32_SMC(0xC0300E0C); 170 171 if (temp) 172 actual_temp = (temp / 8) - 49; 173 else 174 actual_temp = 0; 175 176 actual_temp = actual_temp * 1000; 177 178 return actual_temp; 179 } 180 181 /* 182 * Indirect registers accessor 183 */ 184 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) 185 { 186 unsigned long flags; 187 u32 r; 188 189 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 190 WREG32(PCIE_INDEX, reg); 191 (void)RREG32(PCIE_INDEX); 192 r = RREG32(PCIE_DATA); 193 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 194 return r; 195 } 196 197 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) 198 { 199 unsigned long flags; 200 201 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 202 WREG32(PCIE_INDEX, reg); 203 (void)RREG32(PCIE_INDEX); 204 WREG32(PCIE_DATA, v); 205 (void)RREG32(PCIE_DATA); 206 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 207 } 208 209 static const u32 spectre_rlc_save_restore_register_list[] = 210 { 211 (0x0e00 << 16) | (0xc12c >> 2), 212 0x00000000, 213 (0x0e00 << 16) | (0xc140 >> 2), 214 0x00000000, 215 (0x0e00 << 16) | (0xc150 >> 2), 216 0x00000000, 217 (0x0e00 << 16) | (0xc15c >> 2), 218 0x00000000, 219 (0x0e00 << 16) | (0xc168 >> 2), 220 0x00000000, 221 (0x0e00 << 16) | (0xc170 >> 2), 222 0x00000000, 223 (0x0e00 << 16) | (0xc178 >> 2), 224 0x00000000, 225 (0x0e00 << 16) | (0xc204 >> 2), 226 0x00000000, 227 (0x0e00 << 16) | (0xc2b4 >> 2), 228 0x00000000, 229 (0x0e00 << 16) | (0xc2b8 >> 2), 230 0x00000000, 231 (0x0e00 << 16) | (0xc2bc >> 2), 232 0x00000000, 233 (0x0e00 << 16) | (0xc2c0 >> 2), 234 0x00000000, 235 (0x0e00 << 16) | (0x8228 >> 2), 236 0x00000000, 237 (0x0e00 << 16) | (0x829c >> 2), 238 0x00000000, 239 (0x0e00 << 16) | (0x869c >> 2), 240 0x00000000, 241 (0x0600 << 16) | (0x98f4 >> 2), 242 0x00000000, 243 (0x0e00 << 16) | (0x98f8 >> 2), 244 0x00000000, 245 (0x0e00 << 16) | (0x9900 >> 2), 246 0x00000000, 247 (0x0e00 << 16) | (0xc260 >> 2), 248 0x00000000, 249 (0x0e00 << 16) | (0x90e8 >> 2), 250 0x00000000, 251 (0x0e00 << 16) | (0x3c000 >> 2), 252 0x00000000, 253 (0x0e00 << 16) | (0x3c00c >> 2), 254 0x00000000, 255 (0x0e00 << 16) | (0x8c1c >> 2), 256 0x00000000, 257 (0x0e00 << 16) | (0x9700 >> 2), 258 0x00000000, 259 (0x0e00 << 16) | (0xcd20 >> 2), 260 0x00000000, 261 (0x4e00 << 16) | (0xcd20 >> 2), 262 0x00000000, 263 (0x5e00 << 16) | (0xcd20 >> 2), 264 0x00000000, 265 (0x6e00 << 16) | (0xcd20 >> 2), 266 0x00000000, 267 (0x7e00 << 16) | (0xcd20 >> 2), 268 0x00000000, 269 (0x8e00 << 16) | (0xcd20 >> 2), 270 0x00000000, 271 (0x9e00 << 16) | (0xcd20 >> 2), 272 0x00000000, 273 (0xae00 << 16) | (0xcd20 >> 2), 274 0x00000000, 275 (0xbe00 << 16) | (0xcd20 >> 2), 276 0x00000000, 277 (0x0e00 << 16) | (0x89bc >> 2), 278 0x00000000, 279 (0x0e00 << 16) | (0x8900 >> 2), 280 0x00000000, 281 0x3, 282 (0x0e00 << 16) | (0xc130 >> 2), 283 0x00000000, 284 (0x0e00 << 16) | (0xc134 >> 2), 285 0x00000000, 286 (0x0e00 << 16) | (0xc1fc >> 2), 287 0x00000000, 288 (0x0e00 << 16) | (0xc208 >> 2), 289 0x00000000, 290 (0x0e00 << 16) | (0xc264 >> 2), 291 0x00000000, 292 (0x0e00 << 16) | (0xc268 >> 2), 293 0x00000000, 294 (0x0e00 << 16) | (0xc26c >> 2), 295 0x00000000, 296 (0x0e00 << 16) | (0xc270 >> 2), 297 0x00000000, 298 (0x0e00 << 16) | (0xc274 >> 2), 299 0x00000000, 300 (0x0e00 << 16) | (0xc278 >> 2), 301 0x00000000, 302 (0x0e00 << 16) | (0xc27c >> 2), 303 0x00000000, 304 (0x0e00 << 16) | (0xc280 >> 2), 305 0x00000000, 306 (0x0e00 << 16) | (0xc284 >> 2), 307 0x00000000, 308 (0x0e00 << 16) | (0xc288 >> 2), 309 0x00000000, 310 (0x0e00 << 16) | (0xc28c >> 2), 311 0x00000000, 312 (0x0e00 << 16) | (0xc290 >> 2), 313 0x00000000, 314 (0x0e00 << 16) | (0xc294 >> 2), 315 0x00000000, 316 (0x0e00 << 16) | (0xc298 >> 2), 317 0x00000000, 318 (0x0e00 << 16) | (0xc29c >> 2), 319 0x00000000, 320 (0x0e00 << 16) | (0xc2a0 >> 2), 321 0x00000000, 322 (0x0e00 << 16) | (0xc2a4 >> 2), 323 0x00000000, 324 (0x0e00 << 16) | (0xc2a8 >> 2), 325 0x00000000, 326 (0x0e00 << 16) | (0xc2ac >> 2), 327 0x00000000, 328 (0x0e00 << 16) | (0xc2b0 >> 2), 329 0x00000000, 330 (0x0e00 << 16) | (0x301d0 >> 2), 331 0x00000000, 332 (0x0e00 << 16) | (0x30238 >> 2), 333 0x00000000, 334 (0x0e00 << 16) | (0x30250 >> 2), 335 0x00000000, 336 (0x0e00 << 16) | (0x30254 >> 2), 337 0x00000000, 338 (0x0e00 << 16) | (0x30258 >> 2), 339 0x00000000, 340 (0x0e00 << 16) | (0x3025c >> 2), 341 0x00000000, 342 (0x4e00 << 16) | (0xc900 >> 2), 343 0x00000000, 344 (0x5e00 << 16) | (0xc900 >> 2), 345 0x00000000, 346 (0x6e00 << 16) | (0xc900 >> 2), 347 0x00000000, 348 (0x7e00 << 16) | (0xc900 >> 2), 349 0x00000000, 350 (0x8e00 << 16) | (0xc900 >> 2), 351 0x00000000, 352 (0x9e00 << 16) | (0xc900 >> 2), 353 0x00000000, 354 (0xae00 << 16) | (0xc900 >> 2), 355 0x00000000, 356 (0xbe00 << 16) | (0xc900 >> 2), 357 0x00000000, 358 (0x4e00 << 16) | (0xc904 >> 2), 359 0x00000000, 360 (0x5e00 << 16) | (0xc904 >> 2), 361 0x00000000, 362 (0x6e00 << 16) | (0xc904 >> 2), 363 0x00000000, 364 (0x7e00 << 16) | (0xc904 >> 2), 365 0x00000000, 366 (0x8e00 << 16) | (0xc904 >> 2), 367 0x00000000, 368 (0x9e00 << 16) | (0xc904 >> 2), 369 0x00000000, 370 (0xae00 << 16) | (0xc904 >> 2), 371 0x00000000, 372 (0xbe00 << 16) | (0xc904 >> 2), 373 0x00000000, 374 (0x4e00 << 16) | (0xc908 >> 2), 375 0x00000000, 376 (0x5e00 << 16) | (0xc908 >> 2), 377 0x00000000, 378 (0x6e00 << 16) | (0xc908 >> 2), 379 0x00000000, 380 (0x7e00 << 16) | (0xc908 >> 2), 381 0x00000000, 382 (0x8e00 << 16) | (0xc908 >> 2), 383 0x00000000, 384 (0x9e00 << 16) | (0xc908 >> 2), 385 0x00000000, 386 (0xae00 << 16) | (0xc908 >> 2), 387 0x00000000, 388 (0xbe00 << 16) | (0xc908 >> 2), 389 0x00000000, 390 (0x4e00 << 16) | (0xc90c >> 2), 391 0x00000000, 392 (0x5e00 << 16) | (0xc90c >> 2), 393 0x00000000, 394 (0x6e00 << 16) | (0xc90c >> 2), 395 0x00000000, 396 (0x7e00 << 16) | (0xc90c >> 2), 397 0x00000000, 398 (0x8e00 << 16) | (0xc90c >> 2), 399 0x00000000, 400 (0x9e00 << 16) | (0xc90c >> 2), 401 0x00000000, 402 (0xae00 << 16) | (0xc90c >> 2), 403 0x00000000, 404 (0xbe00 << 16) | (0xc90c >> 2), 405 0x00000000, 406 (0x4e00 << 16) | (0xc910 >> 2), 407 0x00000000, 408 (0x5e00 << 16) | (0xc910 >> 2), 409 0x00000000, 410 (0x6e00 << 16) | (0xc910 >> 2), 411 0x00000000, 412 (0x7e00 << 16) | (0xc910 >> 2), 413 0x00000000, 414 (0x8e00 << 16) | (0xc910 >> 2), 415 0x00000000, 416 (0x9e00 << 16) | (0xc910 >> 2), 417 0x00000000, 418 (0xae00 << 16) | (0xc910 >> 2), 419 0x00000000, 420 (0xbe00 << 16) | (0xc910 >> 2), 421 0x00000000, 422 (0x0e00 << 16) | (0xc99c >> 2), 423 0x00000000, 424 (0x0e00 << 16) | (0x9834 >> 2), 425 0x00000000, 426 (0x0000 << 16) | (0x30f00 >> 2), 427 0x00000000, 428 (0x0001 << 16) | (0x30f00 >> 2), 429 0x00000000, 430 (0x0000 << 16) | (0x30f04 >> 2), 431 0x00000000, 432 (0x0001 << 16) | (0x30f04 >> 2), 433 0x00000000, 434 (0x0000 << 16) | (0x30f08 >> 2), 435 0x00000000, 436 (0x0001 << 16) | (0x30f08 >> 2), 437 0x00000000, 438 (0x0000 << 16) | (0x30f0c >> 2), 439 0x00000000, 440 (0x0001 << 16) | (0x30f0c >> 2), 441 0x00000000, 442 (0x0600 << 16) | (0x9b7c >> 2), 443 0x00000000, 444 (0x0e00 << 16) | (0x8a14 >> 2), 445 0x00000000, 446 (0x0e00 << 16) | (0x8a18 >> 2), 447 0x00000000, 448 (0x0600 << 16) | (0x30a00 >> 2), 449 0x00000000, 450 (0x0e00 << 16) | (0x8bf0 >> 2), 451 0x00000000, 452 (0x0e00 << 16) | (0x8bcc >> 2), 453 0x00000000, 454 (0x0e00 << 16) | (0x8b24 >> 2), 455 0x00000000, 456 (0x0e00 << 16) | (0x30a04 >> 2), 457 0x00000000, 458 (0x0600 << 16) | (0x30a10 >> 2), 459 0x00000000, 460 (0x0600 << 16) | (0x30a14 >> 2), 461 0x00000000, 462 (0x0600 << 16) | (0x30a18 >> 2), 463 0x00000000, 464 (0x0600 << 16) | (0x30a2c >> 2), 465 0x00000000, 466 (0x0e00 << 16) | (0xc700 >> 2), 467 0x00000000, 468 (0x0e00 << 16) | (0xc704 >> 2), 469 0x00000000, 470 (0x0e00 << 16) | (0xc708 >> 2), 471 0x00000000, 472 (0x0e00 << 16) | (0xc768 >> 2), 473 0x00000000, 474 (0x0400 << 16) | (0xc770 >> 2), 475 0x00000000, 476 (0x0400 << 16) | (0xc774 >> 2), 477 0x00000000, 478 (0x0400 << 16) | (0xc778 >> 2), 479 0x00000000, 480 (0x0400 << 16) | (0xc77c >> 2), 481 0x00000000, 482 (0x0400 << 16) | (0xc780 >> 2), 483 0x00000000, 484 (0x0400 << 16) | (0xc784 >> 2), 485 0x00000000, 486 (0x0400 << 16) | (0xc788 >> 2), 487 0x00000000, 488 (0x0400 << 16) | (0xc78c >> 2), 489 0x00000000, 490 (0x0400 << 16) | (0xc798 >> 2), 491 0x00000000, 492 (0x0400 << 16) | (0xc79c >> 2), 493 0x00000000, 494 (0x0400 << 16) | (0xc7a0 >> 2), 495 0x00000000, 496 (0x0400 << 16) | (0xc7a4 >> 2), 497 0x00000000, 498 (0x0400 << 16) | (0xc7a8 >> 2), 499 0x00000000, 500 (0x0400 << 16) | (0xc7ac >> 2), 501 0x00000000, 502 (0x0400 << 16) | (0xc7b0 >> 2), 503 0x00000000, 504 (0x0400 << 16) | (0xc7b4 >> 2), 505 0x00000000, 506 (0x0e00 << 16) | (0x9100 >> 2), 507 0x00000000, 508 (0x0e00 << 16) | (0x3c010 >> 2), 509 0x00000000, 510 (0x0e00 << 16) | (0x92a8 >> 2), 511 0x00000000, 512 (0x0e00 << 16) | (0x92ac >> 2), 513 0x00000000, 514 (0x0e00 << 16) | (0x92b4 >> 2), 515 0x00000000, 516 (0x0e00 << 16) | (0x92b8 >> 2), 517 0x00000000, 518 (0x0e00 << 16) | (0x92bc >> 2), 519 0x00000000, 520 (0x0e00 << 16) | (0x92c0 >> 2), 521 0x00000000, 522 (0x0e00 << 16) | (0x92c4 >> 2), 523 0x00000000, 524 (0x0e00 << 16) | (0x92c8 >> 2), 525 0x00000000, 526 (0x0e00 << 16) | (0x92cc >> 2), 527 0x00000000, 528 (0x0e00 << 16) | (0x92d0 >> 2), 529 0x00000000, 530 (0x0e00 << 16) | (0x8c00 >> 2), 531 0x00000000, 532 (0x0e00 << 16) | (0x8c04 >> 2), 533 0x00000000, 534 (0x0e00 << 16) | (0x8c20 >> 2), 535 0x00000000, 536 (0x0e00 << 16) | (0x8c38 >> 2), 537 0x00000000, 538 (0x0e00 << 16) | (0x8c3c >> 2), 539 0x00000000, 540 (0x0e00 << 16) | (0xae00 >> 2), 541 0x00000000, 542 (0x0e00 << 16) | (0x9604 >> 2), 543 0x00000000, 544 (0x0e00 << 16) | (0xac08 >> 2), 545 0x00000000, 546 (0x0e00 << 16) | (0xac0c >> 2), 547 0x00000000, 548 (0x0e00 << 16) | (0xac10 >> 2), 549 0x00000000, 550 (0x0e00 << 16) | (0xac14 >> 2), 551 0x00000000, 552 (0x0e00 << 16) | (0xac58 >> 2), 553 0x00000000, 554 (0x0e00 << 16) | (0xac68 >> 2), 555 0x00000000, 556 (0x0e00 << 16) | (0xac6c >> 2), 557 0x00000000, 558 (0x0e00 << 16) | (0xac70 >> 2), 559 0x00000000, 560 (0x0e00 << 16) | (0xac74 >> 2), 561 0x00000000, 562 (0x0e00 << 16) | (0xac78 >> 2), 563 0x00000000, 564 (0x0e00 << 16) | (0xac7c >> 2), 565 0x00000000, 566 (0x0e00 << 16) | (0xac80 >> 2), 567 0x00000000, 568 (0x0e00 << 16) | (0xac84 >> 2), 569 0x00000000, 570 (0x0e00 << 16) | (0xac88 >> 2), 571 0x00000000, 572 (0x0e00 << 16) | (0xac8c >> 2), 573 0x00000000, 574 (0x0e00 << 16) | (0x970c >> 2), 575 0x00000000, 576 (0x0e00 << 16) | (0x9714 >> 2), 577 0x00000000, 578 (0x0e00 << 16) | (0x9718 >> 2), 579 0x00000000, 580 (0x0e00 << 16) | (0x971c >> 2), 581 0x00000000, 582 (0x0e00 << 16) | (0x31068 >> 2), 583 0x00000000, 584 (0x4e00 << 16) | (0x31068 >> 2), 585 0x00000000, 586 (0x5e00 << 16) | (0x31068 >> 2), 587 0x00000000, 588 (0x6e00 << 16) | (0x31068 >> 2), 589 0x00000000, 590 (0x7e00 << 16) | (0x31068 >> 2), 591 0x00000000, 592 (0x8e00 << 16) | (0x31068 >> 2), 593 0x00000000, 594 (0x9e00 << 16) | (0x31068 >> 2), 595 0x00000000, 596 (0xae00 << 16) | (0x31068 >> 2), 597 0x00000000, 598 (0xbe00 << 16) | (0x31068 >> 2), 599 0x00000000, 600 (0x0e00 << 16) | (0xcd10 >> 2), 601 0x00000000, 602 (0x0e00 << 16) | (0xcd14 >> 2), 603 0x00000000, 604 (0x0e00 << 16) | (0x88b0 >> 2), 605 0x00000000, 606 (0x0e00 << 16) | (0x88b4 >> 2), 607 0x00000000, 608 (0x0e00 << 16) | (0x88b8 >> 2), 609 0x00000000, 610 (0x0e00 << 16) | (0x88bc >> 2), 611 0x00000000, 612 (0x0400 << 16) | (0x89c0 >> 2), 613 0x00000000, 614 (0x0e00 << 16) | (0x88c4 >> 2), 615 0x00000000, 616 (0x0e00 << 16) | (0x88c8 >> 2), 617 0x00000000, 618 (0x0e00 << 16) | (0x88d0 >> 2), 619 0x00000000, 620 (0x0e00 << 16) | (0x88d4 >> 2), 621 0x00000000, 622 (0x0e00 << 16) | (0x88d8 >> 2), 623 0x00000000, 624 (0x0e00 << 16) | (0x8980 >> 2), 625 0x00000000, 626 (0x0e00 << 16) | (0x30938 >> 2), 627 0x00000000, 628 (0x0e00 << 16) | (0x3093c >> 2), 629 0x00000000, 630 (0x0e00 << 16) | (0x30940 >> 2), 631 0x00000000, 632 (0x0e00 << 16) | (0x89a0 >> 2), 633 0x00000000, 634 (0x0e00 << 16) | (0x30900 >> 2), 635 0x00000000, 636 (0x0e00 << 16) | (0x30904 >> 2), 637 0x00000000, 638 (0x0e00 << 16) | (0x89b4 >> 2), 639 0x00000000, 640 (0x0e00 << 16) | (0x3c210 >> 2), 641 0x00000000, 642 (0x0e00 << 16) | (0x3c214 >> 2), 643 0x00000000, 644 (0x0e00 << 16) | (0x3c218 >> 2), 645 0x00000000, 646 (0x0e00 << 16) | (0x8904 >> 2), 647 0x00000000, 648 0x5, 649 (0x0e00 << 16) | (0x8c28 >> 2), 650 (0x0e00 << 16) | (0x8c2c >> 2), 651 (0x0e00 << 16) | (0x8c30 >> 2), 652 (0x0e00 << 16) | (0x8c34 >> 2), 653 (0x0e00 << 16) | (0x9600 >> 2), 654 }; 655 656 static const u32 kalindi_rlc_save_restore_register_list[] = 657 { 658 (0x0e00 << 16) | (0xc12c >> 2), 659 0x00000000, 660 (0x0e00 << 16) | (0xc140 >> 2), 661 0x00000000, 662 (0x0e00 << 16) | (0xc150 >> 2), 663 0x00000000, 664 (0x0e00 << 16) | (0xc15c >> 2), 665 0x00000000, 666 (0x0e00 << 16) | (0xc168 >> 2), 667 0x00000000, 668 (0x0e00 << 16) | (0xc170 >> 2), 669 0x00000000, 670 (0x0e00 << 16) | (0xc204 >> 2), 671 0x00000000, 672 (0x0e00 << 16) | (0xc2b4 >> 2), 673 0x00000000, 674 (0x0e00 << 16) | (0xc2b8 >> 2), 675 0x00000000, 676 (0x0e00 << 16) | (0xc2bc >> 2), 677 0x00000000, 678 (0x0e00 << 16) | (0xc2c0 >> 2), 679 0x00000000, 680 (0x0e00 << 16) | (0x8228 >> 2), 681 0x00000000, 682 (0x0e00 << 16) | (0x829c >> 2), 683 0x00000000, 684 (0x0e00 << 16) | (0x869c >> 2), 685 0x00000000, 686 (0x0600 << 16) | (0x98f4 >> 2), 687 0x00000000, 688 (0x0e00 << 16) | (0x98f8 >> 2), 689 0x00000000, 690 (0x0e00 << 16) | (0x9900 >> 2), 691 0x00000000, 692 (0x0e00 << 16) | (0xc260 >> 2), 693 0x00000000, 694 (0x0e00 << 16) | (0x90e8 >> 2), 695 0x00000000, 696 (0x0e00 << 16) | (0x3c000 >> 2), 697 0x00000000, 698 (0x0e00 << 16) | (0x3c00c >> 2), 699 0x00000000, 700 (0x0e00 << 16) | (0x8c1c >> 2), 701 0x00000000, 702 (0x0e00 << 16) | (0x9700 >> 2), 703 0x00000000, 704 (0x0e00 << 16) | (0xcd20 >> 2), 705 0x00000000, 706 (0x4e00 << 16) | (0xcd20 >> 2), 707 0x00000000, 708 (0x5e00 << 16) | (0xcd20 >> 2), 709 0x00000000, 710 (0x6e00 << 16) | (0xcd20 >> 2), 711 0x00000000, 712 (0x7e00 << 16) | (0xcd20 >> 2), 713 0x00000000, 714 (0x0e00 << 16) | (0x89bc >> 2), 715 0x00000000, 716 (0x0e00 << 16) | (0x8900 >> 2), 717 0x00000000, 718 0x3, 719 (0x0e00 << 16) | (0xc130 >> 2), 720 0x00000000, 721 (0x0e00 << 16) | (0xc134 >> 2), 722 0x00000000, 723 (0x0e00 << 16) | (0xc1fc >> 2), 724 0x00000000, 725 (0x0e00 << 16) | (0xc208 >> 2), 726 0x00000000, 727 (0x0e00 << 16) | (0xc264 >> 2), 728 0x00000000, 729 (0x0e00 << 16) | (0xc268 >> 2), 730 0x00000000, 731 (0x0e00 << 16) | (0xc26c >> 2), 732 0x00000000, 733 (0x0e00 << 16) | (0xc270 >> 2), 734 0x00000000, 735 (0x0e00 << 16) | (0xc274 >> 2), 736 0x00000000, 737 (0x0e00 << 16) | (0xc28c >> 2), 738 0x00000000, 739 (0x0e00 << 16) | (0xc290 >> 2), 740 0x00000000, 741 (0x0e00 << 16) | (0xc294 >> 2), 742 0x00000000, 743 (0x0e00 << 16) | (0xc298 >> 2), 744 0x00000000, 745 (0x0e00 << 16) | (0xc2a0 >> 2), 746 0x00000000, 747 (0x0e00 << 16) | (0xc2a4 >> 2), 748 0x00000000, 749 (0x0e00 << 16) | (0xc2a8 >> 2), 750 0x00000000, 751 (0x0e00 << 16) | (0xc2ac >> 2), 752 0x00000000, 753 (0x0e00 << 16) | (0x301d0 >> 2), 754 0x00000000, 755 (0x0e00 << 16) | (0x30238 >> 2), 756 0x00000000, 757 (0x0e00 << 16) | (0x30250 >> 2), 758 0x00000000, 759 (0x0e00 << 16) | (0x30254 >> 2), 760 0x00000000, 761 (0x0e00 << 16) | (0x30258 >> 2), 762 0x00000000, 763 (0x0e00 << 16) | (0x3025c >> 2), 764 0x00000000, 765 (0x4e00 << 16) | (0xc900 >> 2), 766 0x00000000, 767 (0x5e00 << 16) | (0xc900 >> 2), 768 0x00000000, 769 (0x6e00 << 16) | (0xc900 >> 2), 770 0x00000000, 771 (0x7e00 << 16) | (0xc900 >> 2), 772 0x00000000, 773 (0x4e00 << 16) | (0xc904 >> 2), 774 0x00000000, 775 (0x5e00 << 16) | (0xc904 >> 2), 776 0x00000000, 777 (0x6e00 << 16) | (0xc904 >> 2), 778 0x00000000, 779 (0x7e00 << 16) | (0xc904 >> 2), 780 0x00000000, 781 (0x4e00 << 16) | (0xc908 >> 2), 782 0x00000000, 783 (0x5e00 << 16) | (0xc908 >> 2), 784 0x00000000, 785 (0x6e00 << 16) | (0xc908 >> 2), 786 0x00000000, 787 (0x7e00 << 16) | (0xc908 >> 2), 788 0x00000000, 789 (0x4e00 << 16) | (0xc90c >> 2), 790 0x00000000, 791 (0x5e00 << 16) | (0xc90c >> 2), 792 0x00000000, 793 (0x6e00 << 16) | (0xc90c >> 2), 794 0x00000000, 795 (0x7e00 << 16) | (0xc90c >> 2), 796 0x00000000, 797 (0x4e00 << 16) | (0xc910 >> 2), 798 0x00000000, 799 (0x5e00 << 16) | (0xc910 >> 2), 800 0x00000000, 801 (0x6e00 << 16) | (0xc910 >> 2), 802 0x00000000, 803 (0x7e00 << 16) | (0xc910 >> 2), 804 0x00000000, 805 (0x0e00 << 16) | (0xc99c >> 2), 806 0x00000000, 807 (0x0e00 << 16) | (0x9834 >> 2), 808 0x00000000, 809 (0x0000 << 16) | (0x30f00 >> 2), 810 0x00000000, 811 (0x0000 << 16) | (0x30f04 >> 2), 812 0x00000000, 813 (0x0000 << 16) | (0x30f08 >> 2), 814 0x00000000, 815 (0x0000 << 16) | (0x30f0c >> 2), 816 0x00000000, 817 (0x0600 << 16) | (0x9b7c >> 2), 818 0x00000000, 819 (0x0e00 << 16) | (0x8a14 >> 2), 820 0x00000000, 821 (0x0e00 << 16) | (0x8a18 >> 2), 822 0x00000000, 823 (0x0600 << 16) | (0x30a00 >> 2), 824 0x00000000, 825 (0x0e00 << 16) | (0x8bf0 >> 2), 826 0x00000000, 827 (0x0e00 << 16) | (0x8bcc >> 2), 828 0x00000000, 829 (0x0e00 << 16) | (0x8b24 >> 2), 830 0x00000000, 831 (0x0e00 << 16) | (0x30a04 >> 2), 832 0x00000000, 833 (0x0600 << 16) | (0x30a10 >> 2), 834 0x00000000, 835 (0x0600 << 16) | (0x30a14 >> 2), 836 0x00000000, 837 (0x0600 << 16) | (0x30a18 >> 2), 838 0x00000000, 839 (0x0600 << 16) | (0x30a2c >> 2), 840 0x00000000, 841 (0x0e00 << 16) | (0xc700 >> 2), 842 0x00000000, 843 (0x0e00 << 16) | (0xc704 >> 2), 844 0x00000000, 845 (0x0e00 << 16) | (0xc708 >> 2), 846 0x00000000, 847 (0x0e00 << 16) | (0xc768 >> 2), 848 0x00000000, 849 (0x0400 << 16) | (0xc770 >> 2), 850 0x00000000, 851 (0x0400 << 16) | (0xc774 >> 2), 852 0x00000000, 853 (0x0400 << 16) | (0xc798 >> 2), 854 0x00000000, 855 (0x0400 << 16) | (0xc79c >> 2), 856 0x00000000, 857 (0x0e00 << 16) | (0x9100 >> 2), 858 0x00000000, 859 (0x0e00 << 16) | (0x3c010 >> 2), 860 0x00000000, 861 (0x0e00 << 16) | (0x8c00 >> 2), 862 0x00000000, 863 (0x0e00 << 16) | (0x8c04 >> 2), 864 0x00000000, 865 (0x0e00 << 16) | (0x8c20 >> 2), 866 0x00000000, 867 (0x0e00 << 16) | (0x8c38 >> 2), 868 0x00000000, 869 (0x0e00 << 16) | (0x8c3c >> 2), 870 0x00000000, 871 (0x0e00 << 16) | (0xae00 >> 2), 872 0x00000000, 873 (0x0e00 << 16) | (0x9604 >> 2), 874 0x00000000, 875 (0x0e00 << 16) | (0xac08 >> 2), 876 0x00000000, 877 (0x0e00 << 16) | (0xac0c >> 2), 878 0x00000000, 879 (0x0e00 << 16) | (0xac10 >> 2), 880 0x00000000, 881 (0x0e00 << 16) | (0xac14 >> 2), 882 0x00000000, 883 (0x0e00 << 16) | (0xac58 >> 2), 884 0x00000000, 885 (0x0e00 << 16) | (0xac68 >> 2), 886 0x00000000, 887 (0x0e00 << 16) | (0xac6c >> 2), 888 0x00000000, 889 (0x0e00 << 16) | (0xac70 >> 2), 890 0x00000000, 891 (0x0e00 << 16) | (0xac74 >> 2), 892 0x00000000, 893 (0x0e00 << 16) | (0xac78 >> 2), 894 0x00000000, 895 (0x0e00 << 16) | (0xac7c >> 2), 896 0x00000000, 897 (0x0e00 << 16) | (0xac80 >> 2), 898 0x00000000, 899 (0x0e00 << 16) | (0xac84 >> 2), 900 0x00000000, 901 (0x0e00 << 16) | (0xac88 >> 2), 902 0x00000000, 903 (0x0e00 << 16) | (0xac8c >> 2), 904 0x00000000, 905 (0x0e00 << 16) | (0x970c >> 2), 906 0x00000000, 907 (0x0e00 << 16) | (0x9714 >> 2), 908 0x00000000, 909 (0x0e00 << 16) | (0x9718 >> 2), 910 0x00000000, 911 (0x0e00 << 16) | (0x971c >> 2), 912 0x00000000, 913 (0x0e00 << 16) | (0x31068 >> 2), 914 0x00000000, 915 (0x4e00 << 16) | (0x31068 >> 2), 916 0x00000000, 917 (0x5e00 << 16) | (0x31068 >> 2), 918 0x00000000, 919 (0x6e00 << 16) | (0x31068 >> 2), 920 0x00000000, 921 (0x7e00 << 16) | (0x31068 >> 2), 922 0x00000000, 923 (0x0e00 << 16) | (0xcd10 >> 2), 924 0x00000000, 925 (0x0e00 << 16) | (0xcd14 >> 2), 926 0x00000000, 927 (0x0e00 << 16) | (0x88b0 >> 2), 928 0x00000000, 929 (0x0e00 << 16) | (0x88b4 >> 2), 930 0x00000000, 931 (0x0e00 << 16) | (0x88b8 >> 2), 932 0x00000000, 933 (0x0e00 << 16) | (0x88bc >> 2), 934 0x00000000, 935 (0x0400 << 16) | (0x89c0 >> 2), 936 0x00000000, 937 (0x0e00 << 16) | (0x88c4 >> 2), 938 0x00000000, 939 (0x0e00 << 16) | (0x88c8 >> 2), 940 0x00000000, 941 (0x0e00 << 16) | (0x88d0 >> 2), 942 0x00000000, 943 (0x0e00 << 16) | (0x88d4 >> 2), 944 0x00000000, 945 (0x0e00 << 16) | (0x88d8 >> 2), 946 0x00000000, 947 (0x0e00 << 16) | (0x8980 >> 2), 948 0x00000000, 949 (0x0e00 << 16) | (0x30938 >> 2), 950 0x00000000, 951 (0x0e00 << 16) | (0x3093c >> 2), 952 0x00000000, 953 (0x0e00 << 16) | (0x30940 >> 2), 954 0x00000000, 955 (0x0e00 << 16) | (0x89a0 >> 2), 956 0x00000000, 957 (0x0e00 << 16) | (0x30900 >> 2), 958 0x00000000, 959 (0x0e00 << 16) | (0x30904 >> 2), 960 0x00000000, 961 (0x0e00 << 16) | (0x89b4 >> 2), 962 0x00000000, 963 (0x0e00 << 16) | (0x3e1fc >> 2), 964 0x00000000, 965 (0x0e00 << 16) | (0x3c210 >> 2), 966 0x00000000, 967 (0x0e00 << 16) | (0x3c214 >> 2), 968 0x00000000, 969 (0x0e00 << 16) | (0x3c218 >> 2), 970 0x00000000, 971 (0x0e00 << 16) | (0x8904 >> 2), 972 0x00000000, 973 0x5, 974 (0x0e00 << 16) | (0x8c28 >> 2), 975 (0x0e00 << 16) | (0x8c2c >> 2), 976 (0x0e00 << 16) | (0x8c30 >> 2), 977 (0x0e00 << 16) | (0x8c34 >> 2), 978 (0x0e00 << 16) | (0x9600 >> 2), 979 }; 980 981 static const u32 bonaire_golden_spm_registers[] = 982 { 983 0x30800, 0xe0ffffff, 0xe0000000 984 }; 985 986 static const u32 bonaire_golden_common_registers[] = 987 { 988 0xc770, 0xffffffff, 0x00000800, 989 0xc774, 0xffffffff, 0x00000800, 990 0xc798, 0xffffffff, 0x00007fbf, 991 0xc79c, 0xffffffff, 0x00007faf 992 }; 993 994 static const u32 bonaire_golden_registers[] = 995 { 996 0x3354, 0x00000333, 0x00000333, 997 0x3350, 0x000c0fc0, 0x00040200, 998 0x9a10, 0x00010000, 0x00058208, 999 0x3c000, 0xffff1fff, 0x00140000, 1000 0x3c200, 0xfdfc0fff, 0x00000100, 1001 0x3c234, 0x40000000, 0x40000200, 1002 0x9830, 0xffffffff, 0x00000000, 1003 0x9834, 0xf00fffff, 0x00000400, 1004 0x9838, 0x0002021c, 0x00020200, 1005 0xc78, 0x00000080, 0x00000000, 1006 0x5bb0, 0x000000f0, 0x00000070, 1007 0x5bc0, 0xf0311fff, 0x80300000, 1008 0x98f8, 0x73773777, 0x12010001, 1009 0x350c, 0x00810000, 0x408af000, 1010 0x7030, 0x31000111, 0x00000011, 1011 0x2f48, 0x73773777, 0x12010001, 1012 0x220c, 0x00007fb6, 0x0021a1b1, 1013 0x2210, 0x00007fb6, 0x002021b1, 1014 0x2180, 0x00007fb6, 0x00002191, 1015 0x2218, 0x00007fb6, 0x002121b1, 1016 0x221c, 0x00007fb6, 0x002021b1, 1017 0x21dc, 0x00007fb6, 0x00002191, 1018 0x21e0, 0x00007fb6, 0x00002191, 1019 0x3628, 0x0000003f, 0x0000000a, 1020 0x362c, 0x0000003f, 0x0000000a, 1021 0x2ae4, 0x00073ffe, 0x000022a2, 1022 0x240c, 0x000007ff, 0x00000000, 1023 0x8a14, 0xf000003f, 0x00000007, 1024 0x8bf0, 0x00002001, 0x00000001, 1025 0x8b24, 0xffffffff, 0x00ffffff, 1026 0x30a04, 0x0000ff0f, 0x00000000, 1027 0x28a4c, 0x07ffffff, 0x06000000, 1028 0x4d8, 0x00000fff, 0x00000100, 1029 0x3e78, 0x00000001, 0x00000002, 1030 0x9100, 0x03000000, 0x0362c688, 1031 0x8c00, 0x000000ff, 0x00000001, 1032 0xe40, 0x00001fff, 0x00001fff, 1033 0x9060, 0x0000007f, 0x00000020, 1034 0x9508, 0x00010000, 0x00010000, 1035 0xac14, 0x000003ff, 0x000000f3, 1036 0xac0c, 0xffffffff, 0x00001032 1037 }; 1038 1039 static const u32 bonaire_mgcg_cgcg_init[] = 1040 { 1041 0xc420, 0xffffffff, 0xfffffffc, 1042 0x30800, 0xffffffff, 0xe0000000, 1043 0x3c2a0, 0xffffffff, 0x00000100, 1044 0x3c208, 0xffffffff, 0x00000100, 1045 0x3c2c0, 0xffffffff, 0xc0000100, 1046 0x3c2c8, 0xffffffff, 0xc0000100, 1047 0x3c2c4, 0xffffffff, 0xc0000100, 1048 0x55e4, 0xffffffff, 0x00600100, 1049 0x3c280, 0xffffffff, 0x00000100, 1050 0x3c214, 0xffffffff, 0x06000100, 1051 0x3c220, 0xffffffff, 0x00000100, 1052 0x3c218, 0xffffffff, 0x06000100, 1053 0x3c204, 0xffffffff, 0x00000100, 1054 0x3c2e0, 0xffffffff, 0x00000100, 1055 0x3c224, 0xffffffff, 0x00000100, 1056 0x3c200, 0xffffffff, 0x00000100, 1057 0x3c230, 0xffffffff, 0x00000100, 1058 0x3c234, 0xffffffff, 0x00000100, 1059 0x3c250, 0xffffffff, 0x00000100, 1060 0x3c254, 0xffffffff, 0x00000100, 1061 0x3c258, 0xffffffff, 0x00000100, 1062 0x3c25c, 0xffffffff, 0x00000100, 1063 0x3c260, 0xffffffff, 0x00000100, 1064 0x3c27c, 0xffffffff, 0x00000100, 1065 0x3c278, 0xffffffff, 0x00000100, 1066 0x3c210, 0xffffffff, 0x06000100, 1067 0x3c290, 0xffffffff, 0x00000100, 1068 0x3c274, 0xffffffff, 0x00000100, 1069 0x3c2b4, 0xffffffff, 0x00000100, 1070 0x3c2b0, 0xffffffff, 0x00000100, 1071 0x3c270, 0xffffffff, 0x00000100, 1072 0x30800, 0xffffffff, 0xe0000000, 1073 0x3c020, 0xffffffff, 0x00010000, 1074 0x3c024, 0xffffffff, 0x00030002, 1075 0x3c028, 0xffffffff, 0x00040007, 1076 0x3c02c, 0xffffffff, 0x00060005, 1077 0x3c030, 0xffffffff, 0x00090008, 1078 0x3c034, 0xffffffff, 0x00010000, 1079 0x3c038, 0xffffffff, 0x00030002, 1080 0x3c03c, 0xffffffff, 0x00040007, 1081 0x3c040, 0xffffffff, 0x00060005, 1082 0x3c044, 0xffffffff, 0x00090008, 1083 0x3c048, 0xffffffff, 0x00010000, 1084 0x3c04c, 0xffffffff, 0x00030002, 1085 0x3c050, 0xffffffff, 0x00040007, 1086 0x3c054, 0xffffffff, 0x00060005, 1087 0x3c058, 0xffffffff, 0x00090008, 1088 0x3c05c, 0xffffffff, 0x00010000, 1089 0x3c060, 0xffffffff, 0x00030002, 1090 0x3c064, 0xffffffff, 0x00040007, 1091 0x3c068, 0xffffffff, 0x00060005, 1092 0x3c06c, 0xffffffff, 0x00090008, 1093 0x3c070, 0xffffffff, 0x00010000, 1094 0x3c074, 0xffffffff, 0x00030002, 1095 0x3c078, 0xffffffff, 0x00040007, 1096 0x3c07c, 0xffffffff, 0x00060005, 1097 0x3c080, 0xffffffff, 0x00090008, 1098 0x3c084, 0xffffffff, 0x00010000, 1099 0x3c088, 0xffffffff, 0x00030002, 1100 0x3c08c, 0xffffffff, 0x00040007, 1101 0x3c090, 0xffffffff, 0x00060005, 1102 0x3c094, 0xffffffff, 0x00090008, 1103 0x3c098, 0xffffffff, 0x00010000, 1104 0x3c09c, 0xffffffff, 0x00030002, 1105 0x3c0a0, 0xffffffff, 0x00040007, 1106 0x3c0a4, 0xffffffff, 0x00060005, 1107 0x3c0a8, 0xffffffff, 0x00090008, 1108 0x3c000, 0xffffffff, 0x96e00200, 1109 0x8708, 0xffffffff, 0x00900100, 1110 0xc424, 0xffffffff, 0x0020003f, 1111 0x38, 0xffffffff, 0x0140001c, 1112 0x3c, 0x000f0000, 0x000f0000, 1113 0x220, 0xffffffff, 0xC060000C, 1114 0x224, 0xc0000fff, 0x00000100, 1115 0xf90, 0xffffffff, 0x00000100, 1116 0xf98, 0x00000101, 0x00000000, 1117 0x20a8, 0xffffffff, 0x00000104, 1118 0x55e4, 0xff000fff, 0x00000100, 1119 0x30cc, 0xc0000fff, 0x00000104, 1120 0xc1e4, 0x00000001, 0x00000001, 1121 0xd00c, 0xff000ff0, 0x00000100, 1122 0xd80c, 0xff000ff0, 0x00000100 1123 }; 1124 1125 static const u32 spectre_golden_spm_registers[] = 1126 { 1127 0x30800, 0xe0ffffff, 0xe0000000 1128 }; 1129 1130 static const u32 spectre_golden_common_registers[] = 1131 { 1132 0xc770, 0xffffffff, 0x00000800, 1133 0xc774, 0xffffffff, 0x00000800, 1134 0xc798, 0xffffffff, 0x00007fbf, 1135 0xc79c, 0xffffffff, 0x00007faf 1136 }; 1137 1138 static const u32 spectre_golden_registers[] = 1139 { 1140 0x3c000, 0xffff1fff, 0x96940200, 1141 0x3c00c, 0xffff0001, 0xff000000, 1142 0x3c200, 0xfffc0fff, 0x00000100, 1143 0x6ed8, 0x00010101, 0x00010000, 1144 0x9834, 0xf00fffff, 0x00000400, 1145 0x9838, 0xfffffffc, 0x00020200, 1146 0x5bb0, 0x000000f0, 0x00000070, 1147 0x5bc0, 0xf0311fff, 0x80300000, 1148 0x98f8, 0x73773777, 0x12010001, 1149 0x9b7c, 0x00ff0000, 0x00fc0000, 1150 0x2f48, 0x73773777, 0x12010001, 1151 0x8a14, 0xf000003f, 0x00000007, 1152 0x8b24, 0xffffffff, 0x00ffffff, 1153 0x28350, 0x3f3f3fff, 0x00000082, 1154 0x28354, 0x0000003f, 0x00000000, 1155 0x3e78, 0x00000001, 0x00000002, 1156 0x913c, 0xffff03df, 0x00000004, 1157 0xc768, 0x00000008, 0x00000008, 1158 0x8c00, 0x000008ff, 0x00000800, 1159 0x9508, 0x00010000, 0x00010000, 1160 0xac0c, 0xffffffff, 0x54763210, 1161 0x214f8, 0x01ff01ff, 0x00000002, 1162 0x21498, 0x007ff800, 0x00200000, 1163 0x2015c, 0xffffffff, 0x00000f40, 1164 0x30934, 0xffffffff, 0x00000001 1165 }; 1166 1167 static const u32 spectre_mgcg_cgcg_init[] = 1168 { 1169 0xc420, 0xffffffff, 0xfffffffc, 1170 0x30800, 0xffffffff, 0xe0000000, 1171 0x3c2a0, 0xffffffff, 0x00000100, 1172 0x3c208, 0xffffffff, 0x00000100, 1173 0x3c2c0, 0xffffffff, 0x00000100, 1174 0x3c2c8, 0xffffffff, 0x00000100, 1175 0x3c2c4, 0xffffffff, 0x00000100, 1176 0x55e4, 0xffffffff, 0x00600100, 1177 0x3c280, 0xffffffff, 0x00000100, 1178 0x3c214, 0xffffffff, 0x06000100, 1179 0x3c220, 0xffffffff, 0x00000100, 1180 0x3c218, 0xffffffff, 0x06000100, 1181 0x3c204, 0xffffffff, 0x00000100, 1182 0x3c2e0, 0xffffffff, 0x00000100, 1183 0x3c224, 0xffffffff, 0x00000100, 1184 0x3c200, 0xffffffff, 0x00000100, 1185 0x3c230, 0xffffffff, 0x00000100, 1186 0x3c234, 0xffffffff, 0x00000100, 1187 0x3c250, 0xffffffff, 0x00000100, 1188 0x3c254, 0xffffffff, 0x00000100, 1189 0x3c258, 0xffffffff, 0x00000100, 1190 0x3c25c, 0xffffffff, 0x00000100, 1191 0x3c260, 0xffffffff, 0x00000100, 1192 0x3c27c, 0xffffffff, 0x00000100, 1193 0x3c278, 0xffffffff, 0x00000100, 1194 0x3c210, 0xffffffff, 0x06000100, 1195 0x3c290, 0xffffffff, 0x00000100, 1196 0x3c274, 0xffffffff, 0x00000100, 1197 0x3c2b4, 0xffffffff, 0x00000100, 1198 0x3c2b0, 0xffffffff, 0x00000100, 1199 0x3c270, 0xffffffff, 0x00000100, 1200 0x30800, 0xffffffff, 0xe0000000, 1201 0x3c020, 0xffffffff, 0x00010000, 1202 0x3c024, 0xffffffff, 0x00030002, 1203 0x3c028, 0xffffffff, 0x00040007, 1204 0x3c02c, 0xffffffff, 0x00060005, 1205 0x3c030, 0xffffffff, 0x00090008, 1206 0x3c034, 0xffffffff, 0x00010000, 1207 0x3c038, 0xffffffff, 0x00030002, 1208 0x3c03c, 0xffffffff, 0x00040007, 1209 0x3c040, 0xffffffff, 0x00060005, 1210 0x3c044, 0xffffffff, 0x00090008, 1211 0x3c048, 0xffffffff, 0x00010000, 1212 0x3c04c, 0xffffffff, 0x00030002, 1213 0x3c050, 0xffffffff, 0x00040007, 1214 0x3c054, 0xffffffff, 0x00060005, 1215 0x3c058, 0xffffffff, 0x00090008, 1216 0x3c05c, 0xffffffff, 0x00010000, 1217 0x3c060, 0xffffffff, 0x00030002, 1218 0x3c064, 0xffffffff, 0x00040007, 1219 0x3c068, 0xffffffff, 0x00060005, 1220 0x3c06c, 0xffffffff, 0x00090008, 1221 0x3c070, 0xffffffff, 0x00010000, 1222 0x3c074, 0xffffffff, 0x00030002, 1223 0x3c078, 0xffffffff, 0x00040007, 1224 0x3c07c, 0xffffffff, 0x00060005, 1225 0x3c080, 0xffffffff, 0x00090008, 1226 0x3c084, 0xffffffff, 0x00010000, 1227 0x3c088, 0xffffffff, 0x00030002, 1228 0x3c08c, 0xffffffff, 0x00040007, 1229 0x3c090, 0xffffffff, 0x00060005, 1230 0x3c094, 0xffffffff, 0x00090008, 1231 0x3c098, 0xffffffff, 0x00010000, 1232 0x3c09c, 0xffffffff, 0x00030002, 1233 0x3c0a0, 0xffffffff, 0x00040007, 1234 0x3c0a4, 0xffffffff, 0x00060005, 1235 0x3c0a8, 0xffffffff, 0x00090008, 1236 0x3c0ac, 0xffffffff, 0x00010000, 1237 0x3c0b0, 0xffffffff, 0x00030002, 1238 0x3c0b4, 0xffffffff, 0x00040007, 1239 0x3c0b8, 0xffffffff, 0x00060005, 1240 0x3c0bc, 0xffffffff, 0x00090008, 1241 0x3c000, 0xffffffff, 0x96e00200, 1242 0x8708, 0xffffffff, 0x00900100, 1243 0xc424, 0xffffffff, 0x0020003f, 1244 0x38, 0xffffffff, 0x0140001c, 1245 0x3c, 0x000f0000, 0x000f0000, 1246 0x220, 0xffffffff, 0xC060000C, 1247 0x224, 0xc0000fff, 0x00000100, 1248 0xf90, 0xffffffff, 0x00000100, 1249 0xf98, 0x00000101, 0x00000000, 1250 0x20a8, 0xffffffff, 0x00000104, 1251 0x55e4, 0xff000fff, 0x00000100, 1252 0x30cc, 0xc0000fff, 0x00000104, 1253 0xc1e4, 0x00000001, 0x00000001, 1254 0xd00c, 0xff000ff0, 0x00000100, 1255 0xd80c, 0xff000ff0, 0x00000100 1256 }; 1257 1258 static const u32 kalindi_golden_spm_registers[] = 1259 { 1260 0x30800, 0xe0ffffff, 0xe0000000 1261 }; 1262 1263 static const u32 kalindi_golden_common_registers[] = 1264 { 1265 0xc770, 0xffffffff, 0x00000800, 1266 0xc774, 0xffffffff, 0x00000800, 1267 0xc798, 0xffffffff, 0x00007fbf, 1268 0xc79c, 0xffffffff, 0x00007faf 1269 }; 1270 1271 static const u32 kalindi_golden_registers[] = 1272 { 1273 0x3c000, 0xffffdfff, 0x6e944040, 1274 0x55e4, 0xff607fff, 0xfc000100, 1275 0x3c220, 0xff000fff, 0x00000100, 1276 0x3c224, 0xff000fff, 0x00000100, 1277 0x3c200, 0xfffc0fff, 0x00000100, 1278 0x6ed8, 0x00010101, 0x00010000, 1279 0x9830, 0xffffffff, 0x00000000, 1280 0x9834, 0xf00fffff, 0x00000400, 1281 0x5bb0, 0x000000f0, 0x00000070, 1282 0x5bc0, 0xf0311fff, 0x80300000, 1283 0x98f8, 0x73773777, 0x12010001, 1284 0x98fc, 0xffffffff, 0x00000010, 1285 0x9b7c, 0x00ff0000, 0x00fc0000, 1286 0x8030, 0x00001f0f, 0x0000100a, 1287 0x2f48, 0x73773777, 0x12010001, 1288 0x2408, 0x000fffff, 0x000c007f, 1289 0x8a14, 0xf000003f, 0x00000007, 1290 0x8b24, 0x3fff3fff, 0x00ffcfff, 1291 0x30a04, 0x0000ff0f, 0x00000000, 1292 0x28a4c, 0x07ffffff, 0x06000000, 1293 0x4d8, 0x00000fff, 0x00000100, 1294 0x3e78, 0x00000001, 0x00000002, 1295 0xc768, 0x00000008, 0x00000008, 1296 0x8c00, 0x000000ff, 0x00000003, 1297 0x214f8, 0x01ff01ff, 0x00000002, 1298 0x21498, 0x007ff800, 0x00200000, 1299 0x2015c, 0xffffffff, 0x00000f40, 1300 0x88c4, 0x001f3ae3, 0x00000082, 1301 0x88d4, 0x0000001f, 0x00000010, 1302 0x30934, 0xffffffff, 0x00000000 1303 }; 1304 1305 static const u32 kalindi_mgcg_cgcg_init[] = 1306 { 1307 0xc420, 0xffffffff, 0xfffffffc, 1308 0x30800, 0xffffffff, 0xe0000000, 1309 0x3c2a0, 0xffffffff, 0x00000100, 1310 0x3c208, 0xffffffff, 0x00000100, 1311 0x3c2c0, 0xffffffff, 0x00000100, 1312 0x3c2c8, 0xffffffff, 0x00000100, 1313 0x3c2c4, 0xffffffff, 0x00000100, 1314 0x55e4, 0xffffffff, 0x00600100, 1315 0x3c280, 0xffffffff, 0x00000100, 1316 0x3c214, 0xffffffff, 0x06000100, 1317 0x3c220, 0xffffffff, 0x00000100, 1318 0x3c218, 0xffffffff, 0x06000100, 1319 0x3c204, 0xffffffff, 0x00000100, 1320 0x3c2e0, 0xffffffff, 0x00000100, 1321 0x3c224, 0xffffffff, 0x00000100, 1322 0x3c200, 0xffffffff, 0x00000100, 1323 0x3c230, 0xffffffff, 0x00000100, 1324 0x3c234, 0xffffffff, 0x00000100, 1325 0x3c250, 0xffffffff, 0x00000100, 1326 0x3c254, 0xffffffff, 0x00000100, 1327 0x3c258, 0xffffffff, 0x00000100, 1328 0x3c25c, 0xffffffff, 0x00000100, 1329 0x3c260, 0xffffffff, 0x00000100, 1330 0x3c27c, 0xffffffff, 0x00000100, 1331 0x3c278, 0xffffffff, 0x00000100, 1332 0x3c210, 0xffffffff, 0x06000100, 1333 0x3c290, 0xffffffff, 0x00000100, 1334 0x3c274, 0xffffffff, 0x00000100, 1335 0x3c2b4, 0xffffffff, 0x00000100, 1336 0x3c2b0, 0xffffffff, 0x00000100, 1337 0x3c270, 0xffffffff, 0x00000100, 1338 0x30800, 0xffffffff, 0xe0000000, 1339 0x3c020, 0xffffffff, 0x00010000, 1340 0x3c024, 0xffffffff, 0x00030002, 1341 0x3c028, 0xffffffff, 0x00040007, 1342 0x3c02c, 0xffffffff, 0x00060005, 1343 0x3c030, 0xffffffff, 0x00090008, 1344 0x3c034, 0xffffffff, 0x00010000, 1345 0x3c038, 0xffffffff, 0x00030002, 1346 0x3c03c, 0xffffffff, 0x00040007, 1347 0x3c040, 0xffffffff, 0x00060005, 1348 0x3c044, 0xffffffff, 0x00090008, 1349 0x3c000, 0xffffffff, 0x96e00200, 1350 0x8708, 0xffffffff, 0x00900100, 1351 0xc424, 0xffffffff, 0x0020003f, 1352 0x38, 0xffffffff, 0x0140001c, 1353 0x3c, 0x000f0000, 0x000f0000, 1354 0x220, 0xffffffff, 0xC060000C, 1355 0x224, 0xc0000fff, 0x00000100, 1356 0x20a8, 0xffffffff, 0x00000104, 1357 0x55e4, 0xff000fff, 0x00000100, 1358 0x30cc, 0xc0000fff, 0x00000104, 1359 0xc1e4, 0x00000001, 0x00000001, 1360 0xd00c, 0xff000ff0, 0x00000100, 1361 0xd80c, 0xff000ff0, 0x00000100 1362 }; 1363 1364 static const u32 hawaii_golden_spm_registers[] = 1365 { 1366 0x30800, 0xe0ffffff, 0xe0000000 1367 }; 1368 1369 static const u32 hawaii_golden_common_registers[] = 1370 { 1371 0x30800, 0xffffffff, 0xe0000000, 1372 0x28350, 0xffffffff, 0x3a00161a, 1373 0x28354, 0xffffffff, 0x0000002e, 1374 0x9a10, 0xffffffff, 0x00018208, 1375 0x98f8, 0xffffffff, 0x12011003 1376 }; 1377 1378 static const u32 hawaii_golden_registers[] = 1379 { 1380 0x3354, 0x00000333, 0x00000333, 1381 0x9a10, 0x00010000, 0x00058208, 1382 0x9830, 0xffffffff, 0x00000000, 1383 0x9834, 0xf00fffff, 0x00000400, 1384 0x9838, 0x0002021c, 0x00020200, 1385 0xc78, 0x00000080, 0x00000000, 1386 0x5bb0, 0x000000f0, 0x00000070, 1387 0x5bc0, 0xf0311fff, 0x80300000, 1388 0x350c, 0x00810000, 0x408af000, 1389 0x7030, 0x31000111, 0x00000011, 1390 0x2f48, 0x73773777, 0x12010001, 1391 0x2120, 0x0000007f, 0x0000001b, 1392 0x21dc, 0x00007fb6, 0x00002191, 1393 0x3628, 0x0000003f, 0x0000000a, 1394 0x362c, 0x0000003f, 0x0000000a, 1395 0x2ae4, 0x00073ffe, 0x000022a2, 1396 0x240c, 0x000007ff, 0x00000000, 1397 0x8bf0, 0x00002001, 0x00000001, 1398 0x8b24, 0xffffffff, 0x00ffffff, 1399 0x30a04, 0x0000ff0f, 0x00000000, 1400 0x28a4c, 0x07ffffff, 0x06000000, 1401 0x3e78, 0x00000001, 0x00000002, 1402 0xc768, 0x00000008, 0x00000008, 1403 0xc770, 0x00000f00, 0x00000800, 1404 0xc774, 0x00000f00, 0x00000800, 1405 0xc798, 0x00ffffff, 0x00ff7fbf, 1406 0xc79c, 0x00ffffff, 0x00ff7faf, 1407 0x8c00, 0x000000ff, 0x00000800, 1408 0xe40, 0x00001fff, 0x00001fff, 1409 0x9060, 0x0000007f, 0x00000020, 1410 0x9508, 0x00010000, 0x00010000, 1411 0xae00, 0x00100000, 0x000ff07c, 1412 0xac14, 0x000003ff, 0x0000000f, 1413 0xac10, 0xffffffff, 0x7564fdec, 1414 0xac0c, 0xffffffff, 0x3120b9a8, 1415 0xac08, 0x20000000, 0x0f9c0000 1416 }; 1417 1418 static const u32 hawaii_mgcg_cgcg_init[] = 1419 { 1420 0xc420, 0xffffffff, 0xfffffffd, 1421 0x30800, 0xffffffff, 0xe0000000, 1422 0x3c2a0, 0xffffffff, 0x00000100, 1423 0x3c208, 0xffffffff, 0x00000100, 1424 0x3c2c0, 0xffffffff, 0x00000100, 1425 0x3c2c8, 0xffffffff, 0x00000100, 1426 0x3c2c4, 0xffffffff, 0x00000100, 1427 0x55e4, 0xffffffff, 0x00200100, 1428 0x3c280, 0xffffffff, 0x00000100, 1429 0x3c214, 0xffffffff, 0x06000100, 1430 0x3c220, 0xffffffff, 0x00000100, 1431 0x3c218, 0xffffffff, 0x06000100, 1432 0x3c204, 0xffffffff, 0x00000100, 1433 0x3c2e0, 0xffffffff, 0x00000100, 1434 0x3c224, 0xffffffff, 0x00000100, 1435 0x3c200, 0xffffffff, 0x00000100, 1436 0x3c230, 0xffffffff, 0x00000100, 1437 0x3c234, 0xffffffff, 0x00000100, 1438 0x3c250, 0xffffffff, 0x00000100, 1439 0x3c254, 0xffffffff, 0x00000100, 1440 0x3c258, 0xffffffff, 0x00000100, 1441 0x3c25c, 0xffffffff, 0x00000100, 1442 0x3c260, 0xffffffff, 0x00000100, 1443 0x3c27c, 0xffffffff, 0x00000100, 1444 0x3c278, 0xffffffff, 0x00000100, 1445 0x3c210, 0xffffffff, 0x06000100, 1446 0x3c290, 0xffffffff, 0x00000100, 1447 0x3c274, 0xffffffff, 0x00000100, 1448 0x3c2b4, 0xffffffff, 0x00000100, 1449 0x3c2b0, 0xffffffff, 0x00000100, 1450 0x3c270, 0xffffffff, 0x00000100, 1451 0x30800, 0xffffffff, 0xe0000000, 1452 0x3c020, 0xffffffff, 0x00010000, 1453 0x3c024, 0xffffffff, 0x00030002, 1454 0x3c028, 0xffffffff, 0x00040007, 1455 0x3c02c, 0xffffffff, 0x00060005, 1456 0x3c030, 0xffffffff, 0x00090008, 1457 0x3c034, 0xffffffff, 0x00010000, 1458 0x3c038, 0xffffffff, 0x00030002, 1459 0x3c03c, 0xffffffff, 0x00040007, 1460 0x3c040, 0xffffffff, 0x00060005, 1461 0x3c044, 0xffffffff, 0x00090008, 1462 0x3c048, 0xffffffff, 0x00010000, 1463 0x3c04c, 0xffffffff, 0x00030002, 1464 0x3c050, 0xffffffff, 0x00040007, 1465 0x3c054, 0xffffffff, 0x00060005, 1466 0x3c058, 0xffffffff, 0x00090008, 1467 0x3c05c, 0xffffffff, 0x00010000, 1468 0x3c060, 0xffffffff, 0x00030002, 1469 0x3c064, 0xffffffff, 0x00040007, 1470 0x3c068, 0xffffffff, 0x00060005, 1471 0x3c06c, 0xffffffff, 0x00090008, 1472 0x3c070, 0xffffffff, 0x00010000, 1473 0x3c074, 0xffffffff, 0x00030002, 1474 0x3c078, 0xffffffff, 0x00040007, 1475 0x3c07c, 0xffffffff, 0x00060005, 1476 0x3c080, 0xffffffff, 0x00090008, 1477 0x3c084, 0xffffffff, 0x00010000, 1478 0x3c088, 0xffffffff, 0x00030002, 1479 0x3c08c, 0xffffffff, 0x00040007, 1480 0x3c090, 0xffffffff, 0x00060005, 1481 0x3c094, 0xffffffff, 0x00090008, 1482 0x3c098, 0xffffffff, 0x00010000, 1483 0x3c09c, 0xffffffff, 0x00030002, 1484 0x3c0a0, 0xffffffff, 0x00040007, 1485 0x3c0a4, 0xffffffff, 0x00060005, 1486 0x3c0a8, 0xffffffff, 0x00090008, 1487 0x3c0ac, 0xffffffff, 0x00010000, 1488 0x3c0b0, 0xffffffff, 0x00030002, 1489 0x3c0b4, 0xffffffff, 0x00040007, 1490 0x3c0b8, 0xffffffff, 0x00060005, 1491 0x3c0bc, 0xffffffff, 0x00090008, 1492 0x3c0c0, 0xffffffff, 0x00010000, 1493 0x3c0c4, 0xffffffff, 0x00030002, 1494 0x3c0c8, 0xffffffff, 0x00040007, 1495 0x3c0cc, 0xffffffff, 0x00060005, 1496 0x3c0d0, 0xffffffff, 0x00090008, 1497 0x3c0d4, 0xffffffff, 0x00010000, 1498 0x3c0d8, 0xffffffff, 0x00030002, 1499 0x3c0dc, 0xffffffff, 0x00040007, 1500 0x3c0e0, 0xffffffff, 0x00060005, 1501 0x3c0e4, 0xffffffff, 0x00090008, 1502 0x3c0e8, 0xffffffff, 0x00010000, 1503 0x3c0ec, 0xffffffff, 0x00030002, 1504 0x3c0f0, 0xffffffff, 0x00040007, 1505 0x3c0f4, 0xffffffff, 0x00060005, 1506 0x3c0f8, 0xffffffff, 0x00090008, 1507 0xc318, 0xffffffff, 0x00020200, 1508 0x3350, 0xffffffff, 0x00000200, 1509 0x15c0, 0xffffffff, 0x00000400, 1510 0x55e8, 0xffffffff, 0x00000000, 1511 0x2f50, 0xffffffff, 0x00000902, 1512 0x3c000, 0xffffffff, 0x96940200, 1513 0x8708, 0xffffffff, 0x00900100, 1514 0xc424, 0xffffffff, 0x0020003f, 1515 0x38, 0xffffffff, 0x0140001c, 1516 0x3c, 0x000f0000, 0x000f0000, 1517 0x220, 0xffffffff, 0xc060000c, 1518 0x224, 0xc0000fff, 0x00000100, 1519 0xf90, 0xffffffff, 0x00000100, 1520 0xf98, 0x00000101, 0x00000000, 1521 0x20a8, 0xffffffff, 0x00000104, 1522 0x55e4, 0xff000fff, 0x00000100, 1523 0x30cc, 0xc0000fff, 0x00000104, 1524 0xc1e4, 0x00000001, 0x00000001, 1525 0xd00c, 0xff000ff0, 0x00000100, 1526 0xd80c, 0xff000ff0, 0x00000100 1527 }; 1528 1529 static const u32 godavari_golden_registers[] = 1530 { 1531 0x55e4, 0xff607fff, 0xfc000100, 1532 0x6ed8, 0x00010101, 0x00010000, 1533 0x9830, 0xffffffff, 0x00000000, 1534 0x98302, 0xf00fffff, 0x00000400, 1535 0x6130, 0xffffffff, 0x00010000, 1536 0x5bb0, 0x000000f0, 0x00000070, 1537 0x5bc0, 0xf0311fff, 0x80300000, 1538 0x98f8, 0x73773777, 0x12010001, 1539 0x98fc, 0xffffffff, 0x00000010, 1540 0x8030, 0x00001f0f, 0x0000100a, 1541 0x2f48, 0x73773777, 0x12010001, 1542 0x2408, 0x000fffff, 0x000c007f, 1543 0x8a14, 0xf000003f, 0x00000007, 1544 0x8b24, 0xffffffff, 0x00ff0fff, 1545 0x30a04, 0x0000ff0f, 0x00000000, 1546 0x28a4c, 0x07ffffff, 0x06000000, 1547 0x4d8, 0x00000fff, 0x00000100, 1548 0xd014, 0x00010000, 0x00810001, 1549 0xd814, 0x00010000, 0x00810001, 1550 0x3e78, 0x00000001, 0x00000002, 1551 0xc768, 0x00000008, 0x00000008, 1552 0xc770, 0x00000f00, 0x00000800, 1553 0xc774, 0x00000f00, 0x00000800, 1554 0xc798, 0x00ffffff, 0x00ff7fbf, 1555 0xc79c, 0x00ffffff, 0x00ff7faf, 1556 0x8c00, 0x000000ff, 0x00000001, 1557 0x214f8, 0x01ff01ff, 0x00000002, 1558 0x21498, 0x007ff800, 0x00200000, 1559 0x2015c, 0xffffffff, 0x00000f40, 1560 0x88c4, 0x001f3ae3, 0x00000082, 1561 0x88d4, 0x0000001f, 0x00000010, 1562 0x30934, 0xffffffff, 0x00000000 1563 }; 1564 1565 1566 static void cik_init_golden_registers(struct radeon_device *rdev) 1567 { 1568 /* Some of the registers might be dependent on GRBM_GFX_INDEX */ 1569 mutex_lock(&rdev->grbm_idx_mutex); 1570 switch (rdev->family) { 1571 case CHIP_BONAIRE: 1572 radeon_program_register_sequence(rdev, 1573 bonaire_mgcg_cgcg_init, 1574 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 1575 radeon_program_register_sequence(rdev, 1576 bonaire_golden_registers, 1577 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 1578 radeon_program_register_sequence(rdev, 1579 bonaire_golden_common_registers, 1580 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 1581 radeon_program_register_sequence(rdev, 1582 bonaire_golden_spm_registers, 1583 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 1584 break; 1585 case CHIP_KABINI: 1586 radeon_program_register_sequence(rdev, 1587 kalindi_mgcg_cgcg_init, 1588 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1589 radeon_program_register_sequence(rdev, 1590 kalindi_golden_registers, 1591 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 1592 radeon_program_register_sequence(rdev, 1593 kalindi_golden_common_registers, 1594 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1595 radeon_program_register_sequence(rdev, 1596 kalindi_golden_spm_registers, 1597 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1598 break; 1599 case CHIP_MULLINS: 1600 radeon_program_register_sequence(rdev, 1601 kalindi_mgcg_cgcg_init, 1602 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1603 radeon_program_register_sequence(rdev, 1604 godavari_golden_registers, 1605 (const u32)ARRAY_SIZE(godavari_golden_registers)); 1606 radeon_program_register_sequence(rdev, 1607 kalindi_golden_common_registers, 1608 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1609 radeon_program_register_sequence(rdev, 1610 kalindi_golden_spm_registers, 1611 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1612 break; 1613 case CHIP_KAVERI: 1614 radeon_program_register_sequence(rdev, 1615 spectre_mgcg_cgcg_init, 1616 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 1617 radeon_program_register_sequence(rdev, 1618 spectre_golden_registers, 1619 (const u32)ARRAY_SIZE(spectre_golden_registers)); 1620 radeon_program_register_sequence(rdev, 1621 spectre_golden_common_registers, 1622 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 1623 radeon_program_register_sequence(rdev, 1624 spectre_golden_spm_registers, 1625 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 1626 break; 1627 case CHIP_HAWAII: 1628 radeon_program_register_sequence(rdev, 1629 hawaii_mgcg_cgcg_init, 1630 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 1631 radeon_program_register_sequence(rdev, 1632 hawaii_golden_registers, 1633 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 1634 radeon_program_register_sequence(rdev, 1635 hawaii_golden_common_registers, 1636 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 1637 radeon_program_register_sequence(rdev, 1638 hawaii_golden_spm_registers, 1639 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 1640 break; 1641 default: 1642 break; 1643 } 1644 mutex_unlock(&rdev->grbm_idx_mutex); 1645 } 1646 1647 /** 1648 * cik_get_xclk - get the xclk 1649 * 1650 * @rdev: radeon_device pointer 1651 * 1652 * Returns the reference clock used by the gfx engine 1653 * (CIK). 1654 */ 1655 u32 cik_get_xclk(struct radeon_device *rdev) 1656 { 1657 u32 reference_clock = rdev->clock.spll.reference_freq; 1658 1659 if (rdev->flags & RADEON_IS_IGP) { 1660 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) 1661 return reference_clock / 2; 1662 } else { 1663 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) 1664 return reference_clock / 4; 1665 } 1666 return reference_clock; 1667 } 1668 1669 /** 1670 * cik_mm_rdoorbell - read a doorbell dword 1671 * 1672 * @rdev: radeon_device pointer 1673 * @index: doorbell index 1674 * 1675 * Returns the value in the doorbell aperture at the 1676 * requested doorbell index (CIK). 1677 */ 1678 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) 1679 { 1680 if (index < rdev->doorbell.num_doorbells) { 1681 return readl(rdev->doorbell.ptr + index); 1682 } else { 1683 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 1684 return 0; 1685 } 1686 } 1687 1688 /** 1689 * cik_mm_wdoorbell - write a doorbell dword 1690 * 1691 * @rdev: radeon_device pointer 1692 * @index: doorbell index 1693 * @v: value to write 1694 * 1695 * Writes @v to the doorbell aperture at the 1696 * requested doorbell index (CIK). 1697 */ 1698 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) 1699 { 1700 if (index < rdev->doorbell.num_doorbells) { 1701 writel(v, rdev->doorbell.ptr + index); 1702 } else { 1703 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 1704 } 1705 } 1706 1707 #define BONAIRE_IO_MC_REGS_SIZE 36 1708 1709 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = 1710 { 1711 {0x00000070, 0x04400000}, 1712 {0x00000071, 0x80c01803}, 1713 {0x00000072, 0x00004004}, 1714 {0x00000073, 0x00000100}, 1715 {0x00000074, 0x00ff0000}, 1716 {0x00000075, 0x34000000}, 1717 {0x00000076, 0x08000014}, 1718 {0x00000077, 0x00cc08ec}, 1719 {0x00000078, 0x00000400}, 1720 {0x00000079, 0x00000000}, 1721 {0x0000007a, 0x04090000}, 1722 {0x0000007c, 0x00000000}, 1723 {0x0000007e, 0x4408a8e8}, 1724 {0x0000007f, 0x00000304}, 1725 {0x00000080, 0x00000000}, 1726 {0x00000082, 0x00000001}, 1727 {0x00000083, 0x00000002}, 1728 {0x00000084, 0xf3e4f400}, 1729 {0x00000085, 0x052024e3}, 1730 {0x00000087, 0x00000000}, 1731 {0x00000088, 0x01000000}, 1732 {0x0000008a, 0x1c0a0000}, 1733 {0x0000008b, 0xff010000}, 1734 {0x0000008d, 0xffffefff}, 1735 {0x0000008e, 0xfff3efff}, 1736 {0x0000008f, 0xfff3efbf}, 1737 {0x00000092, 0xf7ffffff}, 1738 {0x00000093, 0xffffff7f}, 1739 {0x00000095, 0x00101101}, 1740 {0x00000096, 0x00000fff}, 1741 {0x00000097, 0x00116fff}, 1742 {0x00000098, 0x60010000}, 1743 {0x00000099, 0x10010000}, 1744 {0x0000009a, 0x00006000}, 1745 {0x0000009b, 0x00001000}, 1746 {0x0000009f, 0x00b48000} 1747 }; 1748 1749 #define HAWAII_IO_MC_REGS_SIZE 22 1750 1751 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = 1752 { 1753 {0x0000007d, 0x40000000}, 1754 {0x0000007e, 0x40180304}, 1755 {0x0000007f, 0x0000ff00}, 1756 {0x00000081, 0x00000000}, 1757 {0x00000083, 0x00000800}, 1758 {0x00000086, 0x00000000}, 1759 {0x00000087, 0x00000100}, 1760 {0x00000088, 0x00020100}, 1761 {0x00000089, 0x00000000}, 1762 {0x0000008b, 0x00040000}, 1763 {0x0000008c, 0x00000100}, 1764 {0x0000008e, 0xff010000}, 1765 {0x00000090, 0xffffefff}, 1766 {0x00000091, 0xfff3efff}, 1767 {0x00000092, 0xfff3efbf}, 1768 {0x00000093, 0xf7ffffff}, 1769 {0x00000094, 0xffffff7f}, 1770 {0x00000095, 0x00000fff}, 1771 {0x00000096, 0x00116fff}, 1772 {0x00000097, 0x60010000}, 1773 {0x00000098, 0x10010000}, 1774 {0x0000009f, 0x00c79000} 1775 }; 1776 1777 1778 /** 1779 * cik_srbm_select - select specific register instances 1780 * 1781 * @rdev: radeon_device pointer 1782 * @me: selected ME (micro engine) 1783 * @pipe: pipe 1784 * @queue: queue 1785 * @vmid: VMID 1786 * 1787 * Switches the currently active registers instances. Some 1788 * registers are instanced per VMID, others are instanced per 1789 * me/pipe/queue combination. 1790 */ 1791 static void cik_srbm_select(struct radeon_device *rdev, 1792 u32 me, u32 pipe, u32 queue, u32 vmid) 1793 { 1794 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | 1795 MEID(me & 0x3) | 1796 VMID(vmid & 0xf) | 1797 QUEUEID(queue & 0x7)); 1798 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); 1799 } 1800 1801 /* ucode loading */ 1802 /** 1803 * ci_mc_load_microcode - load MC ucode into the hw 1804 * 1805 * @rdev: radeon_device pointer 1806 * 1807 * Load the GDDR MC ucode into the hw (CIK). 1808 * Returns 0 on success, error on failure. 1809 */ 1810 int ci_mc_load_microcode(struct radeon_device *rdev) 1811 { 1812 const __be32 *fw_data = NULL; 1813 const __le32 *new_fw_data = NULL; 1814 u32 running, blackout = 0, tmp; 1815 u32 *io_mc_regs = NULL; 1816 const __le32 *new_io_mc_regs = NULL; 1817 int i, regs_size, ucode_size; 1818 1819 if (!rdev->mc_fw) 1820 return -EINVAL; 1821 1822 if (rdev->new_fw) { 1823 const struct mc_firmware_header_v1_0 *hdr = 1824 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; 1825 1826 radeon_ucode_print_mc_hdr(&hdr->header); 1827 1828 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); 1829 new_io_mc_regs = (const __le32 *) 1830 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); 1831 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1832 new_fw_data = (const __le32 *) 1833 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1834 } else { 1835 ucode_size = rdev->mc_fw->size / 4; 1836 1837 switch (rdev->family) { 1838 case CHIP_BONAIRE: 1839 io_mc_regs = (u32 *)&bonaire_io_mc_regs; 1840 regs_size = BONAIRE_IO_MC_REGS_SIZE; 1841 break; 1842 case CHIP_HAWAII: 1843 io_mc_regs = (u32 *)&hawaii_io_mc_regs; 1844 regs_size = HAWAII_IO_MC_REGS_SIZE; 1845 break; 1846 default: 1847 return -EINVAL; 1848 } 1849 fw_data = (const __be32 *)rdev->mc_fw->data; 1850 } 1851 1852 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1853 1854 if (running == 0) { 1855 if (running) { 1856 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1857 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1858 } 1859 1860 /* reset the engine and set to writable */ 1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1863 1864 /* load mc io regs */ 1865 for (i = 0; i < regs_size; i++) { 1866 if (rdev->new_fw) { 1867 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); 1868 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); 1869 } else { 1870 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1871 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1872 } 1873 } 1874 1875 tmp = RREG32(MC_SEQ_MISC0); 1876 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { 1877 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5); 1878 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023); 1879 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9); 1880 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0); 1881 } 1882 1883 /* load the MC ucode */ 1884 for (i = 0; i < ucode_size; i++) { 1885 if (rdev->new_fw) 1886 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); 1887 else 1888 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1889 } 1890 1891 /* put the engine back into the active state */ 1892 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1893 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1894 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1895 1896 /* wait for training to complete */ 1897 for (i = 0; i < rdev->usec_timeout; i++) { 1898 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1899 break; 1900 udelay(1); 1901 } 1902 for (i = 0; i < rdev->usec_timeout; i++) { 1903 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1904 break; 1905 udelay(1); 1906 } 1907 1908 if (running) 1909 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1910 } 1911 1912 return 0; 1913 } 1914 1915 /** 1916 * cik_init_microcode - load ucode images from disk 1917 * 1918 * @rdev: radeon_device pointer 1919 * 1920 * Use the firmware interface to load the ucode images into 1921 * the driver (not loaded into hw). 1922 * Returns 0 on success, error on failure. 1923 */ 1924 static int cik_init_microcode(struct radeon_device *rdev) 1925 { 1926 const char *chip_name; 1927 const char *new_chip_name; 1928 size_t pfp_req_size, me_req_size, ce_req_size, 1929 mec_req_size, rlc_req_size, mc_req_size = 0, 1930 sdma_req_size, smc_req_size = 0, mc2_req_size = 0; 1931 char fw_name[30]; 1932 int new_fw = 0; 1933 int err; 1934 int num_fw; 1935 1936 DRM_DEBUG("\n"); 1937 1938 switch (rdev->family) { 1939 case CHIP_BONAIRE: 1940 chip_name = "BONAIRE"; 1941 new_chip_name = "bonaire"; 1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1943 me_req_size = CIK_ME_UCODE_SIZE * 4; 1944 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1947 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4; 1948 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4; 1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1950 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4); 1951 num_fw = 8; 1952 break; 1953 case CHIP_HAWAII: 1954 chip_name = "HAWAII"; 1955 new_chip_name = "hawaii"; 1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1957 me_req_size = CIK_ME_UCODE_SIZE * 4; 1958 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1960 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1961 mc_req_size = HAWAII_MC_UCODE_SIZE * 4; 1962 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4; 1963 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1964 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4); 1965 num_fw = 8; 1966 break; 1967 case CHIP_KAVERI: 1968 chip_name = "KAVERI"; 1969 new_chip_name = "kaveri"; 1970 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1971 me_req_size = CIK_ME_UCODE_SIZE * 4; 1972 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1973 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1974 rlc_req_size = KV_RLC_UCODE_SIZE * 4; 1975 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1976 num_fw = 7; 1977 break; 1978 case CHIP_KABINI: 1979 chip_name = "KABINI"; 1980 new_chip_name = "kabini"; 1981 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1982 me_req_size = CIK_ME_UCODE_SIZE * 4; 1983 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1984 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1985 rlc_req_size = KB_RLC_UCODE_SIZE * 4; 1986 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1987 num_fw = 6; 1988 break; 1989 case CHIP_MULLINS: 1990 chip_name = "MULLINS"; 1991 new_chip_name = "mullins"; 1992 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1993 me_req_size = CIK_ME_UCODE_SIZE * 4; 1994 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1995 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1996 rlc_req_size = ML_RLC_UCODE_SIZE * 4; 1997 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1998 num_fw = 6; 1999 break; 2000 default: BUG(); 2001 } 2002 2003 DRM_INFO("Loading %s Microcode\n", new_chip_name); 2004 2005 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name); 2006 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2007 if (err) { 2008 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 2009 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2010 if (err) 2011 goto out; 2012 if (rdev->pfp_fw->size != pfp_req_size) { 2013 printk(KERN_ERR 2014 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2015 rdev->pfp_fw->size, fw_name); 2016 err = -EINVAL; 2017 goto out; 2018 } 2019 } else { 2020 err = radeon_ucode_validate(rdev->pfp_fw); 2021 if (err) { 2022 printk(KERN_ERR 2023 "cik_fw: validation failed for firmware \"%s\"\n", 2024 fw_name); 2025 goto out; 2026 } else { 2027 new_fw++; 2028 } 2029 } 2030 2031 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name); 2032 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2033 if (err) { 2034 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 2035 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2036 if (err) 2037 goto out; 2038 if (rdev->me_fw->size != me_req_size) { 2039 printk(KERN_ERR 2040 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2041 rdev->me_fw->size, fw_name); 2042 err = -EINVAL; 2043 } 2044 } else { 2045 err = radeon_ucode_validate(rdev->me_fw); 2046 if (err) { 2047 printk(KERN_ERR 2048 "cik_fw: validation failed for firmware \"%s\"\n", 2049 fw_name); 2050 goto out; 2051 } else { 2052 new_fw++; 2053 } 2054 } 2055 2056 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name); 2057 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2058 if (err) { 2059 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 2060 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2061 if (err) 2062 goto out; 2063 if (rdev->ce_fw->size != ce_req_size) { 2064 printk(KERN_ERR 2065 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2066 rdev->ce_fw->size, fw_name); 2067 err = -EINVAL; 2068 } 2069 } else { 2070 err = radeon_ucode_validate(rdev->ce_fw); 2071 if (err) { 2072 printk(KERN_ERR 2073 "cik_fw: validation failed for firmware \"%s\"\n", 2074 fw_name); 2075 goto out; 2076 } else { 2077 new_fw++; 2078 } 2079 } 2080 2081 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name); 2082 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2083 if (err) { 2084 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name); 2085 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2086 if (err) 2087 goto out; 2088 if (rdev->mec_fw->size != mec_req_size) { 2089 printk(KERN_ERR 2090 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2091 rdev->mec_fw->size, fw_name); 2092 err = -EINVAL; 2093 } 2094 } else { 2095 err = radeon_ucode_validate(rdev->mec_fw); 2096 if (err) { 2097 printk(KERN_ERR 2098 "cik_fw: validation failed for firmware \"%s\"\n", 2099 fw_name); 2100 goto out; 2101 } else { 2102 new_fw++; 2103 } 2104 } 2105 2106 if (rdev->family == CHIP_KAVERI) { 2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name); 2108 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); 2109 if (err) { 2110 goto out; 2111 } else { 2112 err = radeon_ucode_validate(rdev->mec2_fw); 2113 if (err) { 2114 goto out; 2115 } else { 2116 new_fw++; 2117 } 2118 } 2119 } 2120 2121 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name); 2122 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2123 if (err) { 2124 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); 2125 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2126 if (err) 2127 goto out; 2128 if (rdev->rlc_fw->size != rlc_req_size) { 2129 printk(KERN_ERR 2130 "cik_rlc: Bogus length %zu in firmware \"%s\"\n", 2131 rdev->rlc_fw->size, fw_name); 2132 err = -EINVAL; 2133 } 2134 } else { 2135 err = radeon_ucode_validate(rdev->rlc_fw); 2136 if (err) { 2137 printk(KERN_ERR 2138 "cik_fw: validation failed for firmware \"%s\"\n", 2139 fw_name); 2140 goto out; 2141 } else { 2142 new_fw++; 2143 } 2144 } 2145 2146 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name); 2147 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2148 if (err) { 2149 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name); 2150 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2151 if (err) 2152 goto out; 2153 if (rdev->sdma_fw->size != sdma_req_size) { 2154 printk(KERN_ERR 2155 "cik_sdma: Bogus length %zu in firmware \"%s\"\n", 2156 rdev->sdma_fw->size, fw_name); 2157 err = -EINVAL; 2158 } 2159 } else { 2160 err = radeon_ucode_validate(rdev->sdma_fw); 2161 if (err) { 2162 printk(KERN_ERR 2163 "cik_fw: validation failed for firmware \"%s\"\n", 2164 fw_name); 2165 goto out; 2166 } else { 2167 new_fw++; 2168 } 2169 } 2170 2171 /* No SMC, MC ucode on APUs */ 2172 if (!(rdev->flags & RADEON_IS_IGP)) { 2173 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name); 2174 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2175 if (err) { 2176 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name); 2177 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2178 if (err) { 2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 2180 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2181 if (err) 2182 goto out; 2183 } 2184 if ((rdev->mc_fw->size != mc_req_size) && 2185 (rdev->mc_fw->size != mc2_req_size)){ 2186 printk(KERN_ERR 2187 "cik_mc: Bogus length %zu in firmware \"%s\"\n", 2188 rdev->mc_fw->size, fw_name); 2189 err = -EINVAL; 2190 } 2191 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); 2192 } else { 2193 err = radeon_ucode_validate(rdev->mc_fw); 2194 if (err) { 2195 printk(KERN_ERR 2196 "cik_fw: validation failed for firmware \"%s\"\n", 2197 fw_name); 2198 goto out; 2199 } else { 2200 new_fw++; 2201 } 2202 } 2203 2204 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name); 2205 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2206 if (err) { 2207 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 2208 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2209 if (err) { 2210 printk(KERN_ERR 2211 "smc: error loading firmware \"%s\"\n", 2212 fw_name); 2213 release_firmware(rdev->smc_fw); 2214 rdev->smc_fw = NULL; 2215 err = 0; 2216 } else if (rdev->smc_fw->size != smc_req_size) { 2217 printk(KERN_ERR 2218 "cik_smc: Bogus length %zu in firmware \"%s\"\n", 2219 rdev->smc_fw->size, fw_name); 2220 err = -EINVAL; 2221 } 2222 } else { 2223 err = radeon_ucode_validate(rdev->smc_fw); 2224 if (err) { 2225 printk(KERN_ERR 2226 "cik_fw: validation failed for firmware \"%s\"\n", 2227 fw_name); 2228 goto out; 2229 } else { 2230 new_fw++; 2231 } 2232 } 2233 } 2234 2235 if (new_fw == 0) { 2236 rdev->new_fw = false; 2237 } else if (new_fw < num_fw) { 2238 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n"); 2239 err = -EINVAL; 2240 } else { 2241 rdev->new_fw = true; 2242 } 2243 2244 out: 2245 if (err) { 2246 if (err != -EINVAL) 2247 printk(KERN_ERR 2248 "cik_cp: Failed to load firmware \"%s\"\n", 2249 fw_name); 2250 release_firmware(rdev->pfp_fw); 2251 rdev->pfp_fw = NULL; 2252 release_firmware(rdev->me_fw); 2253 rdev->me_fw = NULL; 2254 release_firmware(rdev->ce_fw); 2255 rdev->ce_fw = NULL; 2256 release_firmware(rdev->mec_fw); 2257 rdev->mec_fw = NULL; 2258 release_firmware(rdev->mec2_fw); 2259 rdev->mec2_fw = NULL; 2260 release_firmware(rdev->rlc_fw); 2261 rdev->rlc_fw = NULL; 2262 release_firmware(rdev->sdma_fw); 2263 rdev->sdma_fw = NULL; 2264 release_firmware(rdev->mc_fw); 2265 rdev->mc_fw = NULL; 2266 release_firmware(rdev->smc_fw); 2267 rdev->smc_fw = NULL; 2268 } 2269 return err; 2270 } 2271 2272 /* 2273 * Core functions 2274 */ 2275 /** 2276 * cik_tiling_mode_table_init - init the hw tiling table 2277 * 2278 * @rdev: radeon_device pointer 2279 * 2280 * Starting with SI, the tiling setup is done globally in a 2281 * set of 32 tiling modes. Rather than selecting each set of 2282 * parameters per surface as on older asics, we just select 2283 * which index in the tiling table we want to use, and the 2284 * surface uses those parameters (CIK). 2285 */ 2286 static void cik_tiling_mode_table_init(struct radeon_device *rdev) 2287 { 2288 const u32 num_tile_mode_states = 32; 2289 const u32 num_secondary_tile_mode_states = 16; 2290 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2291 u32 num_pipe_configs; 2292 u32 num_rbs = rdev->config.cik.max_backends_per_se * 2293 rdev->config.cik.max_shader_engines; 2294 2295 switch (rdev->config.cik.mem_row_size_in_kb) { 2296 case 1: 2297 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2298 break; 2299 case 2: 2300 default: 2301 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2302 break; 2303 case 4: 2304 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2305 break; 2306 } 2307 2308 num_pipe_configs = rdev->config.cik.max_tile_pipes; 2309 if (num_pipe_configs > 8) 2310 num_pipe_configs = 16; 2311 2312 if (num_pipe_configs == 16) { 2313 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2314 switch (reg_offset) { 2315 case 0: 2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2317 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2320 break; 2321 case 1: 2322 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2323 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2324 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2325 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2326 break; 2327 case 2: 2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2329 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2332 break; 2333 case 3: 2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2335 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2336 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2338 break; 2339 case 4: 2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2343 TILE_SPLIT(split_equal_to_row_size)); 2344 break; 2345 case 5: 2346 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2347 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2348 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2349 break; 2350 case 6: 2351 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2355 break; 2356 case 7: 2357 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2360 TILE_SPLIT(split_equal_to_row_size)); 2361 break; 2362 case 8: 2363 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2365 break; 2366 case 9: 2367 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2370 break; 2371 case 10: 2372 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2373 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2374 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2376 break; 2377 case 11: 2378 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2381 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2382 break; 2383 case 12: 2384 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2385 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2387 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2388 break; 2389 case 13: 2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2392 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2393 break; 2394 case 14: 2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2396 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2397 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2398 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2399 break; 2400 case 16: 2401 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2402 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2404 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2405 break; 2406 case 17: 2407 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2408 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2410 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2411 break; 2412 case 27: 2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2416 break; 2417 case 28: 2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2419 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2421 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2422 break; 2423 case 29: 2424 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2425 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2428 break; 2429 case 30: 2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2434 break; 2435 default: 2436 gb_tile_moden = 0; 2437 break; 2438 } 2439 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2440 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2441 } 2442 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2443 switch (reg_offset) { 2444 case 0: 2445 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2448 NUM_BANKS(ADDR_SURF_16_BANK)); 2449 break; 2450 case 1: 2451 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2454 NUM_BANKS(ADDR_SURF_16_BANK)); 2455 break; 2456 case 2: 2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2460 NUM_BANKS(ADDR_SURF_16_BANK)); 2461 break; 2462 case 3: 2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2466 NUM_BANKS(ADDR_SURF_16_BANK)); 2467 break; 2468 case 4: 2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2472 NUM_BANKS(ADDR_SURF_8_BANK)); 2473 break; 2474 case 5: 2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2478 NUM_BANKS(ADDR_SURF_4_BANK)); 2479 break; 2480 case 6: 2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2484 NUM_BANKS(ADDR_SURF_2_BANK)); 2485 break; 2486 case 8: 2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2490 NUM_BANKS(ADDR_SURF_16_BANK)); 2491 break; 2492 case 9: 2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2496 NUM_BANKS(ADDR_SURF_16_BANK)); 2497 break; 2498 case 10: 2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2502 NUM_BANKS(ADDR_SURF_16_BANK)); 2503 break; 2504 case 11: 2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2508 NUM_BANKS(ADDR_SURF_8_BANK)); 2509 break; 2510 case 12: 2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2514 NUM_BANKS(ADDR_SURF_4_BANK)); 2515 break; 2516 case 13: 2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2520 NUM_BANKS(ADDR_SURF_2_BANK)); 2521 break; 2522 case 14: 2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2526 NUM_BANKS(ADDR_SURF_2_BANK)); 2527 break; 2528 default: 2529 gb_tile_moden = 0; 2530 break; 2531 } 2532 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2533 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2534 } 2535 } else if (num_pipe_configs == 8) { 2536 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2537 switch (reg_offset) { 2538 case 0: 2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2543 break; 2544 case 1: 2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2549 break; 2550 case 2: 2551 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2552 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2555 break; 2556 case 3: 2557 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2560 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2561 break; 2562 case 4: 2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2566 TILE_SPLIT(split_equal_to_row_size)); 2567 break; 2568 case 5: 2569 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2572 break; 2573 case 6: 2574 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2575 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2577 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2578 break; 2579 case 7: 2580 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2583 TILE_SPLIT(split_equal_to_row_size)); 2584 break; 2585 case 8: 2586 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2587 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2588 break; 2589 case 9: 2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2592 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2593 break; 2594 case 10: 2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2599 break; 2600 case 11: 2601 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2604 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2605 break; 2606 case 12: 2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2611 break; 2612 case 13: 2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2615 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2616 break; 2617 case 14: 2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2619 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2620 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2622 break; 2623 case 16: 2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2625 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2626 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2628 break; 2629 case 17: 2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2631 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2632 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2634 break; 2635 case 27: 2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2637 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2638 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2639 break; 2640 case 28: 2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2642 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2643 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2645 break; 2646 case 29: 2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2648 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2649 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2651 break; 2652 case 30: 2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2654 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2655 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2657 break; 2658 default: 2659 gb_tile_moden = 0; 2660 break; 2661 } 2662 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2664 } 2665 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2666 switch (reg_offset) { 2667 case 0: 2668 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2671 NUM_BANKS(ADDR_SURF_16_BANK)); 2672 break; 2673 case 1: 2674 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2677 NUM_BANKS(ADDR_SURF_16_BANK)); 2678 break; 2679 case 2: 2680 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2683 NUM_BANKS(ADDR_SURF_16_BANK)); 2684 break; 2685 case 3: 2686 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2689 NUM_BANKS(ADDR_SURF_16_BANK)); 2690 break; 2691 case 4: 2692 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2695 NUM_BANKS(ADDR_SURF_8_BANK)); 2696 break; 2697 case 5: 2698 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2701 NUM_BANKS(ADDR_SURF_4_BANK)); 2702 break; 2703 case 6: 2704 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2705 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2706 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2707 NUM_BANKS(ADDR_SURF_2_BANK)); 2708 break; 2709 case 8: 2710 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2713 NUM_BANKS(ADDR_SURF_16_BANK)); 2714 break; 2715 case 9: 2716 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2719 NUM_BANKS(ADDR_SURF_16_BANK)); 2720 break; 2721 case 10: 2722 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2723 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2724 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2725 NUM_BANKS(ADDR_SURF_16_BANK)); 2726 break; 2727 case 11: 2728 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2731 NUM_BANKS(ADDR_SURF_16_BANK)); 2732 break; 2733 case 12: 2734 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2737 NUM_BANKS(ADDR_SURF_8_BANK)); 2738 break; 2739 case 13: 2740 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2741 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2742 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2743 NUM_BANKS(ADDR_SURF_4_BANK)); 2744 break; 2745 case 14: 2746 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2749 NUM_BANKS(ADDR_SURF_2_BANK)); 2750 break; 2751 default: 2752 gb_tile_moden = 0; 2753 break; 2754 } 2755 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2756 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2757 } 2758 } else if (num_pipe_configs == 4) { 2759 if (num_rbs == 4) { 2760 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2761 switch (reg_offset) { 2762 case 0: 2763 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2765 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2766 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2767 break; 2768 case 1: 2769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2771 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2773 break; 2774 case 2: 2775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2777 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2779 break; 2780 case 3: 2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2785 break; 2786 case 4: 2787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2790 TILE_SPLIT(split_equal_to_row_size)); 2791 break; 2792 case 5: 2793 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2794 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2796 break; 2797 case 6: 2798 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2799 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2800 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2802 break; 2803 case 7: 2804 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2806 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2807 TILE_SPLIT(split_equal_to_row_size)); 2808 break; 2809 case 8: 2810 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2811 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2812 break; 2813 case 9: 2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2815 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2817 break; 2818 case 10: 2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2821 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2823 break; 2824 case 11: 2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2827 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2829 break; 2830 case 12: 2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2835 break; 2836 case 13: 2837 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2838 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2839 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2840 break; 2841 case 14: 2842 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2843 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2844 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2845 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2846 break; 2847 case 16: 2848 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2849 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2850 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2852 break; 2853 case 17: 2854 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2855 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2856 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2857 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2858 break; 2859 case 27: 2860 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2861 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2862 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2863 break; 2864 case 28: 2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2866 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2867 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2869 break; 2870 case 29: 2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2872 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2873 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2875 break; 2876 case 30: 2877 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2881 break; 2882 default: 2883 gb_tile_moden = 0; 2884 break; 2885 } 2886 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2887 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2888 } 2889 } else if (num_rbs < 4) { 2890 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2891 switch (reg_offset) { 2892 case 0: 2893 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2895 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2896 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2897 break; 2898 case 1: 2899 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2900 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2901 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2903 break; 2904 case 2: 2905 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2906 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2907 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2909 break; 2910 case 3: 2911 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2912 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2913 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2914 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2915 break; 2916 case 4: 2917 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2920 TILE_SPLIT(split_equal_to_row_size)); 2921 break; 2922 case 5: 2923 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2924 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2926 break; 2927 case 6: 2928 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2929 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2930 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2931 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2932 break; 2933 case 7: 2934 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2935 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2936 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2937 TILE_SPLIT(split_equal_to_row_size)); 2938 break; 2939 case 8: 2940 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2941 PIPE_CONFIG(ADDR_SURF_P4_8x16)); 2942 break; 2943 case 9: 2944 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2945 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2946 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2947 break; 2948 case 10: 2949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2950 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2951 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2953 break; 2954 case 11: 2955 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2956 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2957 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2959 break; 2960 case 12: 2961 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2962 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2964 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2965 break; 2966 case 13: 2967 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2968 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2969 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2970 break; 2971 case 14: 2972 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2973 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2974 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2975 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2976 break; 2977 case 16: 2978 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2979 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2980 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2982 break; 2983 case 17: 2984 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2985 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2986 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2987 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2988 break; 2989 case 27: 2990 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2991 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2992 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2993 break; 2994 case 28: 2995 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2996 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2997 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2998 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2999 break; 3000 case 29: 3001 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3002 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3003 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 3004 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3005 break; 3006 case 30: 3007 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3008 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3009 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 3010 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3011 break; 3012 default: 3013 gb_tile_moden = 0; 3014 break; 3015 } 3016 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3018 } 3019 } 3020 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 3021 switch (reg_offset) { 3022 case 0: 3023 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3026 NUM_BANKS(ADDR_SURF_16_BANK)); 3027 break; 3028 case 1: 3029 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3030 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3031 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3032 NUM_BANKS(ADDR_SURF_16_BANK)); 3033 break; 3034 case 2: 3035 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3038 NUM_BANKS(ADDR_SURF_16_BANK)); 3039 break; 3040 case 3: 3041 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3042 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3043 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3044 NUM_BANKS(ADDR_SURF_16_BANK)); 3045 break; 3046 case 4: 3047 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3048 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3049 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3050 NUM_BANKS(ADDR_SURF_16_BANK)); 3051 break; 3052 case 5: 3053 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3054 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3055 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3056 NUM_BANKS(ADDR_SURF_8_BANK)); 3057 break; 3058 case 6: 3059 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3060 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3061 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3062 NUM_BANKS(ADDR_SURF_4_BANK)); 3063 break; 3064 case 8: 3065 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3066 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3067 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3068 NUM_BANKS(ADDR_SURF_16_BANK)); 3069 break; 3070 case 9: 3071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3074 NUM_BANKS(ADDR_SURF_16_BANK)); 3075 break; 3076 case 10: 3077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3080 NUM_BANKS(ADDR_SURF_16_BANK)); 3081 break; 3082 case 11: 3083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3086 NUM_BANKS(ADDR_SURF_16_BANK)); 3087 break; 3088 case 12: 3089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3092 NUM_BANKS(ADDR_SURF_16_BANK)); 3093 break; 3094 case 13: 3095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3098 NUM_BANKS(ADDR_SURF_8_BANK)); 3099 break; 3100 case 14: 3101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3104 NUM_BANKS(ADDR_SURF_4_BANK)); 3105 break; 3106 default: 3107 gb_tile_moden = 0; 3108 break; 3109 } 3110 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3111 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3112 } 3113 } else if (num_pipe_configs == 2) { 3114 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 3115 switch (reg_offset) { 3116 case 0: 3117 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3119 PIPE_CONFIG(ADDR_SURF_P2) | 3120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 3121 break; 3122 case 1: 3123 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3125 PIPE_CONFIG(ADDR_SURF_P2) | 3126 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 3127 break; 3128 case 2: 3129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3130 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3131 PIPE_CONFIG(ADDR_SURF_P2) | 3132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3133 break; 3134 case 3: 3135 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3137 PIPE_CONFIG(ADDR_SURF_P2) | 3138 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 3139 break; 3140 case 4: 3141 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3142 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3143 PIPE_CONFIG(ADDR_SURF_P2) | 3144 TILE_SPLIT(split_equal_to_row_size)); 3145 break; 3146 case 5: 3147 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3148 PIPE_CONFIG(ADDR_SURF_P2) | 3149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3150 break; 3151 case 6: 3152 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3153 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3154 PIPE_CONFIG(ADDR_SURF_P2) | 3155 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3156 break; 3157 case 7: 3158 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3159 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3160 PIPE_CONFIG(ADDR_SURF_P2) | 3161 TILE_SPLIT(split_equal_to_row_size)); 3162 break; 3163 case 8: 3164 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3165 PIPE_CONFIG(ADDR_SURF_P2); 3166 break; 3167 case 9: 3168 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3170 PIPE_CONFIG(ADDR_SURF_P2)); 3171 break; 3172 case 10: 3173 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3174 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3175 PIPE_CONFIG(ADDR_SURF_P2) | 3176 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3177 break; 3178 case 11: 3179 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3180 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3181 PIPE_CONFIG(ADDR_SURF_P2) | 3182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3183 break; 3184 case 12: 3185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3186 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3187 PIPE_CONFIG(ADDR_SURF_P2) | 3188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3189 break; 3190 case 13: 3191 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3192 PIPE_CONFIG(ADDR_SURF_P2) | 3193 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 3194 break; 3195 case 14: 3196 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3198 PIPE_CONFIG(ADDR_SURF_P2) | 3199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3200 break; 3201 case 16: 3202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3204 PIPE_CONFIG(ADDR_SURF_P2) | 3205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3206 break; 3207 case 17: 3208 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3209 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3210 PIPE_CONFIG(ADDR_SURF_P2) | 3211 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3212 break; 3213 case 27: 3214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3215 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3216 PIPE_CONFIG(ADDR_SURF_P2)); 3217 break; 3218 case 28: 3219 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3220 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3221 PIPE_CONFIG(ADDR_SURF_P2) | 3222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3223 break; 3224 case 29: 3225 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3226 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3227 PIPE_CONFIG(ADDR_SURF_P2) | 3228 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3229 break; 3230 case 30: 3231 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3232 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3233 PIPE_CONFIG(ADDR_SURF_P2) | 3234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3235 break; 3236 default: 3237 gb_tile_moden = 0; 3238 break; 3239 } 3240 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 3241 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3242 } 3243 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 3244 switch (reg_offset) { 3245 case 0: 3246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3249 NUM_BANKS(ADDR_SURF_16_BANK)); 3250 break; 3251 case 1: 3252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3255 NUM_BANKS(ADDR_SURF_16_BANK)); 3256 break; 3257 case 2: 3258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3261 NUM_BANKS(ADDR_SURF_16_BANK)); 3262 break; 3263 case 3: 3264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3267 NUM_BANKS(ADDR_SURF_16_BANK)); 3268 break; 3269 case 4: 3270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3273 NUM_BANKS(ADDR_SURF_16_BANK)); 3274 break; 3275 case 5: 3276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3279 NUM_BANKS(ADDR_SURF_16_BANK)); 3280 break; 3281 case 6: 3282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3285 NUM_BANKS(ADDR_SURF_8_BANK)); 3286 break; 3287 case 8: 3288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3291 NUM_BANKS(ADDR_SURF_16_BANK)); 3292 break; 3293 case 9: 3294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3297 NUM_BANKS(ADDR_SURF_16_BANK)); 3298 break; 3299 case 10: 3300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3303 NUM_BANKS(ADDR_SURF_16_BANK)); 3304 break; 3305 case 11: 3306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3309 NUM_BANKS(ADDR_SURF_16_BANK)); 3310 break; 3311 case 12: 3312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3315 NUM_BANKS(ADDR_SURF_16_BANK)); 3316 break; 3317 case 13: 3318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3321 NUM_BANKS(ADDR_SURF_16_BANK)); 3322 break; 3323 case 14: 3324 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3327 NUM_BANKS(ADDR_SURF_8_BANK)); 3328 break; 3329 default: 3330 gb_tile_moden = 0; 3331 break; 3332 } 3333 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3334 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3335 } 3336 } else 3337 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); 3338 } 3339 3340 /** 3341 * cik_select_se_sh - select which SE, SH to address 3342 * 3343 * @rdev: radeon_device pointer 3344 * @se_num: shader engine to address 3345 * @sh_num: sh block to address 3346 * 3347 * Select which SE, SH combinations to address. Certain 3348 * registers are instanced per SE or SH. 0xffffffff means 3349 * broadcast to all SEs or SHs (CIK). 3350 */ 3351 static void cik_select_se_sh(struct radeon_device *rdev, 3352 u32 se_num, u32 sh_num) 3353 { 3354 u32 data = INSTANCE_BROADCAST_WRITES; 3355 3356 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 3357 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 3358 else if (se_num == 0xffffffff) 3359 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 3360 else if (sh_num == 0xffffffff) 3361 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 3362 else 3363 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 3364 WREG32(GRBM_GFX_INDEX, data); 3365 } 3366 3367 /** 3368 * cik_create_bitmask - create a bitmask 3369 * 3370 * @bit_width: length of the mask 3371 * 3372 * create a variable length bit mask (CIK). 3373 * Returns the bitmask. 3374 */ 3375 static u32 cik_create_bitmask(u32 bit_width) 3376 { 3377 u32 i, mask = 0; 3378 3379 for (i = 0; i < bit_width; i++) { 3380 mask <<= 1; 3381 mask |= 1; 3382 } 3383 return mask; 3384 } 3385 3386 /** 3387 * cik_get_rb_disabled - computes the mask of disabled RBs 3388 * 3389 * @rdev: radeon_device pointer 3390 * @max_rb_num: max RBs (render backends) for the asic 3391 * @se_num: number of SEs (shader engines) for the asic 3392 * @sh_per_se: number of SH blocks per SE for the asic 3393 * 3394 * Calculates the bitmask of disabled RBs (CIK). 3395 * Returns the disabled RB bitmask. 3396 */ 3397 static u32 cik_get_rb_disabled(struct radeon_device *rdev, 3398 u32 max_rb_num_per_se, 3399 u32 sh_per_se) 3400 { 3401 u32 data, mask; 3402 3403 data = RREG32(CC_RB_BACKEND_DISABLE); 3404 if (data & 1) 3405 data &= BACKEND_DISABLE_MASK; 3406 else 3407 data = 0; 3408 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 3409 3410 data >>= BACKEND_DISABLE_SHIFT; 3411 3412 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); 3413 3414 return data & mask; 3415 } 3416 3417 /** 3418 * cik_setup_rb - setup the RBs on the asic 3419 * 3420 * @rdev: radeon_device pointer 3421 * @se_num: number of SEs (shader engines) for the asic 3422 * @sh_per_se: number of SH blocks per SE for the asic 3423 * @max_rb_num: max RBs (render backends) for the asic 3424 * 3425 * Configures per-SE/SH RB registers (CIK). 3426 */ 3427 static void cik_setup_rb(struct radeon_device *rdev, 3428 u32 se_num, u32 sh_per_se, 3429 u32 max_rb_num_per_se) 3430 { 3431 int i, j; 3432 u32 data, mask; 3433 u32 disabled_rbs = 0; 3434 u32 enabled_rbs = 0; 3435 3436 mutex_lock(&rdev->grbm_idx_mutex); 3437 for (i = 0; i < se_num; i++) { 3438 for (j = 0; j < sh_per_se; j++) { 3439 cik_select_se_sh(rdev, i, j); 3440 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3441 if (rdev->family == CHIP_HAWAII) 3442 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); 3443 else 3444 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); 3445 } 3446 } 3447 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3448 mutex_unlock(&rdev->grbm_idx_mutex); 3449 3450 mask = 1; 3451 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3452 if (!(disabled_rbs & mask)) 3453 enabled_rbs |= mask; 3454 mask <<= 1; 3455 } 3456 3457 rdev->config.cik.backend_enable_mask = enabled_rbs; 3458 3459 mutex_lock(&rdev->grbm_idx_mutex); 3460 for (i = 0; i < se_num; i++) { 3461 cik_select_se_sh(rdev, i, 0xffffffff); 3462 data = 0; 3463 for (j = 0; j < sh_per_se; j++) { 3464 switch (enabled_rbs & 3) { 3465 case 0: 3466 if (j == 0) 3467 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); 3468 else 3469 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); 3470 break; 3471 case 1: 3472 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3473 break; 3474 case 2: 3475 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3476 break; 3477 case 3: 3478 default: 3479 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3480 break; 3481 } 3482 enabled_rbs >>= 2; 3483 } 3484 WREG32(PA_SC_RASTER_CONFIG, data); 3485 } 3486 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3487 mutex_unlock(&rdev->grbm_idx_mutex); 3488 } 3489 3490 /** 3491 * cik_gpu_init - setup the 3D engine 3492 * 3493 * @rdev: radeon_device pointer 3494 * 3495 * Configures the 3D engine and tiling configuration 3496 * registers so that the 3D engine is usable. 3497 */ 3498 static void cik_gpu_init(struct radeon_device *rdev) 3499 { 3500 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); 3501 u32 mc_shared_chmap, mc_arb_ramcfg; 3502 u32 hdp_host_path_cntl; 3503 u32 tmp; 3504 int i, j; 3505 3506 switch (rdev->family) { 3507 case CHIP_BONAIRE: 3508 rdev->config.cik.max_shader_engines = 2; 3509 rdev->config.cik.max_tile_pipes = 4; 3510 rdev->config.cik.max_cu_per_sh = 7; 3511 rdev->config.cik.max_sh_per_se = 1; 3512 rdev->config.cik.max_backends_per_se = 2; 3513 rdev->config.cik.max_texture_channel_caches = 4; 3514 rdev->config.cik.max_gprs = 256; 3515 rdev->config.cik.max_gs_threads = 32; 3516 rdev->config.cik.max_hw_contexts = 8; 3517 3518 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3519 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3520 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3521 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3522 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3523 break; 3524 case CHIP_HAWAII: 3525 rdev->config.cik.max_shader_engines = 4; 3526 rdev->config.cik.max_tile_pipes = 16; 3527 rdev->config.cik.max_cu_per_sh = 11; 3528 rdev->config.cik.max_sh_per_se = 1; 3529 rdev->config.cik.max_backends_per_se = 4; 3530 rdev->config.cik.max_texture_channel_caches = 16; 3531 rdev->config.cik.max_gprs = 256; 3532 rdev->config.cik.max_gs_threads = 32; 3533 rdev->config.cik.max_hw_contexts = 8; 3534 3535 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3536 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3537 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3538 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3539 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; 3540 break; 3541 case CHIP_KAVERI: 3542 rdev->config.cik.max_shader_engines = 1; 3543 rdev->config.cik.max_tile_pipes = 4; 3544 if ((rdev->pdev->device == 0x1304) || 3545 (rdev->pdev->device == 0x1305) || 3546 (rdev->pdev->device == 0x130C) || 3547 (rdev->pdev->device == 0x130F) || 3548 (rdev->pdev->device == 0x1310) || 3549 (rdev->pdev->device == 0x1311) || 3550 (rdev->pdev->device == 0x131C)) { 3551 rdev->config.cik.max_cu_per_sh = 8; 3552 rdev->config.cik.max_backends_per_se = 2; 3553 } else if ((rdev->pdev->device == 0x1309) || 3554 (rdev->pdev->device == 0x130A) || 3555 (rdev->pdev->device == 0x130D) || 3556 (rdev->pdev->device == 0x1313) || 3557 (rdev->pdev->device == 0x131D)) { 3558 rdev->config.cik.max_cu_per_sh = 6; 3559 rdev->config.cik.max_backends_per_se = 2; 3560 } else if ((rdev->pdev->device == 0x1306) || 3561 (rdev->pdev->device == 0x1307) || 3562 (rdev->pdev->device == 0x130B) || 3563 (rdev->pdev->device == 0x130E) || 3564 (rdev->pdev->device == 0x1315) || 3565 (rdev->pdev->device == 0x1318) || 3566 (rdev->pdev->device == 0x131B)) { 3567 rdev->config.cik.max_cu_per_sh = 4; 3568 rdev->config.cik.max_backends_per_se = 1; 3569 } else { 3570 rdev->config.cik.max_cu_per_sh = 3; 3571 rdev->config.cik.max_backends_per_se = 1; 3572 } 3573 rdev->config.cik.max_sh_per_se = 1; 3574 rdev->config.cik.max_texture_channel_caches = 4; 3575 rdev->config.cik.max_gprs = 256; 3576 rdev->config.cik.max_gs_threads = 16; 3577 rdev->config.cik.max_hw_contexts = 8; 3578 3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3584 break; 3585 case CHIP_KABINI: 3586 case CHIP_MULLINS: 3587 default: 3588 rdev->config.cik.max_shader_engines = 1; 3589 rdev->config.cik.max_tile_pipes = 2; 3590 rdev->config.cik.max_cu_per_sh = 2; 3591 rdev->config.cik.max_sh_per_se = 1; 3592 rdev->config.cik.max_backends_per_se = 1; 3593 rdev->config.cik.max_texture_channel_caches = 2; 3594 rdev->config.cik.max_gprs = 256; 3595 rdev->config.cik.max_gs_threads = 16; 3596 rdev->config.cik.max_hw_contexts = 8; 3597 3598 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3599 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3600 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3601 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3602 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3603 break; 3604 } 3605 3606 /* Initialize HDP */ 3607 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3608 WREG32((0x2c14 + j), 0x00000000); 3609 WREG32((0x2c18 + j), 0x00000000); 3610 WREG32((0x2c1c + j), 0x00000000); 3611 WREG32((0x2c20 + j), 0x00000000); 3612 WREG32((0x2c24 + j), 0x00000000); 3613 } 3614 3615 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3616 WREG32(SRBM_INT_CNTL, 0x1); 3617 WREG32(SRBM_INT_ACK, 0x1); 3618 3619 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3620 3621 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3622 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3623 3624 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; 3625 rdev->config.cik.mem_max_burst_length_bytes = 256; 3626 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3627 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3628 if (rdev->config.cik.mem_row_size_in_kb > 4) 3629 rdev->config.cik.mem_row_size_in_kb = 4; 3630 /* XXX use MC settings? */ 3631 rdev->config.cik.shader_engine_tile_size = 32; 3632 rdev->config.cik.num_gpus = 1; 3633 rdev->config.cik.multi_gpu_tile_size = 64; 3634 3635 /* fix up row size */ 3636 gb_addr_config &= ~ROW_SIZE_MASK; 3637 switch (rdev->config.cik.mem_row_size_in_kb) { 3638 case 1: 3639 default: 3640 gb_addr_config |= ROW_SIZE(0); 3641 break; 3642 case 2: 3643 gb_addr_config |= ROW_SIZE(1); 3644 break; 3645 case 4: 3646 gb_addr_config |= ROW_SIZE(2); 3647 break; 3648 } 3649 3650 /* setup tiling info dword. gb_addr_config is not adequate since it does 3651 * not have bank info, so create a custom tiling dword. 3652 * bits 3:0 num_pipes 3653 * bits 7:4 num_banks 3654 * bits 11:8 group_size 3655 * bits 15:12 row_size 3656 */ 3657 rdev->config.cik.tile_config = 0; 3658 switch (rdev->config.cik.num_tile_pipes) { 3659 case 1: 3660 rdev->config.cik.tile_config |= (0 << 0); 3661 break; 3662 case 2: 3663 rdev->config.cik.tile_config |= (1 << 0); 3664 break; 3665 case 4: 3666 rdev->config.cik.tile_config |= (2 << 0); 3667 break; 3668 case 8: 3669 default: 3670 /* XXX what about 12? */ 3671 rdev->config.cik.tile_config |= (3 << 0); 3672 break; 3673 } 3674 rdev->config.cik.tile_config |= 3675 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 3676 rdev->config.cik.tile_config |= 3677 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3678 rdev->config.cik.tile_config |= 3679 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3680 3681 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3682 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3683 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3684 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); 3685 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); 3686 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3687 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3688 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3689 3690 cik_tiling_mode_table_init(rdev); 3691 3692 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, 3693 rdev->config.cik.max_sh_per_se, 3694 rdev->config.cik.max_backends_per_se); 3695 3696 rdev->config.cik.active_cus = 0; 3697 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 3698 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 3699 rdev->config.cik.active_cus += 3700 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); 3701 } 3702 } 3703 3704 /* set HW defaults for 3D engine */ 3705 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3706 3707 mutex_lock(&rdev->grbm_idx_mutex); 3708 /* 3709 * making sure that the following register writes will be broadcasted 3710 * to all the shaders 3711 */ 3712 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3713 WREG32(SX_DEBUG_1, 0x20); 3714 3715 WREG32(TA_CNTL_AUX, 0x00010000); 3716 3717 tmp = RREG32(SPI_CONFIG_CNTL); 3718 tmp |= 0x03000000; 3719 WREG32(SPI_CONFIG_CNTL, tmp); 3720 3721 WREG32(SQ_CONFIG, 1); 3722 3723 WREG32(DB_DEBUG, 0); 3724 3725 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; 3726 tmp |= 0x00000400; 3727 WREG32(DB_DEBUG2, tmp); 3728 3729 tmp = RREG32(DB_DEBUG3) & ~0x0002021c; 3730 tmp |= 0x00020200; 3731 WREG32(DB_DEBUG3, tmp); 3732 3733 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; 3734 tmp |= 0x00018208; 3735 WREG32(CB_HW_CONTROL, tmp); 3736 3737 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3738 3739 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | 3740 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | 3741 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | 3742 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); 3743 3744 WREG32(VGT_NUM_INSTANCES, 1); 3745 3746 WREG32(CP_PERFMON_CNTL, 0); 3747 3748 WREG32(SQ_CONFIG, 0); 3749 3750 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3751 FORCE_EOV_MAX_REZ_CNT(255))); 3752 3753 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3754 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3755 3756 WREG32(VGT_GS_VERTEX_REUSE, 16); 3757 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3758 3759 tmp = RREG32(HDP_MISC_CNTL); 3760 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3761 WREG32(HDP_MISC_CNTL, tmp); 3762 3763 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3764 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3765 3766 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3767 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); 3768 mutex_unlock(&rdev->grbm_idx_mutex); 3769 3770 udelay(50); 3771 } 3772 3773 /* 3774 * GPU scratch registers helpers function. 3775 */ 3776 /** 3777 * cik_scratch_init - setup driver info for CP scratch regs 3778 * 3779 * @rdev: radeon_device pointer 3780 * 3781 * Set up the number and offset of the CP scratch registers. 3782 * NOTE: use of CP scratch registers is a legacy inferface and 3783 * is not used by default on newer asics (r6xx+). On newer asics, 3784 * memory buffers are used for fences rather than scratch regs. 3785 */ 3786 static void cik_scratch_init(struct radeon_device *rdev) 3787 { 3788 int i; 3789 3790 rdev->scratch.num_reg = 7; 3791 rdev->scratch.reg_base = SCRATCH_REG0; 3792 for (i = 0; i < rdev->scratch.num_reg; i++) { 3793 rdev->scratch.free[i] = true; 3794 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3795 } 3796 } 3797 3798 /** 3799 * cik_ring_test - basic gfx ring test 3800 * 3801 * @rdev: radeon_device pointer 3802 * @ring: radeon_ring structure holding ring information 3803 * 3804 * Allocate a scratch register and write to it using the gfx ring (CIK). 3805 * Provides a basic gfx ring test to verify that the ring is working. 3806 * Used by cik_cp_gfx_resume(); 3807 * Returns 0 on success, error on failure. 3808 */ 3809 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) 3810 { 3811 uint32_t scratch; 3812 uint32_t tmp = 0; 3813 unsigned i; 3814 int r; 3815 3816 r = radeon_scratch_get(rdev, &scratch); 3817 if (r) { 3818 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); 3819 return r; 3820 } 3821 WREG32(scratch, 0xCAFEDEAD); 3822 r = radeon_ring_lock(rdev, ring, 3); 3823 if (r) { 3824 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); 3825 radeon_scratch_free(rdev, scratch); 3826 return r; 3827 } 3828 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3829 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); 3830 radeon_ring_write(ring, 0xDEADBEEF); 3831 radeon_ring_unlock_commit(rdev, ring, false); 3832 3833 for (i = 0; i < rdev->usec_timeout; i++) { 3834 tmp = RREG32(scratch); 3835 if (tmp == 0xDEADBEEF) 3836 break; 3837 DRM_UDELAY(1); 3838 } 3839 if (i < rdev->usec_timeout) { 3840 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 3841 } else { 3842 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 3843 ring->idx, scratch, tmp); 3844 r = -EINVAL; 3845 } 3846 radeon_scratch_free(rdev, scratch); 3847 return r; 3848 } 3849 3850 /** 3851 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp 3852 * 3853 * @rdev: radeon_device pointer 3854 * @ridx: radeon ring index 3855 * 3856 * Emits an hdp flush on the cp. 3857 */ 3858 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, 3859 int ridx) 3860 { 3861 struct radeon_ring *ring = &rdev->ring[ridx]; 3862 u32 ref_and_mask; 3863 3864 switch (ring->idx) { 3865 case CAYMAN_RING_TYPE_CP1_INDEX: 3866 case CAYMAN_RING_TYPE_CP2_INDEX: 3867 default: 3868 switch (ring->me) { 3869 case 0: 3870 ref_and_mask = CP2 << ring->pipe; 3871 break; 3872 case 1: 3873 ref_and_mask = CP6 << ring->pipe; 3874 break; 3875 default: 3876 return; 3877 } 3878 break; 3879 case RADEON_RING_TYPE_GFX_INDEX: 3880 ref_and_mask = CP0; 3881 break; 3882 } 3883 3884 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 3885 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 3886 WAIT_REG_MEM_FUNCTION(3) | /* == */ 3887 WAIT_REG_MEM_ENGINE(1))); /* pfp */ 3888 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); 3889 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); 3890 radeon_ring_write(ring, ref_and_mask); 3891 radeon_ring_write(ring, ref_and_mask); 3892 radeon_ring_write(ring, 0x20); /* poll interval */ 3893 } 3894 3895 /** 3896 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring 3897 * 3898 * @rdev: radeon_device pointer 3899 * @fence: radeon fence object 3900 * 3901 * Emits a fence sequnce number on the gfx ring and flushes 3902 * GPU caches. 3903 */ 3904 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, 3905 struct radeon_fence *fence) 3906 { 3907 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3908 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3909 3910 /* Workaround for cache flush problems. First send a dummy EOP 3911 * event down the pipe with seq one below. 3912 */ 3913 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3914 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3915 EOP_TC_ACTION_EN | 3916 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3917 EVENT_INDEX(5))); 3918 radeon_ring_write(ring, addr & 0xfffffffc); 3919 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 3920 DATA_SEL(1) | INT_SEL(0)); 3921 radeon_ring_write(ring, fence->seq - 1); 3922 radeon_ring_write(ring, 0); 3923 3924 /* Then send the real EOP event down the pipe. */ 3925 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3926 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3927 EOP_TC_ACTION_EN | 3928 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3929 EVENT_INDEX(5))); 3930 radeon_ring_write(ring, addr & 0xfffffffc); 3931 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); 3932 radeon_ring_write(ring, fence->seq); 3933 radeon_ring_write(ring, 0); 3934 } 3935 3936 /** 3937 * cik_fence_compute_ring_emit - emit a fence on the compute ring 3938 * 3939 * @rdev: radeon_device pointer 3940 * @fence: radeon fence object 3941 * 3942 * Emits a fence sequnce number on the compute ring and flushes 3943 * GPU caches. 3944 */ 3945 void cik_fence_compute_ring_emit(struct radeon_device *rdev, 3946 struct radeon_fence *fence) 3947 { 3948 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3949 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3950 3951 /* RELEASE_MEM - flush caches, send int */ 3952 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 3953 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3954 EOP_TC_ACTION_EN | 3955 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3956 EVENT_INDEX(5))); 3957 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); 3958 radeon_ring_write(ring, addr & 0xfffffffc); 3959 radeon_ring_write(ring, upper_32_bits(addr)); 3960 radeon_ring_write(ring, fence->seq); 3961 radeon_ring_write(ring, 0); 3962 } 3963 3964 /** 3965 * cik_semaphore_ring_emit - emit a semaphore on the CP ring 3966 * 3967 * @rdev: radeon_device pointer 3968 * @ring: radeon ring buffer object 3969 * @semaphore: radeon semaphore object 3970 * @emit_wait: Is this a sempahore wait? 3971 * 3972 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP 3973 * from running ahead of semaphore waits. 3974 */ 3975 bool cik_semaphore_ring_emit(struct radeon_device *rdev, 3976 struct radeon_ring *ring, 3977 struct radeon_semaphore *semaphore, 3978 bool emit_wait) 3979 { 3980 uint64_t addr = semaphore->gpu_addr; 3981 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; 3982 3983 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); 3984 radeon_ring_write(ring, lower_32_bits(addr)); 3985 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); 3986 3987 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) { 3988 /* Prevent the PFP from running ahead of the semaphore wait */ 3989 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3990 radeon_ring_write(ring, 0x0); 3991 } 3992 3993 return true; 3994 } 3995 3996 /** 3997 * cik_copy_cpdma - copy pages using the CP DMA engine 3998 * 3999 * @rdev: radeon_device pointer 4000 * @src_offset: src GPU address 4001 * @dst_offset: dst GPU address 4002 * @num_gpu_pages: number of GPU pages to xfer 4003 * @resv: reservation object to sync to 4004 * 4005 * Copy GPU paging using the CP DMA engine (CIK+). 4006 * Used by the radeon ttm implementation to move pages if 4007 * registered as the asic copy callback. 4008 */ 4009 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, 4010 uint64_t src_offset, uint64_t dst_offset, 4011 unsigned num_gpu_pages, 4012 struct reservation_object *resv) 4013 { 4014 struct radeon_fence *fence; 4015 struct radeon_sync sync; 4016 int ring_index = rdev->asic->copy.blit_ring_index; 4017 struct radeon_ring *ring = &rdev->ring[ring_index]; 4018 u32 size_in_bytes, cur_size_in_bytes, control; 4019 int i, num_loops; 4020 int r = 0; 4021 4022 radeon_sync_create(&sync); 4023 4024 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 4025 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); 4026 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); 4027 if (r) { 4028 DRM_ERROR("radeon: moving bo (%d).\n", r); 4029 radeon_sync_free(rdev, &sync, NULL); 4030 return ERR_PTR(r); 4031 } 4032 4033 radeon_sync_resv(rdev, &sync, resv, false); 4034 radeon_sync_rings(rdev, &sync, ring->idx); 4035 4036 for (i = 0; i < num_loops; i++) { 4037 cur_size_in_bytes = size_in_bytes; 4038 if (cur_size_in_bytes > 0x1fffff) 4039 cur_size_in_bytes = 0x1fffff; 4040 size_in_bytes -= cur_size_in_bytes; 4041 control = 0; 4042 if (size_in_bytes == 0) 4043 control |= PACKET3_DMA_DATA_CP_SYNC; 4044 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4045 radeon_ring_write(ring, control); 4046 radeon_ring_write(ring, lower_32_bits(src_offset)); 4047 radeon_ring_write(ring, upper_32_bits(src_offset)); 4048 radeon_ring_write(ring, lower_32_bits(dst_offset)); 4049 radeon_ring_write(ring, upper_32_bits(dst_offset)); 4050 radeon_ring_write(ring, cur_size_in_bytes); 4051 src_offset += cur_size_in_bytes; 4052 dst_offset += cur_size_in_bytes; 4053 } 4054 4055 r = radeon_fence_emit(rdev, &fence, ring->idx); 4056 if (r) { 4057 radeon_ring_unlock_undo(rdev, ring); 4058 radeon_sync_free(rdev, &sync, NULL); 4059 return ERR_PTR(r); 4060 } 4061 4062 radeon_ring_unlock_commit(rdev, ring, false); 4063 radeon_sync_free(rdev, &sync, fence); 4064 4065 return fence; 4066 } 4067 4068 /* 4069 * IB stuff 4070 */ 4071 /** 4072 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring 4073 * 4074 * @rdev: radeon_device pointer 4075 * @ib: radeon indirect buffer object 4076 * 4077 * Emits an DE (drawing engine) or CE (constant engine) IB 4078 * on the gfx ring. IBs are usually generated by userspace 4079 * acceleration drivers and submitted to the kernel for 4080 * sheduling on the ring. This function schedules the IB 4081 * on the gfx ring for execution by the GPU. 4082 */ 4083 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 4084 { 4085 struct radeon_ring *ring = &rdev->ring[ib->ring]; 4086 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; 4087 u32 header, control = INDIRECT_BUFFER_VALID; 4088 4089 if (ib->is_const_ib) { 4090 /* set switch buffer packet before const IB */ 4091 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 4092 radeon_ring_write(ring, 0); 4093 4094 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 4095 } else { 4096 u32 next_rptr; 4097 if (ring->rptr_save_reg) { 4098 next_rptr = ring->wptr + 3 + 4; 4099 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 4100 radeon_ring_write(ring, ((ring->rptr_save_reg - 4101 PACKET3_SET_UCONFIG_REG_START) >> 2)); 4102 radeon_ring_write(ring, next_rptr); 4103 } else if (rdev->wb.enabled) { 4104 next_rptr = ring->wptr + 5 + 4; 4105 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4106 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); 4107 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 4108 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); 4109 radeon_ring_write(ring, next_rptr); 4110 } 4111 4112 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 4113 } 4114 4115 control |= ib->length_dw | (vm_id << 24); 4116 4117 radeon_ring_write(ring, header); 4118 radeon_ring_write(ring, 4119 #ifdef __BIG_ENDIAN 4120 (2 << 0) | 4121 #endif 4122 (ib->gpu_addr & 0xFFFFFFFC)); 4123 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 4124 radeon_ring_write(ring, control); 4125 } 4126 4127 /** 4128 * cik_ib_test - basic gfx ring IB test 4129 * 4130 * @rdev: radeon_device pointer 4131 * @ring: radeon_ring structure holding ring information 4132 * 4133 * Allocate an IB and execute it on the gfx ring (CIK). 4134 * Provides a basic gfx ring test to verify that IBs are working. 4135 * Returns 0 on success, error on failure. 4136 */ 4137 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) 4138 { 4139 struct radeon_ib ib; 4140 uint32_t scratch; 4141 uint32_t tmp = 0; 4142 unsigned i; 4143 int r; 4144 4145 r = radeon_scratch_get(rdev, &scratch); 4146 if (r) { 4147 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); 4148 return r; 4149 } 4150 WREG32(scratch, 0xCAFEDEAD); 4151 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); 4152 if (r) { 4153 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 4154 radeon_scratch_free(rdev, scratch); 4155 return r; 4156 } 4157 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); 4158 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); 4159 ib.ptr[2] = 0xDEADBEEF; 4160 ib.length_dw = 3; 4161 r = radeon_ib_schedule(rdev, &ib, NULL, false); 4162 if (r) { 4163 radeon_scratch_free(rdev, scratch); 4164 radeon_ib_free(rdev, &ib); 4165 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); 4166 return r; 4167 } 4168 r = radeon_fence_wait(ib.fence, false); 4169 if (r) { 4170 DRM_ERROR("radeon: fence wait failed (%d).\n", r); 4171 radeon_scratch_free(rdev, scratch); 4172 radeon_ib_free(rdev, &ib); 4173 return r; 4174 } 4175 for (i = 0; i < rdev->usec_timeout; i++) { 4176 tmp = RREG32(scratch); 4177 if (tmp == 0xDEADBEEF) 4178 break; 4179 DRM_UDELAY(1); 4180 } 4181 if (i < rdev->usec_timeout) { 4182 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); 4183 } else { 4184 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 4185 scratch, tmp); 4186 r = -EINVAL; 4187 } 4188 radeon_scratch_free(rdev, scratch); 4189 radeon_ib_free(rdev, &ib); 4190 return r; 4191 } 4192 4193 /* 4194 * CP. 4195 * On CIK, gfx and compute now have independant command processors. 4196 * 4197 * GFX 4198 * Gfx consists of a single ring and can process both gfx jobs and 4199 * compute jobs. The gfx CP consists of three microengines (ME): 4200 * PFP - Pre-Fetch Parser 4201 * ME - Micro Engine 4202 * CE - Constant Engine 4203 * The PFP and ME make up what is considered the Drawing Engine (DE). 4204 * The CE is an asynchronous engine used for updating buffer desciptors 4205 * used by the DE so that they can be loaded into cache in parallel 4206 * while the DE is processing state update packets. 4207 * 4208 * Compute 4209 * The compute CP consists of two microengines (ME): 4210 * MEC1 - Compute MicroEngine 1 4211 * MEC2 - Compute MicroEngine 2 4212 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. 4213 * The queues are exposed to userspace and are programmed directly 4214 * by the compute runtime. 4215 */ 4216 /** 4217 * cik_cp_gfx_enable - enable/disable the gfx CP MEs 4218 * 4219 * @rdev: radeon_device pointer 4220 * @enable: enable or disable the MEs 4221 * 4222 * Halts or unhalts the gfx MEs. 4223 */ 4224 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) 4225 { 4226 if (enable) 4227 WREG32(CP_ME_CNTL, 0); 4228 else { 4229 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4230 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 4231 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 4232 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4233 } 4234 udelay(50); 4235 } 4236 4237 /** 4238 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode 4239 * 4240 * @rdev: radeon_device pointer 4241 * 4242 * Loads the gfx PFP, ME, and CE ucode. 4243 * Returns 0 for success, -EINVAL if the ucode is not available. 4244 */ 4245 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) 4246 { 4247 int i; 4248 4249 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 4250 return -EINVAL; 4251 4252 cik_cp_gfx_enable(rdev, false); 4253 4254 if (rdev->new_fw) { 4255 const struct gfx_firmware_header_v1_0 *pfp_hdr = 4256 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 4257 const struct gfx_firmware_header_v1_0 *ce_hdr = 4258 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 4259 const struct gfx_firmware_header_v1_0 *me_hdr = 4260 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 4261 const __le32 *fw_data; 4262 u32 fw_size; 4263 4264 radeon_ucode_print_gfx_hdr(&pfp_hdr->header); 4265 radeon_ucode_print_gfx_hdr(&ce_hdr->header); 4266 radeon_ucode_print_gfx_hdr(&me_hdr->header); 4267 4268 /* PFP */ 4269 fw_data = (const __le32 *) 4270 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 4271 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 4272 WREG32(CP_PFP_UCODE_ADDR, 0); 4273 for (i = 0; i < fw_size; i++) 4274 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 4275 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version)); 4276 4277 /* CE */ 4278 fw_data = (const __le32 *) 4279 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 4280 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 4281 WREG32(CP_CE_UCODE_ADDR, 0); 4282 for (i = 0; i < fw_size; i++) 4283 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 4284 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version)); 4285 4286 /* ME */ 4287 fw_data = (const __be32 *) 4288 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 4289 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 4290 WREG32(CP_ME_RAM_WADDR, 0); 4291 for (i = 0; i < fw_size; i++) 4292 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 4293 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4294 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4295 } else { 4296 const __be32 *fw_data; 4297 4298 /* PFP */ 4299 fw_data = (const __be32 *)rdev->pfp_fw->data; 4300 WREG32(CP_PFP_UCODE_ADDR, 0); 4301 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) 4302 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 4303 WREG32(CP_PFP_UCODE_ADDR, 0); 4304 4305 /* CE */ 4306 fw_data = (const __be32 *)rdev->ce_fw->data; 4307 WREG32(CP_CE_UCODE_ADDR, 0); 4308 for (i = 0; i < CIK_CE_UCODE_SIZE; i++) 4309 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 4310 WREG32(CP_CE_UCODE_ADDR, 0); 4311 4312 /* ME */ 4313 fw_data = (const __be32 *)rdev->me_fw->data; 4314 WREG32(CP_ME_RAM_WADDR, 0); 4315 for (i = 0; i < CIK_ME_UCODE_SIZE; i++) 4316 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 4317 WREG32(CP_ME_RAM_WADDR, 0); 4318 } 4319 4320 return 0; 4321 } 4322 4323 /** 4324 * cik_cp_gfx_start - start the gfx ring 4325 * 4326 * @rdev: radeon_device pointer 4327 * 4328 * Enables the ring and loads the clear state context and other 4329 * packets required to init the ring. 4330 * Returns 0 for success, error for failure. 4331 */ 4332 static int cik_cp_gfx_start(struct radeon_device *rdev) 4333 { 4334 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4335 int r, i; 4336 4337 /* init the CP */ 4338 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); 4339 WREG32(CP_ENDIAN_SWAP, 0); 4340 WREG32(CP_DEVICE_ID, 1); 4341 4342 cik_cp_gfx_enable(rdev, true); 4343 4344 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); 4345 if (r) { 4346 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 4347 return r; 4348 } 4349 4350 /* init the CE partitions. CE only used for gfx on CIK */ 4351 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4352 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4353 radeon_ring_write(ring, 0x8000); 4354 radeon_ring_write(ring, 0x8000); 4355 4356 /* setup clear context state */ 4357 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4358 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4359 4360 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4361 radeon_ring_write(ring, 0x80000000); 4362 radeon_ring_write(ring, 0x80000000); 4363 4364 for (i = 0; i < cik_default_size; i++) 4365 radeon_ring_write(ring, cik_default_state[i]); 4366 4367 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4368 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4369 4370 /* set clear context state */ 4371 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4372 radeon_ring_write(ring, 0); 4373 4374 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4375 radeon_ring_write(ring, 0x00000316); 4376 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 4377 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 4378 4379 radeon_ring_unlock_commit(rdev, ring, false); 4380 4381 return 0; 4382 } 4383 4384 /** 4385 * cik_cp_gfx_fini - stop the gfx ring 4386 * 4387 * @rdev: radeon_device pointer 4388 * 4389 * Stop the gfx ring and tear down the driver ring 4390 * info. 4391 */ 4392 static void cik_cp_gfx_fini(struct radeon_device *rdev) 4393 { 4394 cik_cp_gfx_enable(rdev, false); 4395 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4396 } 4397 4398 /** 4399 * cik_cp_gfx_resume - setup the gfx ring buffer registers 4400 * 4401 * @rdev: radeon_device pointer 4402 * 4403 * Program the location and size of the gfx ring buffer 4404 * and test it to make sure it's working. 4405 * Returns 0 for success, error for failure. 4406 */ 4407 static int cik_cp_gfx_resume(struct radeon_device *rdev) 4408 { 4409 struct radeon_ring *ring; 4410 u32 tmp; 4411 u32 rb_bufsz; 4412 u64 rb_addr; 4413 int r; 4414 4415 WREG32(CP_SEM_WAIT_TIMER, 0x0); 4416 if (rdev->family != CHIP_HAWAII) 4417 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 4418 4419 /* Set the write pointer delay */ 4420 WREG32(CP_RB_WPTR_DELAY, 0); 4421 4422 /* set the RB to use vmid 0 */ 4423 WREG32(CP_RB_VMID, 0); 4424 4425 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 4426 4427 /* ring 0 - compute and gfx */ 4428 /* Set ring buffer size */ 4429 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4430 rb_bufsz = order_base_2(ring->ring_size / 8); 4431 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 4432 #ifdef __BIG_ENDIAN 4433 tmp |= BUF_SWAP_32BIT; 4434 #endif 4435 WREG32(CP_RB0_CNTL, tmp); 4436 4437 /* Initialize the ring buffer's read and write pointers */ 4438 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 4439 ring->wptr = 0; 4440 WREG32(CP_RB0_WPTR, ring->wptr); 4441 4442 /* set the wb address wether it's enabled or not */ 4443 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 4444 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 4445 4446 /* scratch register shadowing is no longer supported */ 4447 WREG32(SCRATCH_UMSK, 0); 4448 4449 if (!rdev->wb.enabled) 4450 tmp |= RB_NO_UPDATE; 4451 4452 mdelay(1); 4453 WREG32(CP_RB0_CNTL, tmp); 4454 4455 rb_addr = ring->gpu_addr >> 8; 4456 WREG32(CP_RB0_BASE, rb_addr); 4457 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4458 4459 /* start the ring */ 4460 cik_cp_gfx_start(rdev); 4461 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 4462 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4463 if (r) { 4464 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4465 return r; 4466 } 4467 4468 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4469 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 4470 4471 return 0; 4472 } 4473 4474 u32 cik_gfx_get_rptr(struct radeon_device *rdev, 4475 struct radeon_ring *ring) 4476 { 4477 u32 rptr; 4478 4479 if (rdev->wb.enabled) 4480 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4481 else 4482 rptr = RREG32(CP_RB0_RPTR); 4483 4484 return rptr; 4485 } 4486 4487 u32 cik_gfx_get_wptr(struct radeon_device *rdev, 4488 struct radeon_ring *ring) 4489 { 4490 u32 wptr; 4491 4492 wptr = RREG32(CP_RB0_WPTR); 4493 4494 return wptr; 4495 } 4496 4497 void cik_gfx_set_wptr(struct radeon_device *rdev, 4498 struct radeon_ring *ring) 4499 { 4500 WREG32(CP_RB0_WPTR, ring->wptr); 4501 (void)RREG32(CP_RB0_WPTR); 4502 } 4503 4504 u32 cik_compute_get_rptr(struct radeon_device *rdev, 4505 struct radeon_ring *ring) 4506 { 4507 u32 rptr; 4508 4509 if (rdev->wb.enabled) { 4510 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4511 } else { 4512 mutex_lock(&rdev->srbm_mutex); 4513 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4514 rptr = RREG32(CP_HQD_PQ_RPTR); 4515 cik_srbm_select(rdev, 0, 0, 0, 0); 4516 mutex_unlock(&rdev->srbm_mutex); 4517 } 4518 4519 return rptr; 4520 } 4521 4522 u32 cik_compute_get_wptr(struct radeon_device *rdev, 4523 struct radeon_ring *ring) 4524 { 4525 u32 wptr; 4526 4527 if (rdev->wb.enabled) { 4528 /* XXX check if swapping is necessary on BE */ 4529 wptr = rdev->wb.wb[ring->wptr_offs/4]; 4530 } else { 4531 mutex_lock(&rdev->srbm_mutex); 4532 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4533 wptr = RREG32(CP_HQD_PQ_WPTR); 4534 cik_srbm_select(rdev, 0, 0, 0, 0); 4535 mutex_unlock(&rdev->srbm_mutex); 4536 } 4537 4538 return wptr; 4539 } 4540 4541 void cik_compute_set_wptr(struct radeon_device *rdev, 4542 struct radeon_ring *ring) 4543 { 4544 /* XXX check if swapping is necessary on BE */ 4545 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; 4546 WDOORBELL32(ring->doorbell_index, ring->wptr); 4547 } 4548 4549 /** 4550 * cik_cp_compute_enable - enable/disable the compute CP MEs 4551 * 4552 * @rdev: radeon_device pointer 4553 * @enable: enable or disable the MEs 4554 * 4555 * Halts or unhalts the compute MEs. 4556 */ 4557 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) 4558 { 4559 if (enable) 4560 WREG32(CP_MEC_CNTL, 0); 4561 else { 4562 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); 4563 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 4564 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 4565 } 4566 udelay(50); 4567 } 4568 4569 /** 4570 * cik_cp_compute_load_microcode - load the compute CP ME ucode 4571 * 4572 * @rdev: radeon_device pointer 4573 * 4574 * Loads the compute MEC1&2 ucode. 4575 * Returns 0 for success, -EINVAL if the ucode is not available. 4576 */ 4577 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) 4578 { 4579 int i; 4580 4581 if (!rdev->mec_fw) 4582 return -EINVAL; 4583 4584 cik_cp_compute_enable(rdev, false); 4585 4586 if (rdev->new_fw) { 4587 const struct gfx_firmware_header_v1_0 *mec_hdr = 4588 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 4589 const __le32 *fw_data; 4590 u32 fw_size; 4591 4592 radeon_ucode_print_gfx_hdr(&mec_hdr->header); 4593 4594 /* MEC1 */ 4595 fw_data = (const __le32 *) 4596 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 4597 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 4598 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4599 for (i = 0; i < fw_size; i++) 4600 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++)); 4601 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version)); 4602 4603 /* MEC2 */ 4604 if (rdev->family == CHIP_KAVERI) { 4605 const struct gfx_firmware_header_v1_0 *mec2_hdr = 4606 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 4607 4608 fw_data = (const __le32 *) 4609 (rdev->mec2_fw->data + 4610 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); 4611 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; 4612 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4613 for (i = 0; i < fw_size; i++) 4614 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++)); 4615 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version)); 4616 } 4617 } else { 4618 const __be32 *fw_data; 4619 4620 /* MEC1 */ 4621 fw_data = (const __be32 *)rdev->mec_fw->data; 4622 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4623 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4624 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); 4625 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4626 4627 if (rdev->family == CHIP_KAVERI) { 4628 /* MEC2 */ 4629 fw_data = (const __be32 *)rdev->mec_fw->data; 4630 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4631 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4632 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); 4633 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4634 } 4635 } 4636 4637 return 0; 4638 } 4639 4640 /** 4641 * cik_cp_compute_start - start the compute queues 4642 * 4643 * @rdev: radeon_device pointer 4644 * 4645 * Enable the compute queues. 4646 * Returns 0 for success, error for failure. 4647 */ 4648 static int cik_cp_compute_start(struct radeon_device *rdev) 4649 { 4650 cik_cp_compute_enable(rdev, true); 4651 4652 return 0; 4653 } 4654 4655 /** 4656 * cik_cp_compute_fini - stop the compute queues 4657 * 4658 * @rdev: radeon_device pointer 4659 * 4660 * Stop the compute queues and tear down the driver queue 4661 * info. 4662 */ 4663 static void cik_cp_compute_fini(struct radeon_device *rdev) 4664 { 4665 int i, idx, r; 4666 4667 cik_cp_compute_enable(rdev, false); 4668 4669 for (i = 0; i < 2; i++) { 4670 if (i == 0) 4671 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4672 else 4673 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4674 4675 if (rdev->ring[idx].mqd_obj) { 4676 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4677 if (unlikely(r != 0)) 4678 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); 4679 4680 radeon_bo_unpin(rdev->ring[idx].mqd_obj); 4681 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4682 4683 radeon_bo_unref(&rdev->ring[idx].mqd_obj); 4684 rdev->ring[idx].mqd_obj = NULL; 4685 } 4686 } 4687 } 4688 4689 static void cik_mec_fini(struct radeon_device *rdev) 4690 { 4691 int r; 4692 4693 if (rdev->mec.hpd_eop_obj) { 4694 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4695 if (unlikely(r != 0)) 4696 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); 4697 radeon_bo_unpin(rdev->mec.hpd_eop_obj); 4698 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4699 4700 radeon_bo_unref(&rdev->mec.hpd_eop_obj); 4701 rdev->mec.hpd_eop_obj = NULL; 4702 } 4703 } 4704 4705 #define MEC_HPD_SIZE 2048 4706 4707 static int cik_mec_init(struct radeon_device *rdev) 4708 { 4709 int r; 4710 u32 *hpd; 4711 4712 /* 4713 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total 4714 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total 4715 * Nonetheless, we assign only 1 pipe because all other pipes will 4716 * be handled by KFD 4717 */ 4718 rdev->mec.num_mec = 1; 4719 rdev->mec.num_pipe = 1; 4720 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; 4721 4722 if (rdev->mec.hpd_eop_obj == NULL) { 4723 r = radeon_bo_create(rdev, 4724 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, 4725 PAGE_SIZE, true, 4726 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, 4727 &rdev->mec.hpd_eop_obj); 4728 if (r) { 4729 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); 4730 return r; 4731 } 4732 } 4733 4734 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4735 if (unlikely(r != 0)) { 4736 cik_mec_fini(rdev); 4737 return r; 4738 } 4739 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, 4740 &rdev->mec.hpd_eop_gpu_addr); 4741 if (r) { 4742 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); 4743 cik_mec_fini(rdev); 4744 return r; 4745 } 4746 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); 4747 if (r) { 4748 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); 4749 cik_mec_fini(rdev); 4750 return r; 4751 } 4752 4753 /* clear memory. Not sure if this is required or not */ 4754 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); 4755 4756 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); 4757 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4758 4759 return 0; 4760 } 4761 4762 struct hqd_registers 4763 { 4764 u32 cp_mqd_base_addr; 4765 u32 cp_mqd_base_addr_hi; 4766 u32 cp_hqd_active; 4767 u32 cp_hqd_vmid; 4768 u32 cp_hqd_persistent_state; 4769 u32 cp_hqd_pipe_priority; 4770 u32 cp_hqd_queue_priority; 4771 u32 cp_hqd_quantum; 4772 u32 cp_hqd_pq_base; 4773 u32 cp_hqd_pq_base_hi; 4774 u32 cp_hqd_pq_rptr; 4775 u32 cp_hqd_pq_rptr_report_addr; 4776 u32 cp_hqd_pq_rptr_report_addr_hi; 4777 u32 cp_hqd_pq_wptr_poll_addr; 4778 u32 cp_hqd_pq_wptr_poll_addr_hi; 4779 u32 cp_hqd_pq_doorbell_control; 4780 u32 cp_hqd_pq_wptr; 4781 u32 cp_hqd_pq_control; 4782 u32 cp_hqd_ib_base_addr; 4783 u32 cp_hqd_ib_base_addr_hi; 4784 u32 cp_hqd_ib_rptr; 4785 u32 cp_hqd_ib_control; 4786 u32 cp_hqd_iq_timer; 4787 u32 cp_hqd_iq_rptr; 4788 u32 cp_hqd_dequeue_request; 4789 u32 cp_hqd_dma_offload; 4790 u32 cp_hqd_sema_cmd; 4791 u32 cp_hqd_msg_type; 4792 u32 cp_hqd_atomic0_preop_lo; 4793 u32 cp_hqd_atomic0_preop_hi; 4794 u32 cp_hqd_atomic1_preop_lo; 4795 u32 cp_hqd_atomic1_preop_hi; 4796 u32 cp_hqd_hq_scheduler0; 4797 u32 cp_hqd_hq_scheduler1; 4798 u32 cp_mqd_control; 4799 }; 4800 4801 struct bonaire_mqd 4802 { 4803 u32 header; 4804 u32 dispatch_initiator; 4805 u32 dimensions[3]; 4806 u32 start_idx[3]; 4807 u32 num_threads[3]; 4808 u32 pipeline_stat_enable; 4809 u32 perf_counter_enable; 4810 u32 pgm[2]; 4811 u32 tba[2]; 4812 u32 tma[2]; 4813 u32 pgm_rsrc[2]; 4814 u32 vmid; 4815 u32 resource_limits; 4816 u32 static_thread_mgmt01[2]; 4817 u32 tmp_ring_size; 4818 u32 static_thread_mgmt23[2]; 4819 u32 restart[3]; 4820 u32 thread_trace_enable; 4821 u32 reserved1; 4822 u32 user_data[16]; 4823 u32 vgtcs_invoke_count[2]; 4824 struct hqd_registers queue_state; 4825 u32 dequeue_cntr; 4826 u32 interrupt_queue[64]; 4827 }; 4828 4829 /** 4830 * cik_cp_compute_resume - setup the compute queue registers 4831 * 4832 * @rdev: radeon_device pointer 4833 * 4834 * Program the compute queues and test them to make sure they 4835 * are working. 4836 * Returns 0 for success, error for failure. 4837 */ 4838 static int cik_cp_compute_resume(struct radeon_device *rdev) 4839 { 4840 int r, i, j, idx; 4841 u32 tmp; 4842 bool use_doorbell = true; 4843 u64 hqd_gpu_addr; 4844 u64 mqd_gpu_addr; 4845 u64 eop_gpu_addr; 4846 u64 wb_gpu_addr; 4847 u32 *buf; 4848 struct bonaire_mqd *mqd; 4849 4850 r = cik_cp_compute_start(rdev); 4851 if (r) 4852 return r; 4853 4854 /* fix up chicken bits */ 4855 tmp = RREG32(CP_CPF_DEBUG); 4856 tmp |= (1 << 23); 4857 WREG32(CP_CPF_DEBUG, tmp); 4858 4859 /* init the pipes */ 4860 mutex_lock(&rdev->srbm_mutex); 4861 4862 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr; 4863 4864 cik_srbm_select(rdev, 0, 0, 0, 0); 4865 4866 /* write the EOP addr */ 4867 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); 4868 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); 4869 4870 /* set the VMID assigned */ 4871 WREG32(CP_HPD_EOP_VMID, 0); 4872 4873 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4874 tmp = RREG32(CP_HPD_EOP_CONTROL); 4875 tmp &= ~EOP_SIZE_MASK; 4876 tmp |= order_base_2(MEC_HPD_SIZE / 8); 4877 WREG32(CP_HPD_EOP_CONTROL, tmp); 4878 4879 mutex_unlock(&rdev->srbm_mutex); 4880 4881 /* init the queues. Just two for now. */ 4882 for (i = 0; i < 2; i++) { 4883 if (i == 0) 4884 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4885 else 4886 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4887 4888 if (rdev->ring[idx].mqd_obj == NULL) { 4889 r = radeon_bo_create(rdev, 4890 sizeof(struct bonaire_mqd), 4891 PAGE_SIZE, true, 4892 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4893 NULL, &rdev->ring[idx].mqd_obj); 4894 if (r) { 4895 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); 4896 return r; 4897 } 4898 } 4899 4900 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4901 if (unlikely(r != 0)) { 4902 cik_cp_compute_fini(rdev); 4903 return r; 4904 } 4905 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, 4906 &mqd_gpu_addr); 4907 if (r) { 4908 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); 4909 cik_cp_compute_fini(rdev); 4910 return r; 4911 } 4912 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); 4913 if (r) { 4914 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); 4915 cik_cp_compute_fini(rdev); 4916 return r; 4917 } 4918 4919 /* init the mqd struct */ 4920 memset(buf, 0, sizeof(struct bonaire_mqd)); 4921 4922 mqd = (struct bonaire_mqd *)buf; 4923 mqd->header = 0xC0310800; 4924 mqd->static_thread_mgmt01[0] = 0xffffffff; 4925 mqd->static_thread_mgmt01[1] = 0xffffffff; 4926 mqd->static_thread_mgmt23[0] = 0xffffffff; 4927 mqd->static_thread_mgmt23[1] = 0xffffffff; 4928 4929 mutex_lock(&rdev->srbm_mutex); 4930 cik_srbm_select(rdev, rdev->ring[idx].me, 4931 rdev->ring[idx].pipe, 4932 rdev->ring[idx].queue, 0); 4933 4934 /* disable wptr polling */ 4935 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4936 tmp &= ~WPTR_POLL_EN; 4937 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4938 4939 /* enable doorbell? */ 4940 mqd->queue_state.cp_hqd_pq_doorbell_control = 4941 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4942 if (use_doorbell) 4943 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4944 else 4945 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; 4946 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4947 mqd->queue_state.cp_hqd_pq_doorbell_control); 4948 4949 /* disable the queue if it's active */ 4950 mqd->queue_state.cp_hqd_dequeue_request = 0; 4951 mqd->queue_state.cp_hqd_pq_rptr = 0; 4952 mqd->queue_state.cp_hqd_pq_wptr= 0; 4953 if (RREG32(CP_HQD_ACTIVE) & 1) { 4954 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4955 for (j = 0; j < rdev->usec_timeout; j++) { 4956 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4957 break; 4958 udelay(1); 4959 } 4960 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); 4961 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); 4962 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4963 } 4964 4965 /* set the pointer to the MQD */ 4966 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; 4967 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 4968 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); 4969 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); 4970 /* set MQD vmid to 0 */ 4971 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); 4972 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; 4973 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); 4974 4975 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4976 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; 4977 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; 4978 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4979 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); 4980 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); 4981 4982 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4983 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); 4984 mqd->queue_state.cp_hqd_pq_control &= 4985 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); 4986 4987 mqd->queue_state.cp_hqd_pq_control |= 4988 order_base_2(rdev->ring[idx].ring_size / 8); 4989 mqd->queue_state.cp_hqd_pq_control |= 4990 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); 4991 #ifdef __BIG_ENDIAN 4992 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; 4993 #endif 4994 mqd->queue_state.cp_hqd_pq_control &= 4995 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); 4996 mqd->queue_state.cp_hqd_pq_control |= 4997 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ 4998 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); 4999 5000 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ 5001 if (i == 0) 5002 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; 5003 else 5004 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; 5005 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; 5006 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 5007 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); 5008 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, 5009 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); 5010 5011 /* set the wb address wether it's enabled or not */ 5012 if (i == 0) 5013 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; 5014 else 5015 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; 5016 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; 5017 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = 5018 upper_32_bits(wb_gpu_addr) & 0xffff; 5019 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, 5020 mqd->queue_state.cp_hqd_pq_rptr_report_addr); 5021 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, 5022 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); 5023 5024 /* enable the doorbell if requested */ 5025 if (use_doorbell) { 5026 mqd->queue_state.cp_hqd_pq_doorbell_control = 5027 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 5028 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; 5029 mqd->queue_state.cp_hqd_pq_doorbell_control |= 5030 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); 5031 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 5032 mqd->queue_state.cp_hqd_pq_doorbell_control &= 5033 ~(DOORBELL_SOURCE | DOORBELL_HIT); 5034 5035 } else { 5036 mqd->queue_state.cp_hqd_pq_doorbell_control = 0; 5037 } 5038 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 5039 mqd->queue_state.cp_hqd_pq_doorbell_control); 5040 5041 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 5042 rdev->ring[idx].wptr = 0; 5043 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; 5044 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 5045 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); 5046 5047 /* set the vmid for the queue */ 5048 mqd->queue_state.cp_hqd_vmid = 0; 5049 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); 5050 5051 /* activate the queue */ 5052 mqd->queue_state.cp_hqd_active = 1; 5053 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); 5054 5055 cik_srbm_select(rdev, 0, 0, 0, 0); 5056 mutex_unlock(&rdev->srbm_mutex); 5057 5058 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); 5059 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 5060 5061 rdev->ring[idx].ready = true; 5062 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); 5063 if (r) 5064 rdev->ring[idx].ready = false; 5065 } 5066 5067 return 0; 5068 } 5069 5070 static void cik_cp_enable(struct radeon_device *rdev, bool enable) 5071 { 5072 cik_cp_gfx_enable(rdev, enable); 5073 cik_cp_compute_enable(rdev, enable); 5074 } 5075 5076 static int cik_cp_load_microcode(struct radeon_device *rdev) 5077 { 5078 int r; 5079 5080 r = cik_cp_gfx_load_microcode(rdev); 5081 if (r) 5082 return r; 5083 r = cik_cp_compute_load_microcode(rdev); 5084 if (r) 5085 return r; 5086 5087 return 0; 5088 } 5089 5090 static void cik_cp_fini(struct radeon_device *rdev) 5091 { 5092 cik_cp_gfx_fini(rdev); 5093 cik_cp_compute_fini(rdev); 5094 } 5095 5096 static int cik_cp_resume(struct radeon_device *rdev) 5097 { 5098 int r; 5099 5100 cik_enable_gui_idle_interrupt(rdev, false); 5101 5102 r = cik_cp_load_microcode(rdev); 5103 if (r) 5104 return r; 5105 5106 r = cik_cp_gfx_resume(rdev); 5107 if (r) 5108 return r; 5109 r = cik_cp_compute_resume(rdev); 5110 if (r) 5111 return r; 5112 5113 cik_enable_gui_idle_interrupt(rdev, true); 5114 5115 return 0; 5116 } 5117 5118 static void cik_print_gpu_status_regs(struct radeon_device *rdev) 5119 { 5120 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 5121 RREG32(GRBM_STATUS)); 5122 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 5123 RREG32(GRBM_STATUS2)); 5124 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 5125 RREG32(GRBM_STATUS_SE0)); 5126 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 5127 RREG32(GRBM_STATUS_SE1)); 5128 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", 5129 RREG32(GRBM_STATUS_SE2)); 5130 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", 5131 RREG32(GRBM_STATUS_SE3)); 5132 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 5133 RREG32(SRBM_STATUS)); 5134 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", 5135 RREG32(SRBM_STATUS2)); 5136 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", 5137 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); 5138 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", 5139 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); 5140 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); 5141 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", 5142 RREG32(CP_STALLED_STAT1)); 5143 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", 5144 RREG32(CP_STALLED_STAT2)); 5145 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", 5146 RREG32(CP_STALLED_STAT3)); 5147 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", 5148 RREG32(CP_CPF_BUSY_STAT)); 5149 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", 5150 RREG32(CP_CPF_STALLED_STAT1)); 5151 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); 5152 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); 5153 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", 5154 RREG32(CP_CPC_STALLED_STAT1)); 5155 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); 5156 } 5157 5158 /** 5159 * cik_gpu_check_soft_reset - check which blocks are busy 5160 * 5161 * @rdev: radeon_device pointer 5162 * 5163 * Check which blocks are busy and return the relevant reset 5164 * mask to be used by cik_gpu_soft_reset(). 5165 * Returns a mask of the blocks to be reset. 5166 */ 5167 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) 5168 { 5169 u32 reset_mask = 0; 5170 u32 tmp; 5171 5172 /* GRBM_STATUS */ 5173 tmp = RREG32(GRBM_STATUS); 5174 if (tmp & (PA_BUSY | SC_BUSY | 5175 BCI_BUSY | SX_BUSY | 5176 TA_BUSY | VGT_BUSY | 5177 DB_BUSY | CB_BUSY | 5178 GDS_BUSY | SPI_BUSY | 5179 IA_BUSY | IA_BUSY_NO_DMA)) 5180 reset_mask |= RADEON_RESET_GFX; 5181 5182 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) 5183 reset_mask |= RADEON_RESET_CP; 5184 5185 /* GRBM_STATUS2 */ 5186 tmp = RREG32(GRBM_STATUS2); 5187 if (tmp & RLC_BUSY) 5188 reset_mask |= RADEON_RESET_RLC; 5189 5190 /* SDMA0_STATUS_REG */ 5191 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); 5192 if (!(tmp & SDMA_IDLE)) 5193 reset_mask |= RADEON_RESET_DMA; 5194 5195 /* SDMA1_STATUS_REG */ 5196 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); 5197 if (!(tmp & SDMA_IDLE)) 5198 reset_mask |= RADEON_RESET_DMA1; 5199 5200 /* SRBM_STATUS2 */ 5201 tmp = RREG32(SRBM_STATUS2); 5202 if (tmp & SDMA_BUSY) 5203 reset_mask |= RADEON_RESET_DMA; 5204 5205 if (tmp & SDMA1_BUSY) 5206 reset_mask |= RADEON_RESET_DMA1; 5207 5208 /* SRBM_STATUS */ 5209 tmp = RREG32(SRBM_STATUS); 5210 5211 if (tmp & IH_BUSY) 5212 reset_mask |= RADEON_RESET_IH; 5213 5214 if (tmp & SEM_BUSY) 5215 reset_mask |= RADEON_RESET_SEM; 5216 5217 if (tmp & GRBM_RQ_PENDING) 5218 reset_mask |= RADEON_RESET_GRBM; 5219 5220 if (tmp & VMC_BUSY) 5221 reset_mask |= RADEON_RESET_VMC; 5222 5223 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 5224 MCC_BUSY | MCD_BUSY)) 5225 reset_mask |= RADEON_RESET_MC; 5226 5227 if (evergreen_is_display_hung(rdev)) 5228 reset_mask |= RADEON_RESET_DISPLAY; 5229 5230 /* Skip MC reset as it's mostly likely not hung, just busy */ 5231 if (reset_mask & RADEON_RESET_MC) { 5232 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 5233 reset_mask &= ~RADEON_RESET_MC; 5234 } 5235 5236 return reset_mask; 5237 } 5238 5239 /** 5240 * cik_gpu_soft_reset - soft reset GPU 5241 * 5242 * @rdev: radeon_device pointer 5243 * @reset_mask: mask of which blocks to reset 5244 * 5245 * Soft reset the blocks specified in @reset_mask. 5246 */ 5247 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 5248 { 5249 struct evergreen_mc_save save; 5250 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 5251 u32 tmp; 5252 5253 if (reset_mask == 0) 5254 return; 5255 5256 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 5257 5258 cik_print_gpu_status_regs(rdev); 5259 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5260 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 5261 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5262 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 5263 5264 /* disable CG/PG */ 5265 cik_fini_pg(rdev); 5266 cik_fini_cg(rdev); 5267 5268 /* stop the rlc */ 5269 cik_rlc_stop(rdev); 5270 5271 /* Disable GFX parsing/prefetching */ 5272 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5273 5274 /* Disable MEC parsing/prefetching */ 5275 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5276 5277 if (reset_mask & RADEON_RESET_DMA) { 5278 /* sdma0 */ 5279 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5280 tmp |= SDMA_HALT; 5281 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5282 } 5283 if (reset_mask & RADEON_RESET_DMA1) { 5284 /* sdma1 */ 5285 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5286 tmp |= SDMA_HALT; 5287 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5288 } 5289 5290 evergreen_mc_stop(rdev, &save); 5291 if (evergreen_mc_wait_for_idle(rdev)) { 5292 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5293 } 5294 5295 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) 5296 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; 5297 5298 if (reset_mask & RADEON_RESET_CP) { 5299 grbm_soft_reset |= SOFT_RESET_CP; 5300 5301 srbm_soft_reset |= SOFT_RESET_GRBM; 5302 } 5303 5304 if (reset_mask & RADEON_RESET_DMA) 5305 srbm_soft_reset |= SOFT_RESET_SDMA; 5306 5307 if (reset_mask & RADEON_RESET_DMA1) 5308 srbm_soft_reset |= SOFT_RESET_SDMA1; 5309 5310 if (reset_mask & RADEON_RESET_DISPLAY) 5311 srbm_soft_reset |= SOFT_RESET_DC; 5312 5313 if (reset_mask & RADEON_RESET_RLC) 5314 grbm_soft_reset |= SOFT_RESET_RLC; 5315 5316 if (reset_mask & RADEON_RESET_SEM) 5317 srbm_soft_reset |= SOFT_RESET_SEM; 5318 5319 if (reset_mask & RADEON_RESET_IH) 5320 srbm_soft_reset |= SOFT_RESET_IH; 5321 5322 if (reset_mask & RADEON_RESET_GRBM) 5323 srbm_soft_reset |= SOFT_RESET_GRBM; 5324 5325 if (reset_mask & RADEON_RESET_VMC) 5326 srbm_soft_reset |= SOFT_RESET_VMC; 5327 5328 if (!(rdev->flags & RADEON_IS_IGP)) { 5329 if (reset_mask & RADEON_RESET_MC) 5330 srbm_soft_reset |= SOFT_RESET_MC; 5331 } 5332 5333 if (grbm_soft_reset) { 5334 tmp = RREG32(GRBM_SOFT_RESET); 5335 tmp |= grbm_soft_reset; 5336 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 5337 WREG32(GRBM_SOFT_RESET, tmp); 5338 tmp = RREG32(GRBM_SOFT_RESET); 5339 5340 udelay(50); 5341 5342 tmp &= ~grbm_soft_reset; 5343 WREG32(GRBM_SOFT_RESET, tmp); 5344 tmp = RREG32(GRBM_SOFT_RESET); 5345 } 5346 5347 if (srbm_soft_reset) { 5348 tmp = RREG32(SRBM_SOFT_RESET); 5349 tmp |= srbm_soft_reset; 5350 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 5351 WREG32(SRBM_SOFT_RESET, tmp); 5352 tmp = RREG32(SRBM_SOFT_RESET); 5353 5354 udelay(50); 5355 5356 tmp &= ~srbm_soft_reset; 5357 WREG32(SRBM_SOFT_RESET, tmp); 5358 tmp = RREG32(SRBM_SOFT_RESET); 5359 } 5360 5361 /* Wait a little for things to settle down */ 5362 udelay(50); 5363 5364 evergreen_mc_resume(rdev, &save); 5365 udelay(50); 5366 5367 cik_print_gpu_status_regs(rdev); 5368 } 5369 5370 struct kv_reset_save_regs { 5371 u32 gmcon_reng_execute; 5372 u32 gmcon_misc; 5373 u32 gmcon_misc3; 5374 }; 5375 5376 static void kv_save_regs_for_reset(struct radeon_device *rdev, 5377 struct kv_reset_save_regs *save) 5378 { 5379 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); 5380 save->gmcon_misc = RREG32(GMCON_MISC); 5381 save->gmcon_misc3 = RREG32(GMCON_MISC3); 5382 5383 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); 5384 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | 5385 STCTRL_STUTTER_EN)); 5386 } 5387 5388 static void kv_restore_regs_for_reset(struct radeon_device *rdev, 5389 struct kv_reset_save_regs *save) 5390 { 5391 int i; 5392 5393 WREG32(GMCON_PGFSM_WRITE, 0); 5394 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); 5395 5396 for (i = 0; i < 5; i++) 5397 WREG32(GMCON_PGFSM_WRITE, 0); 5398 5399 WREG32(GMCON_PGFSM_WRITE, 0); 5400 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); 5401 5402 for (i = 0; i < 5; i++) 5403 WREG32(GMCON_PGFSM_WRITE, 0); 5404 5405 WREG32(GMCON_PGFSM_WRITE, 0x210000); 5406 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); 5407 5408 for (i = 0; i < 5; i++) 5409 WREG32(GMCON_PGFSM_WRITE, 0); 5410 5411 WREG32(GMCON_PGFSM_WRITE, 0x21003); 5412 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); 5413 5414 for (i = 0; i < 5; i++) 5415 WREG32(GMCON_PGFSM_WRITE, 0); 5416 5417 WREG32(GMCON_PGFSM_WRITE, 0x2b00); 5418 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); 5419 5420 for (i = 0; i < 5; i++) 5421 WREG32(GMCON_PGFSM_WRITE, 0); 5422 5423 WREG32(GMCON_PGFSM_WRITE, 0); 5424 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); 5425 5426 for (i = 0; i < 5; i++) 5427 WREG32(GMCON_PGFSM_WRITE, 0); 5428 5429 WREG32(GMCON_PGFSM_WRITE, 0x420000); 5430 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); 5431 5432 for (i = 0; i < 5; i++) 5433 WREG32(GMCON_PGFSM_WRITE, 0); 5434 5435 WREG32(GMCON_PGFSM_WRITE, 0x120202); 5436 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); 5437 5438 for (i = 0; i < 5; i++) 5439 WREG32(GMCON_PGFSM_WRITE, 0); 5440 5441 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); 5442 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); 5443 5444 for (i = 0; i < 5; i++) 5445 WREG32(GMCON_PGFSM_WRITE, 0); 5446 5447 WREG32(GMCON_PGFSM_WRITE, 0x373f3e); 5448 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); 5449 5450 for (i = 0; i < 5; i++) 5451 WREG32(GMCON_PGFSM_WRITE, 0); 5452 5453 WREG32(GMCON_PGFSM_WRITE, 0x3e1332); 5454 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); 5455 5456 WREG32(GMCON_MISC3, save->gmcon_misc3); 5457 WREG32(GMCON_MISC, save->gmcon_misc); 5458 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); 5459 } 5460 5461 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) 5462 { 5463 struct evergreen_mc_save save; 5464 struct kv_reset_save_regs kv_save = { 0 }; 5465 u32 tmp, i; 5466 5467 dev_info(rdev->dev, "GPU pci config reset\n"); 5468 5469 /* disable dpm? */ 5470 5471 /* disable cg/pg */ 5472 cik_fini_pg(rdev); 5473 cik_fini_cg(rdev); 5474 5475 /* Disable GFX parsing/prefetching */ 5476 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5477 5478 /* Disable MEC parsing/prefetching */ 5479 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5480 5481 /* sdma0 */ 5482 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5483 tmp |= SDMA_HALT; 5484 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5485 /* sdma1 */ 5486 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5487 tmp |= SDMA_HALT; 5488 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5489 /* XXX other engines? */ 5490 5491 /* halt the rlc, disable cp internal ints */ 5492 cik_rlc_stop(rdev); 5493 5494 udelay(50); 5495 5496 /* disable mem access */ 5497 evergreen_mc_stop(rdev, &save); 5498 if (evergreen_mc_wait_for_idle(rdev)) { 5499 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 5500 } 5501 5502 if (rdev->flags & RADEON_IS_IGP) 5503 kv_save_regs_for_reset(rdev, &kv_save); 5504 5505 /* disable BM */ 5506 pci_clear_master(rdev->pdev); 5507 /* reset */ 5508 radeon_pci_config_reset(rdev); 5509 5510 udelay(100); 5511 5512 /* wait for asic to come out of reset */ 5513 for (i = 0; i < rdev->usec_timeout; i++) { 5514 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 5515 break; 5516 udelay(1); 5517 } 5518 5519 /* does asic init need to be run first??? */ 5520 if (rdev->flags & RADEON_IS_IGP) 5521 kv_restore_regs_for_reset(rdev, &kv_save); 5522 } 5523 5524 /** 5525 * cik_asic_reset - soft reset GPU 5526 * 5527 * @rdev: radeon_device pointer 5528 * 5529 * Look up which blocks are hung and attempt 5530 * to reset them. 5531 * Returns 0 for success. 5532 */ 5533 int cik_asic_reset(struct radeon_device *rdev) 5534 { 5535 u32 reset_mask; 5536 5537 reset_mask = cik_gpu_check_soft_reset(rdev); 5538 5539 if (reset_mask) 5540 r600_set_bios_scratch_engine_hung(rdev, true); 5541 5542 /* try soft reset */ 5543 cik_gpu_soft_reset(rdev, reset_mask); 5544 5545 reset_mask = cik_gpu_check_soft_reset(rdev); 5546 5547 /* try pci config reset */ 5548 if (reset_mask && radeon_hard_reset) 5549 cik_gpu_pci_config_reset(rdev); 5550 5551 reset_mask = cik_gpu_check_soft_reset(rdev); 5552 5553 if (!reset_mask) 5554 r600_set_bios_scratch_engine_hung(rdev, false); 5555 5556 return 0; 5557 } 5558 5559 /** 5560 * cik_gfx_is_lockup - check if the 3D engine is locked up 5561 * 5562 * @rdev: radeon_device pointer 5563 * @ring: radeon_ring structure holding ring information 5564 * 5565 * Check if the 3D engine is locked up (CIK). 5566 * Returns true if the engine is locked, false if not. 5567 */ 5568 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 5569 { 5570 u32 reset_mask = cik_gpu_check_soft_reset(rdev); 5571 5572 if (!(reset_mask & (RADEON_RESET_GFX | 5573 RADEON_RESET_COMPUTE | 5574 RADEON_RESET_CP))) { 5575 radeon_ring_lockup_update(rdev, ring); 5576 return false; 5577 } 5578 return radeon_ring_test_lockup(rdev, ring); 5579 } 5580 5581 /* MC */ 5582 /** 5583 * cik_mc_program - program the GPU memory controller 5584 * 5585 * @rdev: radeon_device pointer 5586 * 5587 * Set the location of vram, gart, and AGP in the GPU's 5588 * physical address space (CIK). 5589 */ 5590 static void cik_mc_program(struct radeon_device *rdev) 5591 { 5592 struct evergreen_mc_save save; 5593 u32 tmp; 5594 int i, j; 5595 5596 /* Initialize HDP */ 5597 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 5598 WREG32((0x2c14 + j), 0x00000000); 5599 WREG32((0x2c18 + j), 0x00000000); 5600 WREG32((0x2c1c + j), 0x00000000); 5601 WREG32((0x2c20 + j), 0x00000000); 5602 WREG32((0x2c24 + j), 0x00000000); 5603 } 5604 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 5605 5606 evergreen_mc_stop(rdev, &save); 5607 if (radeon_mc_wait_for_idle(rdev)) { 5608 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5609 } 5610 /* Lockout access through VGA aperture*/ 5611 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 5612 /* Update configuration */ 5613 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 5614 rdev->mc.vram_start >> 12); 5615 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 5616 rdev->mc.vram_end >> 12); 5617 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 5618 rdev->vram_scratch.gpu_addr >> 12); 5619 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 5620 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 5621 WREG32(MC_VM_FB_LOCATION, tmp); 5622 /* XXX double check these! */ 5623 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 5624 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 5625 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 5626 WREG32(MC_VM_AGP_BASE, 0); 5627 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 5628 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 5629 if (radeon_mc_wait_for_idle(rdev)) { 5630 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5631 } 5632 evergreen_mc_resume(rdev, &save); 5633 /* we need to own VRAM, so turn off the VGA renderer here 5634 * to stop it overwriting our objects */ 5635 rv515_vga_render_disable(rdev); 5636 } 5637 5638 /** 5639 * cik_mc_init - initialize the memory controller driver params 5640 * 5641 * @rdev: radeon_device pointer 5642 * 5643 * Look up the amount of vram, vram width, and decide how to place 5644 * vram and gart within the GPU's physical address space (CIK). 5645 * Returns 0 for success. 5646 */ 5647 static int cik_mc_init(struct radeon_device *rdev) 5648 { 5649 u32 tmp; 5650 int chansize, numchan; 5651 5652 /* Get VRAM informations */ 5653 rdev->mc.vram_is_ddr = true; 5654 tmp = RREG32(MC_ARB_RAMCFG); 5655 if (tmp & CHANSIZE_MASK) { 5656 chansize = 64; 5657 } else { 5658 chansize = 32; 5659 } 5660 tmp = RREG32(MC_SHARED_CHMAP); 5661 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 5662 case 0: 5663 default: 5664 numchan = 1; 5665 break; 5666 case 1: 5667 numchan = 2; 5668 break; 5669 case 2: 5670 numchan = 4; 5671 break; 5672 case 3: 5673 numchan = 8; 5674 break; 5675 case 4: 5676 numchan = 3; 5677 break; 5678 case 5: 5679 numchan = 6; 5680 break; 5681 case 6: 5682 numchan = 10; 5683 break; 5684 case 7: 5685 numchan = 12; 5686 break; 5687 case 8: 5688 numchan = 16; 5689 break; 5690 } 5691 rdev->mc.vram_width = numchan * chansize; 5692 /* Could aper size report 0 ? */ 5693 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 5694 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 5695 /* size in MB on si */ 5696 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5697 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5698 rdev->mc.visible_vram_size = rdev->mc.aper_size; 5699 si_vram_gtt_location(rdev, &rdev->mc); 5700 radeon_update_bandwidth_info(rdev); 5701 5702 return 0; 5703 } 5704 5705 /* 5706 * GART 5707 * VMID 0 is the physical GPU addresses as used by the kernel. 5708 * VMIDs 1-15 are used for userspace clients and are handled 5709 * by the radeon vm/hsa code. 5710 */ 5711 /** 5712 * cik_pcie_gart_tlb_flush - gart tlb flush callback 5713 * 5714 * @rdev: radeon_device pointer 5715 * 5716 * Flush the TLB for the VMID 0 page table (CIK). 5717 */ 5718 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) 5719 { 5720 /* flush hdp cache */ 5721 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); 5722 5723 /* bits 0-15 are the VM contexts0-15 */ 5724 WREG32(VM_INVALIDATE_REQUEST, 0x1); 5725 } 5726 5727 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev) 5728 { 5729 int i; 5730 uint32_t sh_mem_bases, sh_mem_config; 5731 5732 sh_mem_bases = 0x6000 | 0x6000 << 16; 5733 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED); 5734 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED); 5735 5736 mutex_lock(&rdev->srbm_mutex); 5737 for (i = 8; i < 16; i++) { 5738 cik_srbm_select(rdev, 0, 0, 0, i); 5739 /* CP and shaders */ 5740 WREG32(SH_MEM_CONFIG, sh_mem_config); 5741 WREG32(SH_MEM_APE1_BASE, 1); 5742 WREG32(SH_MEM_APE1_LIMIT, 0); 5743 WREG32(SH_MEM_BASES, sh_mem_bases); 5744 } 5745 cik_srbm_select(rdev, 0, 0, 0, 0); 5746 mutex_unlock(&rdev->srbm_mutex); 5747 } 5748 5749 /** 5750 * cik_pcie_gart_enable - gart enable 5751 * 5752 * @rdev: radeon_device pointer 5753 * 5754 * This sets up the TLBs, programs the page tables for VMID0, 5755 * sets up the hw for VMIDs 1-15 which are allocated on 5756 * demand, and sets up the global locations for the LDS, GDS, 5757 * and GPUVM for FSA64 clients (CIK). 5758 * Returns 0 for success, errors for failure. 5759 */ 5760 static int cik_pcie_gart_enable(struct radeon_device *rdev) 5761 { 5762 int r, i; 5763 5764 if (rdev->gart.robj == NULL) { 5765 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 5766 return -EINVAL; 5767 } 5768 r = radeon_gart_table_vram_pin(rdev); 5769 if (r) 5770 return r; 5771 /* Setup TLB control */ 5772 WREG32(MC_VM_MX_L1_TLB_CNTL, 5773 (0xA << 7) | 5774 ENABLE_L1_TLB | 5775 ENABLE_L1_FRAGMENT_PROCESSING | 5776 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5777 ENABLE_ADVANCED_DRIVER_MODEL | 5778 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5779 /* Setup L2 cache */ 5780 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 5781 ENABLE_L2_FRAGMENT_PROCESSING | 5782 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5783 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5784 EFFECTIVE_L2_QUEUE_SIZE(7) | 5785 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5786 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 5787 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5788 BANK_SELECT(4) | 5789 L2_CACHE_BIGK_FRAGMENT_SIZE(4)); 5790 /* setup context0 */ 5791 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 5792 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 5793 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 5794 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 5795 (u32)(rdev->dummy_page.addr >> 12)); 5796 WREG32(VM_CONTEXT0_CNTL2, 0); 5797 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 5798 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 5799 5800 WREG32(0x15D4, 0); 5801 WREG32(0x15D8, 0); 5802 WREG32(0x15DC, 0); 5803 5804 /* restore context1-15 */ 5805 /* set vm size, must be a multiple of 4 */ 5806 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 5807 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 5808 for (i = 1; i < 16; i++) { 5809 if (i < 8) 5810 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 5811 rdev->vm_manager.saved_table_addr[i]); 5812 else 5813 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 5814 rdev->vm_manager.saved_table_addr[i]); 5815 } 5816 5817 /* enable context1-15 */ 5818 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 5819 (u32)(rdev->dummy_page.addr >> 12)); 5820 WREG32(VM_CONTEXT1_CNTL2, 4); 5821 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 5822 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | 5823 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5824 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5825 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5826 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5827 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 5828 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 5829 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 5830 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 5831 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 5832 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 5833 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5834 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 5835 5836 if (rdev->family == CHIP_KAVERI) { 5837 u32 tmp = RREG32(CHUB_CONTROL); 5838 tmp &= ~BYPASS_VM; 5839 WREG32(CHUB_CONTROL, tmp); 5840 } 5841 5842 /* XXX SH_MEM regs */ 5843 /* where to put LDS, scratch, GPUVM in FSA64 space */ 5844 mutex_lock(&rdev->srbm_mutex); 5845 for (i = 0; i < 16; i++) { 5846 cik_srbm_select(rdev, 0, 0, 0, i); 5847 /* CP and shaders */ 5848 WREG32(SH_MEM_CONFIG, 0); 5849 WREG32(SH_MEM_APE1_BASE, 1); 5850 WREG32(SH_MEM_APE1_LIMIT, 0); 5851 WREG32(SH_MEM_BASES, 0); 5852 /* SDMA GFX */ 5853 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); 5854 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); 5855 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); 5856 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); 5857 /* XXX SDMA RLC - todo */ 5858 } 5859 cik_srbm_select(rdev, 0, 0, 0, 0); 5860 mutex_unlock(&rdev->srbm_mutex); 5861 5862 cik_pcie_init_compute_vmid(rdev); 5863 5864 cik_pcie_gart_tlb_flush(rdev); 5865 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 5866 (unsigned)(rdev->mc.gtt_size >> 20), 5867 (unsigned long long)rdev->gart.table_addr); 5868 rdev->gart.ready = true; 5869 return 0; 5870 } 5871 5872 /** 5873 * cik_pcie_gart_disable - gart disable 5874 * 5875 * @rdev: radeon_device pointer 5876 * 5877 * This disables all VM page table (CIK). 5878 */ 5879 static void cik_pcie_gart_disable(struct radeon_device *rdev) 5880 { 5881 unsigned i; 5882 5883 for (i = 1; i < 16; ++i) { 5884 uint32_t reg; 5885 if (i < 8) 5886 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); 5887 else 5888 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); 5889 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); 5890 } 5891 5892 /* Disable all tables */ 5893 WREG32(VM_CONTEXT0_CNTL, 0); 5894 WREG32(VM_CONTEXT1_CNTL, 0); 5895 /* Setup TLB control */ 5896 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5897 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5898 /* Setup L2 cache */ 5899 WREG32(VM_L2_CNTL, 5900 ENABLE_L2_FRAGMENT_PROCESSING | 5901 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5902 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5903 EFFECTIVE_L2_QUEUE_SIZE(7) | 5904 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5905 WREG32(VM_L2_CNTL2, 0); 5906 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5907 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5908 radeon_gart_table_vram_unpin(rdev); 5909 } 5910 5911 /** 5912 * cik_pcie_gart_fini - vm fini callback 5913 * 5914 * @rdev: radeon_device pointer 5915 * 5916 * Tears down the driver GART/VM setup (CIK). 5917 */ 5918 static void cik_pcie_gart_fini(struct radeon_device *rdev) 5919 { 5920 cik_pcie_gart_disable(rdev); 5921 radeon_gart_table_vram_free(rdev); 5922 radeon_gart_fini(rdev); 5923 } 5924 5925 /* vm parser */ 5926 /** 5927 * cik_ib_parse - vm ib_parse callback 5928 * 5929 * @rdev: radeon_device pointer 5930 * @ib: indirect buffer pointer 5931 * 5932 * CIK uses hw IB checking so this is a nop (CIK). 5933 */ 5934 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 5935 { 5936 return 0; 5937 } 5938 5939 /* 5940 * vm 5941 * VMID 0 is the physical GPU addresses as used by the kernel. 5942 * VMIDs 1-15 are used for userspace clients and are handled 5943 * by the radeon vm/hsa code. 5944 */ 5945 /** 5946 * cik_vm_init - cik vm init callback 5947 * 5948 * @rdev: radeon_device pointer 5949 * 5950 * Inits cik specific vm parameters (number of VMs, base of vram for 5951 * VMIDs 1-15) (CIK). 5952 * Returns 0 for success. 5953 */ 5954 int cik_vm_init(struct radeon_device *rdev) 5955 { 5956 /* 5957 * number of VMs 5958 * VMID 0 is reserved for System 5959 * radeon graphics/compute will use VMIDs 1-7 5960 * amdkfd will use VMIDs 8-15 5961 */ 5962 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS; 5963 /* base offset of vram pages */ 5964 if (rdev->flags & RADEON_IS_IGP) { 5965 u64 tmp = RREG32(MC_VM_FB_OFFSET); 5966 tmp <<= 22; 5967 rdev->vm_manager.vram_base_offset = tmp; 5968 } else 5969 rdev->vm_manager.vram_base_offset = 0; 5970 5971 return 0; 5972 } 5973 5974 /** 5975 * cik_vm_fini - cik vm fini callback 5976 * 5977 * @rdev: radeon_device pointer 5978 * 5979 * Tear down any asic specific VM setup (CIK). 5980 */ 5981 void cik_vm_fini(struct radeon_device *rdev) 5982 { 5983 } 5984 5985 /** 5986 * cik_vm_decode_fault - print human readable fault info 5987 * 5988 * @rdev: radeon_device pointer 5989 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 5990 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 5991 * 5992 * Print human readable fault information (CIK). 5993 */ 5994 static void cik_vm_decode_fault(struct radeon_device *rdev, 5995 u32 status, u32 addr, u32 mc_client) 5996 { 5997 u32 mc_id; 5998 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 5999 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 6000 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, 6001 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; 6002 6003 if (rdev->family == CHIP_HAWAII) 6004 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 6005 else 6006 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 6007 6008 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", 6009 protections, vmid, addr, 6010 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 6011 block, mc_client, mc_id); 6012 } 6013 6014 /** 6015 * cik_vm_flush - cik vm flush using the CP 6016 * 6017 * @rdev: radeon_device pointer 6018 * 6019 * Update the page table base and flush the VM TLB 6020 * using the CP (CIK). 6021 */ 6022 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, 6023 unsigned vm_id, uint64_t pd_addr) 6024 { 6025 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX); 6026 6027 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6028 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6029 WRITE_DATA_DST_SEL(0))); 6030 if (vm_id < 8) { 6031 radeon_ring_write(ring, 6032 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); 6033 } else { 6034 radeon_ring_write(ring, 6035 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); 6036 } 6037 radeon_ring_write(ring, 0); 6038 radeon_ring_write(ring, pd_addr >> 12); 6039 6040 /* update SH_MEM_* regs */ 6041 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6042 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6043 WRITE_DATA_DST_SEL(0))); 6044 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 6045 radeon_ring_write(ring, 0); 6046 radeon_ring_write(ring, VMID(vm_id)); 6047 6048 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); 6049 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6050 WRITE_DATA_DST_SEL(0))); 6051 radeon_ring_write(ring, SH_MEM_BASES >> 2); 6052 radeon_ring_write(ring, 0); 6053 6054 radeon_ring_write(ring, 0); /* SH_MEM_BASES */ 6055 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */ 6056 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ 6057 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ 6058 6059 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6060 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6061 WRITE_DATA_DST_SEL(0))); 6062 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 6063 radeon_ring_write(ring, 0); 6064 radeon_ring_write(ring, VMID(0)); 6065 6066 /* HDP flush */ 6067 cik_hdp_flush_cp_ring_emit(rdev, ring->idx); 6068 6069 /* bits 0-15 are the VM contexts0-15 */ 6070 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6071 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6072 WRITE_DATA_DST_SEL(0))); 6073 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 6074 radeon_ring_write(ring, 0); 6075 radeon_ring_write(ring, 1 << vm_id); 6076 6077 /* wait for the invalidate to complete */ 6078 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6079 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 6080 WAIT_REG_MEM_FUNCTION(0) | /* always */ 6081 WAIT_REG_MEM_ENGINE(0))); /* me */ 6082 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 6083 radeon_ring_write(ring, 0); 6084 radeon_ring_write(ring, 0); /* ref */ 6085 radeon_ring_write(ring, 0); /* mask */ 6086 radeon_ring_write(ring, 0x20); /* poll interval */ 6087 6088 /* compute doesn't have PFP */ 6089 if (usepfp) { 6090 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6091 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6092 radeon_ring_write(ring, 0x0); 6093 } 6094 } 6095 6096 /* 6097 * RLC 6098 * The RLC is a multi-purpose microengine that handles a 6099 * variety of functions, the most important of which is 6100 * the interrupt controller. 6101 */ 6102 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 6103 bool enable) 6104 { 6105 u32 tmp = RREG32(CP_INT_CNTL_RING0); 6106 6107 if (enable) 6108 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6109 else 6110 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6111 WREG32(CP_INT_CNTL_RING0, tmp); 6112 } 6113 6114 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) 6115 { 6116 u32 tmp; 6117 6118 tmp = RREG32(RLC_LB_CNTL); 6119 if (enable) 6120 tmp |= LOAD_BALANCE_ENABLE; 6121 else 6122 tmp &= ~LOAD_BALANCE_ENABLE; 6123 WREG32(RLC_LB_CNTL, tmp); 6124 } 6125 6126 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) 6127 { 6128 u32 i, j, k; 6129 u32 mask; 6130 6131 mutex_lock(&rdev->grbm_idx_mutex); 6132 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6133 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6134 cik_select_se_sh(rdev, i, j); 6135 for (k = 0; k < rdev->usec_timeout; k++) { 6136 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) 6137 break; 6138 udelay(1); 6139 } 6140 } 6141 } 6142 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6143 mutex_unlock(&rdev->grbm_idx_mutex); 6144 6145 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; 6146 for (k = 0; k < rdev->usec_timeout; k++) { 6147 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 6148 break; 6149 udelay(1); 6150 } 6151 } 6152 6153 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) 6154 { 6155 u32 tmp; 6156 6157 tmp = RREG32(RLC_CNTL); 6158 if (tmp != rlc) 6159 WREG32(RLC_CNTL, rlc); 6160 } 6161 6162 static u32 cik_halt_rlc(struct radeon_device *rdev) 6163 { 6164 u32 data, orig; 6165 6166 orig = data = RREG32(RLC_CNTL); 6167 6168 if (data & RLC_ENABLE) { 6169 u32 i; 6170 6171 data &= ~RLC_ENABLE; 6172 WREG32(RLC_CNTL, data); 6173 6174 for (i = 0; i < rdev->usec_timeout; i++) { 6175 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) 6176 break; 6177 udelay(1); 6178 } 6179 6180 cik_wait_for_rlc_serdes(rdev); 6181 } 6182 6183 return orig; 6184 } 6185 6186 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) 6187 { 6188 u32 tmp, i, mask; 6189 6190 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); 6191 WREG32(RLC_GPR_REG2, tmp); 6192 6193 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; 6194 for (i = 0; i < rdev->usec_timeout; i++) { 6195 if ((RREG32(RLC_GPM_STAT) & mask) == mask) 6196 break; 6197 udelay(1); 6198 } 6199 6200 for (i = 0; i < rdev->usec_timeout; i++) { 6201 if ((RREG32(RLC_GPR_REG2) & REQ) == 0) 6202 break; 6203 udelay(1); 6204 } 6205 } 6206 6207 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) 6208 { 6209 u32 tmp; 6210 6211 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); 6212 WREG32(RLC_GPR_REG2, tmp); 6213 } 6214 6215 /** 6216 * cik_rlc_stop - stop the RLC ME 6217 * 6218 * @rdev: radeon_device pointer 6219 * 6220 * Halt the RLC ME (MicroEngine) (CIK). 6221 */ 6222 static void cik_rlc_stop(struct radeon_device *rdev) 6223 { 6224 WREG32(RLC_CNTL, 0); 6225 6226 cik_enable_gui_idle_interrupt(rdev, false); 6227 6228 cik_wait_for_rlc_serdes(rdev); 6229 } 6230 6231 /** 6232 * cik_rlc_start - start the RLC ME 6233 * 6234 * @rdev: radeon_device pointer 6235 * 6236 * Unhalt the RLC ME (MicroEngine) (CIK). 6237 */ 6238 static void cik_rlc_start(struct radeon_device *rdev) 6239 { 6240 WREG32(RLC_CNTL, RLC_ENABLE); 6241 6242 cik_enable_gui_idle_interrupt(rdev, true); 6243 6244 udelay(50); 6245 } 6246 6247 /** 6248 * cik_rlc_resume - setup the RLC hw 6249 * 6250 * @rdev: radeon_device pointer 6251 * 6252 * Initialize the RLC registers, load the ucode, 6253 * and start the RLC (CIK). 6254 * Returns 0 for success, -EINVAL if the ucode is not available. 6255 */ 6256 static int cik_rlc_resume(struct radeon_device *rdev) 6257 { 6258 u32 i, size, tmp; 6259 6260 if (!rdev->rlc_fw) 6261 return -EINVAL; 6262 6263 cik_rlc_stop(rdev); 6264 6265 /* disable CG */ 6266 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; 6267 WREG32(RLC_CGCG_CGLS_CTRL, tmp); 6268 6269 si_rlc_reset(rdev); 6270 6271 cik_init_pg(rdev); 6272 6273 cik_init_cg(rdev); 6274 6275 WREG32(RLC_LB_CNTR_INIT, 0); 6276 WREG32(RLC_LB_CNTR_MAX, 0x00008000); 6277 6278 mutex_lock(&rdev->grbm_idx_mutex); 6279 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6280 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 6281 WREG32(RLC_LB_PARAMS, 0x00600408); 6282 WREG32(RLC_LB_CNTL, 0x80000004); 6283 mutex_unlock(&rdev->grbm_idx_mutex); 6284 6285 WREG32(RLC_MC_CNTL, 0); 6286 WREG32(RLC_UCODE_CNTL, 0); 6287 6288 if (rdev->new_fw) { 6289 const struct rlc_firmware_header_v1_0 *hdr = 6290 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; 6291 const __le32 *fw_data = (const __le32 *) 6292 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6293 6294 radeon_ucode_print_rlc_hdr(&hdr->header); 6295 6296 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 6297 WREG32(RLC_GPM_UCODE_ADDR, 0); 6298 for (i = 0; i < size; i++) 6299 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 6300 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version)); 6301 } else { 6302 const __be32 *fw_data; 6303 6304 switch (rdev->family) { 6305 case CHIP_BONAIRE: 6306 case CHIP_HAWAII: 6307 default: 6308 size = BONAIRE_RLC_UCODE_SIZE; 6309 break; 6310 case CHIP_KAVERI: 6311 size = KV_RLC_UCODE_SIZE; 6312 break; 6313 case CHIP_KABINI: 6314 size = KB_RLC_UCODE_SIZE; 6315 break; 6316 case CHIP_MULLINS: 6317 size = ML_RLC_UCODE_SIZE; 6318 break; 6319 } 6320 6321 fw_data = (const __be32 *)rdev->rlc_fw->data; 6322 WREG32(RLC_GPM_UCODE_ADDR, 0); 6323 for (i = 0; i < size; i++) 6324 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); 6325 WREG32(RLC_GPM_UCODE_ADDR, 0); 6326 } 6327 6328 /* XXX - find out what chips support lbpw */ 6329 cik_enable_lbpw(rdev, false); 6330 6331 if (rdev->family == CHIP_BONAIRE) 6332 WREG32(RLC_DRIVER_DMA_STATUS, 0); 6333 6334 cik_rlc_start(rdev); 6335 6336 return 0; 6337 } 6338 6339 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) 6340 { 6341 u32 data, orig, tmp, tmp2; 6342 6343 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 6344 6345 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 6346 cik_enable_gui_idle_interrupt(rdev, true); 6347 6348 tmp = cik_halt_rlc(rdev); 6349 6350 mutex_lock(&rdev->grbm_idx_mutex); 6351 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6352 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6353 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6354 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; 6355 WREG32(RLC_SERDES_WR_CTRL, tmp2); 6356 mutex_unlock(&rdev->grbm_idx_mutex); 6357 6358 cik_update_rlc(rdev, tmp); 6359 6360 data |= CGCG_EN | CGLS_EN; 6361 } else { 6362 cik_enable_gui_idle_interrupt(rdev, false); 6363 6364 RREG32(CB_CGTT_SCLK_CTRL); 6365 RREG32(CB_CGTT_SCLK_CTRL); 6366 RREG32(CB_CGTT_SCLK_CTRL); 6367 RREG32(CB_CGTT_SCLK_CTRL); 6368 6369 data &= ~(CGCG_EN | CGLS_EN); 6370 } 6371 6372 if (orig != data) 6373 WREG32(RLC_CGCG_CGLS_CTRL, data); 6374 6375 } 6376 6377 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) 6378 { 6379 u32 data, orig, tmp = 0; 6380 6381 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 6382 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { 6383 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 6384 orig = data = RREG32(CP_MEM_SLP_CNTL); 6385 data |= CP_MEM_LS_EN; 6386 if (orig != data) 6387 WREG32(CP_MEM_SLP_CNTL, data); 6388 } 6389 } 6390 6391 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6392 data |= 0x00000001; 6393 data &= 0xfffffffd; 6394 if (orig != data) 6395 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6396 6397 tmp = cik_halt_rlc(rdev); 6398 6399 mutex_lock(&rdev->grbm_idx_mutex); 6400 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6401 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6402 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6403 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; 6404 WREG32(RLC_SERDES_WR_CTRL, data); 6405 mutex_unlock(&rdev->grbm_idx_mutex); 6406 6407 cik_update_rlc(rdev, tmp); 6408 6409 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { 6410 orig = data = RREG32(CGTS_SM_CTRL_REG); 6411 data &= ~SM_MODE_MASK; 6412 data |= SM_MODE(0x2); 6413 data |= SM_MODE_ENABLE; 6414 data &= ~CGTS_OVERRIDE; 6415 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && 6416 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) 6417 data &= ~CGTS_LS_OVERRIDE; 6418 data &= ~ON_MONITOR_ADD_MASK; 6419 data |= ON_MONITOR_ADD_EN; 6420 data |= ON_MONITOR_ADD(0x96); 6421 if (orig != data) 6422 WREG32(CGTS_SM_CTRL_REG, data); 6423 } 6424 } else { 6425 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6426 data |= 0x00000003; 6427 if (orig != data) 6428 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6429 6430 data = RREG32(RLC_MEM_SLP_CNTL); 6431 if (data & RLC_MEM_LS_EN) { 6432 data &= ~RLC_MEM_LS_EN; 6433 WREG32(RLC_MEM_SLP_CNTL, data); 6434 } 6435 6436 data = RREG32(CP_MEM_SLP_CNTL); 6437 if (data & CP_MEM_LS_EN) { 6438 data &= ~CP_MEM_LS_EN; 6439 WREG32(CP_MEM_SLP_CNTL, data); 6440 } 6441 6442 orig = data = RREG32(CGTS_SM_CTRL_REG); 6443 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; 6444 if (orig != data) 6445 WREG32(CGTS_SM_CTRL_REG, data); 6446 6447 tmp = cik_halt_rlc(rdev); 6448 6449 mutex_lock(&rdev->grbm_idx_mutex); 6450 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6451 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6452 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6453 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; 6454 WREG32(RLC_SERDES_WR_CTRL, data); 6455 mutex_unlock(&rdev->grbm_idx_mutex); 6456 6457 cik_update_rlc(rdev, tmp); 6458 } 6459 } 6460 6461 static const u32 mc_cg_registers[] = 6462 { 6463 MC_HUB_MISC_HUB_CG, 6464 MC_HUB_MISC_SIP_CG, 6465 MC_HUB_MISC_VM_CG, 6466 MC_XPB_CLK_GAT, 6467 ATC_MISC_CG, 6468 MC_CITF_MISC_WR_CG, 6469 MC_CITF_MISC_RD_CG, 6470 MC_CITF_MISC_VM_CG, 6471 VM_L2_CG, 6472 }; 6473 6474 static void cik_enable_mc_ls(struct radeon_device *rdev, 6475 bool enable) 6476 { 6477 int i; 6478 u32 orig, data; 6479 6480 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6481 orig = data = RREG32(mc_cg_registers[i]); 6482 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 6483 data |= MC_LS_ENABLE; 6484 else 6485 data &= ~MC_LS_ENABLE; 6486 if (data != orig) 6487 WREG32(mc_cg_registers[i], data); 6488 } 6489 } 6490 6491 static void cik_enable_mc_mgcg(struct radeon_device *rdev, 6492 bool enable) 6493 { 6494 int i; 6495 u32 orig, data; 6496 6497 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6498 orig = data = RREG32(mc_cg_registers[i]); 6499 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 6500 data |= MC_CG_ENABLE; 6501 else 6502 data &= ~MC_CG_ENABLE; 6503 if (data != orig) 6504 WREG32(mc_cg_registers[i], data); 6505 } 6506 } 6507 6508 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, 6509 bool enable) 6510 { 6511 u32 orig, data; 6512 6513 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 6514 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); 6515 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); 6516 } else { 6517 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); 6518 data |= 0xff000000; 6519 if (data != orig) 6520 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); 6521 6522 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); 6523 data |= 0xff000000; 6524 if (data != orig) 6525 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); 6526 } 6527 } 6528 6529 static void cik_enable_sdma_mgls(struct radeon_device *rdev, 6530 bool enable) 6531 { 6532 u32 orig, data; 6533 6534 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { 6535 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6536 data |= 0x100; 6537 if (orig != data) 6538 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6539 6540 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6541 data |= 0x100; 6542 if (orig != data) 6543 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6544 } else { 6545 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6546 data &= ~0x100; 6547 if (orig != data) 6548 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6549 6550 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6551 data &= ~0x100; 6552 if (orig != data) 6553 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6554 } 6555 } 6556 6557 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, 6558 bool enable) 6559 { 6560 u32 orig, data; 6561 6562 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 6563 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6564 data = 0xfff; 6565 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6566 6567 orig = data = RREG32(UVD_CGC_CTRL); 6568 data |= DCM; 6569 if (orig != data) 6570 WREG32(UVD_CGC_CTRL, data); 6571 } else { 6572 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6573 data &= ~0xfff; 6574 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6575 6576 orig = data = RREG32(UVD_CGC_CTRL); 6577 data &= ~DCM; 6578 if (orig != data) 6579 WREG32(UVD_CGC_CTRL, data); 6580 } 6581 } 6582 6583 static void cik_enable_bif_mgls(struct radeon_device *rdev, 6584 bool enable) 6585 { 6586 u32 orig, data; 6587 6588 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 6589 6590 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 6591 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 6592 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 6593 else 6594 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 6595 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 6596 6597 if (orig != data) 6598 WREG32_PCIE_PORT(PCIE_CNTL2, data); 6599 } 6600 6601 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, 6602 bool enable) 6603 { 6604 u32 orig, data; 6605 6606 orig = data = RREG32(HDP_HOST_PATH_CNTL); 6607 6608 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 6609 data &= ~CLOCK_GATING_DIS; 6610 else 6611 data |= CLOCK_GATING_DIS; 6612 6613 if (orig != data) 6614 WREG32(HDP_HOST_PATH_CNTL, data); 6615 } 6616 6617 static void cik_enable_hdp_ls(struct radeon_device *rdev, 6618 bool enable) 6619 { 6620 u32 orig, data; 6621 6622 orig = data = RREG32(HDP_MEM_POWER_LS); 6623 6624 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 6625 data |= HDP_LS_ENABLE; 6626 else 6627 data &= ~HDP_LS_ENABLE; 6628 6629 if (orig != data) 6630 WREG32(HDP_MEM_POWER_LS, data); 6631 } 6632 6633 void cik_update_cg(struct radeon_device *rdev, 6634 u32 block, bool enable) 6635 { 6636 6637 if (block & RADEON_CG_BLOCK_GFX) { 6638 cik_enable_gui_idle_interrupt(rdev, false); 6639 /* order matters! */ 6640 if (enable) { 6641 cik_enable_mgcg(rdev, true); 6642 cik_enable_cgcg(rdev, true); 6643 } else { 6644 cik_enable_cgcg(rdev, false); 6645 cik_enable_mgcg(rdev, false); 6646 } 6647 cik_enable_gui_idle_interrupt(rdev, true); 6648 } 6649 6650 if (block & RADEON_CG_BLOCK_MC) { 6651 if (!(rdev->flags & RADEON_IS_IGP)) { 6652 cik_enable_mc_mgcg(rdev, enable); 6653 cik_enable_mc_ls(rdev, enable); 6654 } 6655 } 6656 6657 if (block & RADEON_CG_BLOCK_SDMA) { 6658 cik_enable_sdma_mgcg(rdev, enable); 6659 cik_enable_sdma_mgls(rdev, enable); 6660 } 6661 6662 if (block & RADEON_CG_BLOCK_BIF) { 6663 cik_enable_bif_mgls(rdev, enable); 6664 } 6665 6666 if (block & RADEON_CG_BLOCK_UVD) { 6667 if (rdev->has_uvd) 6668 cik_enable_uvd_mgcg(rdev, enable); 6669 } 6670 6671 if (block & RADEON_CG_BLOCK_HDP) { 6672 cik_enable_hdp_mgcg(rdev, enable); 6673 cik_enable_hdp_ls(rdev, enable); 6674 } 6675 6676 if (block & RADEON_CG_BLOCK_VCE) { 6677 vce_v2_0_enable_mgcg(rdev, enable); 6678 } 6679 } 6680 6681 static void cik_init_cg(struct radeon_device *rdev) 6682 { 6683 6684 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); 6685 6686 if (rdev->has_uvd) 6687 si_init_uvd_internal_cg(rdev); 6688 6689 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6690 RADEON_CG_BLOCK_SDMA | 6691 RADEON_CG_BLOCK_BIF | 6692 RADEON_CG_BLOCK_UVD | 6693 RADEON_CG_BLOCK_HDP), true); 6694 } 6695 6696 static void cik_fini_cg(struct radeon_device *rdev) 6697 { 6698 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6699 RADEON_CG_BLOCK_SDMA | 6700 RADEON_CG_BLOCK_BIF | 6701 RADEON_CG_BLOCK_UVD | 6702 RADEON_CG_BLOCK_HDP), false); 6703 6704 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); 6705 } 6706 6707 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, 6708 bool enable) 6709 { 6710 u32 data, orig; 6711 6712 orig = data = RREG32(RLC_PG_CNTL); 6713 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6714 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6715 else 6716 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6717 if (orig != data) 6718 WREG32(RLC_PG_CNTL, data); 6719 } 6720 6721 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, 6722 bool enable) 6723 { 6724 u32 data, orig; 6725 6726 orig = data = RREG32(RLC_PG_CNTL); 6727 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6728 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6729 else 6730 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6731 if (orig != data) 6732 WREG32(RLC_PG_CNTL, data); 6733 } 6734 6735 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) 6736 { 6737 u32 data, orig; 6738 6739 orig = data = RREG32(RLC_PG_CNTL); 6740 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) 6741 data &= ~DISABLE_CP_PG; 6742 else 6743 data |= DISABLE_CP_PG; 6744 if (orig != data) 6745 WREG32(RLC_PG_CNTL, data); 6746 } 6747 6748 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) 6749 { 6750 u32 data, orig; 6751 6752 orig = data = RREG32(RLC_PG_CNTL); 6753 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) 6754 data &= ~DISABLE_GDS_PG; 6755 else 6756 data |= DISABLE_GDS_PG; 6757 if (orig != data) 6758 WREG32(RLC_PG_CNTL, data); 6759 } 6760 6761 #define CP_ME_TABLE_SIZE 96 6762 #define CP_ME_TABLE_OFFSET 2048 6763 #define CP_MEC_TABLE_OFFSET 4096 6764 6765 void cik_init_cp_pg_table(struct radeon_device *rdev) 6766 { 6767 volatile u32 *dst_ptr; 6768 int me, i, max_me = 4; 6769 u32 bo_offset = 0; 6770 u32 table_offset, table_size; 6771 6772 if (rdev->family == CHIP_KAVERI) 6773 max_me = 5; 6774 6775 if (rdev->rlc.cp_table_ptr == NULL) 6776 return; 6777 6778 /* write the cp table buffer */ 6779 dst_ptr = rdev->rlc.cp_table_ptr; 6780 for (me = 0; me < max_me; me++) { 6781 if (rdev->new_fw) { 6782 const __le32 *fw_data; 6783 const struct gfx_firmware_header_v1_0 *hdr; 6784 6785 if (me == 0) { 6786 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 6787 fw_data = (const __le32 *) 6788 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6789 table_offset = le32_to_cpu(hdr->jt_offset); 6790 table_size = le32_to_cpu(hdr->jt_size); 6791 } else if (me == 1) { 6792 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 6793 fw_data = (const __le32 *) 6794 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6795 table_offset = le32_to_cpu(hdr->jt_offset); 6796 table_size = le32_to_cpu(hdr->jt_size); 6797 } else if (me == 2) { 6798 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 6799 fw_data = (const __le32 *) 6800 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6801 table_offset = le32_to_cpu(hdr->jt_offset); 6802 table_size = le32_to_cpu(hdr->jt_size); 6803 } else if (me == 3) { 6804 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 6805 fw_data = (const __le32 *) 6806 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6807 table_offset = le32_to_cpu(hdr->jt_offset); 6808 table_size = le32_to_cpu(hdr->jt_size); 6809 } else { 6810 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 6811 fw_data = (const __le32 *) 6812 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6813 table_offset = le32_to_cpu(hdr->jt_offset); 6814 table_size = le32_to_cpu(hdr->jt_size); 6815 } 6816 6817 for (i = 0; i < table_size; i ++) { 6818 dst_ptr[bo_offset + i] = 6819 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); 6820 } 6821 bo_offset += table_size; 6822 } else { 6823 const __be32 *fw_data; 6824 table_size = CP_ME_TABLE_SIZE; 6825 6826 if (me == 0) { 6827 fw_data = (const __be32 *)rdev->ce_fw->data; 6828 table_offset = CP_ME_TABLE_OFFSET; 6829 } else if (me == 1) { 6830 fw_data = (const __be32 *)rdev->pfp_fw->data; 6831 table_offset = CP_ME_TABLE_OFFSET; 6832 } else if (me == 2) { 6833 fw_data = (const __be32 *)rdev->me_fw->data; 6834 table_offset = CP_ME_TABLE_OFFSET; 6835 } else { 6836 fw_data = (const __be32 *)rdev->mec_fw->data; 6837 table_offset = CP_MEC_TABLE_OFFSET; 6838 } 6839 6840 for (i = 0; i < table_size; i ++) { 6841 dst_ptr[bo_offset + i] = 6842 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); 6843 } 6844 bo_offset += table_size; 6845 } 6846 } 6847 } 6848 6849 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, 6850 bool enable) 6851 { 6852 u32 data, orig; 6853 6854 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 6855 orig = data = RREG32(RLC_PG_CNTL); 6856 data |= GFX_PG_ENABLE; 6857 if (orig != data) 6858 WREG32(RLC_PG_CNTL, data); 6859 6860 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6861 data |= AUTO_PG_EN; 6862 if (orig != data) 6863 WREG32(RLC_AUTO_PG_CTRL, data); 6864 } else { 6865 orig = data = RREG32(RLC_PG_CNTL); 6866 data &= ~GFX_PG_ENABLE; 6867 if (orig != data) 6868 WREG32(RLC_PG_CNTL, data); 6869 6870 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6871 data &= ~AUTO_PG_EN; 6872 if (orig != data) 6873 WREG32(RLC_AUTO_PG_CTRL, data); 6874 6875 data = RREG32(DB_RENDER_CONTROL); 6876 } 6877 } 6878 6879 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 6880 { 6881 u32 mask = 0, tmp, tmp1; 6882 int i; 6883 6884 mutex_lock(&rdev->grbm_idx_mutex); 6885 cik_select_se_sh(rdev, se, sh); 6886 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 6887 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 6888 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6889 mutex_unlock(&rdev->grbm_idx_mutex); 6890 6891 tmp &= 0xffff0000; 6892 6893 tmp |= tmp1; 6894 tmp >>= 16; 6895 6896 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { 6897 mask <<= 1; 6898 mask |= 1; 6899 } 6900 6901 return (~tmp) & mask; 6902 } 6903 6904 static void cik_init_ao_cu_mask(struct radeon_device *rdev) 6905 { 6906 u32 i, j, k, active_cu_number = 0; 6907 u32 mask, counter, cu_bitmap; 6908 u32 tmp = 0; 6909 6910 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6911 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6912 mask = 1; 6913 cu_bitmap = 0; 6914 counter = 0; 6915 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { 6916 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { 6917 if (counter < 2) 6918 cu_bitmap |= mask; 6919 counter ++; 6920 } 6921 mask <<= 1; 6922 } 6923 6924 active_cu_number += counter; 6925 tmp |= (cu_bitmap << (i * 16 + j * 8)); 6926 } 6927 } 6928 6929 WREG32(RLC_PG_AO_CU_MASK, tmp); 6930 6931 tmp = RREG32(RLC_MAX_PG_CU); 6932 tmp &= ~MAX_PU_CU_MASK; 6933 tmp |= MAX_PU_CU(active_cu_number); 6934 WREG32(RLC_MAX_PG_CU, tmp); 6935 } 6936 6937 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, 6938 bool enable) 6939 { 6940 u32 data, orig; 6941 6942 orig = data = RREG32(RLC_PG_CNTL); 6943 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) 6944 data |= STATIC_PER_CU_PG_ENABLE; 6945 else 6946 data &= ~STATIC_PER_CU_PG_ENABLE; 6947 if (orig != data) 6948 WREG32(RLC_PG_CNTL, data); 6949 } 6950 6951 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, 6952 bool enable) 6953 { 6954 u32 data, orig; 6955 6956 orig = data = RREG32(RLC_PG_CNTL); 6957 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) 6958 data |= DYN_PER_CU_PG_ENABLE; 6959 else 6960 data &= ~DYN_PER_CU_PG_ENABLE; 6961 if (orig != data) 6962 WREG32(RLC_PG_CNTL, data); 6963 } 6964 6965 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 6966 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D 6967 6968 static void cik_init_gfx_cgpg(struct radeon_device *rdev) 6969 { 6970 u32 data, orig; 6971 u32 i; 6972 6973 if (rdev->rlc.cs_data) { 6974 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6975 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); 6976 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); 6977 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); 6978 } else { 6979 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6980 for (i = 0; i < 3; i++) 6981 WREG32(RLC_GPM_SCRATCH_DATA, 0); 6982 } 6983 if (rdev->rlc.reg_list) { 6984 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); 6985 for (i = 0; i < rdev->rlc.reg_list_size; i++) 6986 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); 6987 } 6988 6989 orig = data = RREG32(RLC_PG_CNTL); 6990 data |= GFX_PG_SRC; 6991 if (orig != data) 6992 WREG32(RLC_PG_CNTL, data); 6993 6994 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 6995 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); 6996 6997 data = RREG32(CP_RB_WPTR_POLL_CNTL); 6998 data &= ~IDLE_POLL_COUNT_MASK; 6999 data |= IDLE_POLL_COUNT(0x60); 7000 WREG32(CP_RB_WPTR_POLL_CNTL, data); 7001 7002 data = 0x10101010; 7003 WREG32(RLC_PG_DELAY, data); 7004 7005 data = RREG32(RLC_PG_DELAY_2); 7006 data &= ~0xff; 7007 data |= 0x3; 7008 WREG32(RLC_PG_DELAY_2, data); 7009 7010 data = RREG32(RLC_AUTO_PG_CTRL); 7011 data &= ~GRBM_REG_SGIT_MASK; 7012 data |= GRBM_REG_SGIT(0x700); 7013 WREG32(RLC_AUTO_PG_CTRL, data); 7014 7015 } 7016 7017 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) 7018 { 7019 cik_enable_gfx_cgpg(rdev, enable); 7020 cik_enable_gfx_static_mgpg(rdev, enable); 7021 cik_enable_gfx_dynamic_mgpg(rdev, enable); 7022 } 7023 7024 u32 cik_get_csb_size(struct radeon_device *rdev) 7025 { 7026 u32 count = 0; 7027 const struct cs_section_def *sect = NULL; 7028 const struct cs_extent_def *ext = NULL; 7029 7030 if (rdev->rlc.cs_data == NULL) 7031 return 0; 7032 7033 /* begin clear state */ 7034 count += 2; 7035 /* context control state */ 7036 count += 3; 7037 7038 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 7039 for (ext = sect->section; ext->extent != NULL; ++ext) { 7040 if (sect->id == SECT_CONTEXT) 7041 count += 2 + ext->reg_count; 7042 else 7043 return 0; 7044 } 7045 } 7046 /* pa_sc_raster_config/pa_sc_raster_config1 */ 7047 count += 4; 7048 /* end clear state */ 7049 count += 2; 7050 /* clear state */ 7051 count += 2; 7052 7053 return count; 7054 } 7055 7056 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 7057 { 7058 u32 count = 0, i; 7059 const struct cs_section_def *sect = NULL; 7060 const struct cs_extent_def *ext = NULL; 7061 7062 if (rdev->rlc.cs_data == NULL) 7063 return; 7064 if (buffer == NULL) 7065 return; 7066 7067 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 7068 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 7069 7070 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 7071 buffer[count++] = cpu_to_le32(0x80000000); 7072 buffer[count++] = cpu_to_le32(0x80000000); 7073 7074 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 7075 for (ext = sect->section; ext->extent != NULL; ++ext) { 7076 if (sect->id == SECT_CONTEXT) { 7077 buffer[count++] = 7078 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 7079 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 7080 for (i = 0; i < ext->reg_count; i++) 7081 buffer[count++] = cpu_to_le32(ext->extent[i]); 7082 } else { 7083 return; 7084 } 7085 } 7086 } 7087 7088 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 7089 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 7090 switch (rdev->family) { 7091 case CHIP_BONAIRE: 7092 buffer[count++] = cpu_to_le32(0x16000012); 7093 buffer[count++] = cpu_to_le32(0x00000000); 7094 break; 7095 case CHIP_KAVERI: 7096 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 7097 buffer[count++] = cpu_to_le32(0x00000000); 7098 break; 7099 case CHIP_KABINI: 7100 case CHIP_MULLINS: 7101 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 7102 buffer[count++] = cpu_to_le32(0x00000000); 7103 break; 7104 case CHIP_HAWAII: 7105 buffer[count++] = cpu_to_le32(0x3a00161a); 7106 buffer[count++] = cpu_to_le32(0x0000002e); 7107 break; 7108 default: 7109 buffer[count++] = cpu_to_le32(0x00000000); 7110 buffer[count++] = cpu_to_le32(0x00000000); 7111 break; 7112 } 7113 7114 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 7115 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 7116 7117 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 7118 buffer[count++] = cpu_to_le32(0); 7119 } 7120 7121 static void cik_init_pg(struct radeon_device *rdev) 7122 { 7123 if (rdev->pg_flags) { 7124 cik_enable_sck_slowdown_on_pu(rdev, true); 7125 cik_enable_sck_slowdown_on_pd(rdev, true); 7126 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7127 cik_init_gfx_cgpg(rdev); 7128 cik_enable_cp_pg(rdev, true); 7129 cik_enable_gds_pg(rdev, true); 7130 } 7131 cik_init_ao_cu_mask(rdev); 7132 cik_update_gfx_pg(rdev, true); 7133 } 7134 } 7135 7136 static void cik_fini_pg(struct radeon_device *rdev) 7137 { 7138 if (rdev->pg_flags) { 7139 cik_update_gfx_pg(rdev, false); 7140 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7141 cik_enable_cp_pg(rdev, false); 7142 cik_enable_gds_pg(rdev, false); 7143 } 7144 } 7145 } 7146 7147 /* 7148 * Interrupts 7149 * Starting with r6xx, interrupts are handled via a ring buffer. 7150 * Ring buffers are areas of GPU accessible memory that the GPU 7151 * writes interrupt vectors into and the host reads vectors out of. 7152 * There is a rptr (read pointer) that determines where the 7153 * host is currently reading, and a wptr (write pointer) 7154 * which determines where the GPU has written. When the 7155 * pointers are equal, the ring is idle. When the GPU 7156 * writes vectors to the ring buffer, it increments the 7157 * wptr. When there is an interrupt, the host then starts 7158 * fetching commands and processing them until the pointers are 7159 * equal again at which point it updates the rptr. 7160 */ 7161 7162 /** 7163 * cik_enable_interrupts - Enable the interrupt ring buffer 7164 * 7165 * @rdev: radeon_device pointer 7166 * 7167 * Enable the interrupt ring buffer (CIK). 7168 */ 7169 static void cik_enable_interrupts(struct radeon_device *rdev) 7170 { 7171 u32 ih_cntl = RREG32(IH_CNTL); 7172 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7173 7174 ih_cntl |= ENABLE_INTR; 7175 ih_rb_cntl |= IH_RB_ENABLE; 7176 WREG32(IH_CNTL, ih_cntl); 7177 WREG32(IH_RB_CNTL, ih_rb_cntl); 7178 rdev->ih.enabled = true; 7179 } 7180 7181 /** 7182 * cik_disable_interrupts - Disable the interrupt ring buffer 7183 * 7184 * @rdev: radeon_device pointer 7185 * 7186 * Disable the interrupt ring buffer (CIK). 7187 */ 7188 static void cik_disable_interrupts(struct radeon_device *rdev) 7189 { 7190 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7191 u32 ih_cntl = RREG32(IH_CNTL); 7192 7193 ih_rb_cntl &= ~IH_RB_ENABLE; 7194 ih_cntl &= ~ENABLE_INTR; 7195 WREG32(IH_RB_CNTL, ih_rb_cntl); 7196 WREG32(IH_CNTL, ih_cntl); 7197 /* set rptr, wptr to 0 */ 7198 WREG32(IH_RB_RPTR, 0); 7199 WREG32(IH_RB_WPTR, 0); 7200 rdev->ih.enabled = false; 7201 rdev->ih.rptr = 0; 7202 } 7203 7204 /** 7205 * cik_disable_interrupt_state - Disable all interrupt sources 7206 * 7207 * @rdev: radeon_device pointer 7208 * 7209 * Clear all interrupt enable bits used by the driver (CIK). 7210 */ 7211 static void cik_disable_interrupt_state(struct radeon_device *rdev) 7212 { 7213 u32 tmp; 7214 7215 /* gfx ring */ 7216 tmp = RREG32(CP_INT_CNTL_RING0) & 7217 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7218 WREG32(CP_INT_CNTL_RING0, tmp); 7219 /* sdma */ 7220 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7221 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); 7222 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7223 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); 7224 /* compute queues */ 7225 WREG32(CP_ME1_PIPE0_INT_CNTL, 0); 7226 WREG32(CP_ME1_PIPE1_INT_CNTL, 0); 7227 WREG32(CP_ME1_PIPE2_INT_CNTL, 0); 7228 WREG32(CP_ME1_PIPE3_INT_CNTL, 0); 7229 WREG32(CP_ME2_PIPE0_INT_CNTL, 0); 7230 WREG32(CP_ME2_PIPE1_INT_CNTL, 0); 7231 WREG32(CP_ME2_PIPE2_INT_CNTL, 0); 7232 WREG32(CP_ME2_PIPE3_INT_CNTL, 0); 7233 /* grbm */ 7234 WREG32(GRBM_INT_CNTL, 0); 7235 /* SRBM */ 7236 WREG32(SRBM_INT_CNTL, 0); 7237 /* vline/vblank, etc. */ 7238 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7239 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7240 if (rdev->num_crtc >= 4) { 7241 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7242 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7243 } 7244 if (rdev->num_crtc >= 6) { 7245 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7246 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7247 } 7248 /* pflip */ 7249 if (rdev->num_crtc >= 2) { 7250 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7251 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7252 } 7253 if (rdev->num_crtc >= 4) { 7254 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7255 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7256 } 7257 if (rdev->num_crtc >= 6) { 7258 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7259 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7260 } 7261 7262 /* dac hotplug */ 7263 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 7264 7265 /* digital hotplug */ 7266 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7267 WREG32(DC_HPD1_INT_CONTROL, tmp); 7268 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7269 WREG32(DC_HPD2_INT_CONTROL, tmp); 7270 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7271 WREG32(DC_HPD3_INT_CONTROL, tmp); 7272 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7273 WREG32(DC_HPD4_INT_CONTROL, tmp); 7274 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7275 WREG32(DC_HPD5_INT_CONTROL, tmp); 7276 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7277 WREG32(DC_HPD6_INT_CONTROL, tmp); 7278 7279 } 7280 7281 /** 7282 * cik_irq_init - init and enable the interrupt ring 7283 * 7284 * @rdev: radeon_device pointer 7285 * 7286 * Allocate a ring buffer for the interrupt controller, 7287 * enable the RLC, disable interrupts, enable the IH 7288 * ring buffer and enable it (CIK). 7289 * Called at device load and reume. 7290 * Returns 0 for success, errors for failure. 7291 */ 7292 static int cik_irq_init(struct radeon_device *rdev) 7293 { 7294 int ret = 0; 7295 int rb_bufsz; 7296 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 7297 7298 /* allocate ring */ 7299 ret = r600_ih_ring_alloc(rdev); 7300 if (ret) 7301 return ret; 7302 7303 /* disable irqs */ 7304 cik_disable_interrupts(rdev); 7305 7306 /* init rlc */ 7307 ret = cik_rlc_resume(rdev); 7308 if (ret) { 7309 r600_ih_ring_fini(rdev); 7310 return ret; 7311 } 7312 7313 /* setup interrupt control */ 7314 /* XXX this should actually be a bus address, not an MC address. same on older asics */ 7315 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 7316 interrupt_cntl = RREG32(INTERRUPT_CNTL); 7317 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 7318 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 7319 */ 7320 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 7321 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 7322 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 7323 WREG32(INTERRUPT_CNTL, interrupt_cntl); 7324 7325 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 7326 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 7327 7328 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 7329 IH_WPTR_OVERFLOW_CLEAR | 7330 (rb_bufsz << 1)); 7331 7332 if (rdev->wb.enabled) 7333 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 7334 7335 /* set the writeback address whether it's enabled or not */ 7336 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 7337 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 7338 7339 WREG32(IH_RB_CNTL, ih_rb_cntl); 7340 7341 /* set rptr, wptr to 0 */ 7342 WREG32(IH_RB_RPTR, 0); 7343 WREG32(IH_RB_WPTR, 0); 7344 7345 /* Default settings for IH_CNTL (disabled at first) */ 7346 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 7347 /* RPTR_REARM only works if msi's are enabled */ 7348 if (rdev->msi_enabled) 7349 ih_cntl |= RPTR_REARM; 7350 WREG32(IH_CNTL, ih_cntl); 7351 7352 /* force the active interrupt state to all disabled */ 7353 cik_disable_interrupt_state(rdev); 7354 7355 pci_set_master(rdev->pdev); 7356 7357 /* enable irqs */ 7358 cik_enable_interrupts(rdev); 7359 7360 return ret; 7361 } 7362 7363 /** 7364 * cik_irq_set - enable/disable interrupt sources 7365 * 7366 * @rdev: radeon_device pointer 7367 * 7368 * Enable interrupt sources on the GPU (vblanks, hpd, 7369 * etc.) (CIK). 7370 * Returns 0 for success, errors for failure. 7371 */ 7372 int cik_irq_set(struct radeon_device *rdev) 7373 { 7374 u32 cp_int_cntl; 7375 u32 cp_m1p0; 7376 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 7377 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 7378 u32 grbm_int_cntl = 0; 7379 u32 dma_cntl, dma_cntl1; 7380 7381 if (!rdev->irq.installed) { 7382 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 7383 return -EINVAL; 7384 } 7385 /* don't enable anything if the ih is disabled */ 7386 if (!rdev->ih.enabled) { 7387 cik_disable_interrupts(rdev); 7388 /* force the active interrupt state to all disabled */ 7389 cik_disable_interrupt_state(rdev); 7390 return 0; 7391 } 7392 7393 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 7394 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7395 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; 7396 7397 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 7398 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 7399 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 7400 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 7401 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 7402 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 7403 7404 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7405 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7406 7407 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7408 7409 /* enable CP interrupts on all rings */ 7410 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 7411 DRM_DEBUG("cik_irq_set: sw int gfx\n"); 7412 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 7413 } 7414 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 7415 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7416 DRM_DEBUG("si_irq_set: sw int cp1\n"); 7417 if (ring->me == 1) { 7418 switch (ring->pipe) { 7419 case 0: 7420 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7421 break; 7422 default: 7423 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7424 break; 7425 } 7426 } else { 7427 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); 7428 } 7429 } 7430 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 7431 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7432 DRM_DEBUG("si_irq_set: sw int cp2\n"); 7433 if (ring->me == 1) { 7434 switch (ring->pipe) { 7435 case 0: 7436 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7437 break; 7438 default: 7439 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7440 break; 7441 } 7442 } else { 7443 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); 7444 } 7445 } 7446 7447 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 7448 DRM_DEBUG("cik_irq_set: sw int dma\n"); 7449 dma_cntl |= TRAP_ENABLE; 7450 } 7451 7452 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 7453 DRM_DEBUG("cik_irq_set: sw int dma1\n"); 7454 dma_cntl1 |= TRAP_ENABLE; 7455 } 7456 7457 if (rdev->irq.crtc_vblank_int[0] || 7458 atomic_read(&rdev->irq.pflip[0])) { 7459 DRM_DEBUG("cik_irq_set: vblank 0\n"); 7460 crtc1 |= VBLANK_INTERRUPT_MASK; 7461 } 7462 if (rdev->irq.crtc_vblank_int[1] || 7463 atomic_read(&rdev->irq.pflip[1])) { 7464 DRM_DEBUG("cik_irq_set: vblank 1\n"); 7465 crtc2 |= VBLANK_INTERRUPT_MASK; 7466 } 7467 if (rdev->irq.crtc_vblank_int[2] || 7468 atomic_read(&rdev->irq.pflip[2])) { 7469 DRM_DEBUG("cik_irq_set: vblank 2\n"); 7470 crtc3 |= VBLANK_INTERRUPT_MASK; 7471 } 7472 if (rdev->irq.crtc_vblank_int[3] || 7473 atomic_read(&rdev->irq.pflip[3])) { 7474 DRM_DEBUG("cik_irq_set: vblank 3\n"); 7475 crtc4 |= VBLANK_INTERRUPT_MASK; 7476 } 7477 if (rdev->irq.crtc_vblank_int[4] || 7478 atomic_read(&rdev->irq.pflip[4])) { 7479 DRM_DEBUG("cik_irq_set: vblank 4\n"); 7480 crtc5 |= VBLANK_INTERRUPT_MASK; 7481 } 7482 if (rdev->irq.crtc_vblank_int[5] || 7483 atomic_read(&rdev->irq.pflip[5])) { 7484 DRM_DEBUG("cik_irq_set: vblank 5\n"); 7485 crtc6 |= VBLANK_INTERRUPT_MASK; 7486 } 7487 if (rdev->irq.hpd[0]) { 7488 DRM_DEBUG("cik_irq_set: hpd 1\n"); 7489 hpd1 |= DC_HPDx_INT_EN; 7490 } 7491 if (rdev->irq.hpd[1]) { 7492 DRM_DEBUG("cik_irq_set: hpd 2\n"); 7493 hpd2 |= DC_HPDx_INT_EN; 7494 } 7495 if (rdev->irq.hpd[2]) { 7496 DRM_DEBUG("cik_irq_set: hpd 3\n"); 7497 hpd3 |= DC_HPDx_INT_EN; 7498 } 7499 if (rdev->irq.hpd[3]) { 7500 DRM_DEBUG("cik_irq_set: hpd 4\n"); 7501 hpd4 |= DC_HPDx_INT_EN; 7502 } 7503 if (rdev->irq.hpd[4]) { 7504 DRM_DEBUG("cik_irq_set: hpd 5\n"); 7505 hpd5 |= DC_HPDx_INT_EN; 7506 } 7507 if (rdev->irq.hpd[5]) { 7508 DRM_DEBUG("cik_irq_set: hpd 6\n"); 7509 hpd6 |= DC_HPDx_INT_EN; 7510 } 7511 7512 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 7513 7514 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); 7515 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); 7516 7517 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); 7518 7519 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 7520 7521 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 7522 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 7523 if (rdev->num_crtc >= 4) { 7524 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 7525 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 7526 } 7527 if (rdev->num_crtc >= 6) { 7528 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 7529 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 7530 } 7531 7532 if (rdev->num_crtc >= 2) { 7533 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 7534 GRPH_PFLIP_INT_MASK); 7535 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 7536 GRPH_PFLIP_INT_MASK); 7537 } 7538 if (rdev->num_crtc >= 4) { 7539 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 7540 GRPH_PFLIP_INT_MASK); 7541 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 7542 GRPH_PFLIP_INT_MASK); 7543 } 7544 if (rdev->num_crtc >= 6) { 7545 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 7546 GRPH_PFLIP_INT_MASK); 7547 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 7548 GRPH_PFLIP_INT_MASK); 7549 } 7550 7551 WREG32(DC_HPD1_INT_CONTROL, hpd1); 7552 WREG32(DC_HPD2_INT_CONTROL, hpd2); 7553 WREG32(DC_HPD3_INT_CONTROL, hpd3); 7554 WREG32(DC_HPD4_INT_CONTROL, hpd4); 7555 WREG32(DC_HPD5_INT_CONTROL, hpd5); 7556 WREG32(DC_HPD6_INT_CONTROL, hpd6); 7557 7558 /* posting read */ 7559 RREG32(SRBM_STATUS); 7560 7561 return 0; 7562 } 7563 7564 /** 7565 * cik_irq_ack - ack interrupt sources 7566 * 7567 * @rdev: radeon_device pointer 7568 * 7569 * Ack interrupt sources on the GPU (vblanks, hpd, 7570 * etc.) (CIK). Certain interrupts sources are sw 7571 * generated and do not require an explicit ack. 7572 */ 7573 static inline void cik_irq_ack(struct radeon_device *rdev) 7574 { 7575 u32 tmp; 7576 7577 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); 7578 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 7579 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 7580 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 7581 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 7582 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 7583 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); 7584 7585 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + 7586 EVERGREEN_CRTC0_REGISTER_OFFSET); 7587 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + 7588 EVERGREEN_CRTC1_REGISTER_OFFSET); 7589 if (rdev->num_crtc >= 4) { 7590 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + 7591 EVERGREEN_CRTC2_REGISTER_OFFSET); 7592 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + 7593 EVERGREEN_CRTC3_REGISTER_OFFSET); 7594 } 7595 if (rdev->num_crtc >= 6) { 7596 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + 7597 EVERGREEN_CRTC4_REGISTER_OFFSET); 7598 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + 7599 EVERGREEN_CRTC5_REGISTER_OFFSET); 7600 } 7601 7602 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 7603 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, 7604 GRPH_PFLIP_INT_CLEAR); 7605 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 7606 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, 7607 GRPH_PFLIP_INT_CLEAR); 7608 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) 7609 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 7610 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) 7611 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 7612 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 7613 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 7614 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) 7615 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 7616 7617 if (rdev->num_crtc >= 4) { 7618 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 7619 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, 7620 GRPH_PFLIP_INT_CLEAR); 7621 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 7622 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, 7623 GRPH_PFLIP_INT_CLEAR); 7624 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 7625 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 7626 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 7627 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 7628 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 7629 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 7630 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 7631 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 7632 } 7633 7634 if (rdev->num_crtc >= 6) { 7635 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 7636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, 7637 GRPH_PFLIP_INT_CLEAR); 7638 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 7639 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, 7640 GRPH_PFLIP_INT_CLEAR); 7641 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 7642 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 7643 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 7644 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 7645 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 7646 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 7647 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 7648 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 7649 } 7650 7651 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7652 tmp = RREG32(DC_HPD1_INT_CONTROL); 7653 tmp |= DC_HPDx_INT_ACK; 7654 WREG32(DC_HPD1_INT_CONTROL, tmp); 7655 } 7656 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7657 tmp = RREG32(DC_HPD2_INT_CONTROL); 7658 tmp |= DC_HPDx_INT_ACK; 7659 WREG32(DC_HPD2_INT_CONTROL, tmp); 7660 } 7661 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7662 tmp = RREG32(DC_HPD3_INT_CONTROL); 7663 tmp |= DC_HPDx_INT_ACK; 7664 WREG32(DC_HPD3_INT_CONTROL, tmp); 7665 } 7666 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7667 tmp = RREG32(DC_HPD4_INT_CONTROL); 7668 tmp |= DC_HPDx_INT_ACK; 7669 WREG32(DC_HPD4_INT_CONTROL, tmp); 7670 } 7671 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7672 tmp = RREG32(DC_HPD5_INT_CONTROL); 7673 tmp |= DC_HPDx_INT_ACK; 7674 WREG32(DC_HPD5_INT_CONTROL, tmp); 7675 } 7676 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7677 tmp = RREG32(DC_HPD5_INT_CONTROL); 7678 tmp |= DC_HPDx_INT_ACK; 7679 WREG32(DC_HPD6_INT_CONTROL, tmp); 7680 } 7681 } 7682 7683 /** 7684 * cik_irq_disable - disable interrupts 7685 * 7686 * @rdev: radeon_device pointer 7687 * 7688 * Disable interrupts on the hw (CIK). 7689 */ 7690 static void cik_irq_disable(struct radeon_device *rdev) 7691 { 7692 cik_disable_interrupts(rdev); 7693 /* Wait and acknowledge irq */ 7694 mdelay(1); 7695 cik_irq_ack(rdev); 7696 cik_disable_interrupt_state(rdev); 7697 } 7698 7699 /** 7700 * cik_irq_disable - disable interrupts for suspend 7701 * 7702 * @rdev: radeon_device pointer 7703 * 7704 * Disable interrupts and stop the RLC (CIK). 7705 * Used for suspend. 7706 */ 7707 static void cik_irq_suspend(struct radeon_device *rdev) 7708 { 7709 cik_irq_disable(rdev); 7710 cik_rlc_stop(rdev); 7711 } 7712 7713 /** 7714 * cik_irq_fini - tear down interrupt support 7715 * 7716 * @rdev: radeon_device pointer 7717 * 7718 * Disable interrupts on the hw and free the IH ring 7719 * buffer (CIK). 7720 * Used for driver unload. 7721 */ 7722 static void cik_irq_fini(struct radeon_device *rdev) 7723 { 7724 cik_irq_suspend(rdev); 7725 r600_ih_ring_fini(rdev); 7726 } 7727 7728 /** 7729 * cik_get_ih_wptr - get the IH ring buffer wptr 7730 * 7731 * @rdev: radeon_device pointer 7732 * 7733 * Get the IH ring buffer wptr from either the register 7734 * or the writeback memory buffer (CIK). Also check for 7735 * ring buffer overflow and deal with it. 7736 * Used by cik_irq_process(). 7737 * Returns the value of the wptr. 7738 */ 7739 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) 7740 { 7741 u32 wptr, tmp; 7742 7743 if (rdev->wb.enabled) 7744 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 7745 else 7746 wptr = RREG32(IH_RB_WPTR); 7747 7748 if (wptr & RB_OVERFLOW) { 7749 wptr &= ~RB_OVERFLOW; 7750 /* When a ring buffer overflow happen start parsing interrupt 7751 * from the last not overwritten vector (wptr + 16). Hopefully 7752 * this should allow us to catchup. 7753 */ 7754 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 7755 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 7756 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 7757 tmp = RREG32(IH_RB_CNTL); 7758 tmp |= IH_WPTR_OVERFLOW_CLEAR; 7759 WREG32(IH_RB_CNTL, tmp); 7760 } 7761 return (wptr & rdev->ih.ptr_mask); 7762 } 7763 7764 /* CIK IV Ring 7765 * Each IV ring entry is 128 bits: 7766 * [7:0] - interrupt source id 7767 * [31:8] - reserved 7768 * [59:32] - interrupt source data 7769 * [63:60] - reserved 7770 * [71:64] - RINGID 7771 * CP: 7772 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] 7773 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher 7774 * - for gfx, hw shader state (0=PS...5=LS, 6=CS) 7775 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes 7776 * PIPE_ID - ME0 0=3D 7777 * - ME1&2 compute dispatcher (4 pipes each) 7778 * SDMA: 7779 * INSTANCE_ID [1:0], QUEUE_ID[1:0] 7780 * INSTANCE_ID - 0 = sdma0, 1 = sdma1 7781 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 7782 * [79:72] - VMID 7783 * [95:80] - PASID 7784 * [127:96] - reserved 7785 */ 7786 /** 7787 * cik_irq_process - interrupt handler 7788 * 7789 * @rdev: radeon_device pointer 7790 * 7791 * Interrupt hander (CIK). Walk the IH ring, 7792 * ack interrupts and schedule work to handle 7793 * interrupt events. 7794 * Returns irq process return code. 7795 */ 7796 int cik_irq_process(struct radeon_device *rdev) 7797 { 7798 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7799 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7800 u32 wptr; 7801 u32 rptr; 7802 u32 src_id, src_data, ring_id; 7803 u8 me_id, pipe_id, queue_id; 7804 u32 ring_index; 7805 bool queue_hotplug = false; 7806 bool queue_reset = false; 7807 u32 addr, status, mc_client; 7808 bool queue_thermal = false; 7809 7810 if (!rdev->ih.enabled || rdev->shutdown) 7811 return IRQ_NONE; 7812 7813 wptr = cik_get_ih_wptr(rdev); 7814 7815 restart_ih: 7816 /* is somebody else already processing irqs? */ 7817 if (atomic_xchg(&rdev->ih.lock, 1)) 7818 return IRQ_NONE; 7819 7820 rptr = rdev->ih.rptr; 7821 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 7822 7823 /* Order reading of wptr vs. reading of IH ring data */ 7824 rmb(); 7825 7826 /* display interrupts */ 7827 cik_irq_ack(rdev); 7828 7829 while (rptr != wptr) { 7830 /* wptr/rptr are in bytes! */ 7831 ring_index = rptr / 4; 7832 7833 radeon_kfd_interrupt(rdev, 7834 (const void *) &rdev->ih.ring[ring_index]); 7835 7836 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7837 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7838 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7839 7840 switch (src_id) { 7841 case 1: /* D1 vblank/vline */ 7842 switch (src_data) { 7843 case 0: /* D1 vblank */ 7844 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) { 7845 if (rdev->irq.crtc_vblank_int[0]) { 7846 drm_handle_vblank(rdev->ddev, 0); 7847 rdev->pm.vblank_sync = true; 7848 wake_up(&rdev->irq.vblank_queue); 7849 } 7850 if (atomic_read(&rdev->irq.pflip[0])) 7851 radeon_crtc_handle_vblank(rdev, 0); 7852 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 7853 DRM_DEBUG("IH: D1 vblank\n"); 7854 } 7855 break; 7856 case 1: /* D1 vline */ 7857 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) { 7858 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; 7859 DRM_DEBUG("IH: D1 vline\n"); 7860 } 7861 break; 7862 default: 7863 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7864 break; 7865 } 7866 break; 7867 case 2: /* D2 vblank/vline */ 7868 switch (src_data) { 7869 case 0: /* D2 vblank */ 7870 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 7871 if (rdev->irq.crtc_vblank_int[1]) { 7872 drm_handle_vblank(rdev->ddev, 1); 7873 rdev->pm.vblank_sync = true; 7874 wake_up(&rdev->irq.vblank_queue); 7875 } 7876 if (atomic_read(&rdev->irq.pflip[1])) 7877 radeon_crtc_handle_vblank(rdev, 1); 7878 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 7879 DRM_DEBUG("IH: D2 vblank\n"); 7880 } 7881 break; 7882 case 1: /* D2 vline */ 7883 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 7884 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 7885 DRM_DEBUG("IH: D2 vline\n"); 7886 } 7887 break; 7888 default: 7889 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7890 break; 7891 } 7892 break; 7893 case 3: /* D3 vblank/vline */ 7894 switch (src_data) { 7895 case 0: /* D3 vblank */ 7896 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 7897 if (rdev->irq.crtc_vblank_int[2]) { 7898 drm_handle_vblank(rdev->ddev, 2); 7899 rdev->pm.vblank_sync = true; 7900 wake_up(&rdev->irq.vblank_queue); 7901 } 7902 if (atomic_read(&rdev->irq.pflip[2])) 7903 radeon_crtc_handle_vblank(rdev, 2); 7904 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 7905 DRM_DEBUG("IH: D3 vblank\n"); 7906 } 7907 break; 7908 case 1: /* D3 vline */ 7909 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 7910 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 7911 DRM_DEBUG("IH: D3 vline\n"); 7912 } 7913 break; 7914 default: 7915 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7916 break; 7917 } 7918 break; 7919 case 4: /* D4 vblank/vline */ 7920 switch (src_data) { 7921 case 0: /* D4 vblank */ 7922 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 7923 if (rdev->irq.crtc_vblank_int[3]) { 7924 drm_handle_vblank(rdev->ddev, 3); 7925 rdev->pm.vblank_sync = true; 7926 wake_up(&rdev->irq.vblank_queue); 7927 } 7928 if (atomic_read(&rdev->irq.pflip[3])) 7929 radeon_crtc_handle_vblank(rdev, 3); 7930 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 7931 DRM_DEBUG("IH: D4 vblank\n"); 7932 } 7933 break; 7934 case 1: /* D4 vline */ 7935 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 7936 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 7937 DRM_DEBUG("IH: D4 vline\n"); 7938 } 7939 break; 7940 default: 7941 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7942 break; 7943 } 7944 break; 7945 case 5: /* D5 vblank/vline */ 7946 switch (src_data) { 7947 case 0: /* D5 vblank */ 7948 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 7949 if (rdev->irq.crtc_vblank_int[4]) { 7950 drm_handle_vblank(rdev->ddev, 4); 7951 rdev->pm.vblank_sync = true; 7952 wake_up(&rdev->irq.vblank_queue); 7953 } 7954 if (atomic_read(&rdev->irq.pflip[4])) 7955 radeon_crtc_handle_vblank(rdev, 4); 7956 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 7957 DRM_DEBUG("IH: D5 vblank\n"); 7958 } 7959 break; 7960 case 1: /* D5 vline */ 7961 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 7962 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 7963 DRM_DEBUG("IH: D5 vline\n"); 7964 } 7965 break; 7966 default: 7967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7968 break; 7969 } 7970 break; 7971 case 6: /* D6 vblank/vline */ 7972 switch (src_data) { 7973 case 0: /* D6 vblank */ 7974 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 7975 if (rdev->irq.crtc_vblank_int[5]) { 7976 drm_handle_vblank(rdev->ddev, 5); 7977 rdev->pm.vblank_sync = true; 7978 wake_up(&rdev->irq.vblank_queue); 7979 } 7980 if (atomic_read(&rdev->irq.pflip[5])) 7981 radeon_crtc_handle_vblank(rdev, 5); 7982 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 7983 DRM_DEBUG("IH: D6 vblank\n"); 7984 } 7985 break; 7986 case 1: /* D6 vline */ 7987 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 7988 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 7989 DRM_DEBUG("IH: D6 vline\n"); 7990 } 7991 break; 7992 default: 7993 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7994 break; 7995 } 7996 break; 7997 case 8: /* D1 page flip */ 7998 case 10: /* D2 page flip */ 7999 case 12: /* D3 page flip */ 8000 case 14: /* D4 page flip */ 8001 case 16: /* D5 page flip */ 8002 case 18: /* D6 page flip */ 8003 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 8004 if (radeon_use_pflipirq > 0) 8005 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 8006 break; 8007 case 42: /* HPD hotplug */ 8008 switch (src_data) { 8009 case 0: 8010 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 8011 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; 8012 queue_hotplug = true; 8013 DRM_DEBUG("IH: HPD1\n"); 8014 } 8015 break; 8016 case 1: 8017 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 8018 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; 8019 queue_hotplug = true; 8020 DRM_DEBUG("IH: HPD2\n"); 8021 } 8022 break; 8023 case 2: 8024 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 8025 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 8026 queue_hotplug = true; 8027 DRM_DEBUG("IH: HPD3\n"); 8028 } 8029 break; 8030 case 3: 8031 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 8032 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 8033 queue_hotplug = true; 8034 DRM_DEBUG("IH: HPD4\n"); 8035 } 8036 break; 8037 case 4: 8038 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 8039 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 8040 queue_hotplug = true; 8041 DRM_DEBUG("IH: HPD5\n"); 8042 } 8043 break; 8044 case 5: 8045 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 8046 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 8047 queue_hotplug = true; 8048 DRM_DEBUG("IH: HPD6\n"); 8049 } 8050 break; 8051 default: 8052 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8053 break; 8054 } 8055 break; 8056 case 96: 8057 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); 8058 WREG32(SRBM_INT_ACK, 0x1); 8059 break; 8060 case 124: /* UVD */ 8061 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 8062 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 8063 break; 8064 case 146: 8065 case 147: 8066 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 8067 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 8068 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); 8069 /* reset addr and status */ 8070 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 8071 if (addr == 0x0 && status == 0x0) 8072 break; 8073 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 8074 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 8075 addr); 8076 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 8077 status); 8078 cik_vm_decode_fault(rdev, status, addr, mc_client); 8079 break; 8080 case 167: /* VCE */ 8081 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); 8082 switch (src_data) { 8083 case 0: 8084 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); 8085 break; 8086 case 1: 8087 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); 8088 break; 8089 default: 8090 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 8091 break; 8092 } 8093 break; 8094 case 176: /* GFX RB CP_INT */ 8095 case 177: /* GFX IB CP_INT */ 8096 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8097 break; 8098 case 181: /* CP EOP event */ 8099 DRM_DEBUG("IH: CP EOP\n"); 8100 /* XXX check the bitfield order! */ 8101 me_id = (ring_id & 0x60) >> 5; 8102 pipe_id = (ring_id & 0x18) >> 3; 8103 queue_id = (ring_id & 0x7) >> 0; 8104 switch (me_id) { 8105 case 0: 8106 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8107 break; 8108 case 1: 8109 case 2: 8110 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) 8111 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8112 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) 8113 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8114 break; 8115 } 8116 break; 8117 case 184: /* CP Privileged reg access */ 8118 DRM_ERROR("Illegal register access in command stream\n"); 8119 /* XXX check the bitfield order! */ 8120 me_id = (ring_id & 0x60) >> 5; 8121 pipe_id = (ring_id & 0x18) >> 3; 8122 queue_id = (ring_id & 0x7) >> 0; 8123 switch (me_id) { 8124 case 0: 8125 /* This results in a full GPU reset, but all we need to do is soft 8126 * reset the CP for gfx 8127 */ 8128 queue_reset = true; 8129 break; 8130 case 1: 8131 /* XXX compute */ 8132 queue_reset = true; 8133 break; 8134 case 2: 8135 /* XXX compute */ 8136 queue_reset = true; 8137 break; 8138 } 8139 break; 8140 case 185: /* CP Privileged inst */ 8141 DRM_ERROR("Illegal instruction in command stream\n"); 8142 /* XXX check the bitfield order! */ 8143 me_id = (ring_id & 0x60) >> 5; 8144 pipe_id = (ring_id & 0x18) >> 3; 8145 queue_id = (ring_id & 0x7) >> 0; 8146 switch (me_id) { 8147 case 0: 8148 /* This results in a full GPU reset, but all we need to do is soft 8149 * reset the CP for gfx 8150 */ 8151 queue_reset = true; 8152 break; 8153 case 1: 8154 /* XXX compute */ 8155 queue_reset = true; 8156 break; 8157 case 2: 8158 /* XXX compute */ 8159 queue_reset = true; 8160 break; 8161 } 8162 break; 8163 case 224: /* SDMA trap event */ 8164 /* XXX check the bitfield order! */ 8165 me_id = (ring_id & 0x3) >> 0; 8166 queue_id = (ring_id & 0xc) >> 2; 8167 DRM_DEBUG("IH: SDMA trap\n"); 8168 switch (me_id) { 8169 case 0: 8170 switch (queue_id) { 8171 case 0: 8172 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 8173 break; 8174 case 1: 8175 /* XXX compute */ 8176 break; 8177 case 2: 8178 /* XXX compute */ 8179 break; 8180 } 8181 break; 8182 case 1: 8183 switch (queue_id) { 8184 case 0: 8185 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8186 break; 8187 case 1: 8188 /* XXX compute */ 8189 break; 8190 case 2: 8191 /* XXX compute */ 8192 break; 8193 } 8194 break; 8195 } 8196 break; 8197 case 230: /* thermal low to high */ 8198 DRM_DEBUG("IH: thermal low to high\n"); 8199 rdev->pm.dpm.thermal.high_to_low = false; 8200 queue_thermal = true; 8201 break; 8202 case 231: /* thermal high to low */ 8203 DRM_DEBUG("IH: thermal high to low\n"); 8204 rdev->pm.dpm.thermal.high_to_low = true; 8205 queue_thermal = true; 8206 break; 8207 case 233: /* GUI IDLE */ 8208 DRM_DEBUG("IH: GUI idle\n"); 8209 break; 8210 case 241: /* SDMA Privileged inst */ 8211 case 247: /* SDMA Privileged inst */ 8212 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 8213 /* XXX check the bitfield order! */ 8214 me_id = (ring_id & 0x3) >> 0; 8215 queue_id = (ring_id & 0xc) >> 2; 8216 switch (me_id) { 8217 case 0: 8218 switch (queue_id) { 8219 case 0: 8220 queue_reset = true; 8221 break; 8222 case 1: 8223 /* XXX compute */ 8224 queue_reset = true; 8225 break; 8226 case 2: 8227 /* XXX compute */ 8228 queue_reset = true; 8229 break; 8230 } 8231 break; 8232 case 1: 8233 switch (queue_id) { 8234 case 0: 8235 queue_reset = true; 8236 break; 8237 case 1: 8238 /* XXX compute */ 8239 queue_reset = true; 8240 break; 8241 case 2: 8242 /* XXX compute */ 8243 queue_reset = true; 8244 break; 8245 } 8246 break; 8247 } 8248 break; 8249 default: 8250 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8251 break; 8252 } 8253 8254 /* wptr/rptr are in bytes! */ 8255 rptr += 16; 8256 rptr &= rdev->ih.ptr_mask; 8257 WREG32(IH_RB_RPTR, rptr); 8258 } 8259 if (queue_hotplug) 8260 schedule_work(&rdev->hotplug_work); 8261 if (queue_reset) { 8262 rdev->needs_reset = true; 8263 wake_up_all(&rdev->fence_queue); 8264 } 8265 if (queue_thermal) 8266 schedule_work(&rdev->pm.dpm.thermal.work); 8267 rdev->ih.rptr = rptr; 8268 atomic_set(&rdev->ih.lock, 0); 8269 8270 /* make sure wptr hasn't changed while processing */ 8271 wptr = cik_get_ih_wptr(rdev); 8272 if (wptr != rptr) 8273 goto restart_ih; 8274 8275 return IRQ_HANDLED; 8276 } 8277 8278 /* 8279 * startup/shutdown callbacks 8280 */ 8281 /** 8282 * cik_startup - program the asic to a functional state 8283 * 8284 * @rdev: radeon_device pointer 8285 * 8286 * Programs the asic to a functional state (CIK). 8287 * Called by cik_init() and cik_resume(). 8288 * Returns 0 for success, error for failure. 8289 */ 8290 static int cik_startup(struct radeon_device *rdev) 8291 { 8292 struct radeon_ring *ring; 8293 u32 nop; 8294 int r; 8295 8296 /* enable pcie gen2/3 link */ 8297 cik_pcie_gen3_enable(rdev); 8298 /* enable aspm */ 8299 cik_program_aspm(rdev); 8300 8301 /* scratch needs to be initialized before MC */ 8302 r = r600_vram_scratch_init(rdev); 8303 if (r) 8304 return r; 8305 8306 cik_mc_program(rdev); 8307 8308 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { 8309 r = ci_mc_load_microcode(rdev); 8310 if (r) { 8311 DRM_ERROR("Failed to load MC firmware!\n"); 8312 return r; 8313 } 8314 } 8315 8316 r = cik_pcie_gart_enable(rdev); 8317 if (r) 8318 return r; 8319 cik_gpu_init(rdev); 8320 8321 /* allocate rlc buffers */ 8322 if (rdev->flags & RADEON_IS_IGP) { 8323 if (rdev->family == CHIP_KAVERI) { 8324 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; 8325 rdev->rlc.reg_list_size = 8326 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); 8327 } else { 8328 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; 8329 rdev->rlc.reg_list_size = 8330 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); 8331 } 8332 } 8333 rdev->rlc.cs_data = ci_cs_data; 8334 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; 8335 r = sumo_rlc_init(rdev); 8336 if (r) { 8337 DRM_ERROR("Failed to init rlc BOs!\n"); 8338 return r; 8339 } 8340 8341 /* allocate wb buffer */ 8342 r = radeon_wb_init(rdev); 8343 if (r) 8344 return r; 8345 8346 /* allocate mec buffers */ 8347 r = cik_mec_init(rdev); 8348 if (r) { 8349 DRM_ERROR("Failed to init MEC BOs!\n"); 8350 return r; 8351 } 8352 8353 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 8354 if (r) { 8355 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8356 return r; 8357 } 8358 8359 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8360 if (r) { 8361 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8362 return r; 8363 } 8364 8365 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8366 if (r) { 8367 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8368 return r; 8369 } 8370 8371 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 8372 if (r) { 8373 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8374 return r; 8375 } 8376 8377 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8378 if (r) { 8379 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8380 return r; 8381 } 8382 8383 r = radeon_uvd_resume(rdev); 8384 if (!r) { 8385 r = uvd_v4_2_resume(rdev); 8386 if (!r) { 8387 r = radeon_fence_driver_start_ring(rdev, 8388 R600_RING_TYPE_UVD_INDEX); 8389 if (r) 8390 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 8391 } 8392 } 8393 if (r) 8394 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 8395 8396 r = radeon_vce_resume(rdev); 8397 if (!r) { 8398 r = vce_v2_0_resume(rdev); 8399 if (!r) 8400 r = radeon_fence_driver_start_ring(rdev, 8401 TN_RING_TYPE_VCE1_INDEX); 8402 if (!r) 8403 r = radeon_fence_driver_start_ring(rdev, 8404 TN_RING_TYPE_VCE2_INDEX); 8405 } 8406 if (r) { 8407 dev_err(rdev->dev, "VCE init error (%d).\n", r); 8408 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; 8409 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; 8410 } 8411 8412 /* Enable IRQ */ 8413 if (!rdev->irq.installed) { 8414 r = radeon_irq_kms_init(rdev); 8415 if (r) 8416 return r; 8417 } 8418 8419 r = cik_irq_init(rdev); 8420 if (r) { 8421 DRM_ERROR("radeon: IH init failed (%d).\n", r); 8422 radeon_irq_kms_fini(rdev); 8423 return r; 8424 } 8425 cik_irq_set(rdev); 8426 8427 if (rdev->family == CHIP_HAWAII) { 8428 if (rdev->new_fw) 8429 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8430 else 8431 nop = RADEON_CP_PACKET2; 8432 } else { 8433 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8434 } 8435 8436 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8437 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 8438 nop); 8439 if (r) 8440 return r; 8441 8442 /* set up the compute queues */ 8443 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8444 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8445 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 8446 nop); 8447 if (r) 8448 return r; 8449 ring->me = 1; /* first MEC */ 8450 ring->pipe = 0; /* first pipe */ 8451 ring->queue = 0; /* first queue */ 8452 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; 8453 8454 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8455 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8456 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 8457 nop); 8458 if (r) 8459 return r; 8460 /* dGPU only have 1 MEC */ 8461 ring->me = 1; /* first MEC */ 8462 ring->pipe = 0; /* first pipe */ 8463 ring->queue = 1; /* second queue */ 8464 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; 8465 8466 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8467 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 8468 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8469 if (r) 8470 return r; 8471 8472 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8473 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 8474 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8475 if (r) 8476 return r; 8477 8478 r = cik_cp_resume(rdev); 8479 if (r) 8480 return r; 8481 8482 r = cik_sdma_resume(rdev); 8483 if (r) 8484 return r; 8485 8486 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8487 if (ring->ring_size) { 8488 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8489 RADEON_CP_PACKET2); 8490 if (!r) 8491 r = uvd_v1_0_init(rdev); 8492 if (r) 8493 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 8494 } 8495 8496 r = -ENOENT; 8497 8498 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8499 if (ring->ring_size) 8500 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8501 VCE_CMD_NO_OP); 8502 8503 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8504 if (ring->ring_size) 8505 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8506 VCE_CMD_NO_OP); 8507 8508 if (!r) 8509 r = vce_v1_0_init(rdev); 8510 else if (r != -ENOENT) 8511 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); 8512 8513 r = radeon_ib_pool_init(rdev); 8514 if (r) { 8515 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 8516 return r; 8517 } 8518 8519 r = radeon_vm_manager_init(rdev); 8520 if (r) { 8521 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 8522 return r; 8523 } 8524 8525 r = radeon_audio_init(rdev); 8526 if (r) 8527 return r; 8528 8529 r = radeon_kfd_resume(rdev); 8530 if (r) 8531 return r; 8532 8533 return 0; 8534 } 8535 8536 /** 8537 * cik_resume - resume the asic to a functional state 8538 * 8539 * @rdev: radeon_device pointer 8540 * 8541 * Programs the asic to a functional state (CIK). 8542 * Called at resume. 8543 * Returns 0 for success, error for failure. 8544 */ 8545 int cik_resume(struct radeon_device *rdev) 8546 { 8547 int r; 8548 8549 /* post card */ 8550 atom_asic_init(rdev->mode_info.atom_context); 8551 8552 /* init golden registers */ 8553 cik_init_golden_registers(rdev); 8554 8555 if (rdev->pm.pm_method == PM_METHOD_DPM) 8556 radeon_pm_resume(rdev); 8557 8558 rdev->accel_working = true; 8559 r = cik_startup(rdev); 8560 if (r) { 8561 DRM_ERROR("cik startup failed on resume\n"); 8562 rdev->accel_working = false; 8563 return r; 8564 } 8565 8566 return r; 8567 8568 } 8569 8570 /** 8571 * cik_suspend - suspend the asic 8572 * 8573 * @rdev: radeon_device pointer 8574 * 8575 * Bring the chip into a state suitable for suspend (CIK). 8576 * Called at suspend. 8577 * Returns 0 for success. 8578 */ 8579 int cik_suspend(struct radeon_device *rdev) 8580 { 8581 radeon_kfd_suspend(rdev); 8582 radeon_pm_suspend(rdev); 8583 radeon_audio_fini(rdev); 8584 radeon_vm_manager_fini(rdev); 8585 cik_cp_enable(rdev, false); 8586 cik_sdma_enable(rdev, false); 8587 uvd_v1_0_fini(rdev); 8588 radeon_uvd_suspend(rdev); 8589 radeon_vce_suspend(rdev); 8590 cik_fini_pg(rdev); 8591 cik_fini_cg(rdev); 8592 cik_irq_suspend(rdev); 8593 radeon_wb_disable(rdev); 8594 cik_pcie_gart_disable(rdev); 8595 return 0; 8596 } 8597 8598 /* Plan is to move initialization in that function and use 8599 * helper function so that radeon_device_init pretty much 8600 * do nothing more than calling asic specific function. This 8601 * should also allow to remove a bunch of callback function 8602 * like vram_info. 8603 */ 8604 /** 8605 * cik_init - asic specific driver and hw init 8606 * 8607 * @rdev: radeon_device pointer 8608 * 8609 * Setup asic specific driver variables and program the hw 8610 * to a functional state (CIK). 8611 * Called at driver startup. 8612 * Returns 0 for success, errors for failure. 8613 */ 8614 int cik_init(struct radeon_device *rdev) 8615 { 8616 struct radeon_ring *ring; 8617 int r; 8618 8619 /* Read BIOS */ 8620 if (!radeon_get_bios(rdev)) { 8621 if (ASIC_IS_AVIVO(rdev)) 8622 return -EINVAL; 8623 } 8624 /* Must be an ATOMBIOS */ 8625 if (!rdev->is_atom_bios) { 8626 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 8627 return -EINVAL; 8628 } 8629 r = radeon_atombios_init(rdev); 8630 if (r) 8631 return r; 8632 8633 /* Post card if necessary */ 8634 if (!radeon_card_posted(rdev)) { 8635 if (!rdev->bios) { 8636 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 8637 return -EINVAL; 8638 } 8639 DRM_INFO("GPU not posted. posting now...\n"); 8640 atom_asic_init(rdev->mode_info.atom_context); 8641 } 8642 /* init golden registers */ 8643 cik_init_golden_registers(rdev); 8644 /* Initialize scratch registers */ 8645 cik_scratch_init(rdev); 8646 /* Initialize surface registers */ 8647 radeon_surface_init(rdev); 8648 /* Initialize clocks */ 8649 radeon_get_clock_info(rdev->ddev); 8650 8651 /* Fence driver */ 8652 r = radeon_fence_driver_init(rdev); 8653 if (r) 8654 return r; 8655 8656 /* initialize memory controller */ 8657 r = cik_mc_init(rdev); 8658 if (r) 8659 return r; 8660 /* Memory manager */ 8661 r = radeon_bo_init(rdev); 8662 if (r) 8663 return r; 8664 8665 if (rdev->flags & RADEON_IS_IGP) { 8666 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8667 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { 8668 r = cik_init_microcode(rdev); 8669 if (r) { 8670 DRM_ERROR("Failed to load firmware!\n"); 8671 return r; 8672 } 8673 } 8674 } else { 8675 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8676 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || 8677 !rdev->mc_fw) { 8678 r = cik_init_microcode(rdev); 8679 if (r) { 8680 DRM_ERROR("Failed to load firmware!\n"); 8681 return r; 8682 } 8683 } 8684 } 8685 8686 /* Initialize power management */ 8687 radeon_pm_init(rdev); 8688 8689 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8690 ring->ring_obj = NULL; 8691 r600_ring_init(rdev, ring, 1024 * 1024); 8692 8693 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8694 ring->ring_obj = NULL; 8695 r600_ring_init(rdev, ring, 1024 * 1024); 8696 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8697 if (r) 8698 return r; 8699 8700 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8701 ring->ring_obj = NULL; 8702 r600_ring_init(rdev, ring, 1024 * 1024); 8703 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8704 if (r) 8705 return r; 8706 8707 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8708 ring->ring_obj = NULL; 8709 r600_ring_init(rdev, ring, 256 * 1024); 8710 8711 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8712 ring->ring_obj = NULL; 8713 r600_ring_init(rdev, ring, 256 * 1024); 8714 8715 r = radeon_uvd_init(rdev); 8716 if (!r) { 8717 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8718 ring->ring_obj = NULL; 8719 r600_ring_init(rdev, ring, 4096); 8720 } 8721 8722 r = radeon_vce_init(rdev); 8723 if (!r) { 8724 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8725 ring->ring_obj = NULL; 8726 r600_ring_init(rdev, ring, 4096); 8727 8728 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8729 ring->ring_obj = NULL; 8730 r600_ring_init(rdev, ring, 4096); 8731 } 8732 8733 rdev->ih.ring_obj = NULL; 8734 r600_ih_ring_init(rdev, 64 * 1024); 8735 8736 r = r600_pcie_gart_init(rdev); 8737 if (r) 8738 return r; 8739 8740 rdev->accel_working = true; 8741 r = cik_startup(rdev); 8742 if (r) { 8743 dev_err(rdev->dev, "disabling GPU acceleration\n"); 8744 cik_cp_fini(rdev); 8745 cik_sdma_fini(rdev); 8746 cik_irq_fini(rdev); 8747 sumo_rlc_fini(rdev); 8748 cik_mec_fini(rdev); 8749 radeon_wb_fini(rdev); 8750 radeon_ib_pool_fini(rdev); 8751 radeon_vm_manager_fini(rdev); 8752 radeon_irq_kms_fini(rdev); 8753 cik_pcie_gart_fini(rdev); 8754 rdev->accel_working = false; 8755 } 8756 8757 /* Don't start up if the MC ucode is missing. 8758 * The default clocks and voltages before the MC ucode 8759 * is loaded are not suffient for advanced operations. 8760 */ 8761 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 8762 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 8763 return -EINVAL; 8764 } 8765 8766 return 0; 8767 } 8768 8769 /** 8770 * cik_fini - asic specific driver and hw fini 8771 * 8772 * @rdev: radeon_device pointer 8773 * 8774 * Tear down the asic specific driver variables and program the hw 8775 * to an idle state (CIK). 8776 * Called at driver unload. 8777 */ 8778 void cik_fini(struct radeon_device *rdev) 8779 { 8780 radeon_pm_fini(rdev); 8781 cik_cp_fini(rdev); 8782 cik_sdma_fini(rdev); 8783 cik_fini_pg(rdev); 8784 cik_fini_cg(rdev); 8785 cik_irq_fini(rdev); 8786 sumo_rlc_fini(rdev); 8787 cik_mec_fini(rdev); 8788 radeon_wb_fini(rdev); 8789 radeon_vm_manager_fini(rdev); 8790 radeon_ib_pool_fini(rdev); 8791 radeon_irq_kms_fini(rdev); 8792 uvd_v1_0_fini(rdev); 8793 radeon_uvd_fini(rdev); 8794 radeon_vce_fini(rdev); 8795 cik_pcie_gart_fini(rdev); 8796 r600_vram_scratch_fini(rdev); 8797 radeon_gem_fini(rdev); 8798 radeon_fence_driver_fini(rdev); 8799 radeon_bo_fini(rdev); 8800 radeon_atombios_fini(rdev); 8801 kfree(rdev->bios); 8802 rdev->bios = NULL; 8803 } 8804 8805 void dce8_program_fmt(struct drm_encoder *encoder) 8806 { 8807 struct drm_device *dev = encoder->dev; 8808 struct radeon_device *rdev = dev->dev_private; 8809 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 8810 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 8811 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 8812 int bpc = 0; 8813 u32 tmp = 0; 8814 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 8815 8816 if (connector) { 8817 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 8818 bpc = radeon_get_monitor_bpc(connector); 8819 dither = radeon_connector->dither; 8820 } 8821 8822 /* LVDS/eDP FMT is set up by atom */ 8823 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 8824 return; 8825 8826 /* not needed for analog */ 8827 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 8828 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 8829 return; 8830 8831 if (bpc == 0) 8832 return; 8833 8834 switch (bpc) { 8835 case 6: 8836 if (dither == RADEON_FMT_DITHER_ENABLE) 8837 /* XXX sort out optimal dither settings */ 8838 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8839 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); 8840 else 8841 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); 8842 break; 8843 case 8: 8844 if (dither == RADEON_FMT_DITHER_ENABLE) 8845 /* XXX sort out optimal dither settings */ 8846 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8847 FMT_RGB_RANDOM_ENABLE | 8848 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); 8849 else 8850 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); 8851 break; 8852 case 10: 8853 if (dither == RADEON_FMT_DITHER_ENABLE) 8854 /* XXX sort out optimal dither settings */ 8855 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8856 FMT_RGB_RANDOM_ENABLE | 8857 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); 8858 else 8859 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); 8860 break; 8861 default: 8862 /* not needed */ 8863 break; 8864 } 8865 8866 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 8867 } 8868 8869 /* display watermark setup */ 8870 /** 8871 * dce8_line_buffer_adjust - Set up the line buffer 8872 * 8873 * @rdev: radeon_device pointer 8874 * @radeon_crtc: the selected display controller 8875 * @mode: the current display mode on the selected display 8876 * controller 8877 * 8878 * Setup up the line buffer allocation for 8879 * the selected display controller (CIK). 8880 * Returns the line buffer size in pixels. 8881 */ 8882 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, 8883 struct radeon_crtc *radeon_crtc, 8884 struct drm_display_mode *mode) 8885 { 8886 u32 tmp, buffer_alloc, i; 8887 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 8888 /* 8889 * Line Buffer Setup 8890 * There are 6 line buffers, one for each display controllers. 8891 * There are 3 partitions per LB. Select the number of partitions 8892 * to enable based on the display width. For display widths larger 8893 * than 4096, you need use to use 2 display controllers and combine 8894 * them using the stereo blender. 8895 */ 8896 if (radeon_crtc->base.enabled && mode) { 8897 if (mode->crtc_hdisplay < 1920) { 8898 tmp = 1; 8899 buffer_alloc = 2; 8900 } else if (mode->crtc_hdisplay < 2560) { 8901 tmp = 2; 8902 buffer_alloc = 2; 8903 } else if (mode->crtc_hdisplay < 4096) { 8904 tmp = 0; 8905 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8906 } else { 8907 DRM_DEBUG_KMS("Mode too big for LB!\n"); 8908 tmp = 0; 8909 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8910 } 8911 } else { 8912 tmp = 1; 8913 buffer_alloc = 0; 8914 } 8915 8916 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, 8917 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); 8918 8919 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 8920 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 8921 for (i = 0; i < rdev->usec_timeout; i++) { 8922 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 8923 DMIF_BUFFERS_ALLOCATED_COMPLETED) 8924 break; 8925 udelay(1); 8926 } 8927 8928 if (radeon_crtc->base.enabled && mode) { 8929 switch (tmp) { 8930 case 0: 8931 default: 8932 return 4096 * 2; 8933 case 1: 8934 return 1920 * 2; 8935 case 2: 8936 return 2560 * 2; 8937 } 8938 } 8939 8940 /* controller not enabled, so no lb used */ 8941 return 0; 8942 } 8943 8944 /** 8945 * cik_get_number_of_dram_channels - get the number of dram channels 8946 * 8947 * @rdev: radeon_device pointer 8948 * 8949 * Look up the number of video ram channels (CIK). 8950 * Used for display watermark bandwidth calculations 8951 * Returns the number of dram channels 8952 */ 8953 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) 8954 { 8955 u32 tmp = RREG32(MC_SHARED_CHMAP); 8956 8957 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 8958 case 0: 8959 default: 8960 return 1; 8961 case 1: 8962 return 2; 8963 case 2: 8964 return 4; 8965 case 3: 8966 return 8; 8967 case 4: 8968 return 3; 8969 case 5: 8970 return 6; 8971 case 6: 8972 return 10; 8973 case 7: 8974 return 12; 8975 case 8: 8976 return 16; 8977 } 8978 } 8979 8980 struct dce8_wm_params { 8981 u32 dram_channels; /* number of dram channels */ 8982 u32 yclk; /* bandwidth per dram data pin in kHz */ 8983 u32 sclk; /* engine clock in kHz */ 8984 u32 disp_clk; /* display clock in kHz */ 8985 u32 src_width; /* viewport width */ 8986 u32 active_time; /* active display time in ns */ 8987 u32 blank_time; /* blank time in ns */ 8988 bool interlaced; /* mode is interlaced */ 8989 fixed20_12 vsc; /* vertical scale ratio */ 8990 u32 num_heads; /* number of active crtcs */ 8991 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 8992 u32 lb_size; /* line buffer allocated to pipe */ 8993 u32 vtaps; /* vertical scaler taps */ 8994 }; 8995 8996 /** 8997 * dce8_dram_bandwidth - get the dram bandwidth 8998 * 8999 * @wm: watermark calculation data 9000 * 9001 * Calculate the raw dram bandwidth (CIK). 9002 * Used for display watermark bandwidth calculations 9003 * Returns the dram bandwidth in MBytes/s 9004 */ 9005 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) 9006 { 9007 /* Calculate raw DRAM Bandwidth */ 9008 fixed20_12 dram_efficiency; /* 0.7 */ 9009 fixed20_12 yclk, dram_channels, bandwidth; 9010 fixed20_12 a; 9011 9012 a.full = dfixed_const(1000); 9013 yclk.full = dfixed_const(wm->yclk); 9014 yclk.full = dfixed_div(yclk, a); 9015 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9016 a.full = dfixed_const(10); 9017 dram_efficiency.full = dfixed_const(7); 9018 dram_efficiency.full = dfixed_div(dram_efficiency, a); 9019 bandwidth.full = dfixed_mul(dram_channels, yclk); 9020 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 9021 9022 return dfixed_trunc(bandwidth); 9023 } 9024 9025 /** 9026 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display 9027 * 9028 * @wm: watermark calculation data 9029 * 9030 * Calculate the dram bandwidth used for display (CIK). 9031 * Used for display watermark bandwidth calculations 9032 * Returns the dram bandwidth for display in MBytes/s 9033 */ 9034 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9035 { 9036 /* Calculate DRAM Bandwidth and the part allocated to display. */ 9037 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 9038 fixed20_12 yclk, dram_channels, bandwidth; 9039 fixed20_12 a; 9040 9041 a.full = dfixed_const(1000); 9042 yclk.full = dfixed_const(wm->yclk); 9043 yclk.full = dfixed_div(yclk, a); 9044 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9045 a.full = dfixed_const(10); 9046 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 9047 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 9048 bandwidth.full = dfixed_mul(dram_channels, yclk); 9049 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 9050 9051 return dfixed_trunc(bandwidth); 9052 } 9053 9054 /** 9055 * dce8_data_return_bandwidth - get the data return bandwidth 9056 * 9057 * @wm: watermark calculation data 9058 * 9059 * Calculate the data return bandwidth used for display (CIK). 9060 * Used for display watermark bandwidth calculations 9061 * Returns the data return bandwidth in MBytes/s 9062 */ 9063 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) 9064 { 9065 /* Calculate the display Data return Bandwidth */ 9066 fixed20_12 return_efficiency; /* 0.8 */ 9067 fixed20_12 sclk, bandwidth; 9068 fixed20_12 a; 9069 9070 a.full = dfixed_const(1000); 9071 sclk.full = dfixed_const(wm->sclk); 9072 sclk.full = dfixed_div(sclk, a); 9073 a.full = dfixed_const(10); 9074 return_efficiency.full = dfixed_const(8); 9075 return_efficiency.full = dfixed_div(return_efficiency, a); 9076 a.full = dfixed_const(32); 9077 bandwidth.full = dfixed_mul(a, sclk); 9078 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 9079 9080 return dfixed_trunc(bandwidth); 9081 } 9082 9083 /** 9084 * dce8_dmif_request_bandwidth - get the dmif bandwidth 9085 * 9086 * @wm: watermark calculation data 9087 * 9088 * Calculate the dmif bandwidth used for display (CIK). 9089 * Used for display watermark bandwidth calculations 9090 * Returns the dmif bandwidth in MBytes/s 9091 */ 9092 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) 9093 { 9094 /* Calculate the DMIF Request Bandwidth */ 9095 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 9096 fixed20_12 disp_clk, bandwidth; 9097 fixed20_12 a, b; 9098 9099 a.full = dfixed_const(1000); 9100 disp_clk.full = dfixed_const(wm->disp_clk); 9101 disp_clk.full = dfixed_div(disp_clk, a); 9102 a.full = dfixed_const(32); 9103 b.full = dfixed_mul(a, disp_clk); 9104 9105 a.full = dfixed_const(10); 9106 disp_clk_request_efficiency.full = dfixed_const(8); 9107 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 9108 9109 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); 9110 9111 return dfixed_trunc(bandwidth); 9112 } 9113 9114 /** 9115 * dce8_available_bandwidth - get the min available bandwidth 9116 * 9117 * @wm: watermark calculation data 9118 * 9119 * Calculate the min available bandwidth used for display (CIK). 9120 * Used for display watermark bandwidth calculations 9121 * Returns the min available bandwidth in MBytes/s 9122 */ 9123 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) 9124 { 9125 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 9126 u32 dram_bandwidth = dce8_dram_bandwidth(wm); 9127 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); 9128 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); 9129 9130 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 9131 } 9132 9133 /** 9134 * dce8_average_bandwidth - get the average available bandwidth 9135 * 9136 * @wm: watermark calculation data 9137 * 9138 * Calculate the average available bandwidth used for display (CIK). 9139 * Used for display watermark bandwidth calculations 9140 * Returns the average available bandwidth in MBytes/s 9141 */ 9142 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) 9143 { 9144 /* Calculate the display mode Average Bandwidth 9145 * DisplayMode should contain the source and destination dimensions, 9146 * timing, etc. 9147 */ 9148 fixed20_12 bpp; 9149 fixed20_12 line_time; 9150 fixed20_12 src_width; 9151 fixed20_12 bandwidth; 9152 fixed20_12 a; 9153 9154 a.full = dfixed_const(1000); 9155 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 9156 line_time.full = dfixed_div(line_time, a); 9157 bpp.full = dfixed_const(wm->bytes_per_pixel); 9158 src_width.full = dfixed_const(wm->src_width); 9159 bandwidth.full = dfixed_mul(src_width, bpp); 9160 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 9161 bandwidth.full = dfixed_div(bandwidth, line_time); 9162 9163 return dfixed_trunc(bandwidth); 9164 } 9165 9166 /** 9167 * dce8_latency_watermark - get the latency watermark 9168 * 9169 * @wm: watermark calculation data 9170 * 9171 * Calculate the latency watermark (CIK). 9172 * Used for display watermark bandwidth calculations 9173 * Returns the latency watermark in ns 9174 */ 9175 static u32 dce8_latency_watermark(struct dce8_wm_params *wm) 9176 { 9177 /* First calculate the latency in ns */ 9178 u32 mc_latency = 2000; /* 2000 ns. */ 9179 u32 available_bandwidth = dce8_available_bandwidth(wm); 9180 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 9181 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 9182 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 9183 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 9184 (wm->num_heads * cursor_line_pair_return_time); 9185 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 9186 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 9187 u32 tmp, dmif_size = 12288; 9188 fixed20_12 a, b, c; 9189 9190 if (wm->num_heads == 0) 9191 return 0; 9192 9193 a.full = dfixed_const(2); 9194 b.full = dfixed_const(1); 9195 if ((wm->vsc.full > a.full) || 9196 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 9197 (wm->vtaps >= 5) || 9198 ((wm->vsc.full >= a.full) && wm->interlaced)) 9199 max_src_lines_per_dst_line = 4; 9200 else 9201 max_src_lines_per_dst_line = 2; 9202 9203 a.full = dfixed_const(available_bandwidth); 9204 b.full = dfixed_const(wm->num_heads); 9205 a.full = dfixed_div(a, b); 9206 9207 b.full = dfixed_const(mc_latency + 512); 9208 c.full = dfixed_const(wm->disp_clk); 9209 b.full = dfixed_div(b, c); 9210 9211 c.full = dfixed_const(dmif_size); 9212 b.full = dfixed_div(c, b); 9213 9214 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 9215 9216 b.full = dfixed_const(1000); 9217 c.full = dfixed_const(wm->disp_clk); 9218 b.full = dfixed_div(c, b); 9219 c.full = dfixed_const(wm->bytes_per_pixel); 9220 b.full = dfixed_mul(b, c); 9221 9222 lb_fill_bw = min(tmp, dfixed_trunc(b)); 9223 9224 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 9225 b.full = dfixed_const(1000); 9226 c.full = dfixed_const(lb_fill_bw); 9227 b.full = dfixed_div(c, b); 9228 a.full = dfixed_div(a, b); 9229 line_fill_time = dfixed_trunc(a); 9230 9231 if (line_fill_time < wm->active_time) 9232 return latency; 9233 else 9234 return latency + (line_fill_time - wm->active_time); 9235 9236 } 9237 9238 /** 9239 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check 9240 * average and available dram bandwidth 9241 * 9242 * @wm: watermark calculation data 9243 * 9244 * Check if the display average bandwidth fits in the display 9245 * dram bandwidth (CIK). 9246 * Used for display watermark bandwidth calculations 9247 * Returns true if the display fits, false if not. 9248 */ 9249 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9250 { 9251 if (dce8_average_bandwidth(wm) <= 9252 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) 9253 return true; 9254 else 9255 return false; 9256 } 9257 9258 /** 9259 * dce8_average_bandwidth_vs_available_bandwidth - check 9260 * average and available bandwidth 9261 * 9262 * @wm: watermark calculation data 9263 * 9264 * Check if the display average bandwidth fits in the display 9265 * available bandwidth (CIK). 9266 * Used for display watermark bandwidth calculations 9267 * Returns true if the display fits, false if not. 9268 */ 9269 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) 9270 { 9271 if (dce8_average_bandwidth(wm) <= 9272 (dce8_available_bandwidth(wm) / wm->num_heads)) 9273 return true; 9274 else 9275 return false; 9276 } 9277 9278 /** 9279 * dce8_check_latency_hiding - check latency hiding 9280 * 9281 * @wm: watermark calculation data 9282 * 9283 * Check latency hiding (CIK). 9284 * Used for display watermark bandwidth calculations 9285 * Returns true if the display fits, false if not. 9286 */ 9287 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) 9288 { 9289 u32 lb_partitions = wm->lb_size / wm->src_width; 9290 u32 line_time = wm->active_time + wm->blank_time; 9291 u32 latency_tolerant_lines; 9292 u32 latency_hiding; 9293 fixed20_12 a; 9294 9295 a.full = dfixed_const(1); 9296 if (wm->vsc.full > a.full) 9297 latency_tolerant_lines = 1; 9298 else { 9299 if (lb_partitions <= (wm->vtaps + 1)) 9300 latency_tolerant_lines = 1; 9301 else 9302 latency_tolerant_lines = 2; 9303 } 9304 9305 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 9306 9307 if (dce8_latency_watermark(wm) <= latency_hiding) 9308 return true; 9309 else 9310 return false; 9311 } 9312 9313 /** 9314 * dce8_program_watermarks - program display watermarks 9315 * 9316 * @rdev: radeon_device pointer 9317 * @radeon_crtc: the selected display controller 9318 * @lb_size: line buffer size 9319 * @num_heads: number of display controllers in use 9320 * 9321 * Calculate and program the display watermarks for the 9322 * selected display controller (CIK). 9323 */ 9324 static void dce8_program_watermarks(struct radeon_device *rdev, 9325 struct radeon_crtc *radeon_crtc, 9326 u32 lb_size, u32 num_heads) 9327 { 9328 struct drm_display_mode *mode = &radeon_crtc->base.mode; 9329 struct dce8_wm_params wm_low, wm_high; 9330 u32 pixel_period; 9331 u32 line_time = 0; 9332 u32 latency_watermark_a = 0, latency_watermark_b = 0; 9333 u32 tmp, wm_mask; 9334 9335 if (radeon_crtc->base.enabled && num_heads && mode) { 9336 pixel_period = 1000000 / (u32)mode->clock; 9337 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 9338 9339 /* watermark for high clocks */ 9340 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9341 rdev->pm.dpm_enabled) { 9342 wm_high.yclk = 9343 radeon_dpm_get_mclk(rdev, false) * 10; 9344 wm_high.sclk = 9345 radeon_dpm_get_sclk(rdev, false) * 10; 9346 } else { 9347 wm_high.yclk = rdev->pm.current_mclk * 10; 9348 wm_high.sclk = rdev->pm.current_sclk * 10; 9349 } 9350 9351 wm_high.disp_clk = mode->clock; 9352 wm_high.src_width = mode->crtc_hdisplay; 9353 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 9354 wm_high.blank_time = line_time - wm_high.active_time; 9355 wm_high.interlaced = false; 9356 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9357 wm_high.interlaced = true; 9358 wm_high.vsc = radeon_crtc->vsc; 9359 wm_high.vtaps = 1; 9360 if (radeon_crtc->rmx_type != RMX_OFF) 9361 wm_high.vtaps = 2; 9362 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9363 wm_high.lb_size = lb_size; 9364 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); 9365 wm_high.num_heads = num_heads; 9366 9367 /* set for high clocks */ 9368 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); 9369 9370 /* possibly force display priority to high */ 9371 /* should really do this at mode validation time... */ 9372 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 9373 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || 9374 !dce8_check_latency_hiding(&wm_high) || 9375 (rdev->disp_priority == 2)) { 9376 DRM_DEBUG_KMS("force priority to high\n"); 9377 } 9378 9379 /* watermark for low clocks */ 9380 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9381 rdev->pm.dpm_enabled) { 9382 wm_low.yclk = 9383 radeon_dpm_get_mclk(rdev, true) * 10; 9384 wm_low.sclk = 9385 radeon_dpm_get_sclk(rdev, true) * 10; 9386 } else { 9387 wm_low.yclk = rdev->pm.current_mclk * 10; 9388 wm_low.sclk = rdev->pm.current_sclk * 10; 9389 } 9390 9391 wm_low.disp_clk = mode->clock; 9392 wm_low.src_width = mode->crtc_hdisplay; 9393 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 9394 wm_low.blank_time = line_time - wm_low.active_time; 9395 wm_low.interlaced = false; 9396 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9397 wm_low.interlaced = true; 9398 wm_low.vsc = radeon_crtc->vsc; 9399 wm_low.vtaps = 1; 9400 if (radeon_crtc->rmx_type != RMX_OFF) 9401 wm_low.vtaps = 2; 9402 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9403 wm_low.lb_size = lb_size; 9404 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); 9405 wm_low.num_heads = num_heads; 9406 9407 /* set for low clocks */ 9408 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); 9409 9410 /* possibly force display priority to high */ 9411 /* should really do this at mode validation time... */ 9412 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 9413 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || 9414 !dce8_check_latency_hiding(&wm_low) || 9415 (rdev->disp_priority == 2)) { 9416 DRM_DEBUG_KMS("force priority to high\n"); 9417 } 9418 } 9419 9420 /* select wm A */ 9421 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9422 tmp = wm_mask; 9423 tmp &= ~LATENCY_WATERMARK_MASK(3); 9424 tmp |= LATENCY_WATERMARK_MASK(1); 9425 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9426 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9427 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 9428 LATENCY_HIGH_WATERMARK(line_time))); 9429 /* select wm B */ 9430 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9431 tmp &= ~LATENCY_WATERMARK_MASK(3); 9432 tmp |= LATENCY_WATERMARK_MASK(2); 9433 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9434 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9435 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 9436 LATENCY_HIGH_WATERMARK(line_time))); 9437 /* restore original selection */ 9438 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); 9439 9440 /* save values for DPM */ 9441 radeon_crtc->line_time = line_time; 9442 radeon_crtc->wm_high = latency_watermark_a; 9443 radeon_crtc->wm_low = latency_watermark_b; 9444 } 9445 9446 /** 9447 * dce8_bandwidth_update - program display watermarks 9448 * 9449 * @rdev: radeon_device pointer 9450 * 9451 * Calculate and program the display watermarks and line 9452 * buffer allocation (CIK). 9453 */ 9454 void dce8_bandwidth_update(struct radeon_device *rdev) 9455 { 9456 struct drm_display_mode *mode = NULL; 9457 u32 num_heads = 0, lb_size; 9458 int i; 9459 9460 if (!rdev->mode_info.mode_config_initialized) 9461 return; 9462 9463 radeon_update_display_priority(rdev); 9464 9465 for (i = 0; i < rdev->num_crtc; i++) { 9466 if (rdev->mode_info.crtcs[i]->base.enabled) 9467 num_heads++; 9468 } 9469 for (i = 0; i < rdev->num_crtc; i++) { 9470 mode = &rdev->mode_info.crtcs[i]->base.mode; 9471 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); 9472 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 9473 } 9474 } 9475 9476 /** 9477 * cik_get_gpu_clock_counter - return GPU clock counter snapshot 9478 * 9479 * @rdev: radeon_device pointer 9480 * 9481 * Fetches a GPU clock counter snapshot (SI). 9482 * Returns the 64 bit clock counter snapshot. 9483 */ 9484 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) 9485 { 9486 uint64_t clock; 9487 9488 mutex_lock(&rdev->gpu_clock_mutex); 9489 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 9490 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 9491 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 9492 mutex_unlock(&rdev->gpu_clock_mutex); 9493 return clock; 9494 } 9495 9496 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, 9497 u32 cntl_reg, u32 status_reg) 9498 { 9499 int r, i; 9500 struct atom_clock_dividers dividers; 9501 uint32_t tmp; 9502 9503 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9504 clock, false, ÷rs); 9505 if (r) 9506 return r; 9507 9508 tmp = RREG32_SMC(cntl_reg); 9509 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); 9510 tmp |= dividers.post_divider; 9511 WREG32_SMC(cntl_reg, tmp); 9512 9513 for (i = 0; i < 100; i++) { 9514 if (RREG32_SMC(status_reg) & DCLK_STATUS) 9515 break; 9516 mdelay(10); 9517 } 9518 if (i == 100) 9519 return -ETIMEDOUT; 9520 9521 return 0; 9522 } 9523 9524 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 9525 { 9526 int r = 0; 9527 9528 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 9529 if (r) 9530 return r; 9531 9532 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 9533 return r; 9534 } 9535 9536 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) 9537 { 9538 int r, i; 9539 struct atom_clock_dividers dividers; 9540 u32 tmp; 9541 9542 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9543 ecclk, false, ÷rs); 9544 if (r) 9545 return r; 9546 9547 for (i = 0; i < 100; i++) { 9548 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9549 break; 9550 mdelay(10); 9551 } 9552 if (i == 100) 9553 return -ETIMEDOUT; 9554 9555 tmp = RREG32_SMC(CG_ECLK_CNTL); 9556 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); 9557 tmp |= dividers.post_divider; 9558 WREG32_SMC(CG_ECLK_CNTL, tmp); 9559 9560 for (i = 0; i < 100; i++) { 9561 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9562 break; 9563 mdelay(10); 9564 } 9565 if (i == 100) 9566 return -ETIMEDOUT; 9567 9568 return 0; 9569 } 9570 9571 static void cik_pcie_gen3_enable(struct radeon_device *rdev) 9572 { 9573 struct pci_dev *root = rdev->pdev->bus->self; 9574 int bridge_pos, gpu_pos; 9575 u32 speed_cntl, mask, current_data_rate; 9576 int ret, i; 9577 u16 tmp16; 9578 9579 if (pci_is_root_bus(rdev->pdev->bus)) 9580 return; 9581 9582 if (radeon_pcie_gen2 == 0) 9583 return; 9584 9585 if (rdev->flags & RADEON_IS_IGP) 9586 return; 9587 9588 if (!(rdev->flags & RADEON_IS_PCIE)) 9589 return; 9590 9591 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 9592 if (ret != 0) 9593 return; 9594 9595 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 9596 return; 9597 9598 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9599 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 9600 LC_CURRENT_DATA_RATE_SHIFT; 9601 if (mask & DRM_PCIE_SPEED_80) { 9602 if (current_data_rate == 2) { 9603 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 9604 return; 9605 } 9606 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 9607 } else if (mask & DRM_PCIE_SPEED_50) { 9608 if (current_data_rate == 1) { 9609 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 9610 return; 9611 } 9612 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 9613 } 9614 9615 bridge_pos = pci_pcie_cap(root); 9616 if (!bridge_pos) 9617 return; 9618 9619 gpu_pos = pci_pcie_cap(rdev->pdev); 9620 if (!gpu_pos) 9621 return; 9622 9623 if (mask & DRM_PCIE_SPEED_80) { 9624 /* re-try equalization if gen3 is not already enabled */ 9625 if (current_data_rate != 2) { 9626 u16 bridge_cfg, gpu_cfg; 9627 u16 bridge_cfg2, gpu_cfg2; 9628 u32 max_lw, current_lw, tmp; 9629 9630 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9631 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9632 9633 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 9634 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9635 9636 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 9637 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9638 9639 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9640 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 9641 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 9642 9643 if (current_lw < max_lw) { 9644 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9645 if (tmp & LC_RENEGOTIATION_SUPPORT) { 9646 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 9647 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 9648 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 9649 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 9650 } 9651 } 9652 9653 for (i = 0; i < 10; i++) { 9654 /* check status */ 9655 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 9656 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 9657 break; 9658 9659 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9660 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9661 9662 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 9663 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 9664 9665 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9666 tmp |= LC_SET_QUIESCE; 9667 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9668 9669 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9670 tmp |= LC_REDO_EQ; 9671 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9672 9673 mdelay(100); 9674 9675 /* linkctl */ 9676 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 9677 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9678 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 9679 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9680 9681 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 9682 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9683 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 9684 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9685 9686 /* linkctl2 */ 9687 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 9688 tmp16 &= ~((1 << 4) | (7 << 9)); 9689 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 9690 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 9691 9692 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9693 tmp16 &= ~((1 << 4) | (7 << 9)); 9694 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 9695 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9696 9697 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9698 tmp &= ~LC_SET_QUIESCE; 9699 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9700 } 9701 } 9702 } 9703 9704 /* set the link speed */ 9705 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 9706 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 9707 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9708 9709 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9710 tmp16 &= ~0xf; 9711 if (mask & DRM_PCIE_SPEED_80) 9712 tmp16 |= 3; /* gen3 */ 9713 else if (mask & DRM_PCIE_SPEED_50) 9714 tmp16 |= 2; /* gen2 */ 9715 else 9716 tmp16 |= 1; /* gen1 */ 9717 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9718 9719 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9720 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 9721 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9722 9723 for (i = 0; i < rdev->usec_timeout; i++) { 9724 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9725 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 9726 break; 9727 udelay(1); 9728 } 9729 } 9730 9731 static void cik_program_aspm(struct radeon_device *rdev) 9732 { 9733 u32 data, orig; 9734 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 9735 bool disable_clkreq = false; 9736 9737 if (radeon_aspm == 0) 9738 return; 9739 9740 /* XXX double check IGPs */ 9741 if (rdev->flags & RADEON_IS_IGP) 9742 return; 9743 9744 if (!(rdev->flags & RADEON_IS_PCIE)) 9745 return; 9746 9747 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9748 data &= ~LC_XMIT_N_FTS_MASK; 9749 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 9750 if (orig != data) 9751 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 9752 9753 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 9754 data |= LC_GO_TO_RECOVERY; 9755 if (orig != data) 9756 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 9757 9758 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); 9759 data |= P_IGNORE_EDB_ERR; 9760 if (orig != data) 9761 WREG32_PCIE_PORT(PCIE_P_CNTL, data); 9762 9763 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9764 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 9765 data |= LC_PMI_TO_L1_DIS; 9766 if (!disable_l0s) 9767 data |= LC_L0S_INACTIVITY(7); 9768 9769 if (!disable_l1) { 9770 data |= LC_L1_INACTIVITY(7); 9771 data &= ~LC_PMI_TO_L1_DIS; 9772 if (orig != data) 9773 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9774 9775 if (!disable_plloff_in_l1) { 9776 bool clk_req_support; 9777 9778 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); 9779 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9780 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9781 if (orig != data) 9782 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); 9783 9784 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); 9785 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9786 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9787 if (orig != data) 9788 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); 9789 9790 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); 9791 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9792 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9793 if (orig != data) 9794 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); 9795 9796 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); 9797 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9798 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9799 if (orig != data) 9800 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); 9801 9802 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9803 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 9804 data |= LC_DYN_LANES_PWR_STATE(3); 9805 if (orig != data) 9806 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 9807 9808 if (!disable_clkreq && 9809 !pci_is_root_bus(rdev->pdev->bus)) { 9810 struct pci_dev *root = rdev->pdev->bus->self; 9811 u32 lnkcap; 9812 9813 clk_req_support = false; 9814 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 9815 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 9816 clk_req_support = true; 9817 } else { 9818 clk_req_support = false; 9819 } 9820 9821 if (clk_req_support) { 9822 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 9823 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 9824 if (orig != data) 9825 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 9826 9827 orig = data = RREG32_SMC(THM_CLK_CNTL); 9828 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 9829 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 9830 if (orig != data) 9831 WREG32_SMC(THM_CLK_CNTL, data); 9832 9833 orig = data = RREG32_SMC(MISC_CLK_CTRL); 9834 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 9835 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 9836 if (orig != data) 9837 WREG32_SMC(MISC_CLK_CTRL, data); 9838 9839 orig = data = RREG32_SMC(CG_CLKPIN_CNTL); 9840 data &= ~BCLK_AS_XCLK; 9841 if (orig != data) 9842 WREG32_SMC(CG_CLKPIN_CNTL, data); 9843 9844 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); 9845 data &= ~FORCE_BIF_REFCLK_EN; 9846 if (orig != data) 9847 WREG32_SMC(CG_CLKPIN_CNTL_2, data); 9848 9849 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); 9850 data &= ~MPLL_CLKOUT_SEL_MASK; 9851 data |= MPLL_CLKOUT_SEL(4); 9852 if (orig != data) 9853 WREG32_SMC(MPLL_BYPASSCLK_SEL, data); 9854 } 9855 } 9856 } else { 9857 if (orig != data) 9858 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9859 } 9860 9861 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 9862 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 9863 if (orig != data) 9864 WREG32_PCIE_PORT(PCIE_CNTL2, data); 9865 9866 if (!disable_l0s) { 9867 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9868 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 9869 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9870 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 9871 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9872 data &= ~LC_L0S_INACTIVITY_MASK; 9873 if (orig != data) 9874 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9875 } 9876 } 9877 } 9878 } 9879