1 /* 2 * Copyright 2012 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include "drmP.h" 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include "cikd.h" 31 #include "atom.h" 32 #include "cik_blit_shaders.h" 33 #include "radeon_ucode.h" 34 #include "clearstate_ci.h" 35 #include "radeon_kfd.h" 36 37 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 38 MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 39 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); 40 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); 41 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); 42 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin"); 43 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); 44 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); 45 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); 46 47 MODULE_FIRMWARE("radeon/bonaire_pfp.bin"); 48 MODULE_FIRMWARE("radeon/bonaire_me.bin"); 49 MODULE_FIRMWARE("radeon/bonaire_ce.bin"); 50 MODULE_FIRMWARE("radeon/bonaire_mec.bin"); 51 MODULE_FIRMWARE("radeon/bonaire_mc.bin"); 52 MODULE_FIRMWARE("radeon/bonaire_rlc.bin"); 53 MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); 54 MODULE_FIRMWARE("radeon/bonaire_smc.bin"); 55 56 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); 57 MODULE_FIRMWARE("radeon/HAWAII_me.bin"); 58 MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); 59 MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); 60 MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); 61 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin"); 62 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); 63 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); 64 MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); 65 66 MODULE_FIRMWARE("radeon/hawaii_pfp.bin"); 67 MODULE_FIRMWARE("radeon/hawaii_me.bin"); 68 MODULE_FIRMWARE("radeon/hawaii_ce.bin"); 69 MODULE_FIRMWARE("radeon/hawaii_mec.bin"); 70 MODULE_FIRMWARE("radeon/hawaii_mc.bin"); 71 MODULE_FIRMWARE("radeon/hawaii_rlc.bin"); 72 MODULE_FIRMWARE("radeon/hawaii_sdma.bin"); 73 MODULE_FIRMWARE("radeon/hawaii_smc.bin"); 74 75 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); 76 MODULE_FIRMWARE("radeon/KAVERI_me.bin"); 77 MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); 78 MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); 79 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); 80 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); 81 82 MODULE_FIRMWARE("radeon/kaveri_pfp.bin"); 83 MODULE_FIRMWARE("radeon/kaveri_me.bin"); 84 MODULE_FIRMWARE("radeon/kaveri_ce.bin"); 85 MODULE_FIRMWARE("radeon/kaveri_mec.bin"); 86 MODULE_FIRMWARE("radeon/kaveri_mec2.bin"); 87 MODULE_FIRMWARE("radeon/kaveri_rlc.bin"); 88 MODULE_FIRMWARE("radeon/kaveri_sdma.bin"); 89 90 MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); 91 MODULE_FIRMWARE("radeon/KABINI_me.bin"); 92 MODULE_FIRMWARE("radeon/KABINI_ce.bin"); 93 MODULE_FIRMWARE("radeon/KABINI_mec.bin"); 94 MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); 95 MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); 96 97 MODULE_FIRMWARE("radeon/kabini_pfp.bin"); 98 MODULE_FIRMWARE("radeon/kabini_me.bin"); 99 MODULE_FIRMWARE("radeon/kabini_ce.bin"); 100 MODULE_FIRMWARE("radeon/kabini_mec.bin"); 101 MODULE_FIRMWARE("radeon/kabini_rlc.bin"); 102 MODULE_FIRMWARE("radeon/kabini_sdma.bin"); 103 104 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin"); 105 MODULE_FIRMWARE("radeon/MULLINS_me.bin"); 106 MODULE_FIRMWARE("radeon/MULLINS_ce.bin"); 107 MODULE_FIRMWARE("radeon/MULLINS_mec.bin"); 108 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin"); 109 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin"); 110 111 MODULE_FIRMWARE("radeon/mullins_pfp.bin"); 112 MODULE_FIRMWARE("radeon/mullins_me.bin"); 113 MODULE_FIRMWARE("radeon/mullins_ce.bin"); 114 MODULE_FIRMWARE("radeon/mullins_mec.bin"); 115 MODULE_FIRMWARE("radeon/mullins_rlc.bin"); 116 MODULE_FIRMWARE("radeon/mullins_sdma.bin"); 117 118 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 119 extern void r600_ih_ring_fini(struct radeon_device *rdev); 120 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 121 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 122 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 123 extern void sumo_rlc_fini(struct radeon_device *rdev); 124 extern int sumo_rlc_init(struct radeon_device *rdev); 125 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); 126 extern void si_rlc_reset(struct radeon_device *rdev); 127 extern void si_init_uvd_internal_cg(struct radeon_device *rdev); 128 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); 129 extern int cik_sdma_resume(struct radeon_device *rdev); 130 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable); 131 extern void cik_sdma_fini(struct radeon_device *rdev); 132 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable); 133 static void cik_rlc_stop(struct radeon_device *rdev); 134 static void cik_pcie_gen3_enable(struct radeon_device *rdev); 135 static void cik_program_aspm(struct radeon_device *rdev); 136 static void cik_init_pg(struct radeon_device *rdev); 137 static void cik_init_cg(struct radeon_device *rdev); 138 static void cik_fini_pg(struct radeon_device *rdev); 139 static void cik_fini_cg(struct radeon_device *rdev); 140 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 141 bool enable); 142 143 /* get temperature in millidegrees */ 144 int ci_get_temp(struct radeon_device *rdev) 145 { 146 u32 temp; 147 int actual_temp = 0; 148 149 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 150 CTF_TEMP_SHIFT; 151 152 if (temp & 0x200) 153 actual_temp = 255; 154 else 155 actual_temp = temp & 0x1ff; 156 157 actual_temp = actual_temp * 1000; 158 159 return actual_temp; 160 } 161 162 /* get temperature in millidegrees */ 163 int kv_get_temp(struct radeon_device *rdev) 164 { 165 u32 temp; 166 int actual_temp = 0; 167 168 temp = RREG32_SMC(0xC0300E0C); 169 170 if (temp) 171 actual_temp = (temp / 8) - 49; 172 else 173 actual_temp = 0; 174 175 actual_temp = actual_temp * 1000; 176 177 return actual_temp; 178 } 179 180 /* 181 * Indirect registers accessor 182 */ 183 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) 184 { 185 unsigned long flags; 186 u32 r; 187 188 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 189 WREG32(PCIE_INDEX, reg); 190 (void)RREG32(PCIE_INDEX); 191 r = RREG32(PCIE_DATA); 192 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 193 return r; 194 } 195 196 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) 197 { 198 unsigned long flags; 199 200 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 201 WREG32(PCIE_INDEX, reg); 202 (void)RREG32(PCIE_INDEX); 203 WREG32(PCIE_DATA, v); 204 (void)RREG32(PCIE_DATA); 205 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 206 } 207 208 static const u32 spectre_rlc_save_restore_register_list[] = 209 { 210 (0x0e00 << 16) | (0xc12c >> 2), 211 0x00000000, 212 (0x0e00 << 16) | (0xc140 >> 2), 213 0x00000000, 214 (0x0e00 << 16) | (0xc150 >> 2), 215 0x00000000, 216 (0x0e00 << 16) | (0xc15c >> 2), 217 0x00000000, 218 (0x0e00 << 16) | (0xc168 >> 2), 219 0x00000000, 220 (0x0e00 << 16) | (0xc170 >> 2), 221 0x00000000, 222 (0x0e00 << 16) | (0xc178 >> 2), 223 0x00000000, 224 (0x0e00 << 16) | (0xc204 >> 2), 225 0x00000000, 226 (0x0e00 << 16) | (0xc2b4 >> 2), 227 0x00000000, 228 (0x0e00 << 16) | (0xc2b8 >> 2), 229 0x00000000, 230 (0x0e00 << 16) | (0xc2bc >> 2), 231 0x00000000, 232 (0x0e00 << 16) | (0xc2c0 >> 2), 233 0x00000000, 234 (0x0e00 << 16) | (0x8228 >> 2), 235 0x00000000, 236 (0x0e00 << 16) | (0x829c >> 2), 237 0x00000000, 238 (0x0e00 << 16) | (0x869c >> 2), 239 0x00000000, 240 (0x0600 << 16) | (0x98f4 >> 2), 241 0x00000000, 242 (0x0e00 << 16) | (0x98f8 >> 2), 243 0x00000000, 244 (0x0e00 << 16) | (0x9900 >> 2), 245 0x00000000, 246 (0x0e00 << 16) | (0xc260 >> 2), 247 0x00000000, 248 (0x0e00 << 16) | (0x90e8 >> 2), 249 0x00000000, 250 (0x0e00 << 16) | (0x3c000 >> 2), 251 0x00000000, 252 (0x0e00 << 16) | (0x3c00c >> 2), 253 0x00000000, 254 (0x0e00 << 16) | (0x8c1c >> 2), 255 0x00000000, 256 (0x0e00 << 16) | (0x9700 >> 2), 257 0x00000000, 258 (0x0e00 << 16) | (0xcd20 >> 2), 259 0x00000000, 260 (0x4e00 << 16) | (0xcd20 >> 2), 261 0x00000000, 262 (0x5e00 << 16) | (0xcd20 >> 2), 263 0x00000000, 264 (0x6e00 << 16) | (0xcd20 >> 2), 265 0x00000000, 266 (0x7e00 << 16) | (0xcd20 >> 2), 267 0x00000000, 268 (0x8e00 << 16) | (0xcd20 >> 2), 269 0x00000000, 270 (0x9e00 << 16) | (0xcd20 >> 2), 271 0x00000000, 272 (0xae00 << 16) | (0xcd20 >> 2), 273 0x00000000, 274 (0xbe00 << 16) | (0xcd20 >> 2), 275 0x00000000, 276 (0x0e00 << 16) | (0x89bc >> 2), 277 0x00000000, 278 (0x0e00 << 16) | (0x8900 >> 2), 279 0x00000000, 280 0x3, 281 (0x0e00 << 16) | (0xc130 >> 2), 282 0x00000000, 283 (0x0e00 << 16) | (0xc134 >> 2), 284 0x00000000, 285 (0x0e00 << 16) | (0xc1fc >> 2), 286 0x00000000, 287 (0x0e00 << 16) | (0xc208 >> 2), 288 0x00000000, 289 (0x0e00 << 16) | (0xc264 >> 2), 290 0x00000000, 291 (0x0e00 << 16) | (0xc268 >> 2), 292 0x00000000, 293 (0x0e00 << 16) | (0xc26c >> 2), 294 0x00000000, 295 (0x0e00 << 16) | (0xc270 >> 2), 296 0x00000000, 297 (0x0e00 << 16) | (0xc274 >> 2), 298 0x00000000, 299 (0x0e00 << 16) | (0xc278 >> 2), 300 0x00000000, 301 (0x0e00 << 16) | (0xc27c >> 2), 302 0x00000000, 303 (0x0e00 << 16) | (0xc280 >> 2), 304 0x00000000, 305 (0x0e00 << 16) | (0xc284 >> 2), 306 0x00000000, 307 (0x0e00 << 16) | (0xc288 >> 2), 308 0x00000000, 309 (0x0e00 << 16) | (0xc28c >> 2), 310 0x00000000, 311 (0x0e00 << 16) | (0xc290 >> 2), 312 0x00000000, 313 (0x0e00 << 16) | (0xc294 >> 2), 314 0x00000000, 315 (0x0e00 << 16) | (0xc298 >> 2), 316 0x00000000, 317 (0x0e00 << 16) | (0xc29c >> 2), 318 0x00000000, 319 (0x0e00 << 16) | (0xc2a0 >> 2), 320 0x00000000, 321 (0x0e00 << 16) | (0xc2a4 >> 2), 322 0x00000000, 323 (0x0e00 << 16) | (0xc2a8 >> 2), 324 0x00000000, 325 (0x0e00 << 16) | (0xc2ac >> 2), 326 0x00000000, 327 (0x0e00 << 16) | (0xc2b0 >> 2), 328 0x00000000, 329 (0x0e00 << 16) | (0x301d0 >> 2), 330 0x00000000, 331 (0x0e00 << 16) | (0x30238 >> 2), 332 0x00000000, 333 (0x0e00 << 16) | (0x30250 >> 2), 334 0x00000000, 335 (0x0e00 << 16) | (0x30254 >> 2), 336 0x00000000, 337 (0x0e00 << 16) | (0x30258 >> 2), 338 0x00000000, 339 (0x0e00 << 16) | (0x3025c >> 2), 340 0x00000000, 341 (0x4e00 << 16) | (0xc900 >> 2), 342 0x00000000, 343 (0x5e00 << 16) | (0xc900 >> 2), 344 0x00000000, 345 (0x6e00 << 16) | (0xc900 >> 2), 346 0x00000000, 347 (0x7e00 << 16) | (0xc900 >> 2), 348 0x00000000, 349 (0x8e00 << 16) | (0xc900 >> 2), 350 0x00000000, 351 (0x9e00 << 16) | (0xc900 >> 2), 352 0x00000000, 353 (0xae00 << 16) | (0xc900 >> 2), 354 0x00000000, 355 (0xbe00 << 16) | (0xc900 >> 2), 356 0x00000000, 357 (0x4e00 << 16) | (0xc904 >> 2), 358 0x00000000, 359 (0x5e00 << 16) | (0xc904 >> 2), 360 0x00000000, 361 (0x6e00 << 16) | (0xc904 >> 2), 362 0x00000000, 363 (0x7e00 << 16) | (0xc904 >> 2), 364 0x00000000, 365 (0x8e00 << 16) | (0xc904 >> 2), 366 0x00000000, 367 (0x9e00 << 16) | (0xc904 >> 2), 368 0x00000000, 369 (0xae00 << 16) | (0xc904 >> 2), 370 0x00000000, 371 (0xbe00 << 16) | (0xc904 >> 2), 372 0x00000000, 373 (0x4e00 << 16) | (0xc908 >> 2), 374 0x00000000, 375 (0x5e00 << 16) | (0xc908 >> 2), 376 0x00000000, 377 (0x6e00 << 16) | (0xc908 >> 2), 378 0x00000000, 379 (0x7e00 << 16) | (0xc908 >> 2), 380 0x00000000, 381 (0x8e00 << 16) | (0xc908 >> 2), 382 0x00000000, 383 (0x9e00 << 16) | (0xc908 >> 2), 384 0x00000000, 385 (0xae00 << 16) | (0xc908 >> 2), 386 0x00000000, 387 (0xbe00 << 16) | (0xc908 >> 2), 388 0x00000000, 389 (0x4e00 << 16) | (0xc90c >> 2), 390 0x00000000, 391 (0x5e00 << 16) | (0xc90c >> 2), 392 0x00000000, 393 (0x6e00 << 16) | (0xc90c >> 2), 394 0x00000000, 395 (0x7e00 << 16) | (0xc90c >> 2), 396 0x00000000, 397 (0x8e00 << 16) | (0xc90c >> 2), 398 0x00000000, 399 (0x9e00 << 16) | (0xc90c >> 2), 400 0x00000000, 401 (0xae00 << 16) | (0xc90c >> 2), 402 0x00000000, 403 (0xbe00 << 16) | (0xc90c >> 2), 404 0x00000000, 405 (0x4e00 << 16) | (0xc910 >> 2), 406 0x00000000, 407 (0x5e00 << 16) | (0xc910 >> 2), 408 0x00000000, 409 (0x6e00 << 16) | (0xc910 >> 2), 410 0x00000000, 411 (0x7e00 << 16) | (0xc910 >> 2), 412 0x00000000, 413 (0x8e00 << 16) | (0xc910 >> 2), 414 0x00000000, 415 (0x9e00 << 16) | (0xc910 >> 2), 416 0x00000000, 417 (0xae00 << 16) | (0xc910 >> 2), 418 0x00000000, 419 (0xbe00 << 16) | (0xc910 >> 2), 420 0x00000000, 421 (0x0e00 << 16) | (0xc99c >> 2), 422 0x00000000, 423 (0x0e00 << 16) | (0x9834 >> 2), 424 0x00000000, 425 (0x0000 << 16) | (0x30f00 >> 2), 426 0x00000000, 427 (0x0001 << 16) | (0x30f00 >> 2), 428 0x00000000, 429 (0x0000 << 16) | (0x30f04 >> 2), 430 0x00000000, 431 (0x0001 << 16) | (0x30f04 >> 2), 432 0x00000000, 433 (0x0000 << 16) | (0x30f08 >> 2), 434 0x00000000, 435 (0x0001 << 16) | (0x30f08 >> 2), 436 0x00000000, 437 (0x0000 << 16) | (0x30f0c >> 2), 438 0x00000000, 439 (0x0001 << 16) | (0x30f0c >> 2), 440 0x00000000, 441 (0x0600 << 16) | (0x9b7c >> 2), 442 0x00000000, 443 (0x0e00 << 16) | (0x8a14 >> 2), 444 0x00000000, 445 (0x0e00 << 16) | (0x8a18 >> 2), 446 0x00000000, 447 (0x0600 << 16) | (0x30a00 >> 2), 448 0x00000000, 449 (0x0e00 << 16) | (0x8bf0 >> 2), 450 0x00000000, 451 (0x0e00 << 16) | (0x8bcc >> 2), 452 0x00000000, 453 (0x0e00 << 16) | (0x8b24 >> 2), 454 0x00000000, 455 (0x0e00 << 16) | (0x30a04 >> 2), 456 0x00000000, 457 (0x0600 << 16) | (0x30a10 >> 2), 458 0x00000000, 459 (0x0600 << 16) | (0x30a14 >> 2), 460 0x00000000, 461 (0x0600 << 16) | (0x30a18 >> 2), 462 0x00000000, 463 (0x0600 << 16) | (0x30a2c >> 2), 464 0x00000000, 465 (0x0e00 << 16) | (0xc700 >> 2), 466 0x00000000, 467 (0x0e00 << 16) | (0xc704 >> 2), 468 0x00000000, 469 (0x0e00 << 16) | (0xc708 >> 2), 470 0x00000000, 471 (0x0e00 << 16) | (0xc768 >> 2), 472 0x00000000, 473 (0x0400 << 16) | (0xc770 >> 2), 474 0x00000000, 475 (0x0400 << 16) | (0xc774 >> 2), 476 0x00000000, 477 (0x0400 << 16) | (0xc778 >> 2), 478 0x00000000, 479 (0x0400 << 16) | (0xc77c >> 2), 480 0x00000000, 481 (0x0400 << 16) | (0xc780 >> 2), 482 0x00000000, 483 (0x0400 << 16) | (0xc784 >> 2), 484 0x00000000, 485 (0x0400 << 16) | (0xc788 >> 2), 486 0x00000000, 487 (0x0400 << 16) | (0xc78c >> 2), 488 0x00000000, 489 (0x0400 << 16) | (0xc798 >> 2), 490 0x00000000, 491 (0x0400 << 16) | (0xc79c >> 2), 492 0x00000000, 493 (0x0400 << 16) | (0xc7a0 >> 2), 494 0x00000000, 495 (0x0400 << 16) | (0xc7a4 >> 2), 496 0x00000000, 497 (0x0400 << 16) | (0xc7a8 >> 2), 498 0x00000000, 499 (0x0400 << 16) | (0xc7ac >> 2), 500 0x00000000, 501 (0x0400 << 16) | (0xc7b0 >> 2), 502 0x00000000, 503 (0x0400 << 16) | (0xc7b4 >> 2), 504 0x00000000, 505 (0x0e00 << 16) | (0x9100 >> 2), 506 0x00000000, 507 (0x0e00 << 16) | (0x3c010 >> 2), 508 0x00000000, 509 (0x0e00 << 16) | (0x92a8 >> 2), 510 0x00000000, 511 (0x0e00 << 16) | (0x92ac >> 2), 512 0x00000000, 513 (0x0e00 << 16) | (0x92b4 >> 2), 514 0x00000000, 515 (0x0e00 << 16) | (0x92b8 >> 2), 516 0x00000000, 517 (0x0e00 << 16) | (0x92bc >> 2), 518 0x00000000, 519 (0x0e00 << 16) | (0x92c0 >> 2), 520 0x00000000, 521 (0x0e00 << 16) | (0x92c4 >> 2), 522 0x00000000, 523 (0x0e00 << 16) | (0x92c8 >> 2), 524 0x00000000, 525 (0x0e00 << 16) | (0x92cc >> 2), 526 0x00000000, 527 (0x0e00 << 16) | (0x92d0 >> 2), 528 0x00000000, 529 (0x0e00 << 16) | (0x8c00 >> 2), 530 0x00000000, 531 (0x0e00 << 16) | (0x8c04 >> 2), 532 0x00000000, 533 (0x0e00 << 16) | (0x8c20 >> 2), 534 0x00000000, 535 (0x0e00 << 16) | (0x8c38 >> 2), 536 0x00000000, 537 (0x0e00 << 16) | (0x8c3c >> 2), 538 0x00000000, 539 (0x0e00 << 16) | (0xae00 >> 2), 540 0x00000000, 541 (0x0e00 << 16) | (0x9604 >> 2), 542 0x00000000, 543 (0x0e00 << 16) | (0xac08 >> 2), 544 0x00000000, 545 (0x0e00 << 16) | (0xac0c >> 2), 546 0x00000000, 547 (0x0e00 << 16) | (0xac10 >> 2), 548 0x00000000, 549 (0x0e00 << 16) | (0xac14 >> 2), 550 0x00000000, 551 (0x0e00 << 16) | (0xac58 >> 2), 552 0x00000000, 553 (0x0e00 << 16) | (0xac68 >> 2), 554 0x00000000, 555 (0x0e00 << 16) | (0xac6c >> 2), 556 0x00000000, 557 (0x0e00 << 16) | (0xac70 >> 2), 558 0x00000000, 559 (0x0e00 << 16) | (0xac74 >> 2), 560 0x00000000, 561 (0x0e00 << 16) | (0xac78 >> 2), 562 0x00000000, 563 (0x0e00 << 16) | (0xac7c >> 2), 564 0x00000000, 565 (0x0e00 << 16) | (0xac80 >> 2), 566 0x00000000, 567 (0x0e00 << 16) | (0xac84 >> 2), 568 0x00000000, 569 (0x0e00 << 16) | (0xac88 >> 2), 570 0x00000000, 571 (0x0e00 << 16) | (0xac8c >> 2), 572 0x00000000, 573 (0x0e00 << 16) | (0x970c >> 2), 574 0x00000000, 575 (0x0e00 << 16) | (0x9714 >> 2), 576 0x00000000, 577 (0x0e00 << 16) | (0x9718 >> 2), 578 0x00000000, 579 (0x0e00 << 16) | (0x971c >> 2), 580 0x00000000, 581 (0x0e00 << 16) | (0x31068 >> 2), 582 0x00000000, 583 (0x4e00 << 16) | (0x31068 >> 2), 584 0x00000000, 585 (0x5e00 << 16) | (0x31068 >> 2), 586 0x00000000, 587 (0x6e00 << 16) | (0x31068 >> 2), 588 0x00000000, 589 (0x7e00 << 16) | (0x31068 >> 2), 590 0x00000000, 591 (0x8e00 << 16) | (0x31068 >> 2), 592 0x00000000, 593 (0x9e00 << 16) | (0x31068 >> 2), 594 0x00000000, 595 (0xae00 << 16) | (0x31068 >> 2), 596 0x00000000, 597 (0xbe00 << 16) | (0x31068 >> 2), 598 0x00000000, 599 (0x0e00 << 16) | (0xcd10 >> 2), 600 0x00000000, 601 (0x0e00 << 16) | (0xcd14 >> 2), 602 0x00000000, 603 (0x0e00 << 16) | (0x88b0 >> 2), 604 0x00000000, 605 (0x0e00 << 16) | (0x88b4 >> 2), 606 0x00000000, 607 (0x0e00 << 16) | (0x88b8 >> 2), 608 0x00000000, 609 (0x0e00 << 16) | (0x88bc >> 2), 610 0x00000000, 611 (0x0400 << 16) | (0x89c0 >> 2), 612 0x00000000, 613 (0x0e00 << 16) | (0x88c4 >> 2), 614 0x00000000, 615 (0x0e00 << 16) | (0x88c8 >> 2), 616 0x00000000, 617 (0x0e00 << 16) | (0x88d0 >> 2), 618 0x00000000, 619 (0x0e00 << 16) | (0x88d4 >> 2), 620 0x00000000, 621 (0x0e00 << 16) | (0x88d8 >> 2), 622 0x00000000, 623 (0x0e00 << 16) | (0x8980 >> 2), 624 0x00000000, 625 (0x0e00 << 16) | (0x30938 >> 2), 626 0x00000000, 627 (0x0e00 << 16) | (0x3093c >> 2), 628 0x00000000, 629 (0x0e00 << 16) | (0x30940 >> 2), 630 0x00000000, 631 (0x0e00 << 16) | (0x89a0 >> 2), 632 0x00000000, 633 (0x0e00 << 16) | (0x30900 >> 2), 634 0x00000000, 635 (0x0e00 << 16) | (0x30904 >> 2), 636 0x00000000, 637 (0x0e00 << 16) | (0x89b4 >> 2), 638 0x00000000, 639 (0x0e00 << 16) | (0x3c210 >> 2), 640 0x00000000, 641 (0x0e00 << 16) | (0x3c214 >> 2), 642 0x00000000, 643 (0x0e00 << 16) | (0x3c218 >> 2), 644 0x00000000, 645 (0x0e00 << 16) | (0x8904 >> 2), 646 0x00000000, 647 0x5, 648 (0x0e00 << 16) | (0x8c28 >> 2), 649 (0x0e00 << 16) | (0x8c2c >> 2), 650 (0x0e00 << 16) | (0x8c30 >> 2), 651 (0x0e00 << 16) | (0x8c34 >> 2), 652 (0x0e00 << 16) | (0x9600 >> 2), 653 }; 654 655 static const u32 kalindi_rlc_save_restore_register_list[] = 656 { 657 (0x0e00 << 16) | (0xc12c >> 2), 658 0x00000000, 659 (0x0e00 << 16) | (0xc140 >> 2), 660 0x00000000, 661 (0x0e00 << 16) | (0xc150 >> 2), 662 0x00000000, 663 (0x0e00 << 16) | (0xc15c >> 2), 664 0x00000000, 665 (0x0e00 << 16) | (0xc168 >> 2), 666 0x00000000, 667 (0x0e00 << 16) | (0xc170 >> 2), 668 0x00000000, 669 (0x0e00 << 16) | (0xc204 >> 2), 670 0x00000000, 671 (0x0e00 << 16) | (0xc2b4 >> 2), 672 0x00000000, 673 (0x0e00 << 16) | (0xc2b8 >> 2), 674 0x00000000, 675 (0x0e00 << 16) | (0xc2bc >> 2), 676 0x00000000, 677 (0x0e00 << 16) | (0xc2c0 >> 2), 678 0x00000000, 679 (0x0e00 << 16) | (0x8228 >> 2), 680 0x00000000, 681 (0x0e00 << 16) | (0x829c >> 2), 682 0x00000000, 683 (0x0e00 << 16) | (0x869c >> 2), 684 0x00000000, 685 (0x0600 << 16) | (0x98f4 >> 2), 686 0x00000000, 687 (0x0e00 << 16) | (0x98f8 >> 2), 688 0x00000000, 689 (0x0e00 << 16) | (0x9900 >> 2), 690 0x00000000, 691 (0x0e00 << 16) | (0xc260 >> 2), 692 0x00000000, 693 (0x0e00 << 16) | (0x90e8 >> 2), 694 0x00000000, 695 (0x0e00 << 16) | (0x3c000 >> 2), 696 0x00000000, 697 (0x0e00 << 16) | (0x3c00c >> 2), 698 0x00000000, 699 (0x0e00 << 16) | (0x8c1c >> 2), 700 0x00000000, 701 (0x0e00 << 16) | (0x9700 >> 2), 702 0x00000000, 703 (0x0e00 << 16) | (0xcd20 >> 2), 704 0x00000000, 705 (0x4e00 << 16) | (0xcd20 >> 2), 706 0x00000000, 707 (0x5e00 << 16) | (0xcd20 >> 2), 708 0x00000000, 709 (0x6e00 << 16) | (0xcd20 >> 2), 710 0x00000000, 711 (0x7e00 << 16) | (0xcd20 >> 2), 712 0x00000000, 713 (0x0e00 << 16) | (0x89bc >> 2), 714 0x00000000, 715 (0x0e00 << 16) | (0x8900 >> 2), 716 0x00000000, 717 0x3, 718 (0x0e00 << 16) | (0xc130 >> 2), 719 0x00000000, 720 (0x0e00 << 16) | (0xc134 >> 2), 721 0x00000000, 722 (0x0e00 << 16) | (0xc1fc >> 2), 723 0x00000000, 724 (0x0e00 << 16) | (0xc208 >> 2), 725 0x00000000, 726 (0x0e00 << 16) | (0xc264 >> 2), 727 0x00000000, 728 (0x0e00 << 16) | (0xc268 >> 2), 729 0x00000000, 730 (0x0e00 << 16) | (0xc26c >> 2), 731 0x00000000, 732 (0x0e00 << 16) | (0xc270 >> 2), 733 0x00000000, 734 (0x0e00 << 16) | (0xc274 >> 2), 735 0x00000000, 736 (0x0e00 << 16) | (0xc28c >> 2), 737 0x00000000, 738 (0x0e00 << 16) | (0xc290 >> 2), 739 0x00000000, 740 (0x0e00 << 16) | (0xc294 >> 2), 741 0x00000000, 742 (0x0e00 << 16) | (0xc298 >> 2), 743 0x00000000, 744 (0x0e00 << 16) | (0xc2a0 >> 2), 745 0x00000000, 746 (0x0e00 << 16) | (0xc2a4 >> 2), 747 0x00000000, 748 (0x0e00 << 16) | (0xc2a8 >> 2), 749 0x00000000, 750 (0x0e00 << 16) | (0xc2ac >> 2), 751 0x00000000, 752 (0x0e00 << 16) | (0x301d0 >> 2), 753 0x00000000, 754 (0x0e00 << 16) | (0x30238 >> 2), 755 0x00000000, 756 (0x0e00 << 16) | (0x30250 >> 2), 757 0x00000000, 758 (0x0e00 << 16) | (0x30254 >> 2), 759 0x00000000, 760 (0x0e00 << 16) | (0x30258 >> 2), 761 0x00000000, 762 (0x0e00 << 16) | (0x3025c >> 2), 763 0x00000000, 764 (0x4e00 << 16) | (0xc900 >> 2), 765 0x00000000, 766 (0x5e00 << 16) | (0xc900 >> 2), 767 0x00000000, 768 (0x6e00 << 16) | (0xc900 >> 2), 769 0x00000000, 770 (0x7e00 << 16) | (0xc900 >> 2), 771 0x00000000, 772 (0x4e00 << 16) | (0xc904 >> 2), 773 0x00000000, 774 (0x5e00 << 16) | (0xc904 >> 2), 775 0x00000000, 776 (0x6e00 << 16) | (0xc904 >> 2), 777 0x00000000, 778 (0x7e00 << 16) | (0xc904 >> 2), 779 0x00000000, 780 (0x4e00 << 16) | (0xc908 >> 2), 781 0x00000000, 782 (0x5e00 << 16) | (0xc908 >> 2), 783 0x00000000, 784 (0x6e00 << 16) | (0xc908 >> 2), 785 0x00000000, 786 (0x7e00 << 16) | (0xc908 >> 2), 787 0x00000000, 788 (0x4e00 << 16) | (0xc90c >> 2), 789 0x00000000, 790 (0x5e00 << 16) | (0xc90c >> 2), 791 0x00000000, 792 (0x6e00 << 16) | (0xc90c >> 2), 793 0x00000000, 794 (0x7e00 << 16) | (0xc90c >> 2), 795 0x00000000, 796 (0x4e00 << 16) | (0xc910 >> 2), 797 0x00000000, 798 (0x5e00 << 16) | (0xc910 >> 2), 799 0x00000000, 800 (0x6e00 << 16) | (0xc910 >> 2), 801 0x00000000, 802 (0x7e00 << 16) | (0xc910 >> 2), 803 0x00000000, 804 (0x0e00 << 16) | (0xc99c >> 2), 805 0x00000000, 806 (0x0e00 << 16) | (0x9834 >> 2), 807 0x00000000, 808 (0x0000 << 16) | (0x30f00 >> 2), 809 0x00000000, 810 (0x0000 << 16) | (0x30f04 >> 2), 811 0x00000000, 812 (0x0000 << 16) | (0x30f08 >> 2), 813 0x00000000, 814 (0x0000 << 16) | (0x30f0c >> 2), 815 0x00000000, 816 (0x0600 << 16) | (0x9b7c >> 2), 817 0x00000000, 818 (0x0e00 << 16) | (0x8a14 >> 2), 819 0x00000000, 820 (0x0e00 << 16) | (0x8a18 >> 2), 821 0x00000000, 822 (0x0600 << 16) | (0x30a00 >> 2), 823 0x00000000, 824 (0x0e00 << 16) | (0x8bf0 >> 2), 825 0x00000000, 826 (0x0e00 << 16) | (0x8bcc >> 2), 827 0x00000000, 828 (0x0e00 << 16) | (0x8b24 >> 2), 829 0x00000000, 830 (0x0e00 << 16) | (0x30a04 >> 2), 831 0x00000000, 832 (0x0600 << 16) | (0x30a10 >> 2), 833 0x00000000, 834 (0x0600 << 16) | (0x30a14 >> 2), 835 0x00000000, 836 (0x0600 << 16) | (0x30a18 >> 2), 837 0x00000000, 838 (0x0600 << 16) | (0x30a2c >> 2), 839 0x00000000, 840 (0x0e00 << 16) | (0xc700 >> 2), 841 0x00000000, 842 (0x0e00 << 16) | (0xc704 >> 2), 843 0x00000000, 844 (0x0e00 << 16) | (0xc708 >> 2), 845 0x00000000, 846 (0x0e00 << 16) | (0xc768 >> 2), 847 0x00000000, 848 (0x0400 << 16) | (0xc770 >> 2), 849 0x00000000, 850 (0x0400 << 16) | (0xc774 >> 2), 851 0x00000000, 852 (0x0400 << 16) | (0xc798 >> 2), 853 0x00000000, 854 (0x0400 << 16) | (0xc79c >> 2), 855 0x00000000, 856 (0x0e00 << 16) | (0x9100 >> 2), 857 0x00000000, 858 (0x0e00 << 16) | (0x3c010 >> 2), 859 0x00000000, 860 (0x0e00 << 16) | (0x8c00 >> 2), 861 0x00000000, 862 (0x0e00 << 16) | (0x8c04 >> 2), 863 0x00000000, 864 (0x0e00 << 16) | (0x8c20 >> 2), 865 0x00000000, 866 (0x0e00 << 16) | (0x8c38 >> 2), 867 0x00000000, 868 (0x0e00 << 16) | (0x8c3c >> 2), 869 0x00000000, 870 (0x0e00 << 16) | (0xae00 >> 2), 871 0x00000000, 872 (0x0e00 << 16) | (0x9604 >> 2), 873 0x00000000, 874 (0x0e00 << 16) | (0xac08 >> 2), 875 0x00000000, 876 (0x0e00 << 16) | (0xac0c >> 2), 877 0x00000000, 878 (0x0e00 << 16) | (0xac10 >> 2), 879 0x00000000, 880 (0x0e00 << 16) | (0xac14 >> 2), 881 0x00000000, 882 (0x0e00 << 16) | (0xac58 >> 2), 883 0x00000000, 884 (0x0e00 << 16) | (0xac68 >> 2), 885 0x00000000, 886 (0x0e00 << 16) | (0xac6c >> 2), 887 0x00000000, 888 (0x0e00 << 16) | (0xac70 >> 2), 889 0x00000000, 890 (0x0e00 << 16) | (0xac74 >> 2), 891 0x00000000, 892 (0x0e00 << 16) | (0xac78 >> 2), 893 0x00000000, 894 (0x0e00 << 16) | (0xac7c >> 2), 895 0x00000000, 896 (0x0e00 << 16) | (0xac80 >> 2), 897 0x00000000, 898 (0x0e00 << 16) | (0xac84 >> 2), 899 0x00000000, 900 (0x0e00 << 16) | (0xac88 >> 2), 901 0x00000000, 902 (0x0e00 << 16) | (0xac8c >> 2), 903 0x00000000, 904 (0x0e00 << 16) | (0x970c >> 2), 905 0x00000000, 906 (0x0e00 << 16) | (0x9714 >> 2), 907 0x00000000, 908 (0x0e00 << 16) | (0x9718 >> 2), 909 0x00000000, 910 (0x0e00 << 16) | (0x971c >> 2), 911 0x00000000, 912 (0x0e00 << 16) | (0x31068 >> 2), 913 0x00000000, 914 (0x4e00 << 16) | (0x31068 >> 2), 915 0x00000000, 916 (0x5e00 << 16) | (0x31068 >> 2), 917 0x00000000, 918 (0x6e00 << 16) | (0x31068 >> 2), 919 0x00000000, 920 (0x7e00 << 16) | (0x31068 >> 2), 921 0x00000000, 922 (0x0e00 << 16) | (0xcd10 >> 2), 923 0x00000000, 924 (0x0e00 << 16) | (0xcd14 >> 2), 925 0x00000000, 926 (0x0e00 << 16) | (0x88b0 >> 2), 927 0x00000000, 928 (0x0e00 << 16) | (0x88b4 >> 2), 929 0x00000000, 930 (0x0e00 << 16) | (0x88b8 >> 2), 931 0x00000000, 932 (0x0e00 << 16) | (0x88bc >> 2), 933 0x00000000, 934 (0x0400 << 16) | (0x89c0 >> 2), 935 0x00000000, 936 (0x0e00 << 16) | (0x88c4 >> 2), 937 0x00000000, 938 (0x0e00 << 16) | (0x88c8 >> 2), 939 0x00000000, 940 (0x0e00 << 16) | (0x88d0 >> 2), 941 0x00000000, 942 (0x0e00 << 16) | (0x88d4 >> 2), 943 0x00000000, 944 (0x0e00 << 16) | (0x88d8 >> 2), 945 0x00000000, 946 (0x0e00 << 16) | (0x8980 >> 2), 947 0x00000000, 948 (0x0e00 << 16) | (0x30938 >> 2), 949 0x00000000, 950 (0x0e00 << 16) | (0x3093c >> 2), 951 0x00000000, 952 (0x0e00 << 16) | (0x30940 >> 2), 953 0x00000000, 954 (0x0e00 << 16) | (0x89a0 >> 2), 955 0x00000000, 956 (0x0e00 << 16) | (0x30900 >> 2), 957 0x00000000, 958 (0x0e00 << 16) | (0x30904 >> 2), 959 0x00000000, 960 (0x0e00 << 16) | (0x89b4 >> 2), 961 0x00000000, 962 (0x0e00 << 16) | (0x3e1fc >> 2), 963 0x00000000, 964 (0x0e00 << 16) | (0x3c210 >> 2), 965 0x00000000, 966 (0x0e00 << 16) | (0x3c214 >> 2), 967 0x00000000, 968 (0x0e00 << 16) | (0x3c218 >> 2), 969 0x00000000, 970 (0x0e00 << 16) | (0x8904 >> 2), 971 0x00000000, 972 0x5, 973 (0x0e00 << 16) | (0x8c28 >> 2), 974 (0x0e00 << 16) | (0x8c2c >> 2), 975 (0x0e00 << 16) | (0x8c30 >> 2), 976 (0x0e00 << 16) | (0x8c34 >> 2), 977 (0x0e00 << 16) | (0x9600 >> 2), 978 }; 979 980 static const u32 bonaire_golden_spm_registers[] = 981 { 982 0x30800, 0xe0ffffff, 0xe0000000 983 }; 984 985 static const u32 bonaire_golden_common_registers[] = 986 { 987 0xc770, 0xffffffff, 0x00000800, 988 0xc774, 0xffffffff, 0x00000800, 989 0xc798, 0xffffffff, 0x00007fbf, 990 0xc79c, 0xffffffff, 0x00007faf 991 }; 992 993 static const u32 bonaire_golden_registers[] = 994 { 995 0x3354, 0x00000333, 0x00000333, 996 0x3350, 0x000c0fc0, 0x00040200, 997 0x9a10, 0x00010000, 0x00058208, 998 0x3c000, 0xffff1fff, 0x00140000, 999 0x3c200, 0xfdfc0fff, 0x00000100, 1000 0x3c234, 0x40000000, 0x40000200, 1001 0x9830, 0xffffffff, 0x00000000, 1002 0x9834, 0xf00fffff, 0x00000400, 1003 0x9838, 0x0002021c, 0x00020200, 1004 0xc78, 0x00000080, 0x00000000, 1005 0x5bb0, 0x000000f0, 0x00000070, 1006 0x5bc0, 0xf0311fff, 0x80300000, 1007 0x98f8, 0x73773777, 0x12010001, 1008 0x350c, 0x00810000, 0x408af000, 1009 0x7030, 0x31000111, 0x00000011, 1010 0x2f48, 0x73773777, 0x12010001, 1011 0x220c, 0x00007fb6, 0x0021a1b1, 1012 0x2210, 0x00007fb6, 0x002021b1, 1013 0x2180, 0x00007fb6, 0x00002191, 1014 0x2218, 0x00007fb6, 0x002121b1, 1015 0x221c, 0x00007fb6, 0x002021b1, 1016 0x21dc, 0x00007fb6, 0x00002191, 1017 0x21e0, 0x00007fb6, 0x00002191, 1018 0x3628, 0x0000003f, 0x0000000a, 1019 0x362c, 0x0000003f, 0x0000000a, 1020 0x2ae4, 0x00073ffe, 0x000022a2, 1021 0x240c, 0x000007ff, 0x00000000, 1022 0x8a14, 0xf000003f, 0x00000007, 1023 0x8bf0, 0x00002001, 0x00000001, 1024 0x8b24, 0xffffffff, 0x00ffffff, 1025 0x30a04, 0x0000ff0f, 0x00000000, 1026 0x28a4c, 0x07ffffff, 0x06000000, 1027 0x4d8, 0x00000fff, 0x00000100, 1028 0x3e78, 0x00000001, 0x00000002, 1029 0x9100, 0x03000000, 0x0362c688, 1030 0x8c00, 0x000000ff, 0x00000001, 1031 0xe40, 0x00001fff, 0x00001fff, 1032 0x9060, 0x0000007f, 0x00000020, 1033 0x9508, 0x00010000, 0x00010000, 1034 0xac14, 0x000003ff, 0x000000f3, 1035 0xac0c, 0xffffffff, 0x00001032 1036 }; 1037 1038 static const u32 bonaire_mgcg_cgcg_init[] = 1039 { 1040 0xc420, 0xffffffff, 0xfffffffc, 1041 0x30800, 0xffffffff, 0xe0000000, 1042 0x3c2a0, 0xffffffff, 0x00000100, 1043 0x3c208, 0xffffffff, 0x00000100, 1044 0x3c2c0, 0xffffffff, 0xc0000100, 1045 0x3c2c8, 0xffffffff, 0xc0000100, 1046 0x3c2c4, 0xffffffff, 0xc0000100, 1047 0x55e4, 0xffffffff, 0x00600100, 1048 0x3c280, 0xffffffff, 0x00000100, 1049 0x3c214, 0xffffffff, 0x06000100, 1050 0x3c220, 0xffffffff, 0x00000100, 1051 0x3c218, 0xffffffff, 0x06000100, 1052 0x3c204, 0xffffffff, 0x00000100, 1053 0x3c2e0, 0xffffffff, 0x00000100, 1054 0x3c224, 0xffffffff, 0x00000100, 1055 0x3c200, 0xffffffff, 0x00000100, 1056 0x3c230, 0xffffffff, 0x00000100, 1057 0x3c234, 0xffffffff, 0x00000100, 1058 0x3c250, 0xffffffff, 0x00000100, 1059 0x3c254, 0xffffffff, 0x00000100, 1060 0x3c258, 0xffffffff, 0x00000100, 1061 0x3c25c, 0xffffffff, 0x00000100, 1062 0x3c260, 0xffffffff, 0x00000100, 1063 0x3c27c, 0xffffffff, 0x00000100, 1064 0x3c278, 0xffffffff, 0x00000100, 1065 0x3c210, 0xffffffff, 0x06000100, 1066 0x3c290, 0xffffffff, 0x00000100, 1067 0x3c274, 0xffffffff, 0x00000100, 1068 0x3c2b4, 0xffffffff, 0x00000100, 1069 0x3c2b0, 0xffffffff, 0x00000100, 1070 0x3c270, 0xffffffff, 0x00000100, 1071 0x30800, 0xffffffff, 0xe0000000, 1072 0x3c020, 0xffffffff, 0x00010000, 1073 0x3c024, 0xffffffff, 0x00030002, 1074 0x3c028, 0xffffffff, 0x00040007, 1075 0x3c02c, 0xffffffff, 0x00060005, 1076 0x3c030, 0xffffffff, 0x00090008, 1077 0x3c034, 0xffffffff, 0x00010000, 1078 0x3c038, 0xffffffff, 0x00030002, 1079 0x3c03c, 0xffffffff, 0x00040007, 1080 0x3c040, 0xffffffff, 0x00060005, 1081 0x3c044, 0xffffffff, 0x00090008, 1082 0x3c048, 0xffffffff, 0x00010000, 1083 0x3c04c, 0xffffffff, 0x00030002, 1084 0x3c050, 0xffffffff, 0x00040007, 1085 0x3c054, 0xffffffff, 0x00060005, 1086 0x3c058, 0xffffffff, 0x00090008, 1087 0x3c05c, 0xffffffff, 0x00010000, 1088 0x3c060, 0xffffffff, 0x00030002, 1089 0x3c064, 0xffffffff, 0x00040007, 1090 0x3c068, 0xffffffff, 0x00060005, 1091 0x3c06c, 0xffffffff, 0x00090008, 1092 0x3c070, 0xffffffff, 0x00010000, 1093 0x3c074, 0xffffffff, 0x00030002, 1094 0x3c078, 0xffffffff, 0x00040007, 1095 0x3c07c, 0xffffffff, 0x00060005, 1096 0x3c080, 0xffffffff, 0x00090008, 1097 0x3c084, 0xffffffff, 0x00010000, 1098 0x3c088, 0xffffffff, 0x00030002, 1099 0x3c08c, 0xffffffff, 0x00040007, 1100 0x3c090, 0xffffffff, 0x00060005, 1101 0x3c094, 0xffffffff, 0x00090008, 1102 0x3c098, 0xffffffff, 0x00010000, 1103 0x3c09c, 0xffffffff, 0x00030002, 1104 0x3c0a0, 0xffffffff, 0x00040007, 1105 0x3c0a4, 0xffffffff, 0x00060005, 1106 0x3c0a8, 0xffffffff, 0x00090008, 1107 0x3c000, 0xffffffff, 0x96e00200, 1108 0x8708, 0xffffffff, 0x00900100, 1109 0xc424, 0xffffffff, 0x0020003f, 1110 0x38, 0xffffffff, 0x0140001c, 1111 0x3c, 0x000f0000, 0x000f0000, 1112 0x220, 0xffffffff, 0xC060000C, 1113 0x224, 0xc0000fff, 0x00000100, 1114 0xf90, 0xffffffff, 0x00000100, 1115 0xf98, 0x00000101, 0x00000000, 1116 0x20a8, 0xffffffff, 0x00000104, 1117 0x55e4, 0xff000fff, 0x00000100, 1118 0x30cc, 0xc0000fff, 0x00000104, 1119 0xc1e4, 0x00000001, 0x00000001, 1120 0xd00c, 0xff000ff0, 0x00000100, 1121 0xd80c, 0xff000ff0, 0x00000100 1122 }; 1123 1124 static const u32 spectre_golden_spm_registers[] = 1125 { 1126 0x30800, 0xe0ffffff, 0xe0000000 1127 }; 1128 1129 static const u32 spectre_golden_common_registers[] = 1130 { 1131 0xc770, 0xffffffff, 0x00000800, 1132 0xc774, 0xffffffff, 0x00000800, 1133 0xc798, 0xffffffff, 0x00007fbf, 1134 0xc79c, 0xffffffff, 0x00007faf 1135 }; 1136 1137 static const u32 spectre_golden_registers[] = 1138 { 1139 0x3c000, 0xffff1fff, 0x96940200, 1140 0x3c00c, 0xffff0001, 0xff000000, 1141 0x3c200, 0xfffc0fff, 0x00000100, 1142 0x6ed8, 0x00010101, 0x00010000, 1143 0x9834, 0xf00fffff, 0x00000400, 1144 0x9838, 0xfffffffc, 0x00020200, 1145 0x5bb0, 0x000000f0, 0x00000070, 1146 0x5bc0, 0xf0311fff, 0x80300000, 1147 0x98f8, 0x73773777, 0x12010001, 1148 0x9b7c, 0x00ff0000, 0x00fc0000, 1149 0x2f48, 0x73773777, 0x12010001, 1150 0x8a14, 0xf000003f, 0x00000007, 1151 0x8b24, 0xffffffff, 0x00ffffff, 1152 0x28350, 0x3f3f3fff, 0x00000082, 1153 0x28354, 0x0000003f, 0x00000000, 1154 0x3e78, 0x00000001, 0x00000002, 1155 0x913c, 0xffff03df, 0x00000004, 1156 0xc768, 0x00000008, 0x00000008, 1157 0x8c00, 0x000008ff, 0x00000800, 1158 0x9508, 0x00010000, 0x00010000, 1159 0xac0c, 0xffffffff, 0x54763210, 1160 0x214f8, 0x01ff01ff, 0x00000002, 1161 0x21498, 0x007ff800, 0x00200000, 1162 0x2015c, 0xffffffff, 0x00000f40, 1163 0x30934, 0xffffffff, 0x00000001 1164 }; 1165 1166 static const u32 spectre_mgcg_cgcg_init[] = 1167 { 1168 0xc420, 0xffffffff, 0xfffffffc, 1169 0x30800, 0xffffffff, 0xe0000000, 1170 0x3c2a0, 0xffffffff, 0x00000100, 1171 0x3c208, 0xffffffff, 0x00000100, 1172 0x3c2c0, 0xffffffff, 0x00000100, 1173 0x3c2c8, 0xffffffff, 0x00000100, 1174 0x3c2c4, 0xffffffff, 0x00000100, 1175 0x55e4, 0xffffffff, 0x00600100, 1176 0x3c280, 0xffffffff, 0x00000100, 1177 0x3c214, 0xffffffff, 0x06000100, 1178 0x3c220, 0xffffffff, 0x00000100, 1179 0x3c218, 0xffffffff, 0x06000100, 1180 0x3c204, 0xffffffff, 0x00000100, 1181 0x3c2e0, 0xffffffff, 0x00000100, 1182 0x3c224, 0xffffffff, 0x00000100, 1183 0x3c200, 0xffffffff, 0x00000100, 1184 0x3c230, 0xffffffff, 0x00000100, 1185 0x3c234, 0xffffffff, 0x00000100, 1186 0x3c250, 0xffffffff, 0x00000100, 1187 0x3c254, 0xffffffff, 0x00000100, 1188 0x3c258, 0xffffffff, 0x00000100, 1189 0x3c25c, 0xffffffff, 0x00000100, 1190 0x3c260, 0xffffffff, 0x00000100, 1191 0x3c27c, 0xffffffff, 0x00000100, 1192 0x3c278, 0xffffffff, 0x00000100, 1193 0x3c210, 0xffffffff, 0x06000100, 1194 0x3c290, 0xffffffff, 0x00000100, 1195 0x3c274, 0xffffffff, 0x00000100, 1196 0x3c2b4, 0xffffffff, 0x00000100, 1197 0x3c2b0, 0xffffffff, 0x00000100, 1198 0x3c270, 0xffffffff, 0x00000100, 1199 0x30800, 0xffffffff, 0xe0000000, 1200 0x3c020, 0xffffffff, 0x00010000, 1201 0x3c024, 0xffffffff, 0x00030002, 1202 0x3c028, 0xffffffff, 0x00040007, 1203 0x3c02c, 0xffffffff, 0x00060005, 1204 0x3c030, 0xffffffff, 0x00090008, 1205 0x3c034, 0xffffffff, 0x00010000, 1206 0x3c038, 0xffffffff, 0x00030002, 1207 0x3c03c, 0xffffffff, 0x00040007, 1208 0x3c040, 0xffffffff, 0x00060005, 1209 0x3c044, 0xffffffff, 0x00090008, 1210 0x3c048, 0xffffffff, 0x00010000, 1211 0x3c04c, 0xffffffff, 0x00030002, 1212 0x3c050, 0xffffffff, 0x00040007, 1213 0x3c054, 0xffffffff, 0x00060005, 1214 0x3c058, 0xffffffff, 0x00090008, 1215 0x3c05c, 0xffffffff, 0x00010000, 1216 0x3c060, 0xffffffff, 0x00030002, 1217 0x3c064, 0xffffffff, 0x00040007, 1218 0x3c068, 0xffffffff, 0x00060005, 1219 0x3c06c, 0xffffffff, 0x00090008, 1220 0x3c070, 0xffffffff, 0x00010000, 1221 0x3c074, 0xffffffff, 0x00030002, 1222 0x3c078, 0xffffffff, 0x00040007, 1223 0x3c07c, 0xffffffff, 0x00060005, 1224 0x3c080, 0xffffffff, 0x00090008, 1225 0x3c084, 0xffffffff, 0x00010000, 1226 0x3c088, 0xffffffff, 0x00030002, 1227 0x3c08c, 0xffffffff, 0x00040007, 1228 0x3c090, 0xffffffff, 0x00060005, 1229 0x3c094, 0xffffffff, 0x00090008, 1230 0x3c098, 0xffffffff, 0x00010000, 1231 0x3c09c, 0xffffffff, 0x00030002, 1232 0x3c0a0, 0xffffffff, 0x00040007, 1233 0x3c0a4, 0xffffffff, 0x00060005, 1234 0x3c0a8, 0xffffffff, 0x00090008, 1235 0x3c0ac, 0xffffffff, 0x00010000, 1236 0x3c0b0, 0xffffffff, 0x00030002, 1237 0x3c0b4, 0xffffffff, 0x00040007, 1238 0x3c0b8, 0xffffffff, 0x00060005, 1239 0x3c0bc, 0xffffffff, 0x00090008, 1240 0x3c000, 0xffffffff, 0x96e00200, 1241 0x8708, 0xffffffff, 0x00900100, 1242 0xc424, 0xffffffff, 0x0020003f, 1243 0x38, 0xffffffff, 0x0140001c, 1244 0x3c, 0x000f0000, 0x000f0000, 1245 0x220, 0xffffffff, 0xC060000C, 1246 0x224, 0xc0000fff, 0x00000100, 1247 0xf90, 0xffffffff, 0x00000100, 1248 0xf98, 0x00000101, 0x00000000, 1249 0x20a8, 0xffffffff, 0x00000104, 1250 0x55e4, 0xff000fff, 0x00000100, 1251 0x30cc, 0xc0000fff, 0x00000104, 1252 0xc1e4, 0x00000001, 0x00000001, 1253 0xd00c, 0xff000ff0, 0x00000100, 1254 0xd80c, 0xff000ff0, 0x00000100 1255 }; 1256 1257 static const u32 kalindi_golden_spm_registers[] = 1258 { 1259 0x30800, 0xe0ffffff, 0xe0000000 1260 }; 1261 1262 static const u32 kalindi_golden_common_registers[] = 1263 { 1264 0xc770, 0xffffffff, 0x00000800, 1265 0xc774, 0xffffffff, 0x00000800, 1266 0xc798, 0xffffffff, 0x00007fbf, 1267 0xc79c, 0xffffffff, 0x00007faf 1268 }; 1269 1270 static const u32 kalindi_golden_registers[] = 1271 { 1272 0x3c000, 0xffffdfff, 0x6e944040, 1273 0x55e4, 0xff607fff, 0xfc000100, 1274 0x3c220, 0xff000fff, 0x00000100, 1275 0x3c224, 0xff000fff, 0x00000100, 1276 0x3c200, 0xfffc0fff, 0x00000100, 1277 0x6ed8, 0x00010101, 0x00010000, 1278 0x9830, 0xffffffff, 0x00000000, 1279 0x9834, 0xf00fffff, 0x00000400, 1280 0x5bb0, 0x000000f0, 0x00000070, 1281 0x5bc0, 0xf0311fff, 0x80300000, 1282 0x98f8, 0x73773777, 0x12010001, 1283 0x98fc, 0xffffffff, 0x00000010, 1284 0x9b7c, 0x00ff0000, 0x00fc0000, 1285 0x8030, 0x00001f0f, 0x0000100a, 1286 0x2f48, 0x73773777, 0x12010001, 1287 0x2408, 0x000fffff, 0x000c007f, 1288 0x8a14, 0xf000003f, 0x00000007, 1289 0x8b24, 0x3fff3fff, 0x00ffcfff, 1290 0x30a04, 0x0000ff0f, 0x00000000, 1291 0x28a4c, 0x07ffffff, 0x06000000, 1292 0x4d8, 0x00000fff, 0x00000100, 1293 0x3e78, 0x00000001, 0x00000002, 1294 0xc768, 0x00000008, 0x00000008, 1295 0x8c00, 0x000000ff, 0x00000003, 1296 0x214f8, 0x01ff01ff, 0x00000002, 1297 0x21498, 0x007ff800, 0x00200000, 1298 0x2015c, 0xffffffff, 0x00000f40, 1299 0x88c4, 0x001f3ae3, 0x00000082, 1300 0x88d4, 0x0000001f, 0x00000010, 1301 0x30934, 0xffffffff, 0x00000000 1302 }; 1303 1304 static const u32 kalindi_mgcg_cgcg_init[] = 1305 { 1306 0xc420, 0xffffffff, 0xfffffffc, 1307 0x30800, 0xffffffff, 0xe0000000, 1308 0x3c2a0, 0xffffffff, 0x00000100, 1309 0x3c208, 0xffffffff, 0x00000100, 1310 0x3c2c0, 0xffffffff, 0x00000100, 1311 0x3c2c8, 0xffffffff, 0x00000100, 1312 0x3c2c4, 0xffffffff, 0x00000100, 1313 0x55e4, 0xffffffff, 0x00600100, 1314 0x3c280, 0xffffffff, 0x00000100, 1315 0x3c214, 0xffffffff, 0x06000100, 1316 0x3c220, 0xffffffff, 0x00000100, 1317 0x3c218, 0xffffffff, 0x06000100, 1318 0x3c204, 0xffffffff, 0x00000100, 1319 0x3c2e0, 0xffffffff, 0x00000100, 1320 0x3c224, 0xffffffff, 0x00000100, 1321 0x3c200, 0xffffffff, 0x00000100, 1322 0x3c230, 0xffffffff, 0x00000100, 1323 0x3c234, 0xffffffff, 0x00000100, 1324 0x3c250, 0xffffffff, 0x00000100, 1325 0x3c254, 0xffffffff, 0x00000100, 1326 0x3c258, 0xffffffff, 0x00000100, 1327 0x3c25c, 0xffffffff, 0x00000100, 1328 0x3c260, 0xffffffff, 0x00000100, 1329 0x3c27c, 0xffffffff, 0x00000100, 1330 0x3c278, 0xffffffff, 0x00000100, 1331 0x3c210, 0xffffffff, 0x06000100, 1332 0x3c290, 0xffffffff, 0x00000100, 1333 0x3c274, 0xffffffff, 0x00000100, 1334 0x3c2b4, 0xffffffff, 0x00000100, 1335 0x3c2b0, 0xffffffff, 0x00000100, 1336 0x3c270, 0xffffffff, 0x00000100, 1337 0x30800, 0xffffffff, 0xe0000000, 1338 0x3c020, 0xffffffff, 0x00010000, 1339 0x3c024, 0xffffffff, 0x00030002, 1340 0x3c028, 0xffffffff, 0x00040007, 1341 0x3c02c, 0xffffffff, 0x00060005, 1342 0x3c030, 0xffffffff, 0x00090008, 1343 0x3c034, 0xffffffff, 0x00010000, 1344 0x3c038, 0xffffffff, 0x00030002, 1345 0x3c03c, 0xffffffff, 0x00040007, 1346 0x3c040, 0xffffffff, 0x00060005, 1347 0x3c044, 0xffffffff, 0x00090008, 1348 0x3c000, 0xffffffff, 0x96e00200, 1349 0x8708, 0xffffffff, 0x00900100, 1350 0xc424, 0xffffffff, 0x0020003f, 1351 0x38, 0xffffffff, 0x0140001c, 1352 0x3c, 0x000f0000, 0x000f0000, 1353 0x220, 0xffffffff, 0xC060000C, 1354 0x224, 0xc0000fff, 0x00000100, 1355 0x20a8, 0xffffffff, 0x00000104, 1356 0x55e4, 0xff000fff, 0x00000100, 1357 0x30cc, 0xc0000fff, 0x00000104, 1358 0xc1e4, 0x00000001, 0x00000001, 1359 0xd00c, 0xff000ff0, 0x00000100, 1360 0xd80c, 0xff000ff0, 0x00000100 1361 }; 1362 1363 static const u32 hawaii_golden_spm_registers[] = 1364 { 1365 0x30800, 0xe0ffffff, 0xe0000000 1366 }; 1367 1368 static const u32 hawaii_golden_common_registers[] = 1369 { 1370 0x30800, 0xffffffff, 0xe0000000, 1371 0x28350, 0xffffffff, 0x3a00161a, 1372 0x28354, 0xffffffff, 0x0000002e, 1373 0x9a10, 0xffffffff, 0x00018208, 1374 0x98f8, 0xffffffff, 0x12011003 1375 }; 1376 1377 static const u32 hawaii_golden_registers[] = 1378 { 1379 0x3354, 0x00000333, 0x00000333, 1380 0x9a10, 0x00010000, 0x00058208, 1381 0x9830, 0xffffffff, 0x00000000, 1382 0x9834, 0xf00fffff, 0x00000400, 1383 0x9838, 0x0002021c, 0x00020200, 1384 0xc78, 0x00000080, 0x00000000, 1385 0x5bb0, 0x000000f0, 0x00000070, 1386 0x5bc0, 0xf0311fff, 0x80300000, 1387 0x350c, 0x00810000, 0x408af000, 1388 0x7030, 0x31000111, 0x00000011, 1389 0x2f48, 0x73773777, 0x12010001, 1390 0x2120, 0x0000007f, 0x0000001b, 1391 0x21dc, 0x00007fb6, 0x00002191, 1392 0x3628, 0x0000003f, 0x0000000a, 1393 0x362c, 0x0000003f, 0x0000000a, 1394 0x2ae4, 0x00073ffe, 0x000022a2, 1395 0x240c, 0x000007ff, 0x00000000, 1396 0x8bf0, 0x00002001, 0x00000001, 1397 0x8b24, 0xffffffff, 0x00ffffff, 1398 0x30a04, 0x0000ff0f, 0x00000000, 1399 0x28a4c, 0x07ffffff, 0x06000000, 1400 0x3e78, 0x00000001, 0x00000002, 1401 0xc768, 0x00000008, 0x00000008, 1402 0xc770, 0x00000f00, 0x00000800, 1403 0xc774, 0x00000f00, 0x00000800, 1404 0xc798, 0x00ffffff, 0x00ff7fbf, 1405 0xc79c, 0x00ffffff, 0x00ff7faf, 1406 0x8c00, 0x000000ff, 0x00000800, 1407 0xe40, 0x00001fff, 0x00001fff, 1408 0x9060, 0x0000007f, 0x00000020, 1409 0x9508, 0x00010000, 0x00010000, 1410 0xae00, 0x00100000, 0x000ff07c, 1411 0xac14, 0x000003ff, 0x0000000f, 1412 0xac10, 0xffffffff, 0x7564fdec, 1413 0xac0c, 0xffffffff, 0x3120b9a8, 1414 0xac08, 0x20000000, 0x0f9c0000 1415 }; 1416 1417 static const u32 hawaii_mgcg_cgcg_init[] = 1418 { 1419 0xc420, 0xffffffff, 0xfffffffd, 1420 0x30800, 0xffffffff, 0xe0000000, 1421 0x3c2a0, 0xffffffff, 0x00000100, 1422 0x3c208, 0xffffffff, 0x00000100, 1423 0x3c2c0, 0xffffffff, 0x00000100, 1424 0x3c2c8, 0xffffffff, 0x00000100, 1425 0x3c2c4, 0xffffffff, 0x00000100, 1426 0x55e4, 0xffffffff, 0x00200100, 1427 0x3c280, 0xffffffff, 0x00000100, 1428 0x3c214, 0xffffffff, 0x06000100, 1429 0x3c220, 0xffffffff, 0x00000100, 1430 0x3c218, 0xffffffff, 0x06000100, 1431 0x3c204, 0xffffffff, 0x00000100, 1432 0x3c2e0, 0xffffffff, 0x00000100, 1433 0x3c224, 0xffffffff, 0x00000100, 1434 0x3c200, 0xffffffff, 0x00000100, 1435 0x3c230, 0xffffffff, 0x00000100, 1436 0x3c234, 0xffffffff, 0x00000100, 1437 0x3c250, 0xffffffff, 0x00000100, 1438 0x3c254, 0xffffffff, 0x00000100, 1439 0x3c258, 0xffffffff, 0x00000100, 1440 0x3c25c, 0xffffffff, 0x00000100, 1441 0x3c260, 0xffffffff, 0x00000100, 1442 0x3c27c, 0xffffffff, 0x00000100, 1443 0x3c278, 0xffffffff, 0x00000100, 1444 0x3c210, 0xffffffff, 0x06000100, 1445 0x3c290, 0xffffffff, 0x00000100, 1446 0x3c274, 0xffffffff, 0x00000100, 1447 0x3c2b4, 0xffffffff, 0x00000100, 1448 0x3c2b0, 0xffffffff, 0x00000100, 1449 0x3c270, 0xffffffff, 0x00000100, 1450 0x30800, 0xffffffff, 0xe0000000, 1451 0x3c020, 0xffffffff, 0x00010000, 1452 0x3c024, 0xffffffff, 0x00030002, 1453 0x3c028, 0xffffffff, 0x00040007, 1454 0x3c02c, 0xffffffff, 0x00060005, 1455 0x3c030, 0xffffffff, 0x00090008, 1456 0x3c034, 0xffffffff, 0x00010000, 1457 0x3c038, 0xffffffff, 0x00030002, 1458 0x3c03c, 0xffffffff, 0x00040007, 1459 0x3c040, 0xffffffff, 0x00060005, 1460 0x3c044, 0xffffffff, 0x00090008, 1461 0x3c048, 0xffffffff, 0x00010000, 1462 0x3c04c, 0xffffffff, 0x00030002, 1463 0x3c050, 0xffffffff, 0x00040007, 1464 0x3c054, 0xffffffff, 0x00060005, 1465 0x3c058, 0xffffffff, 0x00090008, 1466 0x3c05c, 0xffffffff, 0x00010000, 1467 0x3c060, 0xffffffff, 0x00030002, 1468 0x3c064, 0xffffffff, 0x00040007, 1469 0x3c068, 0xffffffff, 0x00060005, 1470 0x3c06c, 0xffffffff, 0x00090008, 1471 0x3c070, 0xffffffff, 0x00010000, 1472 0x3c074, 0xffffffff, 0x00030002, 1473 0x3c078, 0xffffffff, 0x00040007, 1474 0x3c07c, 0xffffffff, 0x00060005, 1475 0x3c080, 0xffffffff, 0x00090008, 1476 0x3c084, 0xffffffff, 0x00010000, 1477 0x3c088, 0xffffffff, 0x00030002, 1478 0x3c08c, 0xffffffff, 0x00040007, 1479 0x3c090, 0xffffffff, 0x00060005, 1480 0x3c094, 0xffffffff, 0x00090008, 1481 0x3c098, 0xffffffff, 0x00010000, 1482 0x3c09c, 0xffffffff, 0x00030002, 1483 0x3c0a0, 0xffffffff, 0x00040007, 1484 0x3c0a4, 0xffffffff, 0x00060005, 1485 0x3c0a8, 0xffffffff, 0x00090008, 1486 0x3c0ac, 0xffffffff, 0x00010000, 1487 0x3c0b0, 0xffffffff, 0x00030002, 1488 0x3c0b4, 0xffffffff, 0x00040007, 1489 0x3c0b8, 0xffffffff, 0x00060005, 1490 0x3c0bc, 0xffffffff, 0x00090008, 1491 0x3c0c0, 0xffffffff, 0x00010000, 1492 0x3c0c4, 0xffffffff, 0x00030002, 1493 0x3c0c8, 0xffffffff, 0x00040007, 1494 0x3c0cc, 0xffffffff, 0x00060005, 1495 0x3c0d0, 0xffffffff, 0x00090008, 1496 0x3c0d4, 0xffffffff, 0x00010000, 1497 0x3c0d8, 0xffffffff, 0x00030002, 1498 0x3c0dc, 0xffffffff, 0x00040007, 1499 0x3c0e0, 0xffffffff, 0x00060005, 1500 0x3c0e4, 0xffffffff, 0x00090008, 1501 0x3c0e8, 0xffffffff, 0x00010000, 1502 0x3c0ec, 0xffffffff, 0x00030002, 1503 0x3c0f0, 0xffffffff, 0x00040007, 1504 0x3c0f4, 0xffffffff, 0x00060005, 1505 0x3c0f8, 0xffffffff, 0x00090008, 1506 0xc318, 0xffffffff, 0x00020200, 1507 0x3350, 0xffffffff, 0x00000200, 1508 0x15c0, 0xffffffff, 0x00000400, 1509 0x55e8, 0xffffffff, 0x00000000, 1510 0x2f50, 0xffffffff, 0x00000902, 1511 0x3c000, 0xffffffff, 0x96940200, 1512 0x8708, 0xffffffff, 0x00900100, 1513 0xc424, 0xffffffff, 0x0020003f, 1514 0x38, 0xffffffff, 0x0140001c, 1515 0x3c, 0x000f0000, 0x000f0000, 1516 0x220, 0xffffffff, 0xc060000c, 1517 0x224, 0xc0000fff, 0x00000100, 1518 0xf90, 0xffffffff, 0x00000100, 1519 0xf98, 0x00000101, 0x00000000, 1520 0x20a8, 0xffffffff, 0x00000104, 1521 0x55e4, 0xff000fff, 0x00000100, 1522 0x30cc, 0xc0000fff, 0x00000104, 1523 0xc1e4, 0x00000001, 0x00000001, 1524 0xd00c, 0xff000ff0, 0x00000100, 1525 0xd80c, 0xff000ff0, 0x00000100 1526 }; 1527 1528 static const u32 godavari_golden_registers[] = 1529 { 1530 0x55e4, 0xff607fff, 0xfc000100, 1531 0x6ed8, 0x00010101, 0x00010000, 1532 0x9830, 0xffffffff, 0x00000000, 1533 0x98302, 0xf00fffff, 0x00000400, 1534 0x6130, 0xffffffff, 0x00010000, 1535 0x5bb0, 0x000000f0, 0x00000070, 1536 0x5bc0, 0xf0311fff, 0x80300000, 1537 0x98f8, 0x73773777, 0x12010001, 1538 0x98fc, 0xffffffff, 0x00000010, 1539 0x8030, 0x00001f0f, 0x0000100a, 1540 0x2f48, 0x73773777, 0x12010001, 1541 0x2408, 0x000fffff, 0x000c007f, 1542 0x8a14, 0xf000003f, 0x00000007, 1543 0x8b24, 0xffffffff, 0x00ff0fff, 1544 0x30a04, 0x0000ff0f, 0x00000000, 1545 0x28a4c, 0x07ffffff, 0x06000000, 1546 0x4d8, 0x00000fff, 0x00000100, 1547 0xd014, 0x00010000, 0x00810001, 1548 0xd814, 0x00010000, 0x00810001, 1549 0x3e78, 0x00000001, 0x00000002, 1550 0xc768, 0x00000008, 0x00000008, 1551 0xc770, 0x00000f00, 0x00000800, 1552 0xc774, 0x00000f00, 0x00000800, 1553 0xc798, 0x00ffffff, 0x00ff7fbf, 1554 0xc79c, 0x00ffffff, 0x00ff7faf, 1555 0x8c00, 0x000000ff, 0x00000001, 1556 0x214f8, 0x01ff01ff, 0x00000002, 1557 0x21498, 0x007ff800, 0x00200000, 1558 0x2015c, 0xffffffff, 0x00000f40, 1559 0x88c4, 0x001f3ae3, 0x00000082, 1560 0x88d4, 0x0000001f, 0x00000010, 1561 0x30934, 0xffffffff, 0x00000000 1562 }; 1563 1564 1565 static void cik_init_golden_registers(struct radeon_device *rdev) 1566 { 1567 /* Some of the registers might be dependent on GRBM_GFX_INDEX */ 1568 mutex_lock(&rdev->grbm_idx_mutex); 1569 switch (rdev->family) { 1570 case CHIP_BONAIRE: 1571 radeon_program_register_sequence(rdev, 1572 bonaire_mgcg_cgcg_init, 1573 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 1574 radeon_program_register_sequence(rdev, 1575 bonaire_golden_registers, 1576 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 1577 radeon_program_register_sequence(rdev, 1578 bonaire_golden_common_registers, 1579 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 1580 radeon_program_register_sequence(rdev, 1581 bonaire_golden_spm_registers, 1582 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 1583 break; 1584 case CHIP_KABINI: 1585 radeon_program_register_sequence(rdev, 1586 kalindi_mgcg_cgcg_init, 1587 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1588 radeon_program_register_sequence(rdev, 1589 kalindi_golden_registers, 1590 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 1591 radeon_program_register_sequence(rdev, 1592 kalindi_golden_common_registers, 1593 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1594 radeon_program_register_sequence(rdev, 1595 kalindi_golden_spm_registers, 1596 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1597 break; 1598 case CHIP_MULLINS: 1599 radeon_program_register_sequence(rdev, 1600 kalindi_mgcg_cgcg_init, 1601 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1602 radeon_program_register_sequence(rdev, 1603 godavari_golden_registers, 1604 (const u32)ARRAY_SIZE(godavari_golden_registers)); 1605 radeon_program_register_sequence(rdev, 1606 kalindi_golden_common_registers, 1607 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1608 radeon_program_register_sequence(rdev, 1609 kalindi_golden_spm_registers, 1610 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1611 break; 1612 case CHIP_KAVERI: 1613 radeon_program_register_sequence(rdev, 1614 spectre_mgcg_cgcg_init, 1615 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 1616 radeon_program_register_sequence(rdev, 1617 spectre_golden_registers, 1618 (const u32)ARRAY_SIZE(spectre_golden_registers)); 1619 radeon_program_register_sequence(rdev, 1620 spectre_golden_common_registers, 1621 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 1622 radeon_program_register_sequence(rdev, 1623 spectre_golden_spm_registers, 1624 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 1625 break; 1626 case CHIP_HAWAII: 1627 radeon_program_register_sequence(rdev, 1628 hawaii_mgcg_cgcg_init, 1629 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 1630 radeon_program_register_sequence(rdev, 1631 hawaii_golden_registers, 1632 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 1633 radeon_program_register_sequence(rdev, 1634 hawaii_golden_common_registers, 1635 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 1636 radeon_program_register_sequence(rdev, 1637 hawaii_golden_spm_registers, 1638 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 1639 break; 1640 default: 1641 break; 1642 } 1643 mutex_unlock(&rdev->grbm_idx_mutex); 1644 } 1645 1646 /** 1647 * cik_get_xclk - get the xclk 1648 * 1649 * @rdev: radeon_device pointer 1650 * 1651 * Returns the reference clock used by the gfx engine 1652 * (CIK). 1653 */ 1654 u32 cik_get_xclk(struct radeon_device *rdev) 1655 { 1656 u32 reference_clock = rdev->clock.spll.reference_freq; 1657 1658 if (rdev->flags & RADEON_IS_IGP) { 1659 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) 1660 return reference_clock / 2; 1661 } else { 1662 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) 1663 return reference_clock / 4; 1664 } 1665 return reference_clock; 1666 } 1667 1668 /** 1669 * cik_mm_rdoorbell - read a doorbell dword 1670 * 1671 * @rdev: radeon_device pointer 1672 * @index: doorbell index 1673 * 1674 * Returns the value in the doorbell aperture at the 1675 * requested doorbell index (CIK). 1676 */ 1677 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) 1678 { 1679 if (index < rdev->doorbell.num_doorbells) { 1680 return readl(rdev->doorbell.ptr + index); 1681 } else { 1682 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 1683 return 0; 1684 } 1685 } 1686 1687 /** 1688 * cik_mm_wdoorbell - write a doorbell dword 1689 * 1690 * @rdev: radeon_device pointer 1691 * @index: doorbell index 1692 * @v: value to write 1693 * 1694 * Writes @v to the doorbell aperture at the 1695 * requested doorbell index (CIK). 1696 */ 1697 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) 1698 { 1699 if (index < rdev->doorbell.num_doorbells) { 1700 writel(v, rdev->doorbell.ptr + index); 1701 } else { 1702 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 1703 } 1704 } 1705 1706 #define BONAIRE_IO_MC_REGS_SIZE 36 1707 1708 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = 1709 { 1710 {0x00000070, 0x04400000}, 1711 {0x00000071, 0x80c01803}, 1712 {0x00000072, 0x00004004}, 1713 {0x00000073, 0x00000100}, 1714 {0x00000074, 0x00ff0000}, 1715 {0x00000075, 0x34000000}, 1716 {0x00000076, 0x08000014}, 1717 {0x00000077, 0x00cc08ec}, 1718 {0x00000078, 0x00000400}, 1719 {0x00000079, 0x00000000}, 1720 {0x0000007a, 0x04090000}, 1721 {0x0000007c, 0x00000000}, 1722 {0x0000007e, 0x4408a8e8}, 1723 {0x0000007f, 0x00000304}, 1724 {0x00000080, 0x00000000}, 1725 {0x00000082, 0x00000001}, 1726 {0x00000083, 0x00000002}, 1727 {0x00000084, 0xf3e4f400}, 1728 {0x00000085, 0x052024e3}, 1729 {0x00000087, 0x00000000}, 1730 {0x00000088, 0x01000000}, 1731 {0x0000008a, 0x1c0a0000}, 1732 {0x0000008b, 0xff010000}, 1733 {0x0000008d, 0xffffefff}, 1734 {0x0000008e, 0xfff3efff}, 1735 {0x0000008f, 0xfff3efbf}, 1736 {0x00000092, 0xf7ffffff}, 1737 {0x00000093, 0xffffff7f}, 1738 {0x00000095, 0x00101101}, 1739 {0x00000096, 0x00000fff}, 1740 {0x00000097, 0x00116fff}, 1741 {0x00000098, 0x60010000}, 1742 {0x00000099, 0x10010000}, 1743 {0x0000009a, 0x00006000}, 1744 {0x0000009b, 0x00001000}, 1745 {0x0000009f, 0x00b48000} 1746 }; 1747 1748 #define HAWAII_IO_MC_REGS_SIZE 22 1749 1750 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = 1751 { 1752 {0x0000007d, 0x40000000}, 1753 {0x0000007e, 0x40180304}, 1754 {0x0000007f, 0x0000ff00}, 1755 {0x00000081, 0x00000000}, 1756 {0x00000083, 0x00000800}, 1757 {0x00000086, 0x00000000}, 1758 {0x00000087, 0x00000100}, 1759 {0x00000088, 0x00020100}, 1760 {0x00000089, 0x00000000}, 1761 {0x0000008b, 0x00040000}, 1762 {0x0000008c, 0x00000100}, 1763 {0x0000008e, 0xff010000}, 1764 {0x00000090, 0xffffefff}, 1765 {0x00000091, 0xfff3efff}, 1766 {0x00000092, 0xfff3efbf}, 1767 {0x00000093, 0xf7ffffff}, 1768 {0x00000094, 0xffffff7f}, 1769 {0x00000095, 0x00000fff}, 1770 {0x00000096, 0x00116fff}, 1771 {0x00000097, 0x60010000}, 1772 {0x00000098, 0x10010000}, 1773 {0x0000009f, 0x00c79000} 1774 }; 1775 1776 1777 /** 1778 * cik_srbm_select - select specific register instances 1779 * 1780 * @rdev: radeon_device pointer 1781 * @me: selected ME (micro engine) 1782 * @pipe: pipe 1783 * @queue: queue 1784 * @vmid: VMID 1785 * 1786 * Switches the currently active registers instances. Some 1787 * registers are instanced per VMID, others are instanced per 1788 * me/pipe/queue combination. 1789 */ 1790 static void cik_srbm_select(struct radeon_device *rdev, 1791 u32 me, u32 pipe, u32 queue, u32 vmid) 1792 { 1793 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | 1794 MEID(me & 0x3) | 1795 VMID(vmid & 0xf) | 1796 QUEUEID(queue & 0x7)); 1797 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); 1798 } 1799 1800 /* ucode loading */ 1801 /** 1802 * ci_mc_load_microcode - load MC ucode into the hw 1803 * 1804 * @rdev: radeon_device pointer 1805 * 1806 * Load the GDDR MC ucode into the hw (CIK). 1807 * Returns 0 on success, error on failure. 1808 */ 1809 int ci_mc_load_microcode(struct radeon_device *rdev) 1810 { 1811 const __be32 *fw_data = NULL; 1812 const __le32 *new_fw_data = NULL; 1813 u32 running, blackout = 0, tmp; 1814 u32 *io_mc_regs = NULL; 1815 const __le32 *new_io_mc_regs = NULL; 1816 int i, regs_size, ucode_size; 1817 1818 if (!rdev->mc_fw) 1819 return -EINVAL; 1820 1821 if (rdev->new_fw) { 1822 const struct mc_firmware_header_v1_0 *hdr = 1823 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; 1824 1825 radeon_ucode_print_mc_hdr(&hdr->header); 1826 1827 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); 1828 new_io_mc_regs = (const __le32 *) 1829 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); 1830 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1831 new_fw_data = (const __le32 *) 1832 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1833 } else { 1834 ucode_size = rdev->mc_fw->size / 4; 1835 1836 switch (rdev->family) { 1837 case CHIP_BONAIRE: 1838 io_mc_regs = (u32 *)&bonaire_io_mc_regs; 1839 regs_size = BONAIRE_IO_MC_REGS_SIZE; 1840 break; 1841 case CHIP_HAWAII: 1842 io_mc_regs = (u32 *)&hawaii_io_mc_regs; 1843 regs_size = HAWAII_IO_MC_REGS_SIZE; 1844 break; 1845 default: 1846 return -EINVAL; 1847 } 1848 fw_data = (const __be32 *)rdev->mc_fw->data; 1849 } 1850 1851 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1852 1853 if (running == 0) { 1854 if (running) { 1855 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1856 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1857 } 1858 1859 /* reset the engine and set to writable */ 1860 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1862 1863 /* load mc io regs */ 1864 for (i = 0; i < regs_size; i++) { 1865 if (rdev->new_fw) { 1866 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); 1867 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); 1868 } else { 1869 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1870 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1871 } 1872 } 1873 1874 tmp = RREG32(MC_SEQ_MISC0); 1875 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { 1876 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5); 1877 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023); 1878 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9); 1879 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0); 1880 } 1881 1882 /* load the MC ucode */ 1883 for (i = 0; i < ucode_size; i++) { 1884 if (rdev->new_fw) 1885 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); 1886 else 1887 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1888 } 1889 1890 /* put the engine back into the active state */ 1891 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1892 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1893 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1894 1895 /* wait for training to complete */ 1896 for (i = 0; i < rdev->usec_timeout; i++) { 1897 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1898 break; 1899 udelay(1); 1900 } 1901 for (i = 0; i < rdev->usec_timeout; i++) { 1902 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1903 break; 1904 udelay(1); 1905 } 1906 1907 if (running) 1908 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1909 } 1910 1911 return 0; 1912 } 1913 1914 /** 1915 * cik_init_microcode - load ucode images from disk 1916 * 1917 * @rdev: radeon_device pointer 1918 * 1919 * Use the firmware interface to load the ucode images into 1920 * the driver (not loaded into hw). 1921 * Returns 0 on success, error on failure. 1922 */ 1923 static int cik_init_microcode(struct radeon_device *rdev) 1924 { 1925 const char *chip_name; 1926 const char *new_chip_name; 1927 size_t pfp_req_size, me_req_size, ce_req_size, 1928 mec_req_size, rlc_req_size, mc_req_size = 0, 1929 sdma_req_size, smc_req_size = 0, mc2_req_size = 0; 1930 char fw_name[30]; 1931 int new_fw = 0; 1932 int err; 1933 int num_fw; 1934 1935 DRM_DEBUG("\n"); 1936 1937 switch (rdev->family) { 1938 case CHIP_BONAIRE: 1939 chip_name = "BONAIRE"; 1940 new_chip_name = "bonaire"; 1941 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1942 me_req_size = CIK_ME_UCODE_SIZE * 4; 1943 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1944 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1945 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1946 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4; 1947 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4; 1948 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1949 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4); 1950 num_fw = 8; 1951 break; 1952 case CHIP_HAWAII: 1953 chip_name = "HAWAII"; 1954 new_chip_name = "hawaii"; 1955 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1956 me_req_size = CIK_ME_UCODE_SIZE * 4; 1957 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1958 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1959 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1960 mc_req_size = HAWAII_MC_UCODE_SIZE * 4; 1961 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4; 1962 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1963 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4); 1964 num_fw = 8; 1965 break; 1966 case CHIP_KAVERI: 1967 chip_name = "KAVERI"; 1968 new_chip_name = "kaveri"; 1969 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1970 me_req_size = CIK_ME_UCODE_SIZE * 4; 1971 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1972 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1973 rlc_req_size = KV_RLC_UCODE_SIZE * 4; 1974 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1975 num_fw = 7; 1976 break; 1977 case CHIP_KABINI: 1978 chip_name = "KABINI"; 1979 new_chip_name = "kabini"; 1980 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1981 me_req_size = CIK_ME_UCODE_SIZE * 4; 1982 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1983 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1984 rlc_req_size = KB_RLC_UCODE_SIZE * 4; 1985 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1986 num_fw = 6; 1987 break; 1988 case CHIP_MULLINS: 1989 chip_name = "MULLINS"; 1990 new_chip_name = "mullins"; 1991 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1992 me_req_size = CIK_ME_UCODE_SIZE * 4; 1993 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1994 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1995 rlc_req_size = ML_RLC_UCODE_SIZE * 4; 1996 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1997 num_fw = 6; 1998 break; 1999 default: BUG(); 2000 } 2001 2002 DRM_INFO("Loading %s Microcode\n", new_chip_name); 2003 2004 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name); 2005 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2006 if (err) { 2007 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 2008 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 2009 if (err) 2010 goto out; 2011 if (rdev->pfp_fw->size != pfp_req_size) { 2012 printk(KERN_ERR 2013 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2014 rdev->pfp_fw->size, fw_name); 2015 err = -EINVAL; 2016 goto out; 2017 } 2018 } else { 2019 err = radeon_ucode_validate(rdev->pfp_fw); 2020 if (err) { 2021 printk(KERN_ERR 2022 "cik_fw: validation failed for firmware \"%s\"\n", 2023 fw_name); 2024 goto out; 2025 } else { 2026 new_fw++; 2027 } 2028 } 2029 2030 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name); 2031 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2032 if (err) { 2033 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 2034 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 2035 if (err) 2036 goto out; 2037 if (rdev->me_fw->size != me_req_size) { 2038 printk(KERN_ERR 2039 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2040 rdev->me_fw->size, fw_name); 2041 err = -EINVAL; 2042 } 2043 } else { 2044 err = radeon_ucode_validate(rdev->me_fw); 2045 if (err) { 2046 printk(KERN_ERR 2047 "cik_fw: validation failed for firmware \"%s\"\n", 2048 fw_name); 2049 goto out; 2050 } else { 2051 new_fw++; 2052 } 2053 } 2054 2055 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name); 2056 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2057 if (err) { 2058 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 2059 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 2060 if (err) 2061 goto out; 2062 if (rdev->ce_fw->size != ce_req_size) { 2063 printk(KERN_ERR 2064 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2065 rdev->ce_fw->size, fw_name); 2066 err = -EINVAL; 2067 } 2068 } else { 2069 err = radeon_ucode_validate(rdev->ce_fw); 2070 if (err) { 2071 printk(KERN_ERR 2072 "cik_fw: validation failed for firmware \"%s\"\n", 2073 fw_name); 2074 goto out; 2075 } else { 2076 new_fw++; 2077 } 2078 } 2079 2080 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name); 2081 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2082 if (err) { 2083 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name); 2084 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 2085 if (err) 2086 goto out; 2087 if (rdev->mec_fw->size != mec_req_size) { 2088 printk(KERN_ERR 2089 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 2090 rdev->mec_fw->size, fw_name); 2091 err = -EINVAL; 2092 } 2093 } else { 2094 err = radeon_ucode_validate(rdev->mec_fw); 2095 if (err) { 2096 printk(KERN_ERR 2097 "cik_fw: validation failed for firmware \"%s\"\n", 2098 fw_name); 2099 goto out; 2100 } else { 2101 new_fw++; 2102 } 2103 } 2104 2105 if (rdev->family == CHIP_KAVERI) { 2106 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name); 2107 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); 2108 if (err) { 2109 goto out; 2110 } else { 2111 err = radeon_ucode_validate(rdev->mec2_fw); 2112 if (err) { 2113 goto out; 2114 } else { 2115 new_fw++; 2116 } 2117 } 2118 } 2119 2120 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name); 2121 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2122 if (err) { 2123 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); 2124 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 2125 if (err) 2126 goto out; 2127 if (rdev->rlc_fw->size != rlc_req_size) { 2128 printk(KERN_ERR 2129 "cik_rlc: Bogus length %zu in firmware \"%s\"\n", 2130 rdev->rlc_fw->size, fw_name); 2131 err = -EINVAL; 2132 } 2133 } else { 2134 err = radeon_ucode_validate(rdev->rlc_fw); 2135 if (err) { 2136 printk(KERN_ERR 2137 "cik_fw: validation failed for firmware \"%s\"\n", 2138 fw_name); 2139 goto out; 2140 } else { 2141 new_fw++; 2142 } 2143 } 2144 2145 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name); 2146 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2147 if (err) { 2148 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name); 2149 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 2150 if (err) 2151 goto out; 2152 if (rdev->sdma_fw->size != sdma_req_size) { 2153 printk(KERN_ERR 2154 "cik_sdma: Bogus length %zu in firmware \"%s\"\n", 2155 rdev->sdma_fw->size, fw_name); 2156 err = -EINVAL; 2157 } 2158 } else { 2159 err = radeon_ucode_validate(rdev->sdma_fw); 2160 if (err) { 2161 printk(KERN_ERR 2162 "cik_fw: validation failed for firmware \"%s\"\n", 2163 fw_name); 2164 goto out; 2165 } else { 2166 new_fw++; 2167 } 2168 } 2169 2170 /* No SMC, MC ucode on APUs */ 2171 if (!(rdev->flags & RADEON_IS_IGP)) { 2172 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name); 2173 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2174 if (err) { 2175 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name); 2176 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2177 if (err) { 2178 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 2179 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 2180 if (err) 2181 goto out; 2182 } 2183 if ((rdev->mc_fw->size != mc_req_size) && 2184 (rdev->mc_fw->size != mc2_req_size)){ 2185 printk(KERN_ERR 2186 "cik_mc: Bogus length %zu in firmware \"%s\"\n", 2187 rdev->mc_fw->size, fw_name); 2188 err = -EINVAL; 2189 } 2190 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); 2191 } else { 2192 err = radeon_ucode_validate(rdev->mc_fw); 2193 if (err) { 2194 printk(KERN_ERR 2195 "cik_fw: validation failed for firmware \"%s\"\n", 2196 fw_name); 2197 goto out; 2198 } else { 2199 new_fw++; 2200 } 2201 } 2202 2203 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name); 2204 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2205 if (err) { 2206 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 2207 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 2208 if (err) { 2209 printk(KERN_ERR 2210 "smc: error loading firmware \"%s\"\n", 2211 fw_name); 2212 release_firmware(rdev->smc_fw); 2213 rdev->smc_fw = NULL; 2214 err = 0; 2215 } else if (rdev->smc_fw->size != smc_req_size) { 2216 printk(KERN_ERR 2217 "cik_smc: Bogus length %zu in firmware \"%s\"\n", 2218 rdev->smc_fw->size, fw_name); 2219 err = -EINVAL; 2220 } 2221 } else { 2222 err = radeon_ucode_validate(rdev->smc_fw); 2223 if (err) { 2224 printk(KERN_ERR 2225 "cik_fw: validation failed for firmware \"%s\"\n", 2226 fw_name); 2227 goto out; 2228 } else { 2229 new_fw++; 2230 } 2231 } 2232 } 2233 2234 if (new_fw == 0) { 2235 rdev->new_fw = false; 2236 } else if (new_fw < num_fw) { 2237 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n"); 2238 err = -EINVAL; 2239 } else { 2240 rdev->new_fw = true; 2241 } 2242 2243 out: 2244 if (err) { 2245 if (err != -EINVAL) 2246 printk(KERN_ERR 2247 "cik_cp: Failed to load firmware \"%s\"\n", 2248 fw_name); 2249 release_firmware(rdev->pfp_fw); 2250 rdev->pfp_fw = NULL; 2251 release_firmware(rdev->me_fw); 2252 rdev->me_fw = NULL; 2253 release_firmware(rdev->ce_fw); 2254 rdev->ce_fw = NULL; 2255 release_firmware(rdev->mec_fw); 2256 rdev->mec_fw = NULL; 2257 release_firmware(rdev->mec2_fw); 2258 rdev->mec2_fw = NULL; 2259 release_firmware(rdev->rlc_fw); 2260 rdev->rlc_fw = NULL; 2261 release_firmware(rdev->sdma_fw); 2262 rdev->sdma_fw = NULL; 2263 release_firmware(rdev->mc_fw); 2264 rdev->mc_fw = NULL; 2265 release_firmware(rdev->smc_fw); 2266 rdev->smc_fw = NULL; 2267 } 2268 return err; 2269 } 2270 2271 /* 2272 * Core functions 2273 */ 2274 /** 2275 * cik_tiling_mode_table_init - init the hw tiling table 2276 * 2277 * @rdev: radeon_device pointer 2278 * 2279 * Starting with SI, the tiling setup is done globally in a 2280 * set of 32 tiling modes. Rather than selecting each set of 2281 * parameters per surface as on older asics, we just select 2282 * which index in the tiling table we want to use, and the 2283 * surface uses those parameters (CIK). 2284 */ 2285 static void cik_tiling_mode_table_init(struct radeon_device *rdev) 2286 { 2287 const u32 num_tile_mode_states = 32; 2288 const u32 num_secondary_tile_mode_states = 16; 2289 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2290 u32 num_pipe_configs; 2291 u32 num_rbs = rdev->config.cik.max_backends_per_se * 2292 rdev->config.cik.max_shader_engines; 2293 2294 switch (rdev->config.cik.mem_row_size_in_kb) { 2295 case 1: 2296 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2297 break; 2298 case 2: 2299 default: 2300 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2301 break; 2302 case 4: 2303 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2304 break; 2305 } 2306 2307 num_pipe_configs = rdev->config.cik.max_tile_pipes; 2308 if (num_pipe_configs > 8) 2309 num_pipe_configs = 16; 2310 2311 if (num_pipe_configs == 16) { 2312 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2313 switch (reg_offset) { 2314 case 0: 2315 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2316 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2317 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2318 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2319 break; 2320 case 1: 2321 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2322 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2323 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2324 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2325 break; 2326 case 2: 2327 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2328 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2329 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2330 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2331 break; 2332 case 3: 2333 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2335 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2336 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2337 break; 2338 case 4: 2339 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2340 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2341 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2342 TILE_SPLIT(split_equal_to_row_size)); 2343 break; 2344 case 5: 2345 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2346 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2347 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2348 break; 2349 case 6: 2350 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2351 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2352 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2353 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2354 break; 2355 case 7: 2356 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2357 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2358 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2359 TILE_SPLIT(split_equal_to_row_size)); 2360 break; 2361 case 8: 2362 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2363 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2364 break; 2365 case 9: 2366 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2367 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2368 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2369 break; 2370 case 10: 2371 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2372 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2373 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2374 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2375 break; 2376 case 11: 2377 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2378 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2379 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2380 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2381 break; 2382 case 12: 2383 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2384 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2385 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2386 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2387 break; 2388 case 13: 2389 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2390 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2391 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2392 break; 2393 case 14: 2394 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2395 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2396 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2397 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2398 break; 2399 case 16: 2400 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2401 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2402 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2403 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2404 break; 2405 case 17: 2406 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2407 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2408 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2409 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2410 break; 2411 case 27: 2412 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2413 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2414 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2415 break; 2416 case 28: 2417 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2418 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2419 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2420 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2421 break; 2422 case 29: 2423 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2424 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2425 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2426 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2427 break; 2428 case 30: 2429 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2430 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2431 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2432 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2433 break; 2434 default: 2435 gb_tile_moden = 0; 2436 break; 2437 } 2438 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2439 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2440 } 2441 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2442 switch (reg_offset) { 2443 case 0: 2444 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2447 NUM_BANKS(ADDR_SURF_16_BANK)); 2448 break; 2449 case 1: 2450 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2451 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2452 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2453 NUM_BANKS(ADDR_SURF_16_BANK)); 2454 break; 2455 case 2: 2456 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2459 NUM_BANKS(ADDR_SURF_16_BANK)); 2460 break; 2461 case 3: 2462 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2463 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2464 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2465 NUM_BANKS(ADDR_SURF_16_BANK)); 2466 break; 2467 case 4: 2468 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2471 NUM_BANKS(ADDR_SURF_8_BANK)); 2472 break; 2473 case 5: 2474 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2475 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2476 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2477 NUM_BANKS(ADDR_SURF_4_BANK)); 2478 break; 2479 case 6: 2480 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2481 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2482 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2483 NUM_BANKS(ADDR_SURF_2_BANK)); 2484 break; 2485 case 8: 2486 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2489 NUM_BANKS(ADDR_SURF_16_BANK)); 2490 break; 2491 case 9: 2492 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2493 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2494 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2495 NUM_BANKS(ADDR_SURF_16_BANK)); 2496 break; 2497 case 10: 2498 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2499 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2500 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2501 NUM_BANKS(ADDR_SURF_16_BANK)); 2502 break; 2503 case 11: 2504 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2507 NUM_BANKS(ADDR_SURF_8_BANK)); 2508 break; 2509 case 12: 2510 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2513 NUM_BANKS(ADDR_SURF_4_BANK)); 2514 break; 2515 case 13: 2516 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2517 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2518 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2519 NUM_BANKS(ADDR_SURF_2_BANK)); 2520 break; 2521 case 14: 2522 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2523 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2524 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2525 NUM_BANKS(ADDR_SURF_2_BANK)); 2526 break; 2527 default: 2528 gb_tile_moden = 0; 2529 break; 2530 } 2531 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2532 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2533 } 2534 } else if (num_pipe_configs == 8) { 2535 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2536 switch (reg_offset) { 2537 case 0: 2538 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2539 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2540 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2541 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2542 break; 2543 case 1: 2544 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2545 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2546 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2547 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2548 break; 2549 case 2: 2550 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2551 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2552 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2553 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2554 break; 2555 case 3: 2556 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2559 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2560 break; 2561 case 4: 2562 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2563 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2564 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2565 TILE_SPLIT(split_equal_to_row_size)); 2566 break; 2567 case 5: 2568 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2571 break; 2572 case 6: 2573 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2574 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2575 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2576 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2577 break; 2578 case 7: 2579 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2580 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2582 TILE_SPLIT(split_equal_to_row_size)); 2583 break; 2584 case 8: 2585 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2586 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2587 break; 2588 case 9: 2589 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2590 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2591 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2592 break; 2593 case 10: 2594 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2595 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2596 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2597 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2598 break; 2599 case 11: 2600 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2601 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2602 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2603 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2604 break; 2605 case 12: 2606 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2607 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2608 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2609 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2610 break; 2611 case 13: 2612 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2613 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2614 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2615 break; 2616 case 14: 2617 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2618 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2619 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2620 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2621 break; 2622 case 16: 2623 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2624 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2625 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2626 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2627 break; 2628 case 17: 2629 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2630 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2631 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2632 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2633 break; 2634 case 27: 2635 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2636 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2637 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2638 break; 2639 case 28: 2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2641 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2642 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2644 break; 2645 case 29: 2646 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2647 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2648 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2649 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2650 break; 2651 case 30: 2652 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2653 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2654 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2655 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2656 break; 2657 default: 2658 gb_tile_moden = 0; 2659 break; 2660 } 2661 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2662 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2663 } 2664 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2665 switch (reg_offset) { 2666 case 0: 2667 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2668 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2669 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2670 NUM_BANKS(ADDR_SURF_16_BANK)); 2671 break; 2672 case 1: 2673 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2674 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2675 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2676 NUM_BANKS(ADDR_SURF_16_BANK)); 2677 break; 2678 case 2: 2679 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2680 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2681 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2682 NUM_BANKS(ADDR_SURF_16_BANK)); 2683 break; 2684 case 3: 2685 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2686 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2687 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2688 NUM_BANKS(ADDR_SURF_16_BANK)); 2689 break; 2690 case 4: 2691 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2692 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2693 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2694 NUM_BANKS(ADDR_SURF_8_BANK)); 2695 break; 2696 case 5: 2697 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2698 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2699 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2700 NUM_BANKS(ADDR_SURF_4_BANK)); 2701 break; 2702 case 6: 2703 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2704 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2705 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2706 NUM_BANKS(ADDR_SURF_2_BANK)); 2707 break; 2708 case 8: 2709 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2710 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2711 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2712 NUM_BANKS(ADDR_SURF_16_BANK)); 2713 break; 2714 case 9: 2715 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2716 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2717 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2718 NUM_BANKS(ADDR_SURF_16_BANK)); 2719 break; 2720 case 10: 2721 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2722 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2723 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2724 NUM_BANKS(ADDR_SURF_16_BANK)); 2725 break; 2726 case 11: 2727 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2728 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2729 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2730 NUM_BANKS(ADDR_SURF_16_BANK)); 2731 break; 2732 case 12: 2733 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2734 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2735 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2736 NUM_BANKS(ADDR_SURF_8_BANK)); 2737 break; 2738 case 13: 2739 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2740 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2741 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2742 NUM_BANKS(ADDR_SURF_4_BANK)); 2743 break; 2744 case 14: 2745 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2746 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2747 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2748 NUM_BANKS(ADDR_SURF_2_BANK)); 2749 break; 2750 default: 2751 gb_tile_moden = 0; 2752 break; 2753 } 2754 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2755 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2756 } 2757 } else if (num_pipe_configs == 4) { 2758 if (num_rbs == 4) { 2759 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2760 switch (reg_offset) { 2761 case 0: 2762 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2763 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2764 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2765 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2766 break; 2767 case 1: 2768 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2769 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2770 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2771 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2772 break; 2773 case 2: 2774 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2775 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2776 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2777 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2778 break; 2779 case 3: 2780 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2782 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2783 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2784 break; 2785 case 4: 2786 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2787 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2788 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2789 TILE_SPLIT(split_equal_to_row_size)); 2790 break; 2791 case 5: 2792 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2793 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2794 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2795 break; 2796 case 6: 2797 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2798 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2799 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2800 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2801 break; 2802 case 7: 2803 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2804 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2805 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2806 TILE_SPLIT(split_equal_to_row_size)); 2807 break; 2808 case 8: 2809 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2810 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2811 break; 2812 case 9: 2813 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2814 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2816 break; 2817 case 10: 2818 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2819 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2820 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2821 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2822 break; 2823 case 11: 2824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2825 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2826 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2827 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2828 break; 2829 case 12: 2830 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2831 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2832 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2833 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2834 break; 2835 case 13: 2836 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2837 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2838 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2839 break; 2840 case 14: 2841 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2842 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2843 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2844 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2845 break; 2846 case 16: 2847 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2848 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2849 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2850 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2851 break; 2852 case 17: 2853 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2854 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2855 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2856 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2857 break; 2858 case 27: 2859 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2860 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2861 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2862 break; 2863 case 28: 2864 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2865 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2866 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2867 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2868 break; 2869 case 29: 2870 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2871 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2872 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2873 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2874 break; 2875 case 30: 2876 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2877 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2878 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2879 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2880 break; 2881 default: 2882 gb_tile_moden = 0; 2883 break; 2884 } 2885 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2886 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2887 } 2888 } else if (num_rbs < 4) { 2889 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2890 switch (reg_offset) { 2891 case 0: 2892 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2893 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2894 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2895 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2896 break; 2897 case 1: 2898 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2899 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2900 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2901 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2902 break; 2903 case 2: 2904 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2905 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2906 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2907 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2908 break; 2909 case 3: 2910 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2912 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2913 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2914 break; 2915 case 4: 2916 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2917 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2918 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2919 TILE_SPLIT(split_equal_to_row_size)); 2920 break; 2921 case 5: 2922 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2923 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2924 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2925 break; 2926 case 6: 2927 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2928 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2929 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2930 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2931 break; 2932 case 7: 2933 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2934 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2935 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2936 TILE_SPLIT(split_equal_to_row_size)); 2937 break; 2938 case 8: 2939 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2940 PIPE_CONFIG(ADDR_SURF_P4_8x16)); 2941 break; 2942 case 9: 2943 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2944 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2945 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2946 break; 2947 case 10: 2948 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2949 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2950 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2951 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2952 break; 2953 case 11: 2954 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2955 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2956 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2957 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2958 break; 2959 case 12: 2960 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2961 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2962 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2963 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2964 break; 2965 case 13: 2966 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2967 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2968 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2969 break; 2970 case 14: 2971 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2972 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2973 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2974 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2975 break; 2976 case 16: 2977 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2978 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2979 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2980 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2981 break; 2982 case 17: 2983 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2984 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2985 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2986 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2987 break; 2988 case 27: 2989 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2990 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2991 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2992 break; 2993 case 28: 2994 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2995 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2996 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2997 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2998 break; 2999 case 29: 3000 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3001 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3002 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 3003 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3004 break; 3005 case 30: 3006 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3007 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3008 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 3009 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3010 break; 3011 default: 3012 gb_tile_moden = 0; 3013 break; 3014 } 3015 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 3016 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3017 } 3018 } 3019 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 3020 switch (reg_offset) { 3021 case 0: 3022 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3023 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3024 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3025 NUM_BANKS(ADDR_SURF_16_BANK)); 3026 break; 3027 case 1: 3028 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3029 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3030 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3031 NUM_BANKS(ADDR_SURF_16_BANK)); 3032 break; 3033 case 2: 3034 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3035 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3036 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3037 NUM_BANKS(ADDR_SURF_16_BANK)); 3038 break; 3039 case 3: 3040 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3041 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3042 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3043 NUM_BANKS(ADDR_SURF_16_BANK)); 3044 break; 3045 case 4: 3046 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3047 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3048 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3049 NUM_BANKS(ADDR_SURF_16_BANK)); 3050 break; 3051 case 5: 3052 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3053 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3054 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3055 NUM_BANKS(ADDR_SURF_8_BANK)); 3056 break; 3057 case 6: 3058 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3059 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3060 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3061 NUM_BANKS(ADDR_SURF_4_BANK)); 3062 break; 3063 case 8: 3064 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3065 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3066 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3067 NUM_BANKS(ADDR_SURF_16_BANK)); 3068 break; 3069 case 9: 3070 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3071 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3072 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3073 NUM_BANKS(ADDR_SURF_16_BANK)); 3074 break; 3075 case 10: 3076 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3077 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3078 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3079 NUM_BANKS(ADDR_SURF_16_BANK)); 3080 break; 3081 case 11: 3082 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3083 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3084 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3085 NUM_BANKS(ADDR_SURF_16_BANK)); 3086 break; 3087 case 12: 3088 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3089 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3090 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3091 NUM_BANKS(ADDR_SURF_16_BANK)); 3092 break; 3093 case 13: 3094 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3095 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3096 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3097 NUM_BANKS(ADDR_SURF_8_BANK)); 3098 break; 3099 case 14: 3100 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3101 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3102 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3103 NUM_BANKS(ADDR_SURF_4_BANK)); 3104 break; 3105 default: 3106 gb_tile_moden = 0; 3107 break; 3108 } 3109 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3110 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3111 } 3112 } else if (num_pipe_configs == 2) { 3113 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 3114 switch (reg_offset) { 3115 case 0: 3116 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3117 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3118 PIPE_CONFIG(ADDR_SURF_P2) | 3119 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 3120 break; 3121 case 1: 3122 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3123 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3124 PIPE_CONFIG(ADDR_SURF_P2) | 3125 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 3126 break; 3127 case 2: 3128 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3129 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3130 PIPE_CONFIG(ADDR_SURF_P2) | 3131 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3132 break; 3133 case 3: 3134 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3135 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3136 PIPE_CONFIG(ADDR_SURF_P2) | 3137 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 3138 break; 3139 case 4: 3140 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3141 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3142 PIPE_CONFIG(ADDR_SURF_P2) | 3143 TILE_SPLIT(split_equal_to_row_size)); 3144 break; 3145 case 5: 3146 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3147 PIPE_CONFIG(ADDR_SURF_P2) | 3148 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3149 break; 3150 case 6: 3151 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3152 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3153 PIPE_CONFIG(ADDR_SURF_P2) | 3154 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 3155 break; 3156 case 7: 3157 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3158 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 3159 PIPE_CONFIG(ADDR_SURF_P2) | 3160 TILE_SPLIT(split_equal_to_row_size)); 3161 break; 3162 case 8: 3163 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3164 PIPE_CONFIG(ADDR_SURF_P2); 3165 break; 3166 case 9: 3167 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3168 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3169 PIPE_CONFIG(ADDR_SURF_P2)); 3170 break; 3171 case 10: 3172 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3173 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3174 PIPE_CONFIG(ADDR_SURF_P2) | 3175 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3176 break; 3177 case 11: 3178 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3179 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3180 PIPE_CONFIG(ADDR_SURF_P2) | 3181 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3182 break; 3183 case 12: 3184 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3185 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3186 PIPE_CONFIG(ADDR_SURF_P2) | 3187 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3188 break; 3189 case 13: 3190 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3191 PIPE_CONFIG(ADDR_SURF_P2) | 3192 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 3193 break; 3194 case 14: 3195 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3196 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3197 PIPE_CONFIG(ADDR_SURF_P2) | 3198 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3199 break; 3200 case 16: 3201 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3202 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3203 PIPE_CONFIG(ADDR_SURF_P2) | 3204 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3205 break; 3206 case 17: 3207 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3208 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3209 PIPE_CONFIG(ADDR_SURF_P2) | 3210 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3211 break; 3212 case 27: 3213 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3214 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3215 PIPE_CONFIG(ADDR_SURF_P2)); 3216 break; 3217 case 28: 3218 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3219 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3220 PIPE_CONFIG(ADDR_SURF_P2) | 3221 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3222 break; 3223 case 29: 3224 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3225 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3226 PIPE_CONFIG(ADDR_SURF_P2) | 3227 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3228 break; 3229 case 30: 3230 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 3231 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3232 PIPE_CONFIG(ADDR_SURF_P2) | 3233 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3234 break; 3235 default: 3236 gb_tile_moden = 0; 3237 break; 3238 } 3239 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 3240 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3241 } 3242 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 3243 switch (reg_offset) { 3244 case 0: 3245 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3246 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3247 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3248 NUM_BANKS(ADDR_SURF_16_BANK)); 3249 break; 3250 case 1: 3251 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3252 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3253 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3254 NUM_BANKS(ADDR_SURF_16_BANK)); 3255 break; 3256 case 2: 3257 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3258 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3259 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3260 NUM_BANKS(ADDR_SURF_16_BANK)); 3261 break; 3262 case 3: 3263 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3264 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3265 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3266 NUM_BANKS(ADDR_SURF_16_BANK)); 3267 break; 3268 case 4: 3269 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3270 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3271 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3272 NUM_BANKS(ADDR_SURF_16_BANK)); 3273 break; 3274 case 5: 3275 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3276 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3277 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3278 NUM_BANKS(ADDR_SURF_16_BANK)); 3279 break; 3280 case 6: 3281 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3282 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3283 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3284 NUM_BANKS(ADDR_SURF_8_BANK)); 3285 break; 3286 case 8: 3287 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3288 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3289 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3290 NUM_BANKS(ADDR_SURF_16_BANK)); 3291 break; 3292 case 9: 3293 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3294 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3295 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3296 NUM_BANKS(ADDR_SURF_16_BANK)); 3297 break; 3298 case 10: 3299 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3300 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3301 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3302 NUM_BANKS(ADDR_SURF_16_BANK)); 3303 break; 3304 case 11: 3305 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3306 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3307 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3308 NUM_BANKS(ADDR_SURF_16_BANK)); 3309 break; 3310 case 12: 3311 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3312 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3313 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3314 NUM_BANKS(ADDR_SURF_16_BANK)); 3315 break; 3316 case 13: 3317 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3318 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3319 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3320 NUM_BANKS(ADDR_SURF_16_BANK)); 3321 break; 3322 case 14: 3323 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3324 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3325 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3326 NUM_BANKS(ADDR_SURF_8_BANK)); 3327 break; 3328 default: 3329 gb_tile_moden = 0; 3330 break; 3331 } 3332 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3333 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3334 } 3335 } else 3336 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); 3337 } 3338 3339 /** 3340 * cik_select_se_sh - select which SE, SH to address 3341 * 3342 * @rdev: radeon_device pointer 3343 * @se_num: shader engine to address 3344 * @sh_num: sh block to address 3345 * 3346 * Select which SE, SH combinations to address. Certain 3347 * registers are instanced per SE or SH. 0xffffffff means 3348 * broadcast to all SEs or SHs (CIK). 3349 */ 3350 static void cik_select_se_sh(struct radeon_device *rdev, 3351 u32 se_num, u32 sh_num) 3352 { 3353 u32 data = INSTANCE_BROADCAST_WRITES; 3354 3355 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 3356 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 3357 else if (se_num == 0xffffffff) 3358 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 3359 else if (sh_num == 0xffffffff) 3360 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 3361 else 3362 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 3363 WREG32(GRBM_GFX_INDEX, data); 3364 } 3365 3366 /** 3367 * cik_create_bitmask - create a bitmask 3368 * 3369 * @bit_width: length of the mask 3370 * 3371 * create a variable length bit mask (CIK). 3372 * Returns the bitmask. 3373 */ 3374 static u32 cik_create_bitmask(u32 bit_width) 3375 { 3376 u32 i, mask = 0; 3377 3378 for (i = 0; i < bit_width; i++) { 3379 mask <<= 1; 3380 mask |= 1; 3381 } 3382 return mask; 3383 } 3384 3385 /** 3386 * cik_get_rb_disabled - computes the mask of disabled RBs 3387 * 3388 * @rdev: radeon_device pointer 3389 * @max_rb_num: max RBs (render backends) for the asic 3390 * @se_num: number of SEs (shader engines) for the asic 3391 * @sh_per_se: number of SH blocks per SE for the asic 3392 * 3393 * Calculates the bitmask of disabled RBs (CIK). 3394 * Returns the disabled RB bitmask. 3395 */ 3396 static u32 cik_get_rb_disabled(struct radeon_device *rdev, 3397 u32 max_rb_num_per_se, 3398 u32 sh_per_se) 3399 { 3400 u32 data, mask; 3401 3402 data = RREG32(CC_RB_BACKEND_DISABLE); 3403 if (data & 1) 3404 data &= BACKEND_DISABLE_MASK; 3405 else 3406 data = 0; 3407 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 3408 3409 data >>= BACKEND_DISABLE_SHIFT; 3410 3411 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); 3412 3413 return data & mask; 3414 } 3415 3416 /** 3417 * cik_setup_rb - setup the RBs on the asic 3418 * 3419 * @rdev: radeon_device pointer 3420 * @se_num: number of SEs (shader engines) for the asic 3421 * @sh_per_se: number of SH blocks per SE for the asic 3422 * @max_rb_num: max RBs (render backends) for the asic 3423 * 3424 * Configures per-SE/SH RB registers (CIK). 3425 */ 3426 static void cik_setup_rb(struct radeon_device *rdev, 3427 u32 se_num, u32 sh_per_se, 3428 u32 max_rb_num_per_se) 3429 { 3430 int i, j; 3431 u32 data, mask; 3432 u32 disabled_rbs = 0; 3433 u32 enabled_rbs = 0; 3434 3435 mutex_lock(&rdev->grbm_idx_mutex); 3436 for (i = 0; i < se_num; i++) { 3437 for (j = 0; j < sh_per_se; j++) { 3438 cik_select_se_sh(rdev, i, j); 3439 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3440 if (rdev->family == CHIP_HAWAII) 3441 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); 3442 else 3443 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); 3444 } 3445 } 3446 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3447 mutex_unlock(&rdev->grbm_idx_mutex); 3448 3449 mask = 1; 3450 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3451 if (!(disabled_rbs & mask)) 3452 enabled_rbs |= mask; 3453 mask <<= 1; 3454 } 3455 3456 rdev->config.cik.backend_enable_mask = enabled_rbs; 3457 3458 mutex_lock(&rdev->grbm_idx_mutex); 3459 for (i = 0; i < se_num; i++) { 3460 cik_select_se_sh(rdev, i, 0xffffffff); 3461 data = 0; 3462 for (j = 0; j < sh_per_se; j++) { 3463 switch (enabled_rbs & 3) { 3464 case 0: 3465 if (j == 0) 3466 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); 3467 else 3468 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); 3469 break; 3470 case 1: 3471 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3472 break; 3473 case 2: 3474 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3475 break; 3476 case 3: 3477 default: 3478 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3479 break; 3480 } 3481 enabled_rbs >>= 2; 3482 } 3483 WREG32(PA_SC_RASTER_CONFIG, data); 3484 } 3485 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3486 mutex_unlock(&rdev->grbm_idx_mutex); 3487 } 3488 3489 /** 3490 * cik_gpu_init - setup the 3D engine 3491 * 3492 * @rdev: radeon_device pointer 3493 * 3494 * Configures the 3D engine and tiling configuration 3495 * registers so that the 3D engine is usable. 3496 */ 3497 static void cik_gpu_init(struct radeon_device *rdev) 3498 { 3499 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); 3500 u32 mc_shared_chmap, mc_arb_ramcfg; 3501 u32 hdp_host_path_cntl; 3502 u32 tmp; 3503 int i, j; 3504 3505 switch (rdev->family) { 3506 case CHIP_BONAIRE: 3507 rdev->config.cik.max_shader_engines = 2; 3508 rdev->config.cik.max_tile_pipes = 4; 3509 rdev->config.cik.max_cu_per_sh = 7; 3510 rdev->config.cik.max_sh_per_se = 1; 3511 rdev->config.cik.max_backends_per_se = 2; 3512 rdev->config.cik.max_texture_channel_caches = 4; 3513 rdev->config.cik.max_gprs = 256; 3514 rdev->config.cik.max_gs_threads = 32; 3515 rdev->config.cik.max_hw_contexts = 8; 3516 3517 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3518 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3519 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3520 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3521 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3522 break; 3523 case CHIP_HAWAII: 3524 rdev->config.cik.max_shader_engines = 4; 3525 rdev->config.cik.max_tile_pipes = 16; 3526 rdev->config.cik.max_cu_per_sh = 11; 3527 rdev->config.cik.max_sh_per_se = 1; 3528 rdev->config.cik.max_backends_per_se = 4; 3529 rdev->config.cik.max_texture_channel_caches = 16; 3530 rdev->config.cik.max_gprs = 256; 3531 rdev->config.cik.max_gs_threads = 32; 3532 rdev->config.cik.max_hw_contexts = 8; 3533 3534 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3535 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3536 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3537 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3538 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; 3539 break; 3540 case CHIP_KAVERI: 3541 rdev->config.cik.max_shader_engines = 1; 3542 rdev->config.cik.max_tile_pipes = 4; 3543 if ((rdev->pdev->device == 0x1304) || 3544 (rdev->pdev->device == 0x1305) || 3545 (rdev->pdev->device == 0x130C) || 3546 (rdev->pdev->device == 0x130F) || 3547 (rdev->pdev->device == 0x1310) || 3548 (rdev->pdev->device == 0x1311) || 3549 (rdev->pdev->device == 0x131C)) { 3550 rdev->config.cik.max_cu_per_sh = 8; 3551 rdev->config.cik.max_backends_per_se = 2; 3552 } else if ((rdev->pdev->device == 0x1309) || 3553 (rdev->pdev->device == 0x130A) || 3554 (rdev->pdev->device == 0x130D) || 3555 (rdev->pdev->device == 0x1313) || 3556 (rdev->pdev->device == 0x131D)) { 3557 rdev->config.cik.max_cu_per_sh = 6; 3558 rdev->config.cik.max_backends_per_se = 2; 3559 } else if ((rdev->pdev->device == 0x1306) || 3560 (rdev->pdev->device == 0x1307) || 3561 (rdev->pdev->device == 0x130B) || 3562 (rdev->pdev->device == 0x130E) || 3563 (rdev->pdev->device == 0x1315) || 3564 (rdev->pdev->device == 0x1318) || 3565 (rdev->pdev->device == 0x131B)) { 3566 rdev->config.cik.max_cu_per_sh = 4; 3567 rdev->config.cik.max_backends_per_se = 1; 3568 } else { 3569 rdev->config.cik.max_cu_per_sh = 3; 3570 rdev->config.cik.max_backends_per_se = 1; 3571 } 3572 rdev->config.cik.max_sh_per_se = 1; 3573 rdev->config.cik.max_texture_channel_caches = 4; 3574 rdev->config.cik.max_gprs = 256; 3575 rdev->config.cik.max_gs_threads = 16; 3576 rdev->config.cik.max_hw_contexts = 8; 3577 3578 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3579 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3580 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3581 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3582 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3583 break; 3584 case CHIP_KABINI: 3585 case CHIP_MULLINS: 3586 default: 3587 rdev->config.cik.max_shader_engines = 1; 3588 rdev->config.cik.max_tile_pipes = 2; 3589 rdev->config.cik.max_cu_per_sh = 2; 3590 rdev->config.cik.max_sh_per_se = 1; 3591 rdev->config.cik.max_backends_per_se = 1; 3592 rdev->config.cik.max_texture_channel_caches = 2; 3593 rdev->config.cik.max_gprs = 256; 3594 rdev->config.cik.max_gs_threads = 16; 3595 rdev->config.cik.max_hw_contexts = 8; 3596 3597 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3598 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3599 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3600 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3601 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3602 break; 3603 } 3604 3605 /* Initialize HDP */ 3606 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3607 WREG32((0x2c14 + j), 0x00000000); 3608 WREG32((0x2c18 + j), 0x00000000); 3609 WREG32((0x2c1c + j), 0x00000000); 3610 WREG32((0x2c20 + j), 0x00000000); 3611 WREG32((0x2c24 + j), 0x00000000); 3612 } 3613 3614 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3615 3616 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3617 3618 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3619 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3620 3621 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; 3622 rdev->config.cik.mem_max_burst_length_bytes = 256; 3623 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3624 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3625 if (rdev->config.cik.mem_row_size_in_kb > 4) 3626 rdev->config.cik.mem_row_size_in_kb = 4; 3627 /* XXX use MC settings? */ 3628 rdev->config.cik.shader_engine_tile_size = 32; 3629 rdev->config.cik.num_gpus = 1; 3630 rdev->config.cik.multi_gpu_tile_size = 64; 3631 3632 /* fix up row size */ 3633 gb_addr_config &= ~ROW_SIZE_MASK; 3634 switch (rdev->config.cik.mem_row_size_in_kb) { 3635 case 1: 3636 default: 3637 gb_addr_config |= ROW_SIZE(0); 3638 break; 3639 case 2: 3640 gb_addr_config |= ROW_SIZE(1); 3641 break; 3642 case 4: 3643 gb_addr_config |= ROW_SIZE(2); 3644 break; 3645 } 3646 3647 /* setup tiling info dword. gb_addr_config is not adequate since it does 3648 * not have bank info, so create a custom tiling dword. 3649 * bits 3:0 num_pipes 3650 * bits 7:4 num_banks 3651 * bits 11:8 group_size 3652 * bits 15:12 row_size 3653 */ 3654 rdev->config.cik.tile_config = 0; 3655 switch (rdev->config.cik.num_tile_pipes) { 3656 case 1: 3657 rdev->config.cik.tile_config |= (0 << 0); 3658 break; 3659 case 2: 3660 rdev->config.cik.tile_config |= (1 << 0); 3661 break; 3662 case 4: 3663 rdev->config.cik.tile_config |= (2 << 0); 3664 break; 3665 case 8: 3666 default: 3667 /* XXX what about 12? */ 3668 rdev->config.cik.tile_config |= (3 << 0); 3669 break; 3670 } 3671 rdev->config.cik.tile_config |= 3672 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 3673 rdev->config.cik.tile_config |= 3674 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3675 rdev->config.cik.tile_config |= 3676 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3677 3678 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3679 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3680 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3681 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); 3682 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); 3683 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3684 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3685 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3686 3687 cik_tiling_mode_table_init(rdev); 3688 3689 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, 3690 rdev->config.cik.max_sh_per_se, 3691 rdev->config.cik.max_backends_per_se); 3692 3693 rdev->config.cik.active_cus = 0; 3694 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 3695 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 3696 rdev->config.cik.active_cus += 3697 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); 3698 } 3699 } 3700 3701 /* set HW defaults for 3D engine */ 3702 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3703 3704 mutex_lock(&rdev->grbm_idx_mutex); 3705 /* 3706 * making sure that the following register writes will be broadcasted 3707 * to all the shaders 3708 */ 3709 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3710 WREG32(SX_DEBUG_1, 0x20); 3711 3712 WREG32(TA_CNTL_AUX, 0x00010000); 3713 3714 tmp = RREG32(SPI_CONFIG_CNTL); 3715 tmp |= 0x03000000; 3716 WREG32(SPI_CONFIG_CNTL, tmp); 3717 3718 WREG32(SQ_CONFIG, 1); 3719 3720 WREG32(DB_DEBUG, 0); 3721 3722 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; 3723 tmp |= 0x00000400; 3724 WREG32(DB_DEBUG2, tmp); 3725 3726 tmp = RREG32(DB_DEBUG3) & ~0x0002021c; 3727 tmp |= 0x00020200; 3728 WREG32(DB_DEBUG3, tmp); 3729 3730 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; 3731 tmp |= 0x00018208; 3732 WREG32(CB_HW_CONTROL, tmp); 3733 3734 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3735 3736 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | 3737 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | 3738 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | 3739 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); 3740 3741 WREG32(VGT_NUM_INSTANCES, 1); 3742 3743 WREG32(CP_PERFMON_CNTL, 0); 3744 3745 WREG32(SQ_CONFIG, 0); 3746 3747 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3748 FORCE_EOV_MAX_REZ_CNT(255))); 3749 3750 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3751 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3752 3753 WREG32(VGT_GS_VERTEX_REUSE, 16); 3754 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3755 3756 tmp = RREG32(HDP_MISC_CNTL); 3757 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3758 WREG32(HDP_MISC_CNTL, tmp); 3759 3760 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3761 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3762 3763 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3764 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); 3765 mutex_unlock(&rdev->grbm_idx_mutex); 3766 3767 udelay(50); 3768 } 3769 3770 /* 3771 * GPU scratch registers helpers function. 3772 */ 3773 /** 3774 * cik_scratch_init - setup driver info for CP scratch regs 3775 * 3776 * @rdev: radeon_device pointer 3777 * 3778 * Set up the number and offset of the CP scratch registers. 3779 * NOTE: use of CP scratch registers is a legacy inferface and 3780 * is not used by default on newer asics (r6xx+). On newer asics, 3781 * memory buffers are used for fences rather than scratch regs. 3782 */ 3783 static void cik_scratch_init(struct radeon_device *rdev) 3784 { 3785 int i; 3786 3787 rdev->scratch.num_reg = 7; 3788 rdev->scratch.reg_base = SCRATCH_REG0; 3789 for (i = 0; i < rdev->scratch.num_reg; i++) { 3790 rdev->scratch.free[i] = true; 3791 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3792 } 3793 } 3794 3795 /** 3796 * cik_ring_test - basic gfx ring test 3797 * 3798 * @rdev: radeon_device pointer 3799 * @ring: radeon_ring structure holding ring information 3800 * 3801 * Allocate a scratch register and write to it using the gfx ring (CIK). 3802 * Provides a basic gfx ring test to verify that the ring is working. 3803 * Used by cik_cp_gfx_resume(); 3804 * Returns 0 on success, error on failure. 3805 */ 3806 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) 3807 { 3808 uint32_t scratch; 3809 uint32_t tmp = 0; 3810 unsigned i; 3811 int r; 3812 3813 r = radeon_scratch_get(rdev, &scratch); 3814 if (r) { 3815 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); 3816 return r; 3817 } 3818 WREG32(scratch, 0xCAFEDEAD); 3819 r = radeon_ring_lock(rdev, ring, 3); 3820 if (r) { 3821 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); 3822 radeon_scratch_free(rdev, scratch); 3823 return r; 3824 } 3825 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3826 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); 3827 radeon_ring_write(ring, 0xDEADBEEF); 3828 radeon_ring_unlock_commit(rdev, ring, false); 3829 3830 for (i = 0; i < rdev->usec_timeout; i++) { 3831 tmp = RREG32(scratch); 3832 if (tmp == 0xDEADBEEF) 3833 break; 3834 DRM_UDELAY(1); 3835 } 3836 if (i < rdev->usec_timeout) { 3837 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 3838 } else { 3839 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 3840 ring->idx, scratch, tmp); 3841 r = -EINVAL; 3842 } 3843 radeon_scratch_free(rdev, scratch); 3844 return r; 3845 } 3846 3847 /** 3848 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp 3849 * 3850 * @rdev: radeon_device pointer 3851 * @ridx: radeon ring index 3852 * 3853 * Emits an hdp flush on the cp. 3854 */ 3855 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, 3856 int ridx) 3857 { 3858 struct radeon_ring *ring = &rdev->ring[ridx]; 3859 u32 ref_and_mask; 3860 3861 switch (ring->idx) { 3862 case CAYMAN_RING_TYPE_CP1_INDEX: 3863 case CAYMAN_RING_TYPE_CP2_INDEX: 3864 default: 3865 switch (ring->me) { 3866 case 0: 3867 ref_and_mask = CP2 << ring->pipe; 3868 break; 3869 case 1: 3870 ref_and_mask = CP6 << ring->pipe; 3871 break; 3872 default: 3873 return; 3874 } 3875 break; 3876 case RADEON_RING_TYPE_GFX_INDEX: 3877 ref_and_mask = CP0; 3878 break; 3879 } 3880 3881 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 3882 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 3883 WAIT_REG_MEM_FUNCTION(3) | /* == */ 3884 WAIT_REG_MEM_ENGINE(1))); /* pfp */ 3885 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); 3886 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); 3887 radeon_ring_write(ring, ref_and_mask); 3888 radeon_ring_write(ring, ref_and_mask); 3889 radeon_ring_write(ring, 0x20); /* poll interval */ 3890 } 3891 3892 /** 3893 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring 3894 * 3895 * @rdev: radeon_device pointer 3896 * @fence: radeon fence object 3897 * 3898 * Emits a fence sequnce number on the gfx ring and flushes 3899 * GPU caches. 3900 */ 3901 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, 3902 struct radeon_fence *fence) 3903 { 3904 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3905 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3906 3907 /* EVENT_WRITE_EOP - flush caches, send int */ 3908 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3909 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3910 EOP_TC_ACTION_EN | 3911 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3912 EVENT_INDEX(5))); 3913 radeon_ring_write(ring, addr & 0xfffffffc); 3914 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); 3915 radeon_ring_write(ring, fence->seq); 3916 radeon_ring_write(ring, 0); 3917 } 3918 3919 /** 3920 * cik_fence_compute_ring_emit - emit a fence on the compute ring 3921 * 3922 * @rdev: radeon_device pointer 3923 * @fence: radeon fence object 3924 * 3925 * Emits a fence sequnce number on the compute ring and flushes 3926 * GPU caches. 3927 */ 3928 void cik_fence_compute_ring_emit(struct radeon_device *rdev, 3929 struct radeon_fence *fence) 3930 { 3931 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3932 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3933 3934 /* RELEASE_MEM - flush caches, send int */ 3935 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 3936 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3937 EOP_TC_ACTION_EN | 3938 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3939 EVENT_INDEX(5))); 3940 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); 3941 radeon_ring_write(ring, addr & 0xfffffffc); 3942 radeon_ring_write(ring, upper_32_bits(addr)); 3943 radeon_ring_write(ring, fence->seq); 3944 radeon_ring_write(ring, 0); 3945 } 3946 3947 /** 3948 * cik_semaphore_ring_emit - emit a semaphore on the CP ring 3949 * 3950 * @rdev: radeon_device pointer 3951 * @ring: radeon ring buffer object 3952 * @semaphore: radeon semaphore object 3953 * @emit_wait: Is this a sempahore wait? 3954 * 3955 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP 3956 * from running ahead of semaphore waits. 3957 */ 3958 bool cik_semaphore_ring_emit(struct radeon_device *rdev, 3959 struct radeon_ring *ring, 3960 struct radeon_semaphore *semaphore, 3961 bool emit_wait) 3962 { 3963 uint64_t addr = semaphore->gpu_addr; 3964 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; 3965 3966 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); 3967 radeon_ring_write(ring, lower_32_bits(addr)); 3968 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); 3969 3970 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) { 3971 /* Prevent the PFP from running ahead of the semaphore wait */ 3972 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3973 radeon_ring_write(ring, 0x0); 3974 } 3975 3976 return true; 3977 } 3978 3979 /** 3980 * cik_copy_cpdma - copy pages using the CP DMA engine 3981 * 3982 * @rdev: radeon_device pointer 3983 * @src_offset: src GPU address 3984 * @dst_offset: dst GPU address 3985 * @num_gpu_pages: number of GPU pages to xfer 3986 * @resv: reservation object to sync to 3987 * 3988 * Copy GPU paging using the CP DMA engine (CIK+). 3989 * Used by the radeon ttm implementation to move pages if 3990 * registered as the asic copy callback. 3991 */ 3992 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, 3993 uint64_t src_offset, uint64_t dst_offset, 3994 unsigned num_gpu_pages, 3995 struct reservation_object *resv) 3996 { 3997 struct radeon_fence *fence; 3998 struct radeon_sync sync; 3999 int ring_index = rdev->asic->copy.blit_ring_index; 4000 struct radeon_ring *ring = &rdev->ring[ring_index]; 4001 u32 size_in_bytes, cur_size_in_bytes, control; 4002 int i, num_loops; 4003 int r = 0; 4004 4005 radeon_sync_create(&sync); 4006 4007 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 4008 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); 4009 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); 4010 if (r) { 4011 DRM_ERROR("radeon: moving bo (%d).\n", r); 4012 radeon_sync_free(rdev, &sync, NULL); 4013 return ERR_PTR(r); 4014 } 4015 4016 radeon_sync_resv(rdev, &sync, resv, false); 4017 radeon_sync_rings(rdev, &sync, ring->idx); 4018 4019 for (i = 0; i < num_loops; i++) { 4020 cur_size_in_bytes = size_in_bytes; 4021 if (cur_size_in_bytes > 0x1fffff) 4022 cur_size_in_bytes = 0x1fffff; 4023 size_in_bytes -= cur_size_in_bytes; 4024 control = 0; 4025 if (size_in_bytes == 0) 4026 control |= PACKET3_DMA_DATA_CP_SYNC; 4027 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4028 radeon_ring_write(ring, control); 4029 radeon_ring_write(ring, lower_32_bits(src_offset)); 4030 radeon_ring_write(ring, upper_32_bits(src_offset)); 4031 radeon_ring_write(ring, lower_32_bits(dst_offset)); 4032 radeon_ring_write(ring, upper_32_bits(dst_offset)); 4033 radeon_ring_write(ring, cur_size_in_bytes); 4034 src_offset += cur_size_in_bytes; 4035 dst_offset += cur_size_in_bytes; 4036 } 4037 4038 r = radeon_fence_emit(rdev, &fence, ring->idx); 4039 if (r) { 4040 radeon_ring_unlock_undo(rdev, ring); 4041 radeon_sync_free(rdev, &sync, NULL); 4042 return ERR_PTR(r); 4043 } 4044 4045 radeon_ring_unlock_commit(rdev, ring, false); 4046 radeon_sync_free(rdev, &sync, fence); 4047 4048 return fence; 4049 } 4050 4051 /* 4052 * IB stuff 4053 */ 4054 /** 4055 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring 4056 * 4057 * @rdev: radeon_device pointer 4058 * @ib: radeon indirect buffer object 4059 * 4060 * Emits an DE (drawing engine) or CE (constant engine) IB 4061 * on the gfx ring. IBs are usually generated by userspace 4062 * acceleration drivers and submitted to the kernel for 4063 * sheduling on the ring. This function schedules the IB 4064 * on the gfx ring for execution by the GPU. 4065 */ 4066 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 4067 { 4068 struct radeon_ring *ring = &rdev->ring[ib->ring]; 4069 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; 4070 u32 header, control = INDIRECT_BUFFER_VALID; 4071 4072 if (ib->is_const_ib) { 4073 /* set switch buffer packet before const IB */ 4074 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 4075 radeon_ring_write(ring, 0); 4076 4077 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 4078 } else { 4079 u32 next_rptr; 4080 if (ring->rptr_save_reg) { 4081 next_rptr = ring->wptr + 3 + 4; 4082 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 4083 radeon_ring_write(ring, ((ring->rptr_save_reg - 4084 PACKET3_SET_UCONFIG_REG_START) >> 2)); 4085 radeon_ring_write(ring, next_rptr); 4086 } else if (rdev->wb.enabled) { 4087 next_rptr = ring->wptr + 5 + 4; 4088 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4089 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); 4090 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 4091 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); 4092 radeon_ring_write(ring, next_rptr); 4093 } 4094 4095 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 4096 } 4097 4098 control |= ib->length_dw | (vm_id << 24); 4099 4100 radeon_ring_write(ring, header); 4101 radeon_ring_write(ring, 4102 #ifdef __BIG_ENDIAN 4103 (2 << 0) | 4104 #endif 4105 (ib->gpu_addr & 0xFFFFFFFC)); 4106 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 4107 radeon_ring_write(ring, control); 4108 } 4109 4110 /** 4111 * cik_ib_test - basic gfx ring IB test 4112 * 4113 * @rdev: radeon_device pointer 4114 * @ring: radeon_ring structure holding ring information 4115 * 4116 * Allocate an IB and execute it on the gfx ring (CIK). 4117 * Provides a basic gfx ring test to verify that IBs are working. 4118 * Returns 0 on success, error on failure. 4119 */ 4120 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) 4121 { 4122 struct radeon_ib ib; 4123 uint32_t scratch; 4124 uint32_t tmp = 0; 4125 unsigned i; 4126 int r; 4127 4128 r = radeon_scratch_get(rdev, &scratch); 4129 if (r) { 4130 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); 4131 return r; 4132 } 4133 WREG32(scratch, 0xCAFEDEAD); 4134 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); 4135 if (r) { 4136 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 4137 radeon_scratch_free(rdev, scratch); 4138 return r; 4139 } 4140 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); 4141 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); 4142 ib.ptr[2] = 0xDEADBEEF; 4143 ib.length_dw = 3; 4144 r = radeon_ib_schedule(rdev, &ib, NULL, false); 4145 if (r) { 4146 radeon_scratch_free(rdev, scratch); 4147 radeon_ib_free(rdev, &ib); 4148 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); 4149 return r; 4150 } 4151 r = radeon_fence_wait(ib.fence, false); 4152 if (r) { 4153 DRM_ERROR("radeon: fence wait failed (%d).\n", r); 4154 radeon_scratch_free(rdev, scratch); 4155 radeon_ib_free(rdev, &ib); 4156 return r; 4157 } 4158 for (i = 0; i < rdev->usec_timeout; i++) { 4159 tmp = RREG32(scratch); 4160 if (tmp == 0xDEADBEEF) 4161 break; 4162 DRM_UDELAY(1); 4163 } 4164 if (i < rdev->usec_timeout) { 4165 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); 4166 } else { 4167 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 4168 scratch, tmp); 4169 r = -EINVAL; 4170 } 4171 radeon_scratch_free(rdev, scratch); 4172 radeon_ib_free(rdev, &ib); 4173 return r; 4174 } 4175 4176 /* 4177 * CP. 4178 * On CIK, gfx and compute now have independant command processors. 4179 * 4180 * GFX 4181 * Gfx consists of a single ring and can process both gfx jobs and 4182 * compute jobs. The gfx CP consists of three microengines (ME): 4183 * PFP - Pre-Fetch Parser 4184 * ME - Micro Engine 4185 * CE - Constant Engine 4186 * The PFP and ME make up what is considered the Drawing Engine (DE). 4187 * The CE is an asynchronous engine used for updating buffer desciptors 4188 * used by the DE so that they can be loaded into cache in parallel 4189 * while the DE is processing state update packets. 4190 * 4191 * Compute 4192 * The compute CP consists of two microengines (ME): 4193 * MEC1 - Compute MicroEngine 1 4194 * MEC2 - Compute MicroEngine 2 4195 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. 4196 * The queues are exposed to userspace and are programmed directly 4197 * by the compute runtime. 4198 */ 4199 /** 4200 * cik_cp_gfx_enable - enable/disable the gfx CP MEs 4201 * 4202 * @rdev: radeon_device pointer 4203 * @enable: enable or disable the MEs 4204 * 4205 * Halts or unhalts the gfx MEs. 4206 */ 4207 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) 4208 { 4209 if (enable) 4210 WREG32(CP_ME_CNTL, 0); 4211 else { 4212 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4213 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 4214 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 4215 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4216 } 4217 udelay(50); 4218 } 4219 4220 /** 4221 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode 4222 * 4223 * @rdev: radeon_device pointer 4224 * 4225 * Loads the gfx PFP, ME, and CE ucode. 4226 * Returns 0 for success, -EINVAL if the ucode is not available. 4227 */ 4228 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) 4229 { 4230 int i; 4231 4232 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 4233 return -EINVAL; 4234 4235 cik_cp_gfx_enable(rdev, false); 4236 4237 if (rdev->new_fw) { 4238 const struct gfx_firmware_header_v1_0 *pfp_hdr = 4239 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 4240 const struct gfx_firmware_header_v1_0 *ce_hdr = 4241 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 4242 const struct gfx_firmware_header_v1_0 *me_hdr = 4243 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 4244 const __le32 *fw_data; 4245 u32 fw_size; 4246 4247 radeon_ucode_print_gfx_hdr(&pfp_hdr->header); 4248 radeon_ucode_print_gfx_hdr(&ce_hdr->header); 4249 radeon_ucode_print_gfx_hdr(&me_hdr->header); 4250 4251 /* PFP */ 4252 fw_data = (const __le32 *) 4253 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 4254 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 4255 WREG32(CP_PFP_UCODE_ADDR, 0); 4256 for (i = 0; i < fw_size; i++) 4257 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 4258 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version)); 4259 4260 /* CE */ 4261 fw_data = (const __le32 *) 4262 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 4263 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 4264 WREG32(CP_CE_UCODE_ADDR, 0); 4265 for (i = 0; i < fw_size; i++) 4266 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 4267 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version)); 4268 4269 /* ME */ 4270 fw_data = (const __be32 *) 4271 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 4272 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 4273 WREG32(CP_ME_RAM_WADDR, 0); 4274 for (i = 0; i < fw_size; i++) 4275 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 4276 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4277 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version)); 4278 } else { 4279 const __be32 *fw_data; 4280 4281 /* PFP */ 4282 fw_data = (const __be32 *)rdev->pfp_fw->data; 4283 WREG32(CP_PFP_UCODE_ADDR, 0); 4284 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) 4285 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 4286 WREG32(CP_PFP_UCODE_ADDR, 0); 4287 4288 /* CE */ 4289 fw_data = (const __be32 *)rdev->ce_fw->data; 4290 WREG32(CP_CE_UCODE_ADDR, 0); 4291 for (i = 0; i < CIK_CE_UCODE_SIZE; i++) 4292 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 4293 WREG32(CP_CE_UCODE_ADDR, 0); 4294 4295 /* ME */ 4296 fw_data = (const __be32 *)rdev->me_fw->data; 4297 WREG32(CP_ME_RAM_WADDR, 0); 4298 for (i = 0; i < CIK_ME_UCODE_SIZE; i++) 4299 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 4300 WREG32(CP_ME_RAM_WADDR, 0); 4301 } 4302 4303 return 0; 4304 } 4305 4306 /** 4307 * cik_cp_gfx_start - start the gfx ring 4308 * 4309 * @rdev: radeon_device pointer 4310 * 4311 * Enables the ring and loads the clear state context and other 4312 * packets required to init the ring. 4313 * Returns 0 for success, error for failure. 4314 */ 4315 static int cik_cp_gfx_start(struct radeon_device *rdev) 4316 { 4317 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4318 int r, i; 4319 4320 /* init the CP */ 4321 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); 4322 WREG32(CP_ENDIAN_SWAP, 0); 4323 WREG32(CP_DEVICE_ID, 1); 4324 4325 cik_cp_gfx_enable(rdev, true); 4326 4327 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); 4328 if (r) { 4329 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 4330 return r; 4331 } 4332 4333 /* init the CE partitions. CE only used for gfx on CIK */ 4334 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4335 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4336 radeon_ring_write(ring, 0x8000); 4337 radeon_ring_write(ring, 0x8000); 4338 4339 /* setup clear context state */ 4340 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4341 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4342 4343 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4344 radeon_ring_write(ring, 0x80000000); 4345 radeon_ring_write(ring, 0x80000000); 4346 4347 for (i = 0; i < cik_default_size; i++) 4348 radeon_ring_write(ring, cik_default_state[i]); 4349 4350 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4351 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4352 4353 /* set clear context state */ 4354 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4355 radeon_ring_write(ring, 0); 4356 4357 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4358 radeon_ring_write(ring, 0x00000316); 4359 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 4360 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 4361 4362 radeon_ring_unlock_commit(rdev, ring, false); 4363 4364 return 0; 4365 } 4366 4367 /** 4368 * cik_cp_gfx_fini - stop the gfx ring 4369 * 4370 * @rdev: radeon_device pointer 4371 * 4372 * Stop the gfx ring and tear down the driver ring 4373 * info. 4374 */ 4375 static void cik_cp_gfx_fini(struct radeon_device *rdev) 4376 { 4377 cik_cp_gfx_enable(rdev, false); 4378 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4379 } 4380 4381 /** 4382 * cik_cp_gfx_resume - setup the gfx ring buffer registers 4383 * 4384 * @rdev: radeon_device pointer 4385 * 4386 * Program the location and size of the gfx ring buffer 4387 * and test it to make sure it's working. 4388 * Returns 0 for success, error for failure. 4389 */ 4390 static int cik_cp_gfx_resume(struct radeon_device *rdev) 4391 { 4392 struct radeon_ring *ring; 4393 u32 tmp; 4394 u32 rb_bufsz; 4395 u64 rb_addr; 4396 int r; 4397 4398 WREG32(CP_SEM_WAIT_TIMER, 0x0); 4399 if (rdev->family != CHIP_HAWAII) 4400 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 4401 4402 /* Set the write pointer delay */ 4403 WREG32(CP_RB_WPTR_DELAY, 0); 4404 4405 /* set the RB to use vmid 0 */ 4406 WREG32(CP_RB_VMID, 0); 4407 4408 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 4409 4410 /* ring 0 - compute and gfx */ 4411 /* Set ring buffer size */ 4412 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4413 rb_bufsz = order_base_2(ring->ring_size / 8); 4414 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 4415 #ifdef __BIG_ENDIAN 4416 tmp |= BUF_SWAP_32BIT; 4417 #endif 4418 WREG32(CP_RB0_CNTL, tmp); 4419 4420 /* Initialize the ring buffer's read and write pointers */ 4421 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 4422 ring->wptr = 0; 4423 WREG32(CP_RB0_WPTR, ring->wptr); 4424 4425 /* set the wb address wether it's enabled or not */ 4426 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 4427 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 4428 4429 /* scratch register shadowing is no longer supported */ 4430 WREG32(SCRATCH_UMSK, 0); 4431 4432 if (!rdev->wb.enabled) 4433 tmp |= RB_NO_UPDATE; 4434 4435 mdelay(1); 4436 WREG32(CP_RB0_CNTL, tmp); 4437 4438 rb_addr = ring->gpu_addr >> 8; 4439 WREG32(CP_RB0_BASE, rb_addr); 4440 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4441 4442 /* start the ring */ 4443 cik_cp_gfx_start(rdev); 4444 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 4445 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4446 if (r) { 4447 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4448 return r; 4449 } 4450 4451 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4452 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 4453 4454 return 0; 4455 } 4456 4457 u32 cik_gfx_get_rptr(struct radeon_device *rdev, 4458 struct radeon_ring *ring) 4459 { 4460 u32 rptr; 4461 4462 if (rdev->wb.enabled) 4463 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4464 else 4465 rptr = RREG32(CP_RB0_RPTR); 4466 4467 return rptr; 4468 } 4469 4470 u32 cik_gfx_get_wptr(struct radeon_device *rdev, 4471 struct radeon_ring *ring) 4472 { 4473 u32 wptr; 4474 4475 wptr = RREG32(CP_RB0_WPTR); 4476 4477 return wptr; 4478 } 4479 4480 void cik_gfx_set_wptr(struct radeon_device *rdev, 4481 struct radeon_ring *ring) 4482 { 4483 WREG32(CP_RB0_WPTR, ring->wptr); 4484 (void)RREG32(CP_RB0_WPTR); 4485 } 4486 4487 u32 cik_compute_get_rptr(struct radeon_device *rdev, 4488 struct radeon_ring *ring) 4489 { 4490 u32 rptr; 4491 4492 if (rdev->wb.enabled) { 4493 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4494 } else { 4495 mutex_lock(&rdev->srbm_mutex); 4496 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4497 rptr = RREG32(CP_HQD_PQ_RPTR); 4498 cik_srbm_select(rdev, 0, 0, 0, 0); 4499 mutex_unlock(&rdev->srbm_mutex); 4500 } 4501 4502 return rptr; 4503 } 4504 4505 u32 cik_compute_get_wptr(struct radeon_device *rdev, 4506 struct radeon_ring *ring) 4507 { 4508 u32 wptr; 4509 4510 if (rdev->wb.enabled) { 4511 /* XXX check if swapping is necessary on BE */ 4512 wptr = rdev->wb.wb[ring->wptr_offs/4]; 4513 } else { 4514 mutex_lock(&rdev->srbm_mutex); 4515 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4516 wptr = RREG32(CP_HQD_PQ_WPTR); 4517 cik_srbm_select(rdev, 0, 0, 0, 0); 4518 mutex_unlock(&rdev->srbm_mutex); 4519 } 4520 4521 return wptr; 4522 } 4523 4524 void cik_compute_set_wptr(struct radeon_device *rdev, 4525 struct radeon_ring *ring) 4526 { 4527 /* XXX check if swapping is necessary on BE */ 4528 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; 4529 WDOORBELL32(ring->doorbell_index, ring->wptr); 4530 } 4531 4532 /** 4533 * cik_cp_compute_enable - enable/disable the compute CP MEs 4534 * 4535 * @rdev: radeon_device pointer 4536 * @enable: enable or disable the MEs 4537 * 4538 * Halts or unhalts the compute MEs. 4539 */ 4540 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) 4541 { 4542 if (enable) 4543 WREG32(CP_MEC_CNTL, 0); 4544 else { 4545 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); 4546 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 4547 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 4548 } 4549 udelay(50); 4550 } 4551 4552 /** 4553 * cik_cp_compute_load_microcode - load the compute CP ME ucode 4554 * 4555 * @rdev: radeon_device pointer 4556 * 4557 * Loads the compute MEC1&2 ucode. 4558 * Returns 0 for success, -EINVAL if the ucode is not available. 4559 */ 4560 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) 4561 { 4562 int i; 4563 4564 if (!rdev->mec_fw) 4565 return -EINVAL; 4566 4567 cik_cp_compute_enable(rdev, false); 4568 4569 if (rdev->new_fw) { 4570 const struct gfx_firmware_header_v1_0 *mec_hdr = 4571 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 4572 const __le32 *fw_data; 4573 u32 fw_size; 4574 4575 radeon_ucode_print_gfx_hdr(&mec_hdr->header); 4576 4577 /* MEC1 */ 4578 fw_data = (const __le32 *) 4579 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 4580 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4; 4581 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4582 for (i = 0; i < fw_size; i++) 4583 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++)); 4584 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version)); 4585 4586 /* MEC2 */ 4587 if (rdev->family == CHIP_KAVERI) { 4588 const struct gfx_firmware_header_v1_0 *mec2_hdr = 4589 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 4590 4591 fw_data = (const __le32 *) 4592 (rdev->mec2_fw->data + 4593 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes)); 4594 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4; 4595 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4596 for (i = 0; i < fw_size; i++) 4597 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++)); 4598 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version)); 4599 } 4600 } else { 4601 const __be32 *fw_data; 4602 4603 /* MEC1 */ 4604 fw_data = (const __be32 *)rdev->mec_fw->data; 4605 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4606 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4607 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); 4608 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4609 4610 if (rdev->family == CHIP_KAVERI) { 4611 /* MEC2 */ 4612 fw_data = (const __be32 *)rdev->mec_fw->data; 4613 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4614 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4615 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); 4616 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4617 } 4618 } 4619 4620 return 0; 4621 } 4622 4623 /** 4624 * cik_cp_compute_start - start the compute queues 4625 * 4626 * @rdev: radeon_device pointer 4627 * 4628 * Enable the compute queues. 4629 * Returns 0 for success, error for failure. 4630 */ 4631 static int cik_cp_compute_start(struct radeon_device *rdev) 4632 { 4633 cik_cp_compute_enable(rdev, true); 4634 4635 return 0; 4636 } 4637 4638 /** 4639 * cik_cp_compute_fini - stop the compute queues 4640 * 4641 * @rdev: radeon_device pointer 4642 * 4643 * Stop the compute queues and tear down the driver queue 4644 * info. 4645 */ 4646 static void cik_cp_compute_fini(struct radeon_device *rdev) 4647 { 4648 int i, idx, r; 4649 4650 cik_cp_compute_enable(rdev, false); 4651 4652 for (i = 0; i < 2; i++) { 4653 if (i == 0) 4654 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4655 else 4656 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4657 4658 if (rdev->ring[idx].mqd_obj) { 4659 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4660 if (unlikely(r != 0)) 4661 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); 4662 4663 radeon_bo_unpin(rdev->ring[idx].mqd_obj); 4664 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4665 4666 radeon_bo_unref(&rdev->ring[idx].mqd_obj); 4667 rdev->ring[idx].mqd_obj = NULL; 4668 } 4669 } 4670 } 4671 4672 static void cik_mec_fini(struct radeon_device *rdev) 4673 { 4674 int r; 4675 4676 if (rdev->mec.hpd_eop_obj) { 4677 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4678 if (unlikely(r != 0)) 4679 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); 4680 radeon_bo_unpin(rdev->mec.hpd_eop_obj); 4681 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4682 4683 radeon_bo_unref(&rdev->mec.hpd_eop_obj); 4684 rdev->mec.hpd_eop_obj = NULL; 4685 } 4686 } 4687 4688 #define MEC_HPD_SIZE 2048 4689 4690 static int cik_mec_init(struct radeon_device *rdev) 4691 { 4692 int r; 4693 u32 *hpd; 4694 4695 /* 4696 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total 4697 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total 4698 * Nonetheless, we assign only 1 pipe because all other pipes will 4699 * be handled by KFD 4700 */ 4701 rdev->mec.num_mec = 1; 4702 rdev->mec.num_pipe = 1; 4703 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; 4704 4705 if (rdev->mec.hpd_eop_obj == NULL) { 4706 r = radeon_bo_create(rdev, 4707 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, 4708 PAGE_SIZE, true, 4709 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL, 4710 &rdev->mec.hpd_eop_obj); 4711 if (r) { 4712 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); 4713 return r; 4714 } 4715 } 4716 4717 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4718 if (unlikely(r != 0)) { 4719 cik_mec_fini(rdev); 4720 return r; 4721 } 4722 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, 4723 &rdev->mec.hpd_eop_gpu_addr); 4724 if (r) { 4725 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); 4726 cik_mec_fini(rdev); 4727 return r; 4728 } 4729 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); 4730 if (r) { 4731 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); 4732 cik_mec_fini(rdev); 4733 return r; 4734 } 4735 4736 /* clear memory. Not sure if this is required or not */ 4737 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); 4738 4739 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); 4740 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4741 4742 return 0; 4743 } 4744 4745 struct hqd_registers 4746 { 4747 u32 cp_mqd_base_addr; 4748 u32 cp_mqd_base_addr_hi; 4749 u32 cp_hqd_active; 4750 u32 cp_hqd_vmid; 4751 u32 cp_hqd_persistent_state; 4752 u32 cp_hqd_pipe_priority; 4753 u32 cp_hqd_queue_priority; 4754 u32 cp_hqd_quantum; 4755 u32 cp_hqd_pq_base; 4756 u32 cp_hqd_pq_base_hi; 4757 u32 cp_hqd_pq_rptr; 4758 u32 cp_hqd_pq_rptr_report_addr; 4759 u32 cp_hqd_pq_rptr_report_addr_hi; 4760 u32 cp_hqd_pq_wptr_poll_addr; 4761 u32 cp_hqd_pq_wptr_poll_addr_hi; 4762 u32 cp_hqd_pq_doorbell_control; 4763 u32 cp_hqd_pq_wptr; 4764 u32 cp_hqd_pq_control; 4765 u32 cp_hqd_ib_base_addr; 4766 u32 cp_hqd_ib_base_addr_hi; 4767 u32 cp_hqd_ib_rptr; 4768 u32 cp_hqd_ib_control; 4769 u32 cp_hqd_iq_timer; 4770 u32 cp_hqd_iq_rptr; 4771 u32 cp_hqd_dequeue_request; 4772 u32 cp_hqd_dma_offload; 4773 u32 cp_hqd_sema_cmd; 4774 u32 cp_hqd_msg_type; 4775 u32 cp_hqd_atomic0_preop_lo; 4776 u32 cp_hqd_atomic0_preop_hi; 4777 u32 cp_hqd_atomic1_preop_lo; 4778 u32 cp_hqd_atomic1_preop_hi; 4779 u32 cp_hqd_hq_scheduler0; 4780 u32 cp_hqd_hq_scheduler1; 4781 u32 cp_mqd_control; 4782 }; 4783 4784 struct bonaire_mqd 4785 { 4786 u32 header; 4787 u32 dispatch_initiator; 4788 u32 dimensions[3]; 4789 u32 start_idx[3]; 4790 u32 num_threads[3]; 4791 u32 pipeline_stat_enable; 4792 u32 perf_counter_enable; 4793 u32 pgm[2]; 4794 u32 tba[2]; 4795 u32 tma[2]; 4796 u32 pgm_rsrc[2]; 4797 u32 vmid; 4798 u32 resource_limits; 4799 u32 static_thread_mgmt01[2]; 4800 u32 tmp_ring_size; 4801 u32 static_thread_mgmt23[2]; 4802 u32 restart[3]; 4803 u32 thread_trace_enable; 4804 u32 reserved1; 4805 u32 user_data[16]; 4806 u32 vgtcs_invoke_count[2]; 4807 struct hqd_registers queue_state; 4808 u32 dequeue_cntr; 4809 u32 interrupt_queue[64]; 4810 }; 4811 4812 /** 4813 * cik_cp_compute_resume - setup the compute queue registers 4814 * 4815 * @rdev: radeon_device pointer 4816 * 4817 * Program the compute queues and test them to make sure they 4818 * are working. 4819 * Returns 0 for success, error for failure. 4820 */ 4821 static int cik_cp_compute_resume(struct radeon_device *rdev) 4822 { 4823 int r, i, j, idx; 4824 u32 tmp; 4825 bool use_doorbell = true; 4826 u64 hqd_gpu_addr; 4827 u64 mqd_gpu_addr; 4828 u64 eop_gpu_addr; 4829 u64 wb_gpu_addr; 4830 u32 *buf; 4831 struct bonaire_mqd *mqd; 4832 4833 r = cik_cp_compute_start(rdev); 4834 if (r) 4835 return r; 4836 4837 /* fix up chicken bits */ 4838 tmp = RREG32(CP_CPF_DEBUG); 4839 tmp |= (1 << 23); 4840 WREG32(CP_CPF_DEBUG, tmp); 4841 4842 /* init the pipes */ 4843 mutex_lock(&rdev->srbm_mutex); 4844 4845 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr; 4846 4847 cik_srbm_select(rdev, 0, 0, 0, 0); 4848 4849 /* write the EOP addr */ 4850 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); 4851 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); 4852 4853 /* set the VMID assigned */ 4854 WREG32(CP_HPD_EOP_VMID, 0); 4855 4856 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4857 tmp = RREG32(CP_HPD_EOP_CONTROL); 4858 tmp &= ~EOP_SIZE_MASK; 4859 tmp |= order_base_2(MEC_HPD_SIZE / 8); 4860 WREG32(CP_HPD_EOP_CONTROL, tmp); 4861 4862 mutex_unlock(&rdev->srbm_mutex); 4863 4864 /* init the queues. Just two for now. */ 4865 for (i = 0; i < 2; i++) { 4866 if (i == 0) 4867 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4868 else 4869 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4870 4871 if (rdev->ring[idx].mqd_obj == NULL) { 4872 r = radeon_bo_create(rdev, 4873 sizeof(struct bonaire_mqd), 4874 PAGE_SIZE, true, 4875 RADEON_GEM_DOMAIN_GTT, 0, NULL, 4876 NULL, &rdev->ring[idx].mqd_obj); 4877 if (r) { 4878 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); 4879 return r; 4880 } 4881 } 4882 4883 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4884 if (unlikely(r != 0)) { 4885 cik_cp_compute_fini(rdev); 4886 return r; 4887 } 4888 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, 4889 &mqd_gpu_addr); 4890 if (r) { 4891 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); 4892 cik_cp_compute_fini(rdev); 4893 return r; 4894 } 4895 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); 4896 if (r) { 4897 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); 4898 cik_cp_compute_fini(rdev); 4899 return r; 4900 } 4901 4902 /* init the mqd struct */ 4903 memset(buf, 0, sizeof(struct bonaire_mqd)); 4904 4905 mqd = (struct bonaire_mqd *)buf; 4906 mqd->header = 0xC0310800; 4907 mqd->static_thread_mgmt01[0] = 0xffffffff; 4908 mqd->static_thread_mgmt01[1] = 0xffffffff; 4909 mqd->static_thread_mgmt23[0] = 0xffffffff; 4910 mqd->static_thread_mgmt23[1] = 0xffffffff; 4911 4912 mutex_lock(&rdev->srbm_mutex); 4913 cik_srbm_select(rdev, rdev->ring[idx].me, 4914 rdev->ring[idx].pipe, 4915 rdev->ring[idx].queue, 0); 4916 4917 /* disable wptr polling */ 4918 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4919 tmp &= ~WPTR_POLL_EN; 4920 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4921 4922 /* enable doorbell? */ 4923 mqd->queue_state.cp_hqd_pq_doorbell_control = 4924 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4925 if (use_doorbell) 4926 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4927 else 4928 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; 4929 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4930 mqd->queue_state.cp_hqd_pq_doorbell_control); 4931 4932 /* disable the queue if it's active */ 4933 mqd->queue_state.cp_hqd_dequeue_request = 0; 4934 mqd->queue_state.cp_hqd_pq_rptr = 0; 4935 mqd->queue_state.cp_hqd_pq_wptr= 0; 4936 if (RREG32(CP_HQD_ACTIVE) & 1) { 4937 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4938 for (j = 0; j < rdev->usec_timeout; j++) { 4939 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4940 break; 4941 udelay(1); 4942 } 4943 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); 4944 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); 4945 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4946 } 4947 4948 /* set the pointer to the MQD */ 4949 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; 4950 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 4951 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); 4952 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); 4953 /* set MQD vmid to 0 */ 4954 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); 4955 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; 4956 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); 4957 4958 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4959 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; 4960 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; 4961 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4962 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); 4963 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); 4964 4965 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4966 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); 4967 mqd->queue_state.cp_hqd_pq_control &= 4968 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); 4969 4970 mqd->queue_state.cp_hqd_pq_control |= 4971 order_base_2(rdev->ring[idx].ring_size / 8); 4972 mqd->queue_state.cp_hqd_pq_control |= 4973 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); 4974 #ifdef __BIG_ENDIAN 4975 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; 4976 #endif 4977 mqd->queue_state.cp_hqd_pq_control &= 4978 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); 4979 mqd->queue_state.cp_hqd_pq_control |= 4980 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ 4981 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); 4982 4983 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ 4984 if (i == 0) 4985 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; 4986 else 4987 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; 4988 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; 4989 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4990 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); 4991 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, 4992 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); 4993 4994 /* set the wb address wether it's enabled or not */ 4995 if (i == 0) 4996 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; 4997 else 4998 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; 4999 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; 5000 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = 5001 upper_32_bits(wb_gpu_addr) & 0xffff; 5002 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, 5003 mqd->queue_state.cp_hqd_pq_rptr_report_addr); 5004 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, 5005 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); 5006 5007 /* enable the doorbell if requested */ 5008 if (use_doorbell) { 5009 mqd->queue_state.cp_hqd_pq_doorbell_control = 5010 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 5011 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; 5012 mqd->queue_state.cp_hqd_pq_doorbell_control |= 5013 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); 5014 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 5015 mqd->queue_state.cp_hqd_pq_doorbell_control &= 5016 ~(DOORBELL_SOURCE | DOORBELL_HIT); 5017 5018 } else { 5019 mqd->queue_state.cp_hqd_pq_doorbell_control = 0; 5020 } 5021 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 5022 mqd->queue_state.cp_hqd_pq_doorbell_control); 5023 5024 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 5025 rdev->ring[idx].wptr = 0; 5026 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; 5027 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 5028 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); 5029 5030 /* set the vmid for the queue */ 5031 mqd->queue_state.cp_hqd_vmid = 0; 5032 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); 5033 5034 /* activate the queue */ 5035 mqd->queue_state.cp_hqd_active = 1; 5036 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); 5037 5038 cik_srbm_select(rdev, 0, 0, 0, 0); 5039 mutex_unlock(&rdev->srbm_mutex); 5040 5041 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); 5042 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 5043 5044 rdev->ring[idx].ready = true; 5045 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); 5046 if (r) 5047 rdev->ring[idx].ready = false; 5048 } 5049 5050 return 0; 5051 } 5052 5053 static void cik_cp_enable(struct radeon_device *rdev, bool enable) 5054 { 5055 cik_cp_gfx_enable(rdev, enable); 5056 cik_cp_compute_enable(rdev, enable); 5057 } 5058 5059 static int cik_cp_load_microcode(struct radeon_device *rdev) 5060 { 5061 int r; 5062 5063 r = cik_cp_gfx_load_microcode(rdev); 5064 if (r) 5065 return r; 5066 r = cik_cp_compute_load_microcode(rdev); 5067 if (r) 5068 return r; 5069 5070 return 0; 5071 } 5072 5073 static void cik_cp_fini(struct radeon_device *rdev) 5074 { 5075 cik_cp_gfx_fini(rdev); 5076 cik_cp_compute_fini(rdev); 5077 } 5078 5079 static int cik_cp_resume(struct radeon_device *rdev) 5080 { 5081 int r; 5082 5083 cik_enable_gui_idle_interrupt(rdev, false); 5084 5085 r = cik_cp_load_microcode(rdev); 5086 if (r) 5087 return r; 5088 5089 r = cik_cp_gfx_resume(rdev); 5090 if (r) 5091 return r; 5092 r = cik_cp_compute_resume(rdev); 5093 if (r) 5094 return r; 5095 5096 cik_enable_gui_idle_interrupt(rdev, true); 5097 5098 return 0; 5099 } 5100 5101 static void cik_print_gpu_status_regs(struct radeon_device *rdev) 5102 { 5103 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 5104 RREG32(GRBM_STATUS)); 5105 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 5106 RREG32(GRBM_STATUS2)); 5107 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 5108 RREG32(GRBM_STATUS_SE0)); 5109 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 5110 RREG32(GRBM_STATUS_SE1)); 5111 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", 5112 RREG32(GRBM_STATUS_SE2)); 5113 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", 5114 RREG32(GRBM_STATUS_SE3)); 5115 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 5116 RREG32(SRBM_STATUS)); 5117 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", 5118 RREG32(SRBM_STATUS2)); 5119 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", 5120 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); 5121 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", 5122 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); 5123 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); 5124 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", 5125 RREG32(CP_STALLED_STAT1)); 5126 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", 5127 RREG32(CP_STALLED_STAT2)); 5128 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", 5129 RREG32(CP_STALLED_STAT3)); 5130 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", 5131 RREG32(CP_CPF_BUSY_STAT)); 5132 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", 5133 RREG32(CP_CPF_STALLED_STAT1)); 5134 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); 5135 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); 5136 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", 5137 RREG32(CP_CPC_STALLED_STAT1)); 5138 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); 5139 } 5140 5141 /** 5142 * cik_gpu_check_soft_reset - check which blocks are busy 5143 * 5144 * @rdev: radeon_device pointer 5145 * 5146 * Check which blocks are busy and return the relevant reset 5147 * mask to be used by cik_gpu_soft_reset(). 5148 * Returns a mask of the blocks to be reset. 5149 */ 5150 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) 5151 { 5152 u32 reset_mask = 0; 5153 u32 tmp; 5154 5155 /* GRBM_STATUS */ 5156 tmp = RREG32(GRBM_STATUS); 5157 if (tmp & (PA_BUSY | SC_BUSY | 5158 BCI_BUSY | SX_BUSY | 5159 TA_BUSY | VGT_BUSY | 5160 DB_BUSY | CB_BUSY | 5161 GDS_BUSY | SPI_BUSY | 5162 IA_BUSY | IA_BUSY_NO_DMA)) 5163 reset_mask |= RADEON_RESET_GFX; 5164 5165 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) 5166 reset_mask |= RADEON_RESET_CP; 5167 5168 /* GRBM_STATUS2 */ 5169 tmp = RREG32(GRBM_STATUS2); 5170 if (tmp & RLC_BUSY) 5171 reset_mask |= RADEON_RESET_RLC; 5172 5173 /* SDMA0_STATUS_REG */ 5174 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); 5175 if (!(tmp & SDMA_IDLE)) 5176 reset_mask |= RADEON_RESET_DMA; 5177 5178 /* SDMA1_STATUS_REG */ 5179 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); 5180 if (!(tmp & SDMA_IDLE)) 5181 reset_mask |= RADEON_RESET_DMA1; 5182 5183 /* SRBM_STATUS2 */ 5184 tmp = RREG32(SRBM_STATUS2); 5185 if (tmp & SDMA_BUSY) 5186 reset_mask |= RADEON_RESET_DMA; 5187 5188 if (tmp & SDMA1_BUSY) 5189 reset_mask |= RADEON_RESET_DMA1; 5190 5191 /* SRBM_STATUS */ 5192 tmp = RREG32(SRBM_STATUS); 5193 5194 if (tmp & IH_BUSY) 5195 reset_mask |= RADEON_RESET_IH; 5196 5197 if (tmp & SEM_BUSY) 5198 reset_mask |= RADEON_RESET_SEM; 5199 5200 if (tmp & GRBM_RQ_PENDING) 5201 reset_mask |= RADEON_RESET_GRBM; 5202 5203 if (tmp & VMC_BUSY) 5204 reset_mask |= RADEON_RESET_VMC; 5205 5206 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 5207 MCC_BUSY | MCD_BUSY)) 5208 reset_mask |= RADEON_RESET_MC; 5209 5210 if (evergreen_is_display_hung(rdev)) 5211 reset_mask |= RADEON_RESET_DISPLAY; 5212 5213 /* Skip MC reset as it's mostly likely not hung, just busy */ 5214 if (reset_mask & RADEON_RESET_MC) { 5215 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 5216 reset_mask &= ~RADEON_RESET_MC; 5217 } 5218 5219 return reset_mask; 5220 } 5221 5222 /** 5223 * cik_gpu_soft_reset - soft reset GPU 5224 * 5225 * @rdev: radeon_device pointer 5226 * @reset_mask: mask of which blocks to reset 5227 * 5228 * Soft reset the blocks specified in @reset_mask. 5229 */ 5230 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 5231 { 5232 struct evergreen_mc_save save; 5233 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 5234 u32 tmp; 5235 5236 if (reset_mask == 0) 5237 return; 5238 5239 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 5240 5241 cik_print_gpu_status_regs(rdev); 5242 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5243 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 5244 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5245 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 5246 5247 /* disable CG/PG */ 5248 cik_fini_pg(rdev); 5249 cik_fini_cg(rdev); 5250 5251 /* stop the rlc */ 5252 cik_rlc_stop(rdev); 5253 5254 /* Disable GFX parsing/prefetching */ 5255 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5256 5257 /* Disable MEC parsing/prefetching */ 5258 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5259 5260 if (reset_mask & RADEON_RESET_DMA) { 5261 /* sdma0 */ 5262 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5263 tmp |= SDMA_HALT; 5264 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5265 } 5266 if (reset_mask & RADEON_RESET_DMA1) { 5267 /* sdma1 */ 5268 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5269 tmp |= SDMA_HALT; 5270 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5271 } 5272 5273 evergreen_mc_stop(rdev, &save); 5274 if (evergreen_mc_wait_for_idle(rdev)) { 5275 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5276 } 5277 5278 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) 5279 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; 5280 5281 if (reset_mask & RADEON_RESET_CP) { 5282 grbm_soft_reset |= SOFT_RESET_CP; 5283 5284 srbm_soft_reset |= SOFT_RESET_GRBM; 5285 } 5286 5287 if (reset_mask & RADEON_RESET_DMA) 5288 srbm_soft_reset |= SOFT_RESET_SDMA; 5289 5290 if (reset_mask & RADEON_RESET_DMA1) 5291 srbm_soft_reset |= SOFT_RESET_SDMA1; 5292 5293 if (reset_mask & RADEON_RESET_DISPLAY) 5294 srbm_soft_reset |= SOFT_RESET_DC; 5295 5296 if (reset_mask & RADEON_RESET_RLC) 5297 grbm_soft_reset |= SOFT_RESET_RLC; 5298 5299 if (reset_mask & RADEON_RESET_SEM) 5300 srbm_soft_reset |= SOFT_RESET_SEM; 5301 5302 if (reset_mask & RADEON_RESET_IH) 5303 srbm_soft_reset |= SOFT_RESET_IH; 5304 5305 if (reset_mask & RADEON_RESET_GRBM) 5306 srbm_soft_reset |= SOFT_RESET_GRBM; 5307 5308 if (reset_mask & RADEON_RESET_VMC) 5309 srbm_soft_reset |= SOFT_RESET_VMC; 5310 5311 if (!(rdev->flags & RADEON_IS_IGP)) { 5312 if (reset_mask & RADEON_RESET_MC) 5313 srbm_soft_reset |= SOFT_RESET_MC; 5314 } 5315 5316 if (grbm_soft_reset) { 5317 tmp = RREG32(GRBM_SOFT_RESET); 5318 tmp |= grbm_soft_reset; 5319 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 5320 WREG32(GRBM_SOFT_RESET, tmp); 5321 tmp = RREG32(GRBM_SOFT_RESET); 5322 5323 udelay(50); 5324 5325 tmp &= ~grbm_soft_reset; 5326 WREG32(GRBM_SOFT_RESET, tmp); 5327 tmp = RREG32(GRBM_SOFT_RESET); 5328 } 5329 5330 if (srbm_soft_reset) { 5331 tmp = RREG32(SRBM_SOFT_RESET); 5332 tmp |= srbm_soft_reset; 5333 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 5334 WREG32(SRBM_SOFT_RESET, tmp); 5335 tmp = RREG32(SRBM_SOFT_RESET); 5336 5337 udelay(50); 5338 5339 tmp &= ~srbm_soft_reset; 5340 WREG32(SRBM_SOFT_RESET, tmp); 5341 tmp = RREG32(SRBM_SOFT_RESET); 5342 } 5343 5344 /* Wait a little for things to settle down */ 5345 udelay(50); 5346 5347 evergreen_mc_resume(rdev, &save); 5348 udelay(50); 5349 5350 cik_print_gpu_status_regs(rdev); 5351 } 5352 5353 struct kv_reset_save_regs { 5354 u32 gmcon_reng_execute; 5355 u32 gmcon_misc; 5356 u32 gmcon_misc3; 5357 }; 5358 5359 static void kv_save_regs_for_reset(struct radeon_device *rdev, 5360 struct kv_reset_save_regs *save) 5361 { 5362 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); 5363 save->gmcon_misc = RREG32(GMCON_MISC); 5364 save->gmcon_misc3 = RREG32(GMCON_MISC3); 5365 5366 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); 5367 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | 5368 STCTRL_STUTTER_EN)); 5369 } 5370 5371 static void kv_restore_regs_for_reset(struct radeon_device *rdev, 5372 struct kv_reset_save_regs *save) 5373 { 5374 int i; 5375 5376 WREG32(GMCON_PGFSM_WRITE, 0); 5377 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); 5378 5379 for (i = 0; i < 5; i++) 5380 WREG32(GMCON_PGFSM_WRITE, 0); 5381 5382 WREG32(GMCON_PGFSM_WRITE, 0); 5383 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); 5384 5385 for (i = 0; i < 5; i++) 5386 WREG32(GMCON_PGFSM_WRITE, 0); 5387 5388 WREG32(GMCON_PGFSM_WRITE, 0x210000); 5389 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); 5390 5391 for (i = 0; i < 5; i++) 5392 WREG32(GMCON_PGFSM_WRITE, 0); 5393 5394 WREG32(GMCON_PGFSM_WRITE, 0x21003); 5395 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); 5396 5397 for (i = 0; i < 5; i++) 5398 WREG32(GMCON_PGFSM_WRITE, 0); 5399 5400 WREG32(GMCON_PGFSM_WRITE, 0x2b00); 5401 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); 5402 5403 for (i = 0; i < 5; i++) 5404 WREG32(GMCON_PGFSM_WRITE, 0); 5405 5406 WREG32(GMCON_PGFSM_WRITE, 0); 5407 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); 5408 5409 for (i = 0; i < 5; i++) 5410 WREG32(GMCON_PGFSM_WRITE, 0); 5411 5412 WREG32(GMCON_PGFSM_WRITE, 0x420000); 5413 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); 5414 5415 for (i = 0; i < 5; i++) 5416 WREG32(GMCON_PGFSM_WRITE, 0); 5417 5418 WREG32(GMCON_PGFSM_WRITE, 0x120202); 5419 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); 5420 5421 for (i = 0; i < 5; i++) 5422 WREG32(GMCON_PGFSM_WRITE, 0); 5423 5424 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); 5425 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); 5426 5427 for (i = 0; i < 5; i++) 5428 WREG32(GMCON_PGFSM_WRITE, 0); 5429 5430 WREG32(GMCON_PGFSM_WRITE, 0x373f3e); 5431 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); 5432 5433 for (i = 0; i < 5; i++) 5434 WREG32(GMCON_PGFSM_WRITE, 0); 5435 5436 WREG32(GMCON_PGFSM_WRITE, 0x3e1332); 5437 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); 5438 5439 WREG32(GMCON_MISC3, save->gmcon_misc3); 5440 WREG32(GMCON_MISC, save->gmcon_misc); 5441 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); 5442 } 5443 5444 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) 5445 { 5446 struct evergreen_mc_save save; 5447 struct kv_reset_save_regs kv_save = { 0 }; 5448 u32 tmp, i; 5449 5450 dev_info(rdev->dev, "GPU pci config reset\n"); 5451 5452 /* disable dpm? */ 5453 5454 /* disable cg/pg */ 5455 cik_fini_pg(rdev); 5456 cik_fini_cg(rdev); 5457 5458 /* Disable GFX parsing/prefetching */ 5459 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5460 5461 /* Disable MEC parsing/prefetching */ 5462 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5463 5464 /* sdma0 */ 5465 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5466 tmp |= SDMA_HALT; 5467 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5468 /* sdma1 */ 5469 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5470 tmp |= SDMA_HALT; 5471 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5472 /* XXX other engines? */ 5473 5474 /* halt the rlc, disable cp internal ints */ 5475 cik_rlc_stop(rdev); 5476 5477 udelay(50); 5478 5479 /* disable mem access */ 5480 evergreen_mc_stop(rdev, &save); 5481 if (evergreen_mc_wait_for_idle(rdev)) { 5482 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 5483 } 5484 5485 if (rdev->flags & RADEON_IS_IGP) 5486 kv_save_regs_for_reset(rdev, &kv_save); 5487 5488 /* disable BM */ 5489 pci_clear_master(rdev->pdev); 5490 /* reset */ 5491 radeon_pci_config_reset(rdev); 5492 5493 udelay(100); 5494 5495 /* wait for asic to come out of reset */ 5496 for (i = 0; i < rdev->usec_timeout; i++) { 5497 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 5498 break; 5499 udelay(1); 5500 } 5501 5502 /* does asic init need to be run first??? */ 5503 if (rdev->flags & RADEON_IS_IGP) 5504 kv_restore_regs_for_reset(rdev, &kv_save); 5505 } 5506 5507 /** 5508 * cik_asic_reset - soft reset GPU 5509 * 5510 * @rdev: radeon_device pointer 5511 * 5512 * Look up which blocks are hung and attempt 5513 * to reset them. 5514 * Returns 0 for success. 5515 */ 5516 int cik_asic_reset(struct radeon_device *rdev) 5517 { 5518 u32 reset_mask; 5519 5520 reset_mask = cik_gpu_check_soft_reset(rdev); 5521 5522 if (reset_mask) 5523 r600_set_bios_scratch_engine_hung(rdev, true); 5524 5525 /* try soft reset */ 5526 cik_gpu_soft_reset(rdev, reset_mask); 5527 5528 reset_mask = cik_gpu_check_soft_reset(rdev); 5529 5530 /* try pci config reset */ 5531 if (reset_mask && radeon_hard_reset) 5532 cik_gpu_pci_config_reset(rdev); 5533 5534 reset_mask = cik_gpu_check_soft_reset(rdev); 5535 5536 if (!reset_mask) 5537 r600_set_bios_scratch_engine_hung(rdev, false); 5538 5539 return 0; 5540 } 5541 5542 /** 5543 * cik_gfx_is_lockup - check if the 3D engine is locked up 5544 * 5545 * @rdev: radeon_device pointer 5546 * @ring: radeon_ring structure holding ring information 5547 * 5548 * Check if the 3D engine is locked up (CIK). 5549 * Returns true if the engine is locked, false if not. 5550 */ 5551 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 5552 { 5553 u32 reset_mask = cik_gpu_check_soft_reset(rdev); 5554 5555 if (!(reset_mask & (RADEON_RESET_GFX | 5556 RADEON_RESET_COMPUTE | 5557 RADEON_RESET_CP))) { 5558 radeon_ring_lockup_update(rdev, ring); 5559 return false; 5560 } 5561 return radeon_ring_test_lockup(rdev, ring); 5562 } 5563 5564 /* MC */ 5565 /** 5566 * cik_mc_program - program the GPU memory controller 5567 * 5568 * @rdev: radeon_device pointer 5569 * 5570 * Set the location of vram, gart, and AGP in the GPU's 5571 * physical address space (CIK). 5572 */ 5573 static void cik_mc_program(struct radeon_device *rdev) 5574 { 5575 struct evergreen_mc_save save; 5576 u32 tmp; 5577 int i, j; 5578 5579 /* Initialize HDP */ 5580 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 5581 WREG32((0x2c14 + j), 0x00000000); 5582 WREG32((0x2c18 + j), 0x00000000); 5583 WREG32((0x2c1c + j), 0x00000000); 5584 WREG32((0x2c20 + j), 0x00000000); 5585 WREG32((0x2c24 + j), 0x00000000); 5586 } 5587 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 5588 5589 evergreen_mc_stop(rdev, &save); 5590 if (radeon_mc_wait_for_idle(rdev)) { 5591 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5592 } 5593 /* Lockout access through VGA aperture*/ 5594 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 5595 /* Update configuration */ 5596 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 5597 rdev->mc.vram_start >> 12); 5598 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 5599 rdev->mc.vram_end >> 12); 5600 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 5601 rdev->vram_scratch.gpu_addr >> 12); 5602 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 5603 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 5604 WREG32(MC_VM_FB_LOCATION, tmp); 5605 /* XXX double check these! */ 5606 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 5607 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 5608 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 5609 WREG32(MC_VM_AGP_BASE, 0); 5610 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 5611 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 5612 if (radeon_mc_wait_for_idle(rdev)) { 5613 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5614 } 5615 evergreen_mc_resume(rdev, &save); 5616 /* we need to own VRAM, so turn off the VGA renderer here 5617 * to stop it overwriting our objects */ 5618 rv515_vga_render_disable(rdev); 5619 } 5620 5621 /** 5622 * cik_mc_init - initialize the memory controller driver params 5623 * 5624 * @rdev: radeon_device pointer 5625 * 5626 * Look up the amount of vram, vram width, and decide how to place 5627 * vram and gart within the GPU's physical address space (CIK). 5628 * Returns 0 for success. 5629 */ 5630 static int cik_mc_init(struct radeon_device *rdev) 5631 { 5632 u32 tmp; 5633 int chansize, numchan; 5634 5635 /* Get VRAM informations */ 5636 rdev->mc.vram_is_ddr = true; 5637 tmp = RREG32(MC_ARB_RAMCFG); 5638 if (tmp & CHANSIZE_MASK) { 5639 chansize = 64; 5640 } else { 5641 chansize = 32; 5642 } 5643 tmp = RREG32(MC_SHARED_CHMAP); 5644 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 5645 case 0: 5646 default: 5647 numchan = 1; 5648 break; 5649 case 1: 5650 numchan = 2; 5651 break; 5652 case 2: 5653 numchan = 4; 5654 break; 5655 case 3: 5656 numchan = 8; 5657 break; 5658 case 4: 5659 numchan = 3; 5660 break; 5661 case 5: 5662 numchan = 6; 5663 break; 5664 case 6: 5665 numchan = 10; 5666 break; 5667 case 7: 5668 numchan = 12; 5669 break; 5670 case 8: 5671 numchan = 16; 5672 break; 5673 } 5674 rdev->mc.vram_width = numchan * chansize; 5675 /* Could aper size report 0 ? */ 5676 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 5677 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 5678 /* size in MB on si */ 5679 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5680 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5681 rdev->mc.visible_vram_size = rdev->mc.aper_size; 5682 si_vram_gtt_location(rdev, &rdev->mc); 5683 radeon_update_bandwidth_info(rdev); 5684 5685 return 0; 5686 } 5687 5688 /* 5689 * GART 5690 * VMID 0 is the physical GPU addresses as used by the kernel. 5691 * VMIDs 1-15 are used for userspace clients and are handled 5692 * by the radeon vm/hsa code. 5693 */ 5694 /** 5695 * cik_pcie_gart_tlb_flush - gart tlb flush callback 5696 * 5697 * @rdev: radeon_device pointer 5698 * 5699 * Flush the TLB for the VMID 0 page table (CIK). 5700 */ 5701 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) 5702 { 5703 /* flush hdp cache */ 5704 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); 5705 5706 /* bits 0-15 are the VM contexts0-15 */ 5707 WREG32(VM_INVALIDATE_REQUEST, 0x1); 5708 } 5709 5710 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev) 5711 { 5712 int i; 5713 uint32_t sh_mem_bases, sh_mem_config; 5714 5715 sh_mem_bases = 0x6000 | 0x6000 << 16; 5716 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED); 5717 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED); 5718 5719 mutex_lock(&rdev->srbm_mutex); 5720 for (i = 8; i < 16; i++) { 5721 cik_srbm_select(rdev, 0, 0, 0, i); 5722 /* CP and shaders */ 5723 WREG32(SH_MEM_CONFIG, sh_mem_config); 5724 WREG32(SH_MEM_APE1_BASE, 1); 5725 WREG32(SH_MEM_APE1_LIMIT, 0); 5726 WREG32(SH_MEM_BASES, sh_mem_bases); 5727 } 5728 cik_srbm_select(rdev, 0, 0, 0, 0); 5729 mutex_unlock(&rdev->srbm_mutex); 5730 } 5731 5732 /** 5733 * cik_pcie_gart_enable - gart enable 5734 * 5735 * @rdev: radeon_device pointer 5736 * 5737 * This sets up the TLBs, programs the page tables for VMID0, 5738 * sets up the hw for VMIDs 1-15 which are allocated on 5739 * demand, and sets up the global locations for the LDS, GDS, 5740 * and GPUVM for FSA64 clients (CIK). 5741 * Returns 0 for success, errors for failure. 5742 */ 5743 static int cik_pcie_gart_enable(struct radeon_device *rdev) 5744 { 5745 int r, i; 5746 5747 if (rdev->gart.robj == NULL) { 5748 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 5749 return -EINVAL; 5750 } 5751 r = radeon_gart_table_vram_pin(rdev); 5752 if (r) 5753 return r; 5754 /* Setup TLB control */ 5755 WREG32(MC_VM_MX_L1_TLB_CNTL, 5756 (0xA << 7) | 5757 ENABLE_L1_TLB | 5758 ENABLE_L1_FRAGMENT_PROCESSING | 5759 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5760 ENABLE_ADVANCED_DRIVER_MODEL | 5761 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5762 /* Setup L2 cache */ 5763 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 5764 ENABLE_L2_FRAGMENT_PROCESSING | 5765 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5766 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5767 EFFECTIVE_L2_QUEUE_SIZE(7) | 5768 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5769 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 5770 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5771 BANK_SELECT(4) | 5772 L2_CACHE_BIGK_FRAGMENT_SIZE(4)); 5773 /* setup context0 */ 5774 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 5775 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 5776 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 5777 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 5778 (u32)(rdev->dummy_page.addr >> 12)); 5779 WREG32(VM_CONTEXT0_CNTL2, 0); 5780 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 5781 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 5782 5783 WREG32(0x15D4, 0); 5784 WREG32(0x15D8, 0); 5785 WREG32(0x15DC, 0); 5786 5787 /* restore context1-15 */ 5788 /* set vm size, must be a multiple of 4 */ 5789 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 5790 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 5791 for (i = 1; i < 16; i++) { 5792 if (i < 8) 5793 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 5794 rdev->vm_manager.saved_table_addr[i]); 5795 else 5796 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 5797 rdev->vm_manager.saved_table_addr[i]); 5798 } 5799 5800 /* enable context1-15 */ 5801 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 5802 (u32)(rdev->dummy_page.addr >> 12)); 5803 WREG32(VM_CONTEXT1_CNTL2, 4); 5804 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 5805 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | 5806 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5807 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5808 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5809 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5810 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 5811 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 5812 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 5813 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 5814 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 5815 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 5816 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5817 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 5818 5819 if (rdev->family == CHIP_KAVERI) { 5820 u32 tmp = RREG32(CHUB_CONTROL); 5821 tmp &= ~BYPASS_VM; 5822 WREG32(CHUB_CONTROL, tmp); 5823 } 5824 5825 /* XXX SH_MEM regs */ 5826 /* where to put LDS, scratch, GPUVM in FSA64 space */ 5827 mutex_lock(&rdev->srbm_mutex); 5828 for (i = 0; i < 16; i++) { 5829 cik_srbm_select(rdev, 0, 0, 0, i); 5830 /* CP and shaders */ 5831 WREG32(SH_MEM_CONFIG, 0); 5832 WREG32(SH_MEM_APE1_BASE, 1); 5833 WREG32(SH_MEM_APE1_LIMIT, 0); 5834 WREG32(SH_MEM_BASES, 0); 5835 /* SDMA GFX */ 5836 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); 5837 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); 5838 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); 5839 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); 5840 /* XXX SDMA RLC - todo */ 5841 } 5842 cik_srbm_select(rdev, 0, 0, 0, 0); 5843 mutex_unlock(&rdev->srbm_mutex); 5844 5845 cik_pcie_init_compute_vmid(rdev); 5846 5847 cik_pcie_gart_tlb_flush(rdev); 5848 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 5849 (unsigned)(rdev->mc.gtt_size >> 20), 5850 (unsigned long long)rdev->gart.table_addr); 5851 rdev->gart.ready = true; 5852 return 0; 5853 } 5854 5855 /** 5856 * cik_pcie_gart_disable - gart disable 5857 * 5858 * @rdev: radeon_device pointer 5859 * 5860 * This disables all VM page table (CIK). 5861 */ 5862 static void cik_pcie_gart_disable(struct radeon_device *rdev) 5863 { 5864 unsigned i; 5865 5866 for (i = 1; i < 16; ++i) { 5867 uint32_t reg; 5868 if (i < 8) 5869 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2); 5870 else 5871 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2); 5872 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); 5873 } 5874 5875 /* Disable all tables */ 5876 WREG32(VM_CONTEXT0_CNTL, 0); 5877 WREG32(VM_CONTEXT1_CNTL, 0); 5878 /* Setup TLB control */ 5879 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5880 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5881 /* Setup L2 cache */ 5882 WREG32(VM_L2_CNTL, 5883 ENABLE_L2_FRAGMENT_PROCESSING | 5884 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5885 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5886 EFFECTIVE_L2_QUEUE_SIZE(7) | 5887 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5888 WREG32(VM_L2_CNTL2, 0); 5889 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5890 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5891 radeon_gart_table_vram_unpin(rdev); 5892 } 5893 5894 /** 5895 * cik_pcie_gart_fini - vm fini callback 5896 * 5897 * @rdev: radeon_device pointer 5898 * 5899 * Tears down the driver GART/VM setup (CIK). 5900 */ 5901 static void cik_pcie_gart_fini(struct radeon_device *rdev) 5902 { 5903 cik_pcie_gart_disable(rdev); 5904 radeon_gart_table_vram_free(rdev); 5905 radeon_gart_fini(rdev); 5906 } 5907 5908 /* vm parser */ 5909 /** 5910 * cik_ib_parse - vm ib_parse callback 5911 * 5912 * @rdev: radeon_device pointer 5913 * @ib: indirect buffer pointer 5914 * 5915 * CIK uses hw IB checking so this is a nop (CIK). 5916 */ 5917 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 5918 { 5919 return 0; 5920 } 5921 5922 /* 5923 * vm 5924 * VMID 0 is the physical GPU addresses as used by the kernel. 5925 * VMIDs 1-15 are used for userspace clients and are handled 5926 * by the radeon vm/hsa code. 5927 */ 5928 /** 5929 * cik_vm_init - cik vm init callback 5930 * 5931 * @rdev: radeon_device pointer 5932 * 5933 * Inits cik specific vm parameters (number of VMs, base of vram for 5934 * VMIDs 1-15) (CIK). 5935 * Returns 0 for success. 5936 */ 5937 int cik_vm_init(struct radeon_device *rdev) 5938 { 5939 /* 5940 * number of VMs 5941 * VMID 0 is reserved for System 5942 * radeon graphics/compute will use VMIDs 1-7 5943 * amdkfd will use VMIDs 8-15 5944 */ 5945 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS; 5946 /* base offset of vram pages */ 5947 if (rdev->flags & RADEON_IS_IGP) { 5948 u64 tmp = RREG32(MC_VM_FB_OFFSET); 5949 tmp <<= 22; 5950 rdev->vm_manager.vram_base_offset = tmp; 5951 } else 5952 rdev->vm_manager.vram_base_offset = 0; 5953 5954 return 0; 5955 } 5956 5957 /** 5958 * cik_vm_fini - cik vm fini callback 5959 * 5960 * @rdev: radeon_device pointer 5961 * 5962 * Tear down any asic specific VM setup (CIK). 5963 */ 5964 void cik_vm_fini(struct radeon_device *rdev) 5965 { 5966 } 5967 5968 /** 5969 * cik_vm_decode_fault - print human readable fault info 5970 * 5971 * @rdev: radeon_device pointer 5972 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 5973 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 5974 * 5975 * Print human readable fault information (CIK). 5976 */ 5977 static void cik_vm_decode_fault(struct radeon_device *rdev, 5978 u32 status, u32 addr, u32 mc_client) 5979 { 5980 u32 mc_id; 5981 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 5982 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 5983 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, 5984 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; 5985 5986 if (rdev->family == CHIP_HAWAII) 5987 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5988 else 5989 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5990 5991 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", 5992 protections, vmid, addr, 5993 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 5994 block, mc_client, mc_id); 5995 } 5996 5997 /** 5998 * cik_vm_flush - cik vm flush using the CP 5999 * 6000 * @rdev: radeon_device pointer 6001 * 6002 * Update the page table base and flush the VM TLB 6003 * using the CP (CIK). 6004 */ 6005 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, 6006 unsigned vm_id, uint64_t pd_addr) 6007 { 6008 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX); 6009 6010 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6011 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6012 WRITE_DATA_DST_SEL(0))); 6013 if (vm_id < 8) { 6014 radeon_ring_write(ring, 6015 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); 6016 } else { 6017 radeon_ring_write(ring, 6018 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); 6019 } 6020 radeon_ring_write(ring, 0); 6021 radeon_ring_write(ring, pd_addr >> 12); 6022 6023 /* update SH_MEM_* regs */ 6024 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6025 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6026 WRITE_DATA_DST_SEL(0))); 6027 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 6028 radeon_ring_write(ring, 0); 6029 radeon_ring_write(ring, VMID(vm_id)); 6030 6031 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); 6032 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6033 WRITE_DATA_DST_SEL(0))); 6034 radeon_ring_write(ring, SH_MEM_BASES >> 2); 6035 radeon_ring_write(ring, 0); 6036 6037 radeon_ring_write(ring, 0); /* SH_MEM_BASES */ 6038 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */ 6039 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ 6040 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ 6041 6042 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6043 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6044 WRITE_DATA_DST_SEL(0))); 6045 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 6046 radeon_ring_write(ring, 0); 6047 radeon_ring_write(ring, VMID(0)); 6048 6049 /* HDP flush */ 6050 cik_hdp_flush_cp_ring_emit(rdev, ring->idx); 6051 6052 /* bits 0-15 are the VM contexts0-15 */ 6053 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6054 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | 6055 WRITE_DATA_DST_SEL(0))); 6056 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 6057 radeon_ring_write(ring, 0); 6058 radeon_ring_write(ring, 1 << vm_id); 6059 6060 /* wait for the invalidate to complete */ 6061 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6062 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 6063 WAIT_REG_MEM_FUNCTION(0) | /* always */ 6064 WAIT_REG_MEM_ENGINE(0))); /* me */ 6065 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 6066 radeon_ring_write(ring, 0); 6067 radeon_ring_write(ring, 0); /* ref */ 6068 radeon_ring_write(ring, 0); /* mask */ 6069 radeon_ring_write(ring, 0x20); /* poll interval */ 6070 6071 /* compute doesn't have PFP */ 6072 if (usepfp) { 6073 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6074 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6075 radeon_ring_write(ring, 0x0); 6076 } 6077 } 6078 6079 /* 6080 * RLC 6081 * The RLC is a multi-purpose microengine that handles a 6082 * variety of functions, the most important of which is 6083 * the interrupt controller. 6084 */ 6085 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 6086 bool enable) 6087 { 6088 u32 tmp = RREG32(CP_INT_CNTL_RING0); 6089 6090 if (enable) 6091 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6092 else 6093 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6094 WREG32(CP_INT_CNTL_RING0, tmp); 6095 } 6096 6097 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) 6098 { 6099 u32 tmp; 6100 6101 tmp = RREG32(RLC_LB_CNTL); 6102 if (enable) 6103 tmp |= LOAD_BALANCE_ENABLE; 6104 else 6105 tmp &= ~LOAD_BALANCE_ENABLE; 6106 WREG32(RLC_LB_CNTL, tmp); 6107 } 6108 6109 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) 6110 { 6111 u32 i, j, k; 6112 u32 mask; 6113 6114 mutex_lock(&rdev->grbm_idx_mutex); 6115 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6116 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6117 cik_select_se_sh(rdev, i, j); 6118 for (k = 0; k < rdev->usec_timeout; k++) { 6119 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) 6120 break; 6121 udelay(1); 6122 } 6123 } 6124 } 6125 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6126 mutex_unlock(&rdev->grbm_idx_mutex); 6127 6128 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; 6129 for (k = 0; k < rdev->usec_timeout; k++) { 6130 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 6131 break; 6132 udelay(1); 6133 } 6134 } 6135 6136 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) 6137 { 6138 u32 tmp; 6139 6140 tmp = RREG32(RLC_CNTL); 6141 if (tmp != rlc) 6142 WREG32(RLC_CNTL, rlc); 6143 } 6144 6145 static u32 cik_halt_rlc(struct radeon_device *rdev) 6146 { 6147 u32 data, orig; 6148 6149 orig = data = RREG32(RLC_CNTL); 6150 6151 if (data & RLC_ENABLE) { 6152 u32 i; 6153 6154 data &= ~RLC_ENABLE; 6155 WREG32(RLC_CNTL, data); 6156 6157 for (i = 0; i < rdev->usec_timeout; i++) { 6158 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) 6159 break; 6160 udelay(1); 6161 } 6162 6163 cik_wait_for_rlc_serdes(rdev); 6164 } 6165 6166 return orig; 6167 } 6168 6169 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) 6170 { 6171 u32 tmp, i, mask; 6172 6173 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); 6174 WREG32(RLC_GPR_REG2, tmp); 6175 6176 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; 6177 for (i = 0; i < rdev->usec_timeout; i++) { 6178 if ((RREG32(RLC_GPM_STAT) & mask) == mask) 6179 break; 6180 udelay(1); 6181 } 6182 6183 for (i = 0; i < rdev->usec_timeout; i++) { 6184 if ((RREG32(RLC_GPR_REG2) & REQ) == 0) 6185 break; 6186 udelay(1); 6187 } 6188 } 6189 6190 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) 6191 { 6192 u32 tmp; 6193 6194 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); 6195 WREG32(RLC_GPR_REG2, tmp); 6196 } 6197 6198 /** 6199 * cik_rlc_stop - stop the RLC ME 6200 * 6201 * @rdev: radeon_device pointer 6202 * 6203 * Halt the RLC ME (MicroEngine) (CIK). 6204 */ 6205 static void cik_rlc_stop(struct radeon_device *rdev) 6206 { 6207 WREG32(RLC_CNTL, 0); 6208 6209 cik_enable_gui_idle_interrupt(rdev, false); 6210 6211 cik_wait_for_rlc_serdes(rdev); 6212 } 6213 6214 /** 6215 * cik_rlc_start - start the RLC ME 6216 * 6217 * @rdev: radeon_device pointer 6218 * 6219 * Unhalt the RLC ME (MicroEngine) (CIK). 6220 */ 6221 static void cik_rlc_start(struct radeon_device *rdev) 6222 { 6223 WREG32(RLC_CNTL, RLC_ENABLE); 6224 6225 cik_enable_gui_idle_interrupt(rdev, true); 6226 6227 udelay(50); 6228 } 6229 6230 /** 6231 * cik_rlc_resume - setup the RLC hw 6232 * 6233 * @rdev: radeon_device pointer 6234 * 6235 * Initialize the RLC registers, load the ucode, 6236 * and start the RLC (CIK). 6237 * Returns 0 for success, -EINVAL if the ucode is not available. 6238 */ 6239 static int cik_rlc_resume(struct radeon_device *rdev) 6240 { 6241 u32 i, size, tmp; 6242 6243 if (!rdev->rlc_fw) 6244 return -EINVAL; 6245 6246 cik_rlc_stop(rdev); 6247 6248 /* disable CG */ 6249 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; 6250 WREG32(RLC_CGCG_CGLS_CTRL, tmp); 6251 6252 si_rlc_reset(rdev); 6253 6254 cik_init_pg(rdev); 6255 6256 cik_init_cg(rdev); 6257 6258 WREG32(RLC_LB_CNTR_INIT, 0); 6259 WREG32(RLC_LB_CNTR_MAX, 0x00008000); 6260 6261 mutex_lock(&rdev->grbm_idx_mutex); 6262 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6263 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 6264 WREG32(RLC_LB_PARAMS, 0x00600408); 6265 WREG32(RLC_LB_CNTL, 0x80000004); 6266 mutex_unlock(&rdev->grbm_idx_mutex); 6267 6268 WREG32(RLC_MC_CNTL, 0); 6269 WREG32(RLC_UCODE_CNTL, 0); 6270 6271 if (rdev->new_fw) { 6272 const struct rlc_firmware_header_v1_0 *hdr = 6273 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; 6274 const __le32 *fw_data = (const __le32 *) 6275 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6276 6277 radeon_ucode_print_rlc_hdr(&hdr->header); 6278 6279 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 6280 WREG32(RLC_GPM_UCODE_ADDR, 0); 6281 for (i = 0; i < size; i++) 6282 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 6283 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version)); 6284 } else { 6285 const __be32 *fw_data; 6286 6287 switch (rdev->family) { 6288 case CHIP_BONAIRE: 6289 case CHIP_HAWAII: 6290 default: 6291 size = BONAIRE_RLC_UCODE_SIZE; 6292 break; 6293 case CHIP_KAVERI: 6294 size = KV_RLC_UCODE_SIZE; 6295 break; 6296 case CHIP_KABINI: 6297 size = KB_RLC_UCODE_SIZE; 6298 break; 6299 case CHIP_MULLINS: 6300 size = ML_RLC_UCODE_SIZE; 6301 break; 6302 } 6303 6304 fw_data = (const __be32 *)rdev->rlc_fw->data; 6305 WREG32(RLC_GPM_UCODE_ADDR, 0); 6306 for (i = 0; i < size; i++) 6307 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); 6308 WREG32(RLC_GPM_UCODE_ADDR, 0); 6309 } 6310 6311 /* XXX - find out what chips support lbpw */ 6312 cik_enable_lbpw(rdev, false); 6313 6314 if (rdev->family == CHIP_BONAIRE) 6315 WREG32(RLC_DRIVER_DMA_STATUS, 0); 6316 6317 cik_rlc_start(rdev); 6318 6319 return 0; 6320 } 6321 6322 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) 6323 { 6324 u32 data, orig, tmp, tmp2; 6325 6326 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 6327 6328 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 6329 cik_enable_gui_idle_interrupt(rdev, true); 6330 6331 tmp = cik_halt_rlc(rdev); 6332 6333 mutex_lock(&rdev->grbm_idx_mutex); 6334 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6335 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6336 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6337 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; 6338 WREG32(RLC_SERDES_WR_CTRL, tmp2); 6339 mutex_unlock(&rdev->grbm_idx_mutex); 6340 6341 cik_update_rlc(rdev, tmp); 6342 6343 data |= CGCG_EN | CGLS_EN; 6344 } else { 6345 cik_enable_gui_idle_interrupt(rdev, false); 6346 6347 RREG32(CB_CGTT_SCLK_CTRL); 6348 RREG32(CB_CGTT_SCLK_CTRL); 6349 RREG32(CB_CGTT_SCLK_CTRL); 6350 RREG32(CB_CGTT_SCLK_CTRL); 6351 6352 data &= ~(CGCG_EN | CGLS_EN); 6353 } 6354 6355 if (orig != data) 6356 WREG32(RLC_CGCG_CGLS_CTRL, data); 6357 6358 } 6359 6360 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) 6361 { 6362 u32 data, orig, tmp = 0; 6363 6364 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 6365 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { 6366 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 6367 orig = data = RREG32(CP_MEM_SLP_CNTL); 6368 data |= CP_MEM_LS_EN; 6369 if (orig != data) 6370 WREG32(CP_MEM_SLP_CNTL, data); 6371 } 6372 } 6373 6374 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6375 data |= 0x00000001; 6376 data &= 0xfffffffd; 6377 if (orig != data) 6378 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6379 6380 tmp = cik_halt_rlc(rdev); 6381 6382 mutex_lock(&rdev->grbm_idx_mutex); 6383 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6384 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6385 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6386 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; 6387 WREG32(RLC_SERDES_WR_CTRL, data); 6388 mutex_unlock(&rdev->grbm_idx_mutex); 6389 6390 cik_update_rlc(rdev, tmp); 6391 6392 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { 6393 orig = data = RREG32(CGTS_SM_CTRL_REG); 6394 data &= ~SM_MODE_MASK; 6395 data |= SM_MODE(0x2); 6396 data |= SM_MODE_ENABLE; 6397 data &= ~CGTS_OVERRIDE; 6398 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && 6399 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) 6400 data &= ~CGTS_LS_OVERRIDE; 6401 data &= ~ON_MONITOR_ADD_MASK; 6402 data |= ON_MONITOR_ADD_EN; 6403 data |= ON_MONITOR_ADD(0x96); 6404 if (orig != data) 6405 WREG32(CGTS_SM_CTRL_REG, data); 6406 } 6407 } else { 6408 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 6409 data |= 0x00000003; 6410 if (orig != data) 6411 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 6412 6413 data = RREG32(RLC_MEM_SLP_CNTL); 6414 if (data & RLC_MEM_LS_EN) { 6415 data &= ~RLC_MEM_LS_EN; 6416 WREG32(RLC_MEM_SLP_CNTL, data); 6417 } 6418 6419 data = RREG32(CP_MEM_SLP_CNTL); 6420 if (data & CP_MEM_LS_EN) { 6421 data &= ~CP_MEM_LS_EN; 6422 WREG32(CP_MEM_SLP_CNTL, data); 6423 } 6424 6425 orig = data = RREG32(CGTS_SM_CTRL_REG); 6426 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; 6427 if (orig != data) 6428 WREG32(CGTS_SM_CTRL_REG, data); 6429 6430 tmp = cik_halt_rlc(rdev); 6431 6432 mutex_lock(&rdev->grbm_idx_mutex); 6433 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6434 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 6435 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 6436 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; 6437 WREG32(RLC_SERDES_WR_CTRL, data); 6438 mutex_unlock(&rdev->grbm_idx_mutex); 6439 6440 cik_update_rlc(rdev, tmp); 6441 } 6442 } 6443 6444 static const u32 mc_cg_registers[] = 6445 { 6446 MC_HUB_MISC_HUB_CG, 6447 MC_HUB_MISC_SIP_CG, 6448 MC_HUB_MISC_VM_CG, 6449 MC_XPB_CLK_GAT, 6450 ATC_MISC_CG, 6451 MC_CITF_MISC_WR_CG, 6452 MC_CITF_MISC_RD_CG, 6453 MC_CITF_MISC_VM_CG, 6454 VM_L2_CG, 6455 }; 6456 6457 static void cik_enable_mc_ls(struct radeon_device *rdev, 6458 bool enable) 6459 { 6460 int i; 6461 u32 orig, data; 6462 6463 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6464 orig = data = RREG32(mc_cg_registers[i]); 6465 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 6466 data |= MC_LS_ENABLE; 6467 else 6468 data &= ~MC_LS_ENABLE; 6469 if (data != orig) 6470 WREG32(mc_cg_registers[i], data); 6471 } 6472 } 6473 6474 static void cik_enable_mc_mgcg(struct radeon_device *rdev, 6475 bool enable) 6476 { 6477 int i; 6478 u32 orig, data; 6479 6480 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 6481 orig = data = RREG32(mc_cg_registers[i]); 6482 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 6483 data |= MC_CG_ENABLE; 6484 else 6485 data &= ~MC_CG_ENABLE; 6486 if (data != orig) 6487 WREG32(mc_cg_registers[i], data); 6488 } 6489 } 6490 6491 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, 6492 bool enable) 6493 { 6494 u32 orig, data; 6495 6496 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 6497 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); 6498 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); 6499 } else { 6500 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); 6501 data |= 0xff000000; 6502 if (data != orig) 6503 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); 6504 6505 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); 6506 data |= 0xff000000; 6507 if (data != orig) 6508 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); 6509 } 6510 } 6511 6512 static void cik_enable_sdma_mgls(struct radeon_device *rdev, 6513 bool enable) 6514 { 6515 u32 orig, data; 6516 6517 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { 6518 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6519 data |= 0x100; 6520 if (orig != data) 6521 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6522 6523 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6524 data |= 0x100; 6525 if (orig != data) 6526 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6527 } else { 6528 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6529 data &= ~0x100; 6530 if (orig != data) 6531 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6532 6533 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6534 data &= ~0x100; 6535 if (orig != data) 6536 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6537 } 6538 } 6539 6540 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, 6541 bool enable) 6542 { 6543 u32 orig, data; 6544 6545 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 6546 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6547 data = 0xfff; 6548 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6549 6550 orig = data = RREG32(UVD_CGC_CTRL); 6551 data |= DCM; 6552 if (orig != data) 6553 WREG32(UVD_CGC_CTRL, data); 6554 } else { 6555 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6556 data &= ~0xfff; 6557 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6558 6559 orig = data = RREG32(UVD_CGC_CTRL); 6560 data &= ~DCM; 6561 if (orig != data) 6562 WREG32(UVD_CGC_CTRL, data); 6563 } 6564 } 6565 6566 static void cik_enable_bif_mgls(struct radeon_device *rdev, 6567 bool enable) 6568 { 6569 u32 orig, data; 6570 6571 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 6572 6573 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 6574 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 6575 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 6576 else 6577 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 6578 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 6579 6580 if (orig != data) 6581 WREG32_PCIE_PORT(PCIE_CNTL2, data); 6582 } 6583 6584 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, 6585 bool enable) 6586 { 6587 u32 orig, data; 6588 6589 orig = data = RREG32(HDP_HOST_PATH_CNTL); 6590 6591 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 6592 data &= ~CLOCK_GATING_DIS; 6593 else 6594 data |= CLOCK_GATING_DIS; 6595 6596 if (orig != data) 6597 WREG32(HDP_HOST_PATH_CNTL, data); 6598 } 6599 6600 static void cik_enable_hdp_ls(struct radeon_device *rdev, 6601 bool enable) 6602 { 6603 u32 orig, data; 6604 6605 orig = data = RREG32(HDP_MEM_POWER_LS); 6606 6607 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 6608 data |= HDP_LS_ENABLE; 6609 else 6610 data &= ~HDP_LS_ENABLE; 6611 6612 if (orig != data) 6613 WREG32(HDP_MEM_POWER_LS, data); 6614 } 6615 6616 void cik_update_cg(struct radeon_device *rdev, 6617 u32 block, bool enable) 6618 { 6619 6620 if (block & RADEON_CG_BLOCK_GFX) { 6621 cik_enable_gui_idle_interrupt(rdev, false); 6622 /* order matters! */ 6623 if (enable) { 6624 cik_enable_mgcg(rdev, true); 6625 cik_enable_cgcg(rdev, true); 6626 } else { 6627 cik_enable_cgcg(rdev, false); 6628 cik_enable_mgcg(rdev, false); 6629 } 6630 cik_enable_gui_idle_interrupt(rdev, true); 6631 } 6632 6633 if (block & RADEON_CG_BLOCK_MC) { 6634 if (!(rdev->flags & RADEON_IS_IGP)) { 6635 cik_enable_mc_mgcg(rdev, enable); 6636 cik_enable_mc_ls(rdev, enable); 6637 } 6638 } 6639 6640 if (block & RADEON_CG_BLOCK_SDMA) { 6641 cik_enable_sdma_mgcg(rdev, enable); 6642 cik_enable_sdma_mgls(rdev, enable); 6643 } 6644 6645 if (block & RADEON_CG_BLOCK_BIF) { 6646 cik_enable_bif_mgls(rdev, enable); 6647 } 6648 6649 if (block & RADEON_CG_BLOCK_UVD) { 6650 if (rdev->has_uvd) 6651 cik_enable_uvd_mgcg(rdev, enable); 6652 } 6653 6654 if (block & RADEON_CG_BLOCK_HDP) { 6655 cik_enable_hdp_mgcg(rdev, enable); 6656 cik_enable_hdp_ls(rdev, enable); 6657 } 6658 6659 if (block & RADEON_CG_BLOCK_VCE) { 6660 vce_v2_0_enable_mgcg(rdev, enable); 6661 } 6662 } 6663 6664 static void cik_init_cg(struct radeon_device *rdev) 6665 { 6666 6667 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); 6668 6669 if (rdev->has_uvd) 6670 si_init_uvd_internal_cg(rdev); 6671 6672 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6673 RADEON_CG_BLOCK_SDMA | 6674 RADEON_CG_BLOCK_BIF | 6675 RADEON_CG_BLOCK_UVD | 6676 RADEON_CG_BLOCK_HDP), true); 6677 } 6678 6679 static void cik_fini_cg(struct radeon_device *rdev) 6680 { 6681 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6682 RADEON_CG_BLOCK_SDMA | 6683 RADEON_CG_BLOCK_BIF | 6684 RADEON_CG_BLOCK_UVD | 6685 RADEON_CG_BLOCK_HDP), false); 6686 6687 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); 6688 } 6689 6690 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, 6691 bool enable) 6692 { 6693 u32 data, orig; 6694 6695 orig = data = RREG32(RLC_PG_CNTL); 6696 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6697 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6698 else 6699 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6700 if (orig != data) 6701 WREG32(RLC_PG_CNTL, data); 6702 } 6703 6704 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, 6705 bool enable) 6706 { 6707 u32 data, orig; 6708 6709 orig = data = RREG32(RLC_PG_CNTL); 6710 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6711 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6712 else 6713 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6714 if (orig != data) 6715 WREG32(RLC_PG_CNTL, data); 6716 } 6717 6718 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) 6719 { 6720 u32 data, orig; 6721 6722 orig = data = RREG32(RLC_PG_CNTL); 6723 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) 6724 data &= ~DISABLE_CP_PG; 6725 else 6726 data |= DISABLE_CP_PG; 6727 if (orig != data) 6728 WREG32(RLC_PG_CNTL, data); 6729 } 6730 6731 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) 6732 { 6733 u32 data, orig; 6734 6735 orig = data = RREG32(RLC_PG_CNTL); 6736 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) 6737 data &= ~DISABLE_GDS_PG; 6738 else 6739 data |= DISABLE_GDS_PG; 6740 if (orig != data) 6741 WREG32(RLC_PG_CNTL, data); 6742 } 6743 6744 #define CP_ME_TABLE_SIZE 96 6745 #define CP_ME_TABLE_OFFSET 2048 6746 #define CP_MEC_TABLE_OFFSET 4096 6747 6748 void cik_init_cp_pg_table(struct radeon_device *rdev) 6749 { 6750 volatile u32 *dst_ptr; 6751 int me, i, max_me = 4; 6752 u32 bo_offset = 0; 6753 u32 table_offset, table_size; 6754 6755 if (rdev->family == CHIP_KAVERI) 6756 max_me = 5; 6757 6758 if (rdev->rlc.cp_table_ptr == NULL) 6759 return; 6760 6761 /* write the cp table buffer */ 6762 dst_ptr = rdev->rlc.cp_table_ptr; 6763 for (me = 0; me < max_me; me++) { 6764 if (rdev->new_fw) { 6765 const __le32 *fw_data; 6766 const struct gfx_firmware_header_v1_0 *hdr; 6767 6768 if (me == 0) { 6769 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 6770 fw_data = (const __le32 *) 6771 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6772 table_offset = le32_to_cpu(hdr->jt_offset); 6773 table_size = le32_to_cpu(hdr->jt_size); 6774 } else if (me == 1) { 6775 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 6776 fw_data = (const __le32 *) 6777 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6778 table_offset = le32_to_cpu(hdr->jt_offset); 6779 table_size = le32_to_cpu(hdr->jt_size); 6780 } else if (me == 2) { 6781 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 6782 fw_data = (const __le32 *) 6783 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6784 table_offset = le32_to_cpu(hdr->jt_offset); 6785 table_size = le32_to_cpu(hdr->jt_size); 6786 } else if (me == 3) { 6787 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; 6788 fw_data = (const __le32 *) 6789 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6790 table_offset = le32_to_cpu(hdr->jt_offset); 6791 table_size = le32_to_cpu(hdr->jt_size); 6792 } else { 6793 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; 6794 fw_data = (const __le32 *) 6795 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 6796 table_offset = le32_to_cpu(hdr->jt_offset); 6797 table_size = le32_to_cpu(hdr->jt_size); 6798 } 6799 6800 for (i = 0; i < table_size; i ++) { 6801 dst_ptr[bo_offset + i] = 6802 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); 6803 } 6804 bo_offset += table_size; 6805 } else { 6806 const __be32 *fw_data; 6807 table_size = CP_ME_TABLE_SIZE; 6808 6809 if (me == 0) { 6810 fw_data = (const __be32 *)rdev->ce_fw->data; 6811 table_offset = CP_ME_TABLE_OFFSET; 6812 } else if (me == 1) { 6813 fw_data = (const __be32 *)rdev->pfp_fw->data; 6814 table_offset = CP_ME_TABLE_OFFSET; 6815 } else if (me == 2) { 6816 fw_data = (const __be32 *)rdev->me_fw->data; 6817 table_offset = CP_ME_TABLE_OFFSET; 6818 } else { 6819 fw_data = (const __be32 *)rdev->mec_fw->data; 6820 table_offset = CP_MEC_TABLE_OFFSET; 6821 } 6822 6823 for (i = 0; i < table_size; i ++) { 6824 dst_ptr[bo_offset + i] = 6825 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); 6826 } 6827 bo_offset += table_size; 6828 } 6829 } 6830 } 6831 6832 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, 6833 bool enable) 6834 { 6835 u32 data, orig; 6836 6837 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 6838 orig = data = RREG32(RLC_PG_CNTL); 6839 data |= GFX_PG_ENABLE; 6840 if (orig != data) 6841 WREG32(RLC_PG_CNTL, data); 6842 6843 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6844 data |= AUTO_PG_EN; 6845 if (orig != data) 6846 WREG32(RLC_AUTO_PG_CTRL, data); 6847 } else { 6848 orig = data = RREG32(RLC_PG_CNTL); 6849 data &= ~GFX_PG_ENABLE; 6850 if (orig != data) 6851 WREG32(RLC_PG_CNTL, data); 6852 6853 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6854 data &= ~AUTO_PG_EN; 6855 if (orig != data) 6856 WREG32(RLC_AUTO_PG_CTRL, data); 6857 6858 data = RREG32(DB_RENDER_CONTROL); 6859 } 6860 } 6861 6862 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 6863 { 6864 u32 mask = 0, tmp, tmp1; 6865 int i; 6866 6867 mutex_lock(&rdev->grbm_idx_mutex); 6868 cik_select_se_sh(rdev, se, sh); 6869 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 6870 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 6871 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6872 mutex_unlock(&rdev->grbm_idx_mutex); 6873 6874 tmp &= 0xffff0000; 6875 6876 tmp |= tmp1; 6877 tmp >>= 16; 6878 6879 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { 6880 mask <<= 1; 6881 mask |= 1; 6882 } 6883 6884 return (~tmp) & mask; 6885 } 6886 6887 static void cik_init_ao_cu_mask(struct radeon_device *rdev) 6888 { 6889 u32 i, j, k, active_cu_number = 0; 6890 u32 mask, counter, cu_bitmap; 6891 u32 tmp = 0; 6892 6893 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6894 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6895 mask = 1; 6896 cu_bitmap = 0; 6897 counter = 0; 6898 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { 6899 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { 6900 if (counter < 2) 6901 cu_bitmap |= mask; 6902 counter ++; 6903 } 6904 mask <<= 1; 6905 } 6906 6907 active_cu_number += counter; 6908 tmp |= (cu_bitmap << (i * 16 + j * 8)); 6909 } 6910 } 6911 6912 WREG32(RLC_PG_AO_CU_MASK, tmp); 6913 6914 tmp = RREG32(RLC_MAX_PG_CU); 6915 tmp &= ~MAX_PU_CU_MASK; 6916 tmp |= MAX_PU_CU(active_cu_number); 6917 WREG32(RLC_MAX_PG_CU, tmp); 6918 } 6919 6920 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, 6921 bool enable) 6922 { 6923 u32 data, orig; 6924 6925 orig = data = RREG32(RLC_PG_CNTL); 6926 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) 6927 data |= STATIC_PER_CU_PG_ENABLE; 6928 else 6929 data &= ~STATIC_PER_CU_PG_ENABLE; 6930 if (orig != data) 6931 WREG32(RLC_PG_CNTL, data); 6932 } 6933 6934 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, 6935 bool enable) 6936 { 6937 u32 data, orig; 6938 6939 orig = data = RREG32(RLC_PG_CNTL); 6940 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) 6941 data |= DYN_PER_CU_PG_ENABLE; 6942 else 6943 data &= ~DYN_PER_CU_PG_ENABLE; 6944 if (orig != data) 6945 WREG32(RLC_PG_CNTL, data); 6946 } 6947 6948 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 6949 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D 6950 6951 static void cik_init_gfx_cgpg(struct radeon_device *rdev) 6952 { 6953 u32 data, orig; 6954 u32 i; 6955 6956 if (rdev->rlc.cs_data) { 6957 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6958 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); 6959 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); 6960 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); 6961 } else { 6962 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6963 for (i = 0; i < 3; i++) 6964 WREG32(RLC_GPM_SCRATCH_DATA, 0); 6965 } 6966 if (rdev->rlc.reg_list) { 6967 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); 6968 for (i = 0; i < rdev->rlc.reg_list_size; i++) 6969 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); 6970 } 6971 6972 orig = data = RREG32(RLC_PG_CNTL); 6973 data |= GFX_PG_SRC; 6974 if (orig != data) 6975 WREG32(RLC_PG_CNTL, data); 6976 6977 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 6978 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); 6979 6980 data = RREG32(CP_RB_WPTR_POLL_CNTL); 6981 data &= ~IDLE_POLL_COUNT_MASK; 6982 data |= IDLE_POLL_COUNT(0x60); 6983 WREG32(CP_RB_WPTR_POLL_CNTL, data); 6984 6985 data = 0x10101010; 6986 WREG32(RLC_PG_DELAY, data); 6987 6988 data = RREG32(RLC_PG_DELAY_2); 6989 data &= ~0xff; 6990 data |= 0x3; 6991 WREG32(RLC_PG_DELAY_2, data); 6992 6993 data = RREG32(RLC_AUTO_PG_CTRL); 6994 data &= ~GRBM_REG_SGIT_MASK; 6995 data |= GRBM_REG_SGIT(0x700); 6996 WREG32(RLC_AUTO_PG_CTRL, data); 6997 6998 } 6999 7000 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) 7001 { 7002 cik_enable_gfx_cgpg(rdev, enable); 7003 cik_enable_gfx_static_mgpg(rdev, enable); 7004 cik_enable_gfx_dynamic_mgpg(rdev, enable); 7005 } 7006 7007 u32 cik_get_csb_size(struct radeon_device *rdev) 7008 { 7009 u32 count = 0; 7010 const struct cs_section_def *sect = NULL; 7011 const struct cs_extent_def *ext = NULL; 7012 7013 if (rdev->rlc.cs_data == NULL) 7014 return 0; 7015 7016 /* begin clear state */ 7017 count += 2; 7018 /* context control state */ 7019 count += 3; 7020 7021 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 7022 for (ext = sect->section; ext->extent != NULL; ++ext) { 7023 if (sect->id == SECT_CONTEXT) 7024 count += 2 + ext->reg_count; 7025 else 7026 return 0; 7027 } 7028 } 7029 /* pa_sc_raster_config/pa_sc_raster_config1 */ 7030 count += 4; 7031 /* end clear state */ 7032 count += 2; 7033 /* clear state */ 7034 count += 2; 7035 7036 return count; 7037 } 7038 7039 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 7040 { 7041 u32 count = 0, i; 7042 const struct cs_section_def *sect = NULL; 7043 const struct cs_extent_def *ext = NULL; 7044 7045 if (rdev->rlc.cs_data == NULL) 7046 return; 7047 if (buffer == NULL) 7048 return; 7049 7050 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 7051 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 7052 7053 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 7054 buffer[count++] = cpu_to_le32(0x80000000); 7055 buffer[count++] = cpu_to_le32(0x80000000); 7056 7057 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 7058 for (ext = sect->section; ext->extent != NULL; ++ext) { 7059 if (sect->id == SECT_CONTEXT) { 7060 buffer[count++] = 7061 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 7062 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 7063 for (i = 0; i < ext->reg_count; i++) 7064 buffer[count++] = cpu_to_le32(ext->extent[i]); 7065 } else { 7066 return; 7067 } 7068 } 7069 } 7070 7071 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 7072 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 7073 switch (rdev->family) { 7074 case CHIP_BONAIRE: 7075 buffer[count++] = cpu_to_le32(0x16000012); 7076 buffer[count++] = cpu_to_le32(0x00000000); 7077 break; 7078 case CHIP_KAVERI: 7079 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 7080 buffer[count++] = cpu_to_le32(0x00000000); 7081 break; 7082 case CHIP_KABINI: 7083 case CHIP_MULLINS: 7084 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 7085 buffer[count++] = cpu_to_le32(0x00000000); 7086 break; 7087 case CHIP_HAWAII: 7088 buffer[count++] = cpu_to_le32(0x3a00161a); 7089 buffer[count++] = cpu_to_le32(0x0000002e); 7090 break; 7091 default: 7092 buffer[count++] = cpu_to_le32(0x00000000); 7093 buffer[count++] = cpu_to_le32(0x00000000); 7094 break; 7095 } 7096 7097 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 7098 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 7099 7100 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 7101 buffer[count++] = cpu_to_le32(0); 7102 } 7103 7104 static void cik_init_pg(struct radeon_device *rdev) 7105 { 7106 if (rdev->pg_flags) { 7107 cik_enable_sck_slowdown_on_pu(rdev, true); 7108 cik_enable_sck_slowdown_on_pd(rdev, true); 7109 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7110 cik_init_gfx_cgpg(rdev); 7111 cik_enable_cp_pg(rdev, true); 7112 cik_enable_gds_pg(rdev, true); 7113 } 7114 cik_init_ao_cu_mask(rdev); 7115 cik_update_gfx_pg(rdev, true); 7116 } 7117 } 7118 7119 static void cik_fini_pg(struct radeon_device *rdev) 7120 { 7121 if (rdev->pg_flags) { 7122 cik_update_gfx_pg(rdev, false); 7123 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 7124 cik_enable_cp_pg(rdev, false); 7125 cik_enable_gds_pg(rdev, false); 7126 } 7127 } 7128 } 7129 7130 /* 7131 * Interrupts 7132 * Starting with r6xx, interrupts are handled via a ring buffer. 7133 * Ring buffers are areas of GPU accessible memory that the GPU 7134 * writes interrupt vectors into and the host reads vectors out of. 7135 * There is a rptr (read pointer) that determines where the 7136 * host is currently reading, and a wptr (write pointer) 7137 * which determines where the GPU has written. When the 7138 * pointers are equal, the ring is idle. When the GPU 7139 * writes vectors to the ring buffer, it increments the 7140 * wptr. When there is an interrupt, the host then starts 7141 * fetching commands and processing them until the pointers are 7142 * equal again at which point it updates the rptr. 7143 */ 7144 7145 /** 7146 * cik_enable_interrupts - Enable the interrupt ring buffer 7147 * 7148 * @rdev: radeon_device pointer 7149 * 7150 * Enable the interrupt ring buffer (CIK). 7151 */ 7152 static void cik_enable_interrupts(struct radeon_device *rdev) 7153 { 7154 u32 ih_cntl = RREG32(IH_CNTL); 7155 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7156 7157 ih_cntl |= ENABLE_INTR; 7158 ih_rb_cntl |= IH_RB_ENABLE; 7159 WREG32(IH_CNTL, ih_cntl); 7160 WREG32(IH_RB_CNTL, ih_rb_cntl); 7161 rdev->ih.enabled = true; 7162 } 7163 7164 /** 7165 * cik_disable_interrupts - Disable the interrupt ring buffer 7166 * 7167 * @rdev: radeon_device pointer 7168 * 7169 * Disable the interrupt ring buffer (CIK). 7170 */ 7171 static void cik_disable_interrupts(struct radeon_device *rdev) 7172 { 7173 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 7174 u32 ih_cntl = RREG32(IH_CNTL); 7175 7176 ih_rb_cntl &= ~IH_RB_ENABLE; 7177 ih_cntl &= ~ENABLE_INTR; 7178 WREG32(IH_RB_CNTL, ih_rb_cntl); 7179 WREG32(IH_CNTL, ih_cntl); 7180 /* set rptr, wptr to 0 */ 7181 WREG32(IH_RB_RPTR, 0); 7182 WREG32(IH_RB_WPTR, 0); 7183 rdev->ih.enabled = false; 7184 rdev->ih.rptr = 0; 7185 } 7186 7187 /** 7188 * cik_disable_interrupt_state - Disable all interrupt sources 7189 * 7190 * @rdev: radeon_device pointer 7191 * 7192 * Clear all interrupt enable bits used by the driver (CIK). 7193 */ 7194 static void cik_disable_interrupt_state(struct radeon_device *rdev) 7195 { 7196 u32 tmp; 7197 7198 /* gfx ring */ 7199 tmp = RREG32(CP_INT_CNTL_RING0) & 7200 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7201 WREG32(CP_INT_CNTL_RING0, tmp); 7202 /* sdma */ 7203 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7204 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); 7205 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7206 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); 7207 /* compute queues */ 7208 WREG32(CP_ME1_PIPE0_INT_CNTL, 0); 7209 WREG32(CP_ME1_PIPE1_INT_CNTL, 0); 7210 WREG32(CP_ME1_PIPE2_INT_CNTL, 0); 7211 WREG32(CP_ME1_PIPE3_INT_CNTL, 0); 7212 WREG32(CP_ME2_PIPE0_INT_CNTL, 0); 7213 WREG32(CP_ME2_PIPE1_INT_CNTL, 0); 7214 WREG32(CP_ME2_PIPE2_INT_CNTL, 0); 7215 WREG32(CP_ME2_PIPE3_INT_CNTL, 0); 7216 /* grbm */ 7217 WREG32(GRBM_INT_CNTL, 0); 7218 /* vline/vblank, etc. */ 7219 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7220 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7221 if (rdev->num_crtc >= 4) { 7222 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7223 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7224 } 7225 if (rdev->num_crtc >= 6) { 7226 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7227 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7228 } 7229 /* pflip */ 7230 if (rdev->num_crtc >= 2) { 7231 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 7232 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 7233 } 7234 if (rdev->num_crtc >= 4) { 7235 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 7236 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 7237 } 7238 if (rdev->num_crtc >= 6) { 7239 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 7240 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 7241 } 7242 7243 /* dac hotplug */ 7244 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 7245 7246 /* digital hotplug */ 7247 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7248 WREG32(DC_HPD1_INT_CONTROL, tmp); 7249 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7250 WREG32(DC_HPD2_INT_CONTROL, tmp); 7251 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7252 WREG32(DC_HPD3_INT_CONTROL, tmp); 7253 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7254 WREG32(DC_HPD4_INT_CONTROL, tmp); 7255 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7256 WREG32(DC_HPD5_INT_CONTROL, tmp); 7257 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 7258 WREG32(DC_HPD6_INT_CONTROL, tmp); 7259 7260 } 7261 7262 /** 7263 * cik_irq_init - init and enable the interrupt ring 7264 * 7265 * @rdev: radeon_device pointer 7266 * 7267 * Allocate a ring buffer for the interrupt controller, 7268 * enable the RLC, disable interrupts, enable the IH 7269 * ring buffer and enable it (CIK). 7270 * Called at device load and reume. 7271 * Returns 0 for success, errors for failure. 7272 */ 7273 static int cik_irq_init(struct radeon_device *rdev) 7274 { 7275 int ret = 0; 7276 int rb_bufsz; 7277 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 7278 7279 /* allocate ring */ 7280 ret = r600_ih_ring_alloc(rdev); 7281 if (ret) 7282 return ret; 7283 7284 /* disable irqs */ 7285 cik_disable_interrupts(rdev); 7286 7287 /* init rlc */ 7288 ret = cik_rlc_resume(rdev); 7289 if (ret) { 7290 r600_ih_ring_fini(rdev); 7291 return ret; 7292 } 7293 7294 /* setup interrupt control */ 7295 /* XXX this should actually be a bus address, not an MC address. same on older asics */ 7296 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 7297 interrupt_cntl = RREG32(INTERRUPT_CNTL); 7298 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 7299 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 7300 */ 7301 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 7302 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 7303 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 7304 WREG32(INTERRUPT_CNTL, interrupt_cntl); 7305 7306 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 7307 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 7308 7309 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 7310 IH_WPTR_OVERFLOW_CLEAR | 7311 (rb_bufsz << 1)); 7312 7313 if (rdev->wb.enabled) 7314 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 7315 7316 /* set the writeback address whether it's enabled or not */ 7317 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 7318 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 7319 7320 WREG32(IH_RB_CNTL, ih_rb_cntl); 7321 7322 /* set rptr, wptr to 0 */ 7323 WREG32(IH_RB_RPTR, 0); 7324 WREG32(IH_RB_WPTR, 0); 7325 7326 /* Default settings for IH_CNTL (disabled at first) */ 7327 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 7328 /* RPTR_REARM only works if msi's are enabled */ 7329 if (rdev->msi_enabled) 7330 ih_cntl |= RPTR_REARM; 7331 WREG32(IH_CNTL, ih_cntl); 7332 7333 /* force the active interrupt state to all disabled */ 7334 cik_disable_interrupt_state(rdev); 7335 7336 pci_set_master(rdev->pdev); 7337 7338 /* enable irqs */ 7339 cik_enable_interrupts(rdev); 7340 7341 return ret; 7342 } 7343 7344 /** 7345 * cik_irq_set - enable/disable interrupt sources 7346 * 7347 * @rdev: radeon_device pointer 7348 * 7349 * Enable interrupt sources on the GPU (vblanks, hpd, 7350 * etc.) (CIK). 7351 * Returns 0 for success, errors for failure. 7352 */ 7353 int cik_irq_set(struct radeon_device *rdev) 7354 { 7355 u32 cp_int_cntl; 7356 u32 cp_m1p0; 7357 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 7358 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 7359 u32 grbm_int_cntl = 0; 7360 u32 dma_cntl, dma_cntl1; 7361 u32 thermal_int; 7362 7363 if (!rdev->irq.installed) { 7364 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 7365 return -EINVAL; 7366 } 7367 /* don't enable anything if the ih is disabled */ 7368 if (!rdev->ih.enabled) { 7369 cik_disable_interrupts(rdev); 7370 /* force the active interrupt state to all disabled */ 7371 cik_disable_interrupt_state(rdev); 7372 return 0; 7373 } 7374 7375 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 7376 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 7377 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; 7378 7379 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 7380 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 7381 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 7382 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 7383 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 7384 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 7385 7386 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 7387 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 7388 7389 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 7390 7391 if (rdev->flags & RADEON_IS_IGP) 7392 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) & 7393 ~(THERM_INTH_MASK | THERM_INTL_MASK); 7394 else 7395 thermal_int = RREG32_SMC(CG_THERMAL_INT) & 7396 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 7397 7398 /* enable CP interrupts on all rings */ 7399 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 7400 DRM_DEBUG("cik_irq_set: sw int gfx\n"); 7401 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 7402 } 7403 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 7404 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7405 DRM_DEBUG("si_irq_set: sw int cp1\n"); 7406 if (ring->me == 1) { 7407 switch (ring->pipe) { 7408 case 0: 7409 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7410 break; 7411 default: 7412 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 7413 break; 7414 } 7415 } else { 7416 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); 7417 } 7418 } 7419 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 7420 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7421 DRM_DEBUG("si_irq_set: sw int cp2\n"); 7422 if (ring->me == 1) { 7423 switch (ring->pipe) { 7424 case 0: 7425 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 7426 break; 7427 default: 7428 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 7429 break; 7430 } 7431 } else { 7432 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); 7433 } 7434 } 7435 7436 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 7437 DRM_DEBUG("cik_irq_set: sw int dma\n"); 7438 dma_cntl |= TRAP_ENABLE; 7439 } 7440 7441 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 7442 DRM_DEBUG("cik_irq_set: sw int dma1\n"); 7443 dma_cntl1 |= TRAP_ENABLE; 7444 } 7445 7446 if (rdev->irq.crtc_vblank_int[0] || 7447 atomic_read(&rdev->irq.pflip[0])) { 7448 DRM_DEBUG("cik_irq_set: vblank 0\n"); 7449 crtc1 |= VBLANK_INTERRUPT_MASK; 7450 } 7451 if (rdev->irq.crtc_vblank_int[1] || 7452 atomic_read(&rdev->irq.pflip[1])) { 7453 DRM_DEBUG("cik_irq_set: vblank 1\n"); 7454 crtc2 |= VBLANK_INTERRUPT_MASK; 7455 } 7456 if (rdev->irq.crtc_vblank_int[2] || 7457 atomic_read(&rdev->irq.pflip[2])) { 7458 DRM_DEBUG("cik_irq_set: vblank 2\n"); 7459 crtc3 |= VBLANK_INTERRUPT_MASK; 7460 } 7461 if (rdev->irq.crtc_vblank_int[3] || 7462 atomic_read(&rdev->irq.pflip[3])) { 7463 DRM_DEBUG("cik_irq_set: vblank 3\n"); 7464 crtc4 |= VBLANK_INTERRUPT_MASK; 7465 } 7466 if (rdev->irq.crtc_vblank_int[4] || 7467 atomic_read(&rdev->irq.pflip[4])) { 7468 DRM_DEBUG("cik_irq_set: vblank 4\n"); 7469 crtc5 |= VBLANK_INTERRUPT_MASK; 7470 } 7471 if (rdev->irq.crtc_vblank_int[5] || 7472 atomic_read(&rdev->irq.pflip[5])) { 7473 DRM_DEBUG("cik_irq_set: vblank 5\n"); 7474 crtc6 |= VBLANK_INTERRUPT_MASK; 7475 } 7476 if (rdev->irq.hpd[0]) { 7477 DRM_DEBUG("cik_irq_set: hpd 1\n"); 7478 hpd1 |= DC_HPDx_INT_EN; 7479 } 7480 if (rdev->irq.hpd[1]) { 7481 DRM_DEBUG("cik_irq_set: hpd 2\n"); 7482 hpd2 |= DC_HPDx_INT_EN; 7483 } 7484 if (rdev->irq.hpd[2]) { 7485 DRM_DEBUG("cik_irq_set: hpd 3\n"); 7486 hpd3 |= DC_HPDx_INT_EN; 7487 } 7488 if (rdev->irq.hpd[3]) { 7489 DRM_DEBUG("cik_irq_set: hpd 4\n"); 7490 hpd4 |= DC_HPDx_INT_EN; 7491 } 7492 if (rdev->irq.hpd[4]) { 7493 DRM_DEBUG("cik_irq_set: hpd 5\n"); 7494 hpd5 |= DC_HPDx_INT_EN; 7495 } 7496 if (rdev->irq.hpd[5]) { 7497 DRM_DEBUG("cik_irq_set: hpd 6\n"); 7498 hpd6 |= DC_HPDx_INT_EN; 7499 } 7500 7501 if (rdev->irq.dpm_thermal) { 7502 DRM_DEBUG("dpm thermal\n"); 7503 if (rdev->flags & RADEON_IS_IGP) 7504 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK; 7505 else 7506 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 7507 } 7508 7509 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 7510 7511 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); 7512 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); 7513 7514 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); 7515 7516 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 7517 7518 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 7519 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 7520 if (rdev->num_crtc >= 4) { 7521 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 7522 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 7523 } 7524 if (rdev->num_crtc >= 6) { 7525 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 7526 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 7527 } 7528 7529 if (rdev->num_crtc >= 2) { 7530 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 7531 GRPH_PFLIP_INT_MASK); 7532 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 7533 GRPH_PFLIP_INT_MASK); 7534 } 7535 if (rdev->num_crtc >= 4) { 7536 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 7537 GRPH_PFLIP_INT_MASK); 7538 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 7539 GRPH_PFLIP_INT_MASK); 7540 } 7541 if (rdev->num_crtc >= 6) { 7542 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 7543 GRPH_PFLIP_INT_MASK); 7544 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 7545 GRPH_PFLIP_INT_MASK); 7546 } 7547 7548 WREG32(DC_HPD1_INT_CONTROL, hpd1); 7549 WREG32(DC_HPD2_INT_CONTROL, hpd2); 7550 WREG32(DC_HPD3_INT_CONTROL, hpd3); 7551 WREG32(DC_HPD4_INT_CONTROL, hpd4); 7552 WREG32(DC_HPD5_INT_CONTROL, hpd5); 7553 WREG32(DC_HPD6_INT_CONTROL, hpd6); 7554 7555 if (rdev->flags & RADEON_IS_IGP) 7556 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int); 7557 else 7558 WREG32_SMC(CG_THERMAL_INT, thermal_int); 7559 7560 return 0; 7561 } 7562 7563 /** 7564 * cik_irq_ack - ack interrupt sources 7565 * 7566 * @rdev: radeon_device pointer 7567 * 7568 * Ack interrupt sources on the GPU (vblanks, hpd, 7569 * etc.) (CIK). Certain interrupts sources are sw 7570 * generated and do not require an explicit ack. 7571 */ 7572 static inline void cik_irq_ack(struct radeon_device *rdev) 7573 { 7574 u32 tmp; 7575 7576 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); 7577 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 7578 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 7579 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 7580 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 7581 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 7582 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); 7583 7584 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + 7585 EVERGREEN_CRTC0_REGISTER_OFFSET); 7586 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + 7587 EVERGREEN_CRTC1_REGISTER_OFFSET); 7588 if (rdev->num_crtc >= 4) { 7589 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + 7590 EVERGREEN_CRTC2_REGISTER_OFFSET); 7591 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + 7592 EVERGREEN_CRTC3_REGISTER_OFFSET); 7593 } 7594 if (rdev->num_crtc >= 6) { 7595 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + 7596 EVERGREEN_CRTC4_REGISTER_OFFSET); 7597 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + 7598 EVERGREEN_CRTC5_REGISTER_OFFSET); 7599 } 7600 7601 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 7602 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, 7603 GRPH_PFLIP_INT_CLEAR); 7604 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 7605 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, 7606 GRPH_PFLIP_INT_CLEAR); 7607 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) 7608 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 7609 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) 7610 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 7611 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 7612 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 7613 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) 7614 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 7615 7616 if (rdev->num_crtc >= 4) { 7617 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 7618 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, 7619 GRPH_PFLIP_INT_CLEAR); 7620 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 7621 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, 7622 GRPH_PFLIP_INT_CLEAR); 7623 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 7624 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 7625 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 7626 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 7627 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 7628 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 7629 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 7630 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 7631 } 7632 7633 if (rdev->num_crtc >= 6) { 7634 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 7635 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, 7636 GRPH_PFLIP_INT_CLEAR); 7637 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 7638 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, 7639 GRPH_PFLIP_INT_CLEAR); 7640 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 7641 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 7642 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 7643 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 7644 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 7645 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 7646 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 7647 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 7648 } 7649 7650 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7651 tmp = RREG32(DC_HPD1_INT_CONTROL); 7652 tmp |= DC_HPDx_INT_ACK; 7653 WREG32(DC_HPD1_INT_CONTROL, tmp); 7654 } 7655 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7656 tmp = RREG32(DC_HPD2_INT_CONTROL); 7657 tmp |= DC_HPDx_INT_ACK; 7658 WREG32(DC_HPD2_INT_CONTROL, tmp); 7659 } 7660 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7661 tmp = RREG32(DC_HPD3_INT_CONTROL); 7662 tmp |= DC_HPDx_INT_ACK; 7663 WREG32(DC_HPD3_INT_CONTROL, tmp); 7664 } 7665 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7666 tmp = RREG32(DC_HPD4_INT_CONTROL); 7667 tmp |= DC_HPDx_INT_ACK; 7668 WREG32(DC_HPD4_INT_CONTROL, tmp); 7669 } 7670 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7671 tmp = RREG32(DC_HPD5_INT_CONTROL); 7672 tmp |= DC_HPDx_INT_ACK; 7673 WREG32(DC_HPD5_INT_CONTROL, tmp); 7674 } 7675 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7676 tmp = RREG32(DC_HPD5_INT_CONTROL); 7677 tmp |= DC_HPDx_INT_ACK; 7678 WREG32(DC_HPD6_INT_CONTROL, tmp); 7679 } 7680 } 7681 7682 /** 7683 * cik_irq_disable - disable interrupts 7684 * 7685 * @rdev: radeon_device pointer 7686 * 7687 * Disable interrupts on the hw (CIK). 7688 */ 7689 static void cik_irq_disable(struct radeon_device *rdev) 7690 { 7691 cik_disable_interrupts(rdev); 7692 /* Wait and acknowledge irq */ 7693 mdelay(1); 7694 cik_irq_ack(rdev); 7695 cik_disable_interrupt_state(rdev); 7696 } 7697 7698 /** 7699 * cik_irq_disable - disable interrupts for suspend 7700 * 7701 * @rdev: radeon_device pointer 7702 * 7703 * Disable interrupts and stop the RLC (CIK). 7704 * Used for suspend. 7705 */ 7706 static void cik_irq_suspend(struct radeon_device *rdev) 7707 { 7708 cik_irq_disable(rdev); 7709 cik_rlc_stop(rdev); 7710 } 7711 7712 /** 7713 * cik_irq_fini - tear down interrupt support 7714 * 7715 * @rdev: radeon_device pointer 7716 * 7717 * Disable interrupts on the hw and free the IH ring 7718 * buffer (CIK). 7719 * Used for driver unload. 7720 */ 7721 static void cik_irq_fini(struct radeon_device *rdev) 7722 { 7723 cik_irq_suspend(rdev); 7724 r600_ih_ring_fini(rdev); 7725 } 7726 7727 /** 7728 * cik_get_ih_wptr - get the IH ring buffer wptr 7729 * 7730 * @rdev: radeon_device pointer 7731 * 7732 * Get the IH ring buffer wptr from either the register 7733 * or the writeback memory buffer (CIK). Also check for 7734 * ring buffer overflow and deal with it. 7735 * Used by cik_irq_process(). 7736 * Returns the value of the wptr. 7737 */ 7738 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) 7739 { 7740 u32 wptr, tmp; 7741 7742 if (rdev->wb.enabled) 7743 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 7744 else 7745 wptr = RREG32(IH_RB_WPTR); 7746 7747 if (wptr & RB_OVERFLOW) { 7748 wptr &= ~RB_OVERFLOW; 7749 /* When a ring buffer overflow happen start parsing interrupt 7750 * from the last not overwritten vector (wptr + 16). Hopefully 7751 * this should allow us to catchup. 7752 */ 7753 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 7754 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 7755 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 7756 tmp = RREG32(IH_RB_CNTL); 7757 tmp |= IH_WPTR_OVERFLOW_CLEAR; 7758 WREG32(IH_RB_CNTL, tmp); 7759 } 7760 return (wptr & rdev->ih.ptr_mask); 7761 } 7762 7763 /* CIK IV Ring 7764 * Each IV ring entry is 128 bits: 7765 * [7:0] - interrupt source id 7766 * [31:8] - reserved 7767 * [59:32] - interrupt source data 7768 * [63:60] - reserved 7769 * [71:64] - RINGID 7770 * CP: 7771 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] 7772 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher 7773 * - for gfx, hw shader state (0=PS...5=LS, 6=CS) 7774 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes 7775 * PIPE_ID - ME0 0=3D 7776 * - ME1&2 compute dispatcher (4 pipes each) 7777 * SDMA: 7778 * INSTANCE_ID [1:0], QUEUE_ID[1:0] 7779 * INSTANCE_ID - 0 = sdma0, 1 = sdma1 7780 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 7781 * [79:72] - VMID 7782 * [95:80] - PASID 7783 * [127:96] - reserved 7784 */ 7785 /** 7786 * cik_irq_process - interrupt handler 7787 * 7788 * @rdev: radeon_device pointer 7789 * 7790 * Interrupt hander (CIK). Walk the IH ring, 7791 * ack interrupts and schedule work to handle 7792 * interrupt events. 7793 * Returns irq process return code. 7794 */ 7795 int cik_irq_process(struct radeon_device *rdev) 7796 { 7797 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7798 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7799 u32 wptr; 7800 u32 rptr; 7801 u32 src_id, src_data, ring_id; 7802 u8 me_id, pipe_id, queue_id; 7803 u32 ring_index; 7804 bool queue_hotplug = false; 7805 bool queue_reset = false; 7806 u32 addr, status, mc_client; 7807 bool queue_thermal = false; 7808 7809 if (!rdev->ih.enabled || rdev->shutdown) 7810 return IRQ_NONE; 7811 7812 wptr = cik_get_ih_wptr(rdev); 7813 7814 restart_ih: 7815 /* is somebody else already processing irqs? */ 7816 if (atomic_xchg(&rdev->ih.lock, 1)) 7817 return IRQ_NONE; 7818 7819 rptr = rdev->ih.rptr; 7820 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 7821 7822 /* Order reading of wptr vs. reading of IH ring data */ 7823 rmb(); 7824 7825 /* display interrupts */ 7826 cik_irq_ack(rdev); 7827 7828 while (rptr != wptr) { 7829 /* wptr/rptr are in bytes! */ 7830 ring_index = rptr / 4; 7831 7832 radeon_kfd_interrupt(rdev, 7833 (const void *) &rdev->ih.ring[ring_index]); 7834 7835 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7836 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7837 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7838 7839 switch (src_id) { 7840 case 1: /* D1 vblank/vline */ 7841 switch (src_data) { 7842 case 0: /* D1 vblank */ 7843 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) { 7844 if (rdev->irq.crtc_vblank_int[0]) { 7845 drm_handle_vblank(rdev->ddev, 0); 7846 rdev->pm.vblank_sync = true; 7847 wake_up(&rdev->irq.vblank_queue); 7848 } 7849 if (atomic_read(&rdev->irq.pflip[0])) 7850 radeon_crtc_handle_vblank(rdev, 0); 7851 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 7852 DRM_DEBUG("IH: D1 vblank\n"); 7853 } 7854 break; 7855 case 1: /* D1 vline */ 7856 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) { 7857 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; 7858 DRM_DEBUG("IH: D1 vline\n"); 7859 } 7860 break; 7861 default: 7862 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7863 break; 7864 } 7865 break; 7866 case 2: /* D2 vblank/vline */ 7867 switch (src_data) { 7868 case 0: /* D2 vblank */ 7869 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 7870 if (rdev->irq.crtc_vblank_int[1]) { 7871 drm_handle_vblank(rdev->ddev, 1); 7872 rdev->pm.vblank_sync = true; 7873 wake_up(&rdev->irq.vblank_queue); 7874 } 7875 if (atomic_read(&rdev->irq.pflip[1])) 7876 radeon_crtc_handle_vblank(rdev, 1); 7877 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 7878 DRM_DEBUG("IH: D2 vblank\n"); 7879 } 7880 break; 7881 case 1: /* D2 vline */ 7882 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 7883 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 7884 DRM_DEBUG("IH: D2 vline\n"); 7885 } 7886 break; 7887 default: 7888 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7889 break; 7890 } 7891 break; 7892 case 3: /* D3 vblank/vline */ 7893 switch (src_data) { 7894 case 0: /* D3 vblank */ 7895 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 7896 if (rdev->irq.crtc_vblank_int[2]) { 7897 drm_handle_vblank(rdev->ddev, 2); 7898 rdev->pm.vblank_sync = true; 7899 wake_up(&rdev->irq.vblank_queue); 7900 } 7901 if (atomic_read(&rdev->irq.pflip[2])) 7902 radeon_crtc_handle_vblank(rdev, 2); 7903 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 7904 DRM_DEBUG("IH: D3 vblank\n"); 7905 } 7906 break; 7907 case 1: /* D3 vline */ 7908 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 7909 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 7910 DRM_DEBUG("IH: D3 vline\n"); 7911 } 7912 break; 7913 default: 7914 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7915 break; 7916 } 7917 break; 7918 case 4: /* D4 vblank/vline */ 7919 switch (src_data) { 7920 case 0: /* D4 vblank */ 7921 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 7922 if (rdev->irq.crtc_vblank_int[3]) { 7923 drm_handle_vblank(rdev->ddev, 3); 7924 rdev->pm.vblank_sync = true; 7925 wake_up(&rdev->irq.vblank_queue); 7926 } 7927 if (atomic_read(&rdev->irq.pflip[3])) 7928 radeon_crtc_handle_vblank(rdev, 3); 7929 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 7930 DRM_DEBUG("IH: D4 vblank\n"); 7931 } 7932 break; 7933 case 1: /* D4 vline */ 7934 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 7935 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 7936 DRM_DEBUG("IH: D4 vline\n"); 7937 } 7938 break; 7939 default: 7940 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7941 break; 7942 } 7943 break; 7944 case 5: /* D5 vblank/vline */ 7945 switch (src_data) { 7946 case 0: /* D5 vblank */ 7947 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 7948 if (rdev->irq.crtc_vblank_int[4]) { 7949 drm_handle_vblank(rdev->ddev, 4); 7950 rdev->pm.vblank_sync = true; 7951 wake_up(&rdev->irq.vblank_queue); 7952 } 7953 if (atomic_read(&rdev->irq.pflip[4])) 7954 radeon_crtc_handle_vblank(rdev, 4); 7955 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 7956 DRM_DEBUG("IH: D5 vblank\n"); 7957 } 7958 break; 7959 case 1: /* D5 vline */ 7960 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 7961 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 7962 DRM_DEBUG("IH: D5 vline\n"); 7963 } 7964 break; 7965 default: 7966 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7967 break; 7968 } 7969 break; 7970 case 6: /* D6 vblank/vline */ 7971 switch (src_data) { 7972 case 0: /* D6 vblank */ 7973 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 7974 if (rdev->irq.crtc_vblank_int[5]) { 7975 drm_handle_vblank(rdev->ddev, 5); 7976 rdev->pm.vblank_sync = true; 7977 wake_up(&rdev->irq.vblank_queue); 7978 } 7979 if (atomic_read(&rdev->irq.pflip[5])) 7980 radeon_crtc_handle_vblank(rdev, 5); 7981 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 7982 DRM_DEBUG("IH: D6 vblank\n"); 7983 } 7984 break; 7985 case 1: /* D6 vline */ 7986 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 7987 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 7988 DRM_DEBUG("IH: D6 vline\n"); 7989 } 7990 break; 7991 default: 7992 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7993 break; 7994 } 7995 break; 7996 case 8: /* D1 page flip */ 7997 case 10: /* D2 page flip */ 7998 case 12: /* D3 page flip */ 7999 case 14: /* D4 page flip */ 8000 case 16: /* D5 page flip */ 8001 case 18: /* D6 page flip */ 8002 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 8003 if (radeon_use_pflipirq > 0) 8004 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 8005 break; 8006 case 42: /* HPD hotplug */ 8007 switch (src_data) { 8008 case 0: 8009 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 8010 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; 8011 queue_hotplug = true; 8012 DRM_DEBUG("IH: HPD1\n"); 8013 } 8014 break; 8015 case 1: 8016 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 8017 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; 8018 queue_hotplug = true; 8019 DRM_DEBUG("IH: HPD2\n"); 8020 } 8021 break; 8022 case 2: 8023 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 8024 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 8025 queue_hotplug = true; 8026 DRM_DEBUG("IH: HPD3\n"); 8027 } 8028 break; 8029 case 3: 8030 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 8031 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 8032 queue_hotplug = true; 8033 DRM_DEBUG("IH: HPD4\n"); 8034 } 8035 break; 8036 case 4: 8037 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 8038 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 8039 queue_hotplug = true; 8040 DRM_DEBUG("IH: HPD5\n"); 8041 } 8042 break; 8043 case 5: 8044 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 8045 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 8046 queue_hotplug = true; 8047 DRM_DEBUG("IH: HPD6\n"); 8048 } 8049 break; 8050 default: 8051 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8052 break; 8053 } 8054 break; 8055 case 124: /* UVD */ 8056 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 8057 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 8058 break; 8059 case 146: 8060 case 147: 8061 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 8062 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 8063 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); 8064 /* reset addr and status */ 8065 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 8066 if (addr == 0x0 && status == 0x0) 8067 break; 8068 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 8069 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 8070 addr); 8071 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 8072 status); 8073 cik_vm_decode_fault(rdev, status, addr, mc_client); 8074 break; 8075 case 167: /* VCE */ 8076 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); 8077 switch (src_data) { 8078 case 0: 8079 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); 8080 break; 8081 case 1: 8082 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); 8083 break; 8084 default: 8085 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 8086 break; 8087 } 8088 break; 8089 case 176: /* GFX RB CP_INT */ 8090 case 177: /* GFX IB CP_INT */ 8091 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8092 break; 8093 case 181: /* CP EOP event */ 8094 DRM_DEBUG("IH: CP EOP\n"); 8095 /* XXX check the bitfield order! */ 8096 me_id = (ring_id & 0x60) >> 5; 8097 pipe_id = (ring_id & 0x18) >> 3; 8098 queue_id = (ring_id & 0x7) >> 0; 8099 switch (me_id) { 8100 case 0: 8101 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 8102 break; 8103 case 1: 8104 case 2: 8105 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) 8106 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8107 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) 8108 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8109 break; 8110 } 8111 break; 8112 case 184: /* CP Privileged reg access */ 8113 DRM_ERROR("Illegal register access in command stream\n"); 8114 /* XXX check the bitfield order! */ 8115 me_id = (ring_id & 0x60) >> 5; 8116 pipe_id = (ring_id & 0x18) >> 3; 8117 queue_id = (ring_id & 0x7) >> 0; 8118 switch (me_id) { 8119 case 0: 8120 /* This results in a full GPU reset, but all we need to do is soft 8121 * reset the CP for gfx 8122 */ 8123 queue_reset = true; 8124 break; 8125 case 1: 8126 /* XXX compute */ 8127 queue_reset = true; 8128 break; 8129 case 2: 8130 /* XXX compute */ 8131 queue_reset = true; 8132 break; 8133 } 8134 break; 8135 case 185: /* CP Privileged inst */ 8136 DRM_ERROR("Illegal instruction in command stream\n"); 8137 /* XXX check the bitfield order! */ 8138 me_id = (ring_id & 0x60) >> 5; 8139 pipe_id = (ring_id & 0x18) >> 3; 8140 queue_id = (ring_id & 0x7) >> 0; 8141 switch (me_id) { 8142 case 0: 8143 /* This results in a full GPU reset, but all we need to do is soft 8144 * reset the CP for gfx 8145 */ 8146 queue_reset = true; 8147 break; 8148 case 1: 8149 /* XXX compute */ 8150 queue_reset = true; 8151 break; 8152 case 2: 8153 /* XXX compute */ 8154 queue_reset = true; 8155 break; 8156 } 8157 break; 8158 case 224: /* SDMA trap event */ 8159 /* XXX check the bitfield order! */ 8160 me_id = (ring_id & 0x3) >> 0; 8161 queue_id = (ring_id & 0xc) >> 2; 8162 DRM_DEBUG("IH: SDMA trap\n"); 8163 switch (me_id) { 8164 case 0: 8165 switch (queue_id) { 8166 case 0: 8167 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 8168 break; 8169 case 1: 8170 /* XXX compute */ 8171 break; 8172 case 2: 8173 /* XXX compute */ 8174 break; 8175 } 8176 break; 8177 case 1: 8178 switch (queue_id) { 8179 case 0: 8180 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8181 break; 8182 case 1: 8183 /* XXX compute */ 8184 break; 8185 case 2: 8186 /* XXX compute */ 8187 break; 8188 } 8189 break; 8190 } 8191 break; 8192 case 230: /* thermal low to high */ 8193 DRM_DEBUG("IH: thermal low to high\n"); 8194 rdev->pm.dpm.thermal.high_to_low = false; 8195 queue_thermal = true; 8196 break; 8197 case 231: /* thermal high to low */ 8198 DRM_DEBUG("IH: thermal high to low\n"); 8199 rdev->pm.dpm.thermal.high_to_low = true; 8200 queue_thermal = true; 8201 break; 8202 case 233: /* GUI IDLE */ 8203 DRM_DEBUG("IH: GUI idle\n"); 8204 break; 8205 case 241: /* SDMA Privileged inst */ 8206 case 247: /* SDMA Privileged inst */ 8207 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 8208 /* XXX check the bitfield order! */ 8209 me_id = (ring_id & 0x3) >> 0; 8210 queue_id = (ring_id & 0xc) >> 2; 8211 switch (me_id) { 8212 case 0: 8213 switch (queue_id) { 8214 case 0: 8215 queue_reset = true; 8216 break; 8217 case 1: 8218 /* XXX compute */ 8219 queue_reset = true; 8220 break; 8221 case 2: 8222 /* XXX compute */ 8223 queue_reset = true; 8224 break; 8225 } 8226 break; 8227 case 1: 8228 switch (queue_id) { 8229 case 0: 8230 queue_reset = true; 8231 break; 8232 case 1: 8233 /* XXX compute */ 8234 queue_reset = true; 8235 break; 8236 case 2: 8237 /* XXX compute */ 8238 queue_reset = true; 8239 break; 8240 } 8241 break; 8242 } 8243 break; 8244 default: 8245 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 8246 break; 8247 } 8248 8249 /* wptr/rptr are in bytes! */ 8250 rptr += 16; 8251 rptr &= rdev->ih.ptr_mask; 8252 WREG32(IH_RB_RPTR, rptr); 8253 } 8254 if (queue_hotplug) 8255 schedule_work(&rdev->hotplug_work); 8256 if (queue_reset) { 8257 rdev->needs_reset = true; 8258 wake_up_all(&rdev->fence_queue); 8259 } 8260 if (queue_thermal) 8261 schedule_work(&rdev->pm.dpm.thermal.work); 8262 rdev->ih.rptr = rptr; 8263 atomic_set(&rdev->ih.lock, 0); 8264 8265 /* make sure wptr hasn't changed while processing */ 8266 wptr = cik_get_ih_wptr(rdev); 8267 if (wptr != rptr) 8268 goto restart_ih; 8269 8270 return IRQ_HANDLED; 8271 } 8272 8273 /* 8274 * startup/shutdown callbacks 8275 */ 8276 /** 8277 * cik_startup - program the asic to a functional state 8278 * 8279 * @rdev: radeon_device pointer 8280 * 8281 * Programs the asic to a functional state (CIK). 8282 * Called by cik_init() and cik_resume(). 8283 * Returns 0 for success, error for failure. 8284 */ 8285 static int cik_startup(struct radeon_device *rdev) 8286 { 8287 struct radeon_ring *ring; 8288 u32 nop; 8289 int r; 8290 8291 /* enable pcie gen2/3 link */ 8292 cik_pcie_gen3_enable(rdev); 8293 /* enable aspm */ 8294 cik_program_aspm(rdev); 8295 8296 /* scratch needs to be initialized before MC */ 8297 r = r600_vram_scratch_init(rdev); 8298 if (r) 8299 return r; 8300 8301 cik_mc_program(rdev); 8302 8303 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { 8304 r = ci_mc_load_microcode(rdev); 8305 if (r) { 8306 DRM_ERROR("Failed to load MC firmware!\n"); 8307 return r; 8308 } 8309 } 8310 8311 r = cik_pcie_gart_enable(rdev); 8312 if (r) 8313 return r; 8314 cik_gpu_init(rdev); 8315 8316 /* allocate rlc buffers */ 8317 if (rdev->flags & RADEON_IS_IGP) { 8318 if (rdev->family == CHIP_KAVERI) { 8319 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; 8320 rdev->rlc.reg_list_size = 8321 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); 8322 } else { 8323 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; 8324 rdev->rlc.reg_list_size = 8325 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); 8326 } 8327 } 8328 rdev->rlc.cs_data = ci_cs_data; 8329 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; 8330 r = sumo_rlc_init(rdev); 8331 if (r) { 8332 DRM_ERROR("Failed to init rlc BOs!\n"); 8333 return r; 8334 } 8335 8336 /* allocate wb buffer */ 8337 r = radeon_wb_init(rdev); 8338 if (r) 8339 return r; 8340 8341 /* allocate mec buffers */ 8342 r = cik_mec_init(rdev); 8343 if (r) { 8344 DRM_ERROR("Failed to init MEC BOs!\n"); 8345 return r; 8346 } 8347 8348 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 8349 if (r) { 8350 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8351 return r; 8352 } 8353 8354 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 8355 if (r) { 8356 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8357 return r; 8358 } 8359 8360 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 8361 if (r) { 8362 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 8363 return r; 8364 } 8365 8366 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 8367 if (r) { 8368 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8369 return r; 8370 } 8371 8372 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 8373 if (r) { 8374 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 8375 return r; 8376 } 8377 8378 r = radeon_uvd_resume(rdev); 8379 if (!r) { 8380 r = uvd_v4_2_resume(rdev); 8381 if (!r) { 8382 r = radeon_fence_driver_start_ring(rdev, 8383 R600_RING_TYPE_UVD_INDEX); 8384 if (r) 8385 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 8386 } 8387 } 8388 if (r) 8389 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 8390 8391 r = radeon_vce_resume(rdev); 8392 if (!r) { 8393 r = vce_v2_0_resume(rdev); 8394 if (!r) 8395 r = radeon_fence_driver_start_ring(rdev, 8396 TN_RING_TYPE_VCE1_INDEX); 8397 if (!r) 8398 r = radeon_fence_driver_start_ring(rdev, 8399 TN_RING_TYPE_VCE2_INDEX); 8400 } 8401 if (r) { 8402 dev_err(rdev->dev, "VCE init error (%d).\n", r); 8403 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; 8404 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; 8405 } 8406 8407 /* Enable IRQ */ 8408 if (!rdev->irq.installed) { 8409 r = radeon_irq_kms_init(rdev); 8410 if (r) 8411 return r; 8412 } 8413 8414 r = cik_irq_init(rdev); 8415 if (r) { 8416 DRM_ERROR("radeon: IH init failed (%d).\n", r); 8417 radeon_irq_kms_fini(rdev); 8418 return r; 8419 } 8420 cik_irq_set(rdev); 8421 8422 if (rdev->family == CHIP_HAWAII) { 8423 if (rdev->new_fw) 8424 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8425 else 8426 nop = RADEON_CP_PACKET2; 8427 } else { 8428 nop = PACKET3(PACKET3_NOP, 0x3FFF); 8429 } 8430 8431 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8432 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 8433 nop); 8434 if (r) 8435 return r; 8436 8437 /* set up the compute queues */ 8438 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8439 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8440 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 8441 nop); 8442 if (r) 8443 return r; 8444 ring->me = 1; /* first MEC */ 8445 ring->pipe = 0; /* first pipe */ 8446 ring->queue = 0; /* first queue */ 8447 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; 8448 8449 /* type-2 packets are deprecated on MEC, use type-3 instead */ 8450 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8451 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 8452 nop); 8453 if (r) 8454 return r; 8455 /* dGPU only have 1 MEC */ 8456 ring->me = 1; /* first MEC */ 8457 ring->pipe = 0; /* first pipe */ 8458 ring->queue = 1; /* second queue */ 8459 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; 8460 8461 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8462 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 8463 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8464 if (r) 8465 return r; 8466 8467 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8468 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 8469 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 8470 if (r) 8471 return r; 8472 8473 r = cik_cp_resume(rdev); 8474 if (r) 8475 return r; 8476 8477 r = cik_sdma_resume(rdev); 8478 if (r) 8479 return r; 8480 8481 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8482 if (ring->ring_size) { 8483 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8484 RADEON_CP_PACKET2); 8485 if (!r) 8486 r = uvd_v1_0_init(rdev); 8487 if (r) 8488 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 8489 } 8490 8491 r = -ENOENT; 8492 8493 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8494 if (ring->ring_size) 8495 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8496 VCE_CMD_NO_OP); 8497 8498 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8499 if (ring->ring_size) 8500 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 8501 VCE_CMD_NO_OP); 8502 8503 if (!r) 8504 r = vce_v1_0_init(rdev); 8505 else if (r != -ENOENT) 8506 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); 8507 8508 r = radeon_ib_pool_init(rdev); 8509 if (r) { 8510 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 8511 return r; 8512 } 8513 8514 r = radeon_vm_manager_init(rdev); 8515 if (r) { 8516 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 8517 return r; 8518 } 8519 8520 r = dce6_audio_init(rdev); 8521 if (r) 8522 return r; 8523 8524 r = radeon_kfd_resume(rdev); 8525 if (r) 8526 return r; 8527 8528 return 0; 8529 } 8530 8531 /** 8532 * cik_resume - resume the asic to a functional state 8533 * 8534 * @rdev: radeon_device pointer 8535 * 8536 * Programs the asic to a functional state (CIK). 8537 * Called at resume. 8538 * Returns 0 for success, error for failure. 8539 */ 8540 int cik_resume(struct radeon_device *rdev) 8541 { 8542 int r; 8543 8544 /* post card */ 8545 atom_asic_init(rdev->mode_info.atom_context); 8546 8547 /* init golden registers */ 8548 cik_init_golden_registers(rdev); 8549 8550 if (rdev->pm.pm_method == PM_METHOD_DPM) 8551 radeon_pm_resume(rdev); 8552 8553 rdev->accel_working = true; 8554 r = cik_startup(rdev); 8555 if (r) { 8556 DRM_ERROR("cik startup failed on resume\n"); 8557 rdev->accel_working = false; 8558 return r; 8559 } 8560 8561 return r; 8562 8563 } 8564 8565 /** 8566 * cik_suspend - suspend the asic 8567 * 8568 * @rdev: radeon_device pointer 8569 * 8570 * Bring the chip into a state suitable for suspend (CIK). 8571 * Called at suspend. 8572 * Returns 0 for success. 8573 */ 8574 int cik_suspend(struct radeon_device *rdev) 8575 { 8576 radeon_kfd_suspend(rdev); 8577 radeon_pm_suspend(rdev); 8578 dce6_audio_fini(rdev); 8579 radeon_vm_manager_fini(rdev); 8580 cik_cp_enable(rdev, false); 8581 cik_sdma_enable(rdev, false); 8582 uvd_v1_0_fini(rdev); 8583 radeon_uvd_suspend(rdev); 8584 radeon_vce_suspend(rdev); 8585 cik_fini_pg(rdev); 8586 cik_fini_cg(rdev); 8587 cik_irq_suspend(rdev); 8588 radeon_wb_disable(rdev); 8589 cik_pcie_gart_disable(rdev); 8590 return 0; 8591 } 8592 8593 /* Plan is to move initialization in that function and use 8594 * helper function so that radeon_device_init pretty much 8595 * do nothing more than calling asic specific function. This 8596 * should also allow to remove a bunch of callback function 8597 * like vram_info. 8598 */ 8599 /** 8600 * cik_init - asic specific driver and hw init 8601 * 8602 * @rdev: radeon_device pointer 8603 * 8604 * Setup asic specific driver variables and program the hw 8605 * to a functional state (CIK). 8606 * Called at driver startup. 8607 * Returns 0 for success, errors for failure. 8608 */ 8609 int cik_init(struct radeon_device *rdev) 8610 { 8611 struct radeon_ring *ring; 8612 int r; 8613 8614 /* Read BIOS */ 8615 if (!radeon_get_bios(rdev)) { 8616 if (ASIC_IS_AVIVO(rdev)) 8617 return -EINVAL; 8618 } 8619 /* Must be an ATOMBIOS */ 8620 if (!rdev->is_atom_bios) { 8621 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 8622 return -EINVAL; 8623 } 8624 r = radeon_atombios_init(rdev); 8625 if (r) 8626 return r; 8627 8628 /* Post card if necessary */ 8629 if (!radeon_card_posted(rdev)) { 8630 if (!rdev->bios) { 8631 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 8632 return -EINVAL; 8633 } 8634 DRM_INFO("GPU not posted. posting now...\n"); 8635 atom_asic_init(rdev->mode_info.atom_context); 8636 } 8637 /* init golden registers */ 8638 cik_init_golden_registers(rdev); 8639 /* Initialize scratch registers */ 8640 cik_scratch_init(rdev); 8641 /* Initialize surface registers */ 8642 radeon_surface_init(rdev); 8643 /* Initialize clocks */ 8644 radeon_get_clock_info(rdev->ddev); 8645 8646 /* Fence driver */ 8647 r = radeon_fence_driver_init(rdev); 8648 if (r) 8649 return r; 8650 8651 /* initialize memory controller */ 8652 r = cik_mc_init(rdev); 8653 if (r) 8654 return r; 8655 /* Memory manager */ 8656 r = radeon_bo_init(rdev); 8657 if (r) 8658 return r; 8659 8660 if (rdev->flags & RADEON_IS_IGP) { 8661 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8662 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { 8663 r = cik_init_microcode(rdev); 8664 if (r) { 8665 DRM_ERROR("Failed to load firmware!\n"); 8666 return r; 8667 } 8668 } 8669 } else { 8670 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8671 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || 8672 !rdev->mc_fw) { 8673 r = cik_init_microcode(rdev); 8674 if (r) { 8675 DRM_ERROR("Failed to load firmware!\n"); 8676 return r; 8677 } 8678 } 8679 } 8680 8681 /* Initialize power management */ 8682 radeon_pm_init(rdev); 8683 8684 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8685 ring->ring_obj = NULL; 8686 r600_ring_init(rdev, ring, 1024 * 1024); 8687 8688 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8689 ring->ring_obj = NULL; 8690 r600_ring_init(rdev, ring, 1024 * 1024); 8691 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8692 if (r) 8693 return r; 8694 8695 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8696 ring->ring_obj = NULL; 8697 r600_ring_init(rdev, ring, 1024 * 1024); 8698 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8699 if (r) 8700 return r; 8701 8702 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8703 ring->ring_obj = NULL; 8704 r600_ring_init(rdev, ring, 256 * 1024); 8705 8706 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8707 ring->ring_obj = NULL; 8708 r600_ring_init(rdev, ring, 256 * 1024); 8709 8710 r = radeon_uvd_init(rdev); 8711 if (!r) { 8712 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8713 ring->ring_obj = NULL; 8714 r600_ring_init(rdev, ring, 4096); 8715 } 8716 8717 r = radeon_vce_init(rdev); 8718 if (!r) { 8719 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8720 ring->ring_obj = NULL; 8721 r600_ring_init(rdev, ring, 4096); 8722 8723 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8724 ring->ring_obj = NULL; 8725 r600_ring_init(rdev, ring, 4096); 8726 } 8727 8728 rdev->ih.ring_obj = NULL; 8729 r600_ih_ring_init(rdev, 64 * 1024); 8730 8731 r = r600_pcie_gart_init(rdev); 8732 if (r) 8733 return r; 8734 8735 rdev->accel_working = true; 8736 r = cik_startup(rdev); 8737 if (r) { 8738 dev_err(rdev->dev, "disabling GPU acceleration\n"); 8739 cik_cp_fini(rdev); 8740 cik_sdma_fini(rdev); 8741 cik_irq_fini(rdev); 8742 sumo_rlc_fini(rdev); 8743 cik_mec_fini(rdev); 8744 radeon_wb_fini(rdev); 8745 radeon_ib_pool_fini(rdev); 8746 radeon_vm_manager_fini(rdev); 8747 radeon_irq_kms_fini(rdev); 8748 cik_pcie_gart_fini(rdev); 8749 rdev->accel_working = false; 8750 } 8751 8752 /* Don't start up if the MC ucode is missing. 8753 * The default clocks and voltages before the MC ucode 8754 * is loaded are not suffient for advanced operations. 8755 */ 8756 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 8757 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 8758 return -EINVAL; 8759 } 8760 8761 return 0; 8762 } 8763 8764 /** 8765 * cik_fini - asic specific driver and hw fini 8766 * 8767 * @rdev: radeon_device pointer 8768 * 8769 * Tear down the asic specific driver variables and program the hw 8770 * to an idle state (CIK). 8771 * Called at driver unload. 8772 */ 8773 void cik_fini(struct radeon_device *rdev) 8774 { 8775 radeon_pm_fini(rdev); 8776 cik_cp_fini(rdev); 8777 cik_sdma_fini(rdev); 8778 cik_fini_pg(rdev); 8779 cik_fini_cg(rdev); 8780 cik_irq_fini(rdev); 8781 sumo_rlc_fini(rdev); 8782 cik_mec_fini(rdev); 8783 radeon_wb_fini(rdev); 8784 radeon_vm_manager_fini(rdev); 8785 radeon_ib_pool_fini(rdev); 8786 radeon_irq_kms_fini(rdev); 8787 uvd_v1_0_fini(rdev); 8788 radeon_uvd_fini(rdev); 8789 radeon_vce_fini(rdev); 8790 cik_pcie_gart_fini(rdev); 8791 r600_vram_scratch_fini(rdev); 8792 radeon_gem_fini(rdev); 8793 radeon_fence_driver_fini(rdev); 8794 radeon_bo_fini(rdev); 8795 radeon_atombios_fini(rdev); 8796 kfree(rdev->bios); 8797 rdev->bios = NULL; 8798 } 8799 8800 void dce8_program_fmt(struct drm_encoder *encoder) 8801 { 8802 struct drm_device *dev = encoder->dev; 8803 struct radeon_device *rdev = dev->dev_private; 8804 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 8805 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 8806 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 8807 int bpc = 0; 8808 u32 tmp = 0; 8809 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 8810 8811 if (connector) { 8812 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 8813 bpc = radeon_get_monitor_bpc(connector); 8814 dither = radeon_connector->dither; 8815 } 8816 8817 /* LVDS/eDP FMT is set up by atom */ 8818 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 8819 return; 8820 8821 /* not needed for analog */ 8822 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 8823 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 8824 return; 8825 8826 if (bpc == 0) 8827 return; 8828 8829 switch (bpc) { 8830 case 6: 8831 if (dither == RADEON_FMT_DITHER_ENABLE) 8832 /* XXX sort out optimal dither settings */ 8833 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8834 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); 8835 else 8836 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); 8837 break; 8838 case 8: 8839 if (dither == RADEON_FMT_DITHER_ENABLE) 8840 /* XXX sort out optimal dither settings */ 8841 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8842 FMT_RGB_RANDOM_ENABLE | 8843 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); 8844 else 8845 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); 8846 break; 8847 case 10: 8848 if (dither == RADEON_FMT_DITHER_ENABLE) 8849 /* XXX sort out optimal dither settings */ 8850 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8851 FMT_RGB_RANDOM_ENABLE | 8852 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); 8853 else 8854 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); 8855 break; 8856 default: 8857 /* not needed */ 8858 break; 8859 } 8860 8861 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 8862 } 8863 8864 /* display watermark setup */ 8865 /** 8866 * dce8_line_buffer_adjust - Set up the line buffer 8867 * 8868 * @rdev: radeon_device pointer 8869 * @radeon_crtc: the selected display controller 8870 * @mode: the current display mode on the selected display 8871 * controller 8872 * 8873 * Setup up the line buffer allocation for 8874 * the selected display controller (CIK). 8875 * Returns the line buffer size in pixels. 8876 */ 8877 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, 8878 struct radeon_crtc *radeon_crtc, 8879 struct drm_display_mode *mode) 8880 { 8881 u32 tmp, buffer_alloc, i; 8882 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 8883 /* 8884 * Line Buffer Setup 8885 * There are 6 line buffers, one for each display controllers. 8886 * There are 3 partitions per LB. Select the number of partitions 8887 * to enable based on the display width. For display widths larger 8888 * than 4096, you need use to use 2 display controllers and combine 8889 * them using the stereo blender. 8890 */ 8891 if (radeon_crtc->base.enabled && mode) { 8892 if (mode->crtc_hdisplay < 1920) { 8893 tmp = 1; 8894 buffer_alloc = 2; 8895 } else if (mode->crtc_hdisplay < 2560) { 8896 tmp = 2; 8897 buffer_alloc = 2; 8898 } else if (mode->crtc_hdisplay < 4096) { 8899 tmp = 0; 8900 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8901 } else { 8902 DRM_DEBUG_KMS("Mode too big for LB!\n"); 8903 tmp = 0; 8904 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8905 } 8906 } else { 8907 tmp = 1; 8908 buffer_alloc = 0; 8909 } 8910 8911 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, 8912 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); 8913 8914 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 8915 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 8916 for (i = 0; i < rdev->usec_timeout; i++) { 8917 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 8918 DMIF_BUFFERS_ALLOCATED_COMPLETED) 8919 break; 8920 udelay(1); 8921 } 8922 8923 if (radeon_crtc->base.enabled && mode) { 8924 switch (tmp) { 8925 case 0: 8926 default: 8927 return 4096 * 2; 8928 case 1: 8929 return 1920 * 2; 8930 case 2: 8931 return 2560 * 2; 8932 } 8933 } 8934 8935 /* controller not enabled, so no lb used */ 8936 return 0; 8937 } 8938 8939 /** 8940 * cik_get_number_of_dram_channels - get the number of dram channels 8941 * 8942 * @rdev: radeon_device pointer 8943 * 8944 * Look up the number of video ram channels (CIK). 8945 * Used for display watermark bandwidth calculations 8946 * Returns the number of dram channels 8947 */ 8948 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) 8949 { 8950 u32 tmp = RREG32(MC_SHARED_CHMAP); 8951 8952 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 8953 case 0: 8954 default: 8955 return 1; 8956 case 1: 8957 return 2; 8958 case 2: 8959 return 4; 8960 case 3: 8961 return 8; 8962 case 4: 8963 return 3; 8964 case 5: 8965 return 6; 8966 case 6: 8967 return 10; 8968 case 7: 8969 return 12; 8970 case 8: 8971 return 16; 8972 } 8973 } 8974 8975 struct dce8_wm_params { 8976 u32 dram_channels; /* number of dram channels */ 8977 u32 yclk; /* bandwidth per dram data pin in kHz */ 8978 u32 sclk; /* engine clock in kHz */ 8979 u32 disp_clk; /* display clock in kHz */ 8980 u32 src_width; /* viewport width */ 8981 u32 active_time; /* active display time in ns */ 8982 u32 blank_time; /* blank time in ns */ 8983 bool interlaced; /* mode is interlaced */ 8984 fixed20_12 vsc; /* vertical scale ratio */ 8985 u32 num_heads; /* number of active crtcs */ 8986 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 8987 u32 lb_size; /* line buffer allocated to pipe */ 8988 u32 vtaps; /* vertical scaler taps */ 8989 }; 8990 8991 /** 8992 * dce8_dram_bandwidth - get the dram bandwidth 8993 * 8994 * @wm: watermark calculation data 8995 * 8996 * Calculate the raw dram bandwidth (CIK). 8997 * Used for display watermark bandwidth calculations 8998 * Returns the dram bandwidth in MBytes/s 8999 */ 9000 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) 9001 { 9002 /* Calculate raw DRAM Bandwidth */ 9003 fixed20_12 dram_efficiency; /* 0.7 */ 9004 fixed20_12 yclk, dram_channels, bandwidth; 9005 fixed20_12 a; 9006 9007 a.full = dfixed_const(1000); 9008 yclk.full = dfixed_const(wm->yclk); 9009 yclk.full = dfixed_div(yclk, a); 9010 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9011 a.full = dfixed_const(10); 9012 dram_efficiency.full = dfixed_const(7); 9013 dram_efficiency.full = dfixed_div(dram_efficiency, a); 9014 bandwidth.full = dfixed_mul(dram_channels, yclk); 9015 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 9016 9017 return dfixed_trunc(bandwidth); 9018 } 9019 9020 /** 9021 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display 9022 * 9023 * @wm: watermark calculation data 9024 * 9025 * Calculate the dram bandwidth used for display (CIK). 9026 * Used for display watermark bandwidth calculations 9027 * Returns the dram bandwidth for display in MBytes/s 9028 */ 9029 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9030 { 9031 /* Calculate DRAM Bandwidth and the part allocated to display. */ 9032 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 9033 fixed20_12 yclk, dram_channels, bandwidth; 9034 fixed20_12 a; 9035 9036 a.full = dfixed_const(1000); 9037 yclk.full = dfixed_const(wm->yclk); 9038 yclk.full = dfixed_div(yclk, a); 9039 dram_channels.full = dfixed_const(wm->dram_channels * 4); 9040 a.full = dfixed_const(10); 9041 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 9042 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 9043 bandwidth.full = dfixed_mul(dram_channels, yclk); 9044 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 9045 9046 return dfixed_trunc(bandwidth); 9047 } 9048 9049 /** 9050 * dce8_data_return_bandwidth - get the data return bandwidth 9051 * 9052 * @wm: watermark calculation data 9053 * 9054 * Calculate the data return bandwidth used for display (CIK). 9055 * Used for display watermark bandwidth calculations 9056 * Returns the data return bandwidth in MBytes/s 9057 */ 9058 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) 9059 { 9060 /* Calculate the display Data return Bandwidth */ 9061 fixed20_12 return_efficiency; /* 0.8 */ 9062 fixed20_12 sclk, bandwidth; 9063 fixed20_12 a; 9064 9065 a.full = dfixed_const(1000); 9066 sclk.full = dfixed_const(wm->sclk); 9067 sclk.full = dfixed_div(sclk, a); 9068 a.full = dfixed_const(10); 9069 return_efficiency.full = dfixed_const(8); 9070 return_efficiency.full = dfixed_div(return_efficiency, a); 9071 a.full = dfixed_const(32); 9072 bandwidth.full = dfixed_mul(a, sclk); 9073 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 9074 9075 return dfixed_trunc(bandwidth); 9076 } 9077 9078 /** 9079 * dce8_dmif_request_bandwidth - get the dmif bandwidth 9080 * 9081 * @wm: watermark calculation data 9082 * 9083 * Calculate the dmif bandwidth used for display (CIK). 9084 * Used for display watermark bandwidth calculations 9085 * Returns the dmif bandwidth in MBytes/s 9086 */ 9087 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) 9088 { 9089 /* Calculate the DMIF Request Bandwidth */ 9090 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 9091 fixed20_12 disp_clk, bandwidth; 9092 fixed20_12 a, b; 9093 9094 a.full = dfixed_const(1000); 9095 disp_clk.full = dfixed_const(wm->disp_clk); 9096 disp_clk.full = dfixed_div(disp_clk, a); 9097 a.full = dfixed_const(32); 9098 b.full = dfixed_mul(a, disp_clk); 9099 9100 a.full = dfixed_const(10); 9101 disp_clk_request_efficiency.full = dfixed_const(8); 9102 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 9103 9104 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); 9105 9106 return dfixed_trunc(bandwidth); 9107 } 9108 9109 /** 9110 * dce8_available_bandwidth - get the min available bandwidth 9111 * 9112 * @wm: watermark calculation data 9113 * 9114 * Calculate the min available bandwidth used for display (CIK). 9115 * Used for display watermark bandwidth calculations 9116 * Returns the min available bandwidth in MBytes/s 9117 */ 9118 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) 9119 { 9120 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 9121 u32 dram_bandwidth = dce8_dram_bandwidth(wm); 9122 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); 9123 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); 9124 9125 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 9126 } 9127 9128 /** 9129 * dce8_average_bandwidth - get the average available bandwidth 9130 * 9131 * @wm: watermark calculation data 9132 * 9133 * Calculate the average available bandwidth used for display (CIK). 9134 * Used for display watermark bandwidth calculations 9135 * Returns the average available bandwidth in MBytes/s 9136 */ 9137 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) 9138 { 9139 /* Calculate the display mode Average Bandwidth 9140 * DisplayMode should contain the source and destination dimensions, 9141 * timing, etc. 9142 */ 9143 fixed20_12 bpp; 9144 fixed20_12 line_time; 9145 fixed20_12 src_width; 9146 fixed20_12 bandwidth; 9147 fixed20_12 a; 9148 9149 a.full = dfixed_const(1000); 9150 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 9151 line_time.full = dfixed_div(line_time, a); 9152 bpp.full = dfixed_const(wm->bytes_per_pixel); 9153 src_width.full = dfixed_const(wm->src_width); 9154 bandwidth.full = dfixed_mul(src_width, bpp); 9155 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 9156 bandwidth.full = dfixed_div(bandwidth, line_time); 9157 9158 return dfixed_trunc(bandwidth); 9159 } 9160 9161 /** 9162 * dce8_latency_watermark - get the latency watermark 9163 * 9164 * @wm: watermark calculation data 9165 * 9166 * Calculate the latency watermark (CIK). 9167 * Used for display watermark bandwidth calculations 9168 * Returns the latency watermark in ns 9169 */ 9170 static u32 dce8_latency_watermark(struct dce8_wm_params *wm) 9171 { 9172 /* First calculate the latency in ns */ 9173 u32 mc_latency = 2000; /* 2000 ns. */ 9174 u32 available_bandwidth = dce8_available_bandwidth(wm); 9175 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 9176 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 9177 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 9178 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 9179 (wm->num_heads * cursor_line_pair_return_time); 9180 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 9181 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 9182 u32 tmp, dmif_size = 12288; 9183 fixed20_12 a, b, c; 9184 9185 if (wm->num_heads == 0) 9186 return 0; 9187 9188 a.full = dfixed_const(2); 9189 b.full = dfixed_const(1); 9190 if ((wm->vsc.full > a.full) || 9191 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 9192 (wm->vtaps >= 5) || 9193 ((wm->vsc.full >= a.full) && wm->interlaced)) 9194 max_src_lines_per_dst_line = 4; 9195 else 9196 max_src_lines_per_dst_line = 2; 9197 9198 a.full = dfixed_const(available_bandwidth); 9199 b.full = dfixed_const(wm->num_heads); 9200 a.full = dfixed_div(a, b); 9201 9202 b.full = dfixed_const(mc_latency + 512); 9203 c.full = dfixed_const(wm->disp_clk); 9204 b.full = dfixed_div(b, c); 9205 9206 c.full = dfixed_const(dmif_size); 9207 b.full = dfixed_div(c, b); 9208 9209 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 9210 9211 b.full = dfixed_const(1000); 9212 c.full = dfixed_const(wm->disp_clk); 9213 b.full = dfixed_div(c, b); 9214 c.full = dfixed_const(wm->bytes_per_pixel); 9215 b.full = dfixed_mul(b, c); 9216 9217 lb_fill_bw = min(tmp, dfixed_trunc(b)); 9218 9219 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 9220 b.full = dfixed_const(1000); 9221 c.full = dfixed_const(lb_fill_bw); 9222 b.full = dfixed_div(c, b); 9223 a.full = dfixed_div(a, b); 9224 line_fill_time = dfixed_trunc(a); 9225 9226 if (line_fill_time < wm->active_time) 9227 return latency; 9228 else 9229 return latency + (line_fill_time - wm->active_time); 9230 9231 } 9232 9233 /** 9234 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check 9235 * average and available dram bandwidth 9236 * 9237 * @wm: watermark calculation data 9238 * 9239 * Check if the display average bandwidth fits in the display 9240 * dram bandwidth (CIK). 9241 * Used for display watermark bandwidth calculations 9242 * Returns true if the display fits, false if not. 9243 */ 9244 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) 9245 { 9246 if (dce8_average_bandwidth(wm) <= 9247 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) 9248 return true; 9249 else 9250 return false; 9251 } 9252 9253 /** 9254 * dce8_average_bandwidth_vs_available_bandwidth - check 9255 * average and available bandwidth 9256 * 9257 * @wm: watermark calculation data 9258 * 9259 * Check if the display average bandwidth fits in the display 9260 * available bandwidth (CIK). 9261 * Used for display watermark bandwidth calculations 9262 * Returns true if the display fits, false if not. 9263 */ 9264 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) 9265 { 9266 if (dce8_average_bandwidth(wm) <= 9267 (dce8_available_bandwidth(wm) / wm->num_heads)) 9268 return true; 9269 else 9270 return false; 9271 } 9272 9273 /** 9274 * dce8_check_latency_hiding - check latency hiding 9275 * 9276 * @wm: watermark calculation data 9277 * 9278 * Check latency hiding (CIK). 9279 * Used for display watermark bandwidth calculations 9280 * Returns true if the display fits, false if not. 9281 */ 9282 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) 9283 { 9284 u32 lb_partitions = wm->lb_size / wm->src_width; 9285 u32 line_time = wm->active_time + wm->blank_time; 9286 u32 latency_tolerant_lines; 9287 u32 latency_hiding; 9288 fixed20_12 a; 9289 9290 a.full = dfixed_const(1); 9291 if (wm->vsc.full > a.full) 9292 latency_tolerant_lines = 1; 9293 else { 9294 if (lb_partitions <= (wm->vtaps + 1)) 9295 latency_tolerant_lines = 1; 9296 else 9297 latency_tolerant_lines = 2; 9298 } 9299 9300 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 9301 9302 if (dce8_latency_watermark(wm) <= latency_hiding) 9303 return true; 9304 else 9305 return false; 9306 } 9307 9308 /** 9309 * dce8_program_watermarks - program display watermarks 9310 * 9311 * @rdev: radeon_device pointer 9312 * @radeon_crtc: the selected display controller 9313 * @lb_size: line buffer size 9314 * @num_heads: number of display controllers in use 9315 * 9316 * Calculate and program the display watermarks for the 9317 * selected display controller (CIK). 9318 */ 9319 static void dce8_program_watermarks(struct radeon_device *rdev, 9320 struct radeon_crtc *radeon_crtc, 9321 u32 lb_size, u32 num_heads) 9322 { 9323 struct drm_display_mode *mode = &radeon_crtc->base.mode; 9324 struct dce8_wm_params wm_low, wm_high; 9325 u32 pixel_period; 9326 u32 line_time = 0; 9327 u32 latency_watermark_a = 0, latency_watermark_b = 0; 9328 u32 tmp, wm_mask; 9329 9330 if (radeon_crtc->base.enabled && num_heads && mode) { 9331 pixel_period = 1000000 / (u32)mode->clock; 9332 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 9333 9334 /* watermark for high clocks */ 9335 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9336 rdev->pm.dpm_enabled) { 9337 wm_high.yclk = 9338 radeon_dpm_get_mclk(rdev, false) * 10; 9339 wm_high.sclk = 9340 radeon_dpm_get_sclk(rdev, false) * 10; 9341 } else { 9342 wm_high.yclk = rdev->pm.current_mclk * 10; 9343 wm_high.sclk = rdev->pm.current_sclk * 10; 9344 } 9345 9346 wm_high.disp_clk = mode->clock; 9347 wm_high.src_width = mode->crtc_hdisplay; 9348 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 9349 wm_high.blank_time = line_time - wm_high.active_time; 9350 wm_high.interlaced = false; 9351 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9352 wm_high.interlaced = true; 9353 wm_high.vsc = radeon_crtc->vsc; 9354 wm_high.vtaps = 1; 9355 if (radeon_crtc->rmx_type != RMX_OFF) 9356 wm_high.vtaps = 2; 9357 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9358 wm_high.lb_size = lb_size; 9359 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); 9360 wm_high.num_heads = num_heads; 9361 9362 /* set for high clocks */ 9363 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); 9364 9365 /* possibly force display priority to high */ 9366 /* should really do this at mode validation time... */ 9367 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 9368 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || 9369 !dce8_check_latency_hiding(&wm_high) || 9370 (rdev->disp_priority == 2)) { 9371 DRM_DEBUG_KMS("force priority to high\n"); 9372 } 9373 9374 /* watermark for low clocks */ 9375 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 9376 rdev->pm.dpm_enabled) { 9377 wm_low.yclk = 9378 radeon_dpm_get_mclk(rdev, true) * 10; 9379 wm_low.sclk = 9380 radeon_dpm_get_sclk(rdev, true) * 10; 9381 } else { 9382 wm_low.yclk = rdev->pm.current_mclk * 10; 9383 wm_low.sclk = rdev->pm.current_sclk * 10; 9384 } 9385 9386 wm_low.disp_clk = mode->clock; 9387 wm_low.src_width = mode->crtc_hdisplay; 9388 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 9389 wm_low.blank_time = line_time - wm_low.active_time; 9390 wm_low.interlaced = false; 9391 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 9392 wm_low.interlaced = true; 9393 wm_low.vsc = radeon_crtc->vsc; 9394 wm_low.vtaps = 1; 9395 if (radeon_crtc->rmx_type != RMX_OFF) 9396 wm_low.vtaps = 2; 9397 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 9398 wm_low.lb_size = lb_size; 9399 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); 9400 wm_low.num_heads = num_heads; 9401 9402 /* set for low clocks */ 9403 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); 9404 9405 /* possibly force display priority to high */ 9406 /* should really do this at mode validation time... */ 9407 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 9408 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || 9409 !dce8_check_latency_hiding(&wm_low) || 9410 (rdev->disp_priority == 2)) { 9411 DRM_DEBUG_KMS("force priority to high\n"); 9412 } 9413 } 9414 9415 /* select wm A */ 9416 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9417 tmp = wm_mask; 9418 tmp &= ~LATENCY_WATERMARK_MASK(3); 9419 tmp |= LATENCY_WATERMARK_MASK(1); 9420 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9421 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9422 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 9423 LATENCY_HIGH_WATERMARK(line_time))); 9424 /* select wm B */ 9425 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 9426 tmp &= ~LATENCY_WATERMARK_MASK(3); 9427 tmp |= LATENCY_WATERMARK_MASK(2); 9428 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 9429 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 9430 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 9431 LATENCY_HIGH_WATERMARK(line_time))); 9432 /* restore original selection */ 9433 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); 9434 9435 /* save values for DPM */ 9436 radeon_crtc->line_time = line_time; 9437 radeon_crtc->wm_high = latency_watermark_a; 9438 radeon_crtc->wm_low = latency_watermark_b; 9439 } 9440 9441 /** 9442 * dce8_bandwidth_update - program display watermarks 9443 * 9444 * @rdev: radeon_device pointer 9445 * 9446 * Calculate and program the display watermarks and line 9447 * buffer allocation (CIK). 9448 */ 9449 void dce8_bandwidth_update(struct radeon_device *rdev) 9450 { 9451 struct drm_display_mode *mode = NULL; 9452 u32 num_heads = 0, lb_size; 9453 int i; 9454 9455 if (!rdev->mode_info.mode_config_initialized) 9456 return; 9457 9458 radeon_update_display_priority(rdev); 9459 9460 for (i = 0; i < rdev->num_crtc; i++) { 9461 if (rdev->mode_info.crtcs[i]->base.enabled) 9462 num_heads++; 9463 } 9464 for (i = 0; i < rdev->num_crtc; i++) { 9465 mode = &rdev->mode_info.crtcs[i]->base.mode; 9466 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); 9467 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 9468 } 9469 } 9470 9471 /** 9472 * cik_get_gpu_clock_counter - return GPU clock counter snapshot 9473 * 9474 * @rdev: radeon_device pointer 9475 * 9476 * Fetches a GPU clock counter snapshot (SI). 9477 * Returns the 64 bit clock counter snapshot. 9478 */ 9479 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) 9480 { 9481 uint64_t clock; 9482 9483 mutex_lock(&rdev->gpu_clock_mutex); 9484 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 9485 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 9486 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 9487 mutex_unlock(&rdev->gpu_clock_mutex); 9488 return clock; 9489 } 9490 9491 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, 9492 u32 cntl_reg, u32 status_reg) 9493 { 9494 int r, i; 9495 struct atom_clock_dividers dividers; 9496 uint32_t tmp; 9497 9498 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9499 clock, false, ÷rs); 9500 if (r) 9501 return r; 9502 9503 tmp = RREG32_SMC(cntl_reg); 9504 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); 9505 tmp |= dividers.post_divider; 9506 WREG32_SMC(cntl_reg, tmp); 9507 9508 for (i = 0; i < 100; i++) { 9509 if (RREG32_SMC(status_reg) & DCLK_STATUS) 9510 break; 9511 mdelay(10); 9512 } 9513 if (i == 100) 9514 return -ETIMEDOUT; 9515 9516 return 0; 9517 } 9518 9519 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 9520 { 9521 int r = 0; 9522 9523 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 9524 if (r) 9525 return r; 9526 9527 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 9528 return r; 9529 } 9530 9531 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) 9532 { 9533 int r, i; 9534 struct atom_clock_dividers dividers; 9535 u32 tmp; 9536 9537 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 9538 ecclk, false, ÷rs); 9539 if (r) 9540 return r; 9541 9542 for (i = 0; i < 100; i++) { 9543 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9544 break; 9545 mdelay(10); 9546 } 9547 if (i == 100) 9548 return -ETIMEDOUT; 9549 9550 tmp = RREG32_SMC(CG_ECLK_CNTL); 9551 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); 9552 tmp |= dividers.post_divider; 9553 WREG32_SMC(CG_ECLK_CNTL, tmp); 9554 9555 for (i = 0; i < 100; i++) { 9556 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 9557 break; 9558 mdelay(10); 9559 } 9560 if (i == 100) 9561 return -ETIMEDOUT; 9562 9563 return 0; 9564 } 9565 9566 static void cik_pcie_gen3_enable(struct radeon_device *rdev) 9567 { 9568 struct pci_dev *root = rdev->pdev->bus->self; 9569 int bridge_pos, gpu_pos; 9570 u32 speed_cntl, mask, current_data_rate; 9571 int ret, i; 9572 u16 tmp16; 9573 9574 if (pci_is_root_bus(rdev->pdev->bus)) 9575 return; 9576 9577 if (radeon_pcie_gen2 == 0) 9578 return; 9579 9580 if (rdev->flags & RADEON_IS_IGP) 9581 return; 9582 9583 if (!(rdev->flags & RADEON_IS_PCIE)) 9584 return; 9585 9586 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 9587 if (ret != 0) 9588 return; 9589 9590 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 9591 return; 9592 9593 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9594 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 9595 LC_CURRENT_DATA_RATE_SHIFT; 9596 if (mask & DRM_PCIE_SPEED_80) { 9597 if (current_data_rate == 2) { 9598 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 9599 return; 9600 } 9601 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 9602 } else if (mask & DRM_PCIE_SPEED_50) { 9603 if (current_data_rate == 1) { 9604 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 9605 return; 9606 } 9607 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 9608 } 9609 9610 bridge_pos = pci_pcie_cap(root); 9611 if (!bridge_pos) 9612 return; 9613 9614 gpu_pos = pci_pcie_cap(rdev->pdev); 9615 if (!gpu_pos) 9616 return; 9617 9618 if (mask & DRM_PCIE_SPEED_80) { 9619 /* re-try equalization if gen3 is not already enabled */ 9620 if (current_data_rate != 2) { 9621 u16 bridge_cfg, gpu_cfg; 9622 u16 bridge_cfg2, gpu_cfg2; 9623 u32 max_lw, current_lw, tmp; 9624 9625 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9626 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9627 9628 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 9629 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9630 9631 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 9632 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9633 9634 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9635 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 9636 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 9637 9638 if (current_lw < max_lw) { 9639 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9640 if (tmp & LC_RENEGOTIATION_SUPPORT) { 9641 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 9642 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 9643 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 9644 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 9645 } 9646 } 9647 9648 for (i = 0; i < 10; i++) { 9649 /* check status */ 9650 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 9651 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 9652 break; 9653 9654 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9655 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9656 9657 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 9658 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 9659 9660 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9661 tmp |= LC_SET_QUIESCE; 9662 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9663 9664 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9665 tmp |= LC_REDO_EQ; 9666 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9667 9668 mdelay(100); 9669 9670 /* linkctl */ 9671 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 9672 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9673 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 9674 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9675 9676 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 9677 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9678 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 9679 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9680 9681 /* linkctl2 */ 9682 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 9683 tmp16 &= ~((1 << 4) | (7 << 9)); 9684 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 9685 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 9686 9687 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9688 tmp16 &= ~((1 << 4) | (7 << 9)); 9689 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 9690 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9691 9692 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9693 tmp &= ~LC_SET_QUIESCE; 9694 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9695 } 9696 } 9697 } 9698 9699 /* set the link speed */ 9700 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 9701 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 9702 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9703 9704 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9705 tmp16 &= ~0xf; 9706 if (mask & DRM_PCIE_SPEED_80) 9707 tmp16 |= 3; /* gen3 */ 9708 else if (mask & DRM_PCIE_SPEED_50) 9709 tmp16 |= 2; /* gen2 */ 9710 else 9711 tmp16 |= 1; /* gen1 */ 9712 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9713 9714 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9715 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 9716 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9717 9718 for (i = 0; i < rdev->usec_timeout; i++) { 9719 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9720 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 9721 break; 9722 udelay(1); 9723 } 9724 } 9725 9726 static void cik_program_aspm(struct radeon_device *rdev) 9727 { 9728 u32 data, orig; 9729 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 9730 bool disable_clkreq = false; 9731 9732 if (radeon_aspm == 0) 9733 return; 9734 9735 /* XXX double check IGPs */ 9736 if (rdev->flags & RADEON_IS_IGP) 9737 return; 9738 9739 if (!(rdev->flags & RADEON_IS_PCIE)) 9740 return; 9741 9742 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9743 data &= ~LC_XMIT_N_FTS_MASK; 9744 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 9745 if (orig != data) 9746 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 9747 9748 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 9749 data |= LC_GO_TO_RECOVERY; 9750 if (orig != data) 9751 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 9752 9753 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); 9754 data |= P_IGNORE_EDB_ERR; 9755 if (orig != data) 9756 WREG32_PCIE_PORT(PCIE_P_CNTL, data); 9757 9758 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9759 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 9760 data |= LC_PMI_TO_L1_DIS; 9761 if (!disable_l0s) 9762 data |= LC_L0S_INACTIVITY(7); 9763 9764 if (!disable_l1) { 9765 data |= LC_L1_INACTIVITY(7); 9766 data &= ~LC_PMI_TO_L1_DIS; 9767 if (orig != data) 9768 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9769 9770 if (!disable_plloff_in_l1) { 9771 bool clk_req_support; 9772 9773 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); 9774 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9775 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9776 if (orig != data) 9777 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); 9778 9779 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); 9780 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9781 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9782 if (orig != data) 9783 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); 9784 9785 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); 9786 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9787 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9788 if (orig != data) 9789 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); 9790 9791 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); 9792 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9793 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9794 if (orig != data) 9795 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); 9796 9797 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9798 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 9799 data |= LC_DYN_LANES_PWR_STATE(3); 9800 if (orig != data) 9801 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 9802 9803 if (!disable_clkreq && 9804 !pci_is_root_bus(rdev->pdev->bus)) { 9805 struct pci_dev *root = rdev->pdev->bus->self; 9806 u32 lnkcap; 9807 9808 clk_req_support = false; 9809 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 9810 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 9811 clk_req_support = true; 9812 } else { 9813 clk_req_support = false; 9814 } 9815 9816 if (clk_req_support) { 9817 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 9818 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 9819 if (orig != data) 9820 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 9821 9822 orig = data = RREG32_SMC(THM_CLK_CNTL); 9823 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 9824 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 9825 if (orig != data) 9826 WREG32_SMC(THM_CLK_CNTL, data); 9827 9828 orig = data = RREG32_SMC(MISC_CLK_CTRL); 9829 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 9830 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 9831 if (orig != data) 9832 WREG32_SMC(MISC_CLK_CTRL, data); 9833 9834 orig = data = RREG32_SMC(CG_CLKPIN_CNTL); 9835 data &= ~BCLK_AS_XCLK; 9836 if (orig != data) 9837 WREG32_SMC(CG_CLKPIN_CNTL, data); 9838 9839 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); 9840 data &= ~FORCE_BIF_REFCLK_EN; 9841 if (orig != data) 9842 WREG32_SMC(CG_CLKPIN_CNTL_2, data); 9843 9844 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); 9845 data &= ~MPLL_CLKOUT_SEL_MASK; 9846 data |= MPLL_CLKOUT_SEL(4); 9847 if (orig != data) 9848 WREG32_SMC(MPLL_BYPASSCLK_SEL, data); 9849 } 9850 } 9851 } else { 9852 if (orig != data) 9853 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9854 } 9855 9856 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 9857 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 9858 if (orig != data) 9859 WREG32_PCIE_PORT(PCIE_CNTL2, data); 9860 9861 if (!disable_l0s) { 9862 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9863 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 9864 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9865 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 9866 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9867 data &= ~LC_L0S_INACTIVITY_MASK; 9868 if (orig != data) 9869 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9870 } 9871 } 9872 } 9873 } 9874