1 /* 2 * Copyright 2012 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include "drmP.h" 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include "cikd.h" 31 #include "atom.h" 32 #include "cik_blit_shaders.h" 33 #include "radeon_ucode.h" 34 #include "clearstate_ci.h" 35 36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin"); 37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin"); 38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin"); 39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin"); 40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin"); 41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin"); 42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin"); 43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin"); 44 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin"); 45 MODULE_FIRMWARE("radeon/HAWAII_me.bin"); 46 MODULE_FIRMWARE("radeon/HAWAII_ce.bin"); 47 MODULE_FIRMWARE("radeon/HAWAII_mec.bin"); 48 MODULE_FIRMWARE("radeon/HAWAII_mc.bin"); 49 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin"); 50 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin"); 51 MODULE_FIRMWARE("radeon/HAWAII_smc.bin"); 52 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin"); 53 MODULE_FIRMWARE("radeon/KAVERI_me.bin"); 54 MODULE_FIRMWARE("radeon/KAVERI_ce.bin"); 55 MODULE_FIRMWARE("radeon/KAVERI_mec.bin"); 56 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin"); 57 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin"); 58 MODULE_FIRMWARE("radeon/KABINI_pfp.bin"); 59 MODULE_FIRMWARE("radeon/KABINI_me.bin"); 60 MODULE_FIRMWARE("radeon/KABINI_ce.bin"); 61 MODULE_FIRMWARE("radeon/KABINI_mec.bin"); 62 MODULE_FIRMWARE("radeon/KABINI_rlc.bin"); 63 MODULE_FIRMWARE("radeon/KABINI_sdma.bin"); 64 65 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 66 extern void r600_ih_ring_fini(struct radeon_device *rdev); 67 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 68 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 69 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 70 extern void sumo_rlc_fini(struct radeon_device *rdev); 71 extern int sumo_rlc_init(struct radeon_device *rdev); 72 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc); 73 extern void si_rlc_reset(struct radeon_device *rdev); 74 extern void si_init_uvd_internal_cg(struct radeon_device *rdev); 75 extern int cik_sdma_resume(struct radeon_device *rdev); 76 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable); 77 extern void cik_sdma_fini(struct radeon_device *rdev); 78 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable); 79 static void cik_rlc_stop(struct radeon_device *rdev); 80 static void cik_pcie_gen3_enable(struct radeon_device *rdev); 81 static void cik_program_aspm(struct radeon_device *rdev); 82 static void cik_init_pg(struct radeon_device *rdev); 83 static void cik_init_cg(struct radeon_device *rdev); 84 static void cik_fini_pg(struct radeon_device *rdev); 85 static void cik_fini_cg(struct radeon_device *rdev); 86 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 87 bool enable); 88 89 /* get temperature in millidegrees */ 90 int ci_get_temp(struct radeon_device *rdev) 91 { 92 u32 temp; 93 int actual_temp = 0; 94 95 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 96 CTF_TEMP_SHIFT; 97 98 if (temp & 0x200) 99 actual_temp = 255; 100 else 101 actual_temp = temp & 0x1ff; 102 103 actual_temp = actual_temp * 1000; 104 105 return actual_temp; 106 } 107 108 /* get temperature in millidegrees */ 109 int kv_get_temp(struct radeon_device *rdev) 110 { 111 u32 temp; 112 int actual_temp = 0; 113 114 temp = RREG32_SMC(0xC0300E0C); 115 116 if (temp) 117 actual_temp = (temp / 8) - 49; 118 else 119 actual_temp = 0; 120 121 actual_temp = actual_temp * 1000; 122 123 return actual_temp; 124 } 125 126 /* 127 * Indirect registers accessor 128 */ 129 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) 130 { 131 unsigned long flags; 132 u32 r; 133 134 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 135 WREG32(PCIE_INDEX, reg); 136 (void)RREG32(PCIE_INDEX); 137 r = RREG32(PCIE_DATA); 138 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 139 return r; 140 } 141 142 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) 143 { 144 unsigned long flags; 145 146 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); 147 WREG32(PCIE_INDEX, reg); 148 (void)RREG32(PCIE_INDEX); 149 WREG32(PCIE_DATA, v); 150 (void)RREG32(PCIE_DATA); 151 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); 152 } 153 154 static const u32 spectre_rlc_save_restore_register_list[] = 155 { 156 (0x0e00 << 16) | (0xc12c >> 2), 157 0x00000000, 158 (0x0e00 << 16) | (0xc140 >> 2), 159 0x00000000, 160 (0x0e00 << 16) | (0xc150 >> 2), 161 0x00000000, 162 (0x0e00 << 16) | (0xc15c >> 2), 163 0x00000000, 164 (0x0e00 << 16) | (0xc168 >> 2), 165 0x00000000, 166 (0x0e00 << 16) | (0xc170 >> 2), 167 0x00000000, 168 (0x0e00 << 16) | (0xc178 >> 2), 169 0x00000000, 170 (0x0e00 << 16) | (0xc204 >> 2), 171 0x00000000, 172 (0x0e00 << 16) | (0xc2b4 >> 2), 173 0x00000000, 174 (0x0e00 << 16) | (0xc2b8 >> 2), 175 0x00000000, 176 (0x0e00 << 16) | (0xc2bc >> 2), 177 0x00000000, 178 (0x0e00 << 16) | (0xc2c0 >> 2), 179 0x00000000, 180 (0x0e00 << 16) | (0x8228 >> 2), 181 0x00000000, 182 (0x0e00 << 16) | (0x829c >> 2), 183 0x00000000, 184 (0x0e00 << 16) | (0x869c >> 2), 185 0x00000000, 186 (0x0600 << 16) | (0x98f4 >> 2), 187 0x00000000, 188 (0x0e00 << 16) | (0x98f8 >> 2), 189 0x00000000, 190 (0x0e00 << 16) | (0x9900 >> 2), 191 0x00000000, 192 (0x0e00 << 16) | (0xc260 >> 2), 193 0x00000000, 194 (0x0e00 << 16) | (0x90e8 >> 2), 195 0x00000000, 196 (0x0e00 << 16) | (0x3c000 >> 2), 197 0x00000000, 198 (0x0e00 << 16) | (0x3c00c >> 2), 199 0x00000000, 200 (0x0e00 << 16) | (0x8c1c >> 2), 201 0x00000000, 202 (0x0e00 << 16) | (0x9700 >> 2), 203 0x00000000, 204 (0x0e00 << 16) | (0xcd20 >> 2), 205 0x00000000, 206 (0x4e00 << 16) | (0xcd20 >> 2), 207 0x00000000, 208 (0x5e00 << 16) | (0xcd20 >> 2), 209 0x00000000, 210 (0x6e00 << 16) | (0xcd20 >> 2), 211 0x00000000, 212 (0x7e00 << 16) | (0xcd20 >> 2), 213 0x00000000, 214 (0x8e00 << 16) | (0xcd20 >> 2), 215 0x00000000, 216 (0x9e00 << 16) | (0xcd20 >> 2), 217 0x00000000, 218 (0xae00 << 16) | (0xcd20 >> 2), 219 0x00000000, 220 (0xbe00 << 16) | (0xcd20 >> 2), 221 0x00000000, 222 (0x0e00 << 16) | (0x89bc >> 2), 223 0x00000000, 224 (0x0e00 << 16) | (0x8900 >> 2), 225 0x00000000, 226 0x3, 227 (0x0e00 << 16) | (0xc130 >> 2), 228 0x00000000, 229 (0x0e00 << 16) | (0xc134 >> 2), 230 0x00000000, 231 (0x0e00 << 16) | (0xc1fc >> 2), 232 0x00000000, 233 (0x0e00 << 16) | (0xc208 >> 2), 234 0x00000000, 235 (0x0e00 << 16) | (0xc264 >> 2), 236 0x00000000, 237 (0x0e00 << 16) | (0xc268 >> 2), 238 0x00000000, 239 (0x0e00 << 16) | (0xc26c >> 2), 240 0x00000000, 241 (0x0e00 << 16) | (0xc270 >> 2), 242 0x00000000, 243 (0x0e00 << 16) | (0xc274 >> 2), 244 0x00000000, 245 (0x0e00 << 16) | (0xc278 >> 2), 246 0x00000000, 247 (0x0e00 << 16) | (0xc27c >> 2), 248 0x00000000, 249 (0x0e00 << 16) | (0xc280 >> 2), 250 0x00000000, 251 (0x0e00 << 16) | (0xc284 >> 2), 252 0x00000000, 253 (0x0e00 << 16) | (0xc288 >> 2), 254 0x00000000, 255 (0x0e00 << 16) | (0xc28c >> 2), 256 0x00000000, 257 (0x0e00 << 16) | (0xc290 >> 2), 258 0x00000000, 259 (0x0e00 << 16) | (0xc294 >> 2), 260 0x00000000, 261 (0x0e00 << 16) | (0xc298 >> 2), 262 0x00000000, 263 (0x0e00 << 16) | (0xc29c >> 2), 264 0x00000000, 265 (0x0e00 << 16) | (0xc2a0 >> 2), 266 0x00000000, 267 (0x0e00 << 16) | (0xc2a4 >> 2), 268 0x00000000, 269 (0x0e00 << 16) | (0xc2a8 >> 2), 270 0x00000000, 271 (0x0e00 << 16) | (0xc2ac >> 2), 272 0x00000000, 273 (0x0e00 << 16) | (0xc2b0 >> 2), 274 0x00000000, 275 (0x0e00 << 16) | (0x301d0 >> 2), 276 0x00000000, 277 (0x0e00 << 16) | (0x30238 >> 2), 278 0x00000000, 279 (0x0e00 << 16) | (0x30250 >> 2), 280 0x00000000, 281 (0x0e00 << 16) | (0x30254 >> 2), 282 0x00000000, 283 (0x0e00 << 16) | (0x30258 >> 2), 284 0x00000000, 285 (0x0e00 << 16) | (0x3025c >> 2), 286 0x00000000, 287 (0x4e00 << 16) | (0xc900 >> 2), 288 0x00000000, 289 (0x5e00 << 16) | (0xc900 >> 2), 290 0x00000000, 291 (0x6e00 << 16) | (0xc900 >> 2), 292 0x00000000, 293 (0x7e00 << 16) | (0xc900 >> 2), 294 0x00000000, 295 (0x8e00 << 16) | (0xc900 >> 2), 296 0x00000000, 297 (0x9e00 << 16) | (0xc900 >> 2), 298 0x00000000, 299 (0xae00 << 16) | (0xc900 >> 2), 300 0x00000000, 301 (0xbe00 << 16) | (0xc900 >> 2), 302 0x00000000, 303 (0x4e00 << 16) | (0xc904 >> 2), 304 0x00000000, 305 (0x5e00 << 16) | (0xc904 >> 2), 306 0x00000000, 307 (0x6e00 << 16) | (0xc904 >> 2), 308 0x00000000, 309 (0x7e00 << 16) | (0xc904 >> 2), 310 0x00000000, 311 (0x8e00 << 16) | (0xc904 >> 2), 312 0x00000000, 313 (0x9e00 << 16) | (0xc904 >> 2), 314 0x00000000, 315 (0xae00 << 16) | (0xc904 >> 2), 316 0x00000000, 317 (0xbe00 << 16) | (0xc904 >> 2), 318 0x00000000, 319 (0x4e00 << 16) | (0xc908 >> 2), 320 0x00000000, 321 (0x5e00 << 16) | (0xc908 >> 2), 322 0x00000000, 323 (0x6e00 << 16) | (0xc908 >> 2), 324 0x00000000, 325 (0x7e00 << 16) | (0xc908 >> 2), 326 0x00000000, 327 (0x8e00 << 16) | (0xc908 >> 2), 328 0x00000000, 329 (0x9e00 << 16) | (0xc908 >> 2), 330 0x00000000, 331 (0xae00 << 16) | (0xc908 >> 2), 332 0x00000000, 333 (0xbe00 << 16) | (0xc908 >> 2), 334 0x00000000, 335 (0x4e00 << 16) | (0xc90c >> 2), 336 0x00000000, 337 (0x5e00 << 16) | (0xc90c >> 2), 338 0x00000000, 339 (0x6e00 << 16) | (0xc90c >> 2), 340 0x00000000, 341 (0x7e00 << 16) | (0xc90c >> 2), 342 0x00000000, 343 (0x8e00 << 16) | (0xc90c >> 2), 344 0x00000000, 345 (0x9e00 << 16) | (0xc90c >> 2), 346 0x00000000, 347 (0xae00 << 16) | (0xc90c >> 2), 348 0x00000000, 349 (0xbe00 << 16) | (0xc90c >> 2), 350 0x00000000, 351 (0x4e00 << 16) | (0xc910 >> 2), 352 0x00000000, 353 (0x5e00 << 16) | (0xc910 >> 2), 354 0x00000000, 355 (0x6e00 << 16) | (0xc910 >> 2), 356 0x00000000, 357 (0x7e00 << 16) | (0xc910 >> 2), 358 0x00000000, 359 (0x8e00 << 16) | (0xc910 >> 2), 360 0x00000000, 361 (0x9e00 << 16) | (0xc910 >> 2), 362 0x00000000, 363 (0xae00 << 16) | (0xc910 >> 2), 364 0x00000000, 365 (0xbe00 << 16) | (0xc910 >> 2), 366 0x00000000, 367 (0x0e00 << 16) | (0xc99c >> 2), 368 0x00000000, 369 (0x0e00 << 16) | (0x9834 >> 2), 370 0x00000000, 371 (0x0000 << 16) | (0x30f00 >> 2), 372 0x00000000, 373 (0x0001 << 16) | (0x30f00 >> 2), 374 0x00000000, 375 (0x0000 << 16) | (0x30f04 >> 2), 376 0x00000000, 377 (0x0001 << 16) | (0x30f04 >> 2), 378 0x00000000, 379 (0x0000 << 16) | (0x30f08 >> 2), 380 0x00000000, 381 (0x0001 << 16) | (0x30f08 >> 2), 382 0x00000000, 383 (0x0000 << 16) | (0x30f0c >> 2), 384 0x00000000, 385 (0x0001 << 16) | (0x30f0c >> 2), 386 0x00000000, 387 (0x0600 << 16) | (0x9b7c >> 2), 388 0x00000000, 389 (0x0e00 << 16) | (0x8a14 >> 2), 390 0x00000000, 391 (0x0e00 << 16) | (0x8a18 >> 2), 392 0x00000000, 393 (0x0600 << 16) | (0x30a00 >> 2), 394 0x00000000, 395 (0x0e00 << 16) | (0x8bf0 >> 2), 396 0x00000000, 397 (0x0e00 << 16) | (0x8bcc >> 2), 398 0x00000000, 399 (0x0e00 << 16) | (0x8b24 >> 2), 400 0x00000000, 401 (0x0e00 << 16) | (0x30a04 >> 2), 402 0x00000000, 403 (0x0600 << 16) | (0x30a10 >> 2), 404 0x00000000, 405 (0x0600 << 16) | (0x30a14 >> 2), 406 0x00000000, 407 (0x0600 << 16) | (0x30a18 >> 2), 408 0x00000000, 409 (0x0600 << 16) | (0x30a2c >> 2), 410 0x00000000, 411 (0x0e00 << 16) | (0xc700 >> 2), 412 0x00000000, 413 (0x0e00 << 16) | (0xc704 >> 2), 414 0x00000000, 415 (0x0e00 << 16) | (0xc708 >> 2), 416 0x00000000, 417 (0x0e00 << 16) | (0xc768 >> 2), 418 0x00000000, 419 (0x0400 << 16) | (0xc770 >> 2), 420 0x00000000, 421 (0x0400 << 16) | (0xc774 >> 2), 422 0x00000000, 423 (0x0400 << 16) | (0xc778 >> 2), 424 0x00000000, 425 (0x0400 << 16) | (0xc77c >> 2), 426 0x00000000, 427 (0x0400 << 16) | (0xc780 >> 2), 428 0x00000000, 429 (0x0400 << 16) | (0xc784 >> 2), 430 0x00000000, 431 (0x0400 << 16) | (0xc788 >> 2), 432 0x00000000, 433 (0x0400 << 16) | (0xc78c >> 2), 434 0x00000000, 435 (0x0400 << 16) | (0xc798 >> 2), 436 0x00000000, 437 (0x0400 << 16) | (0xc79c >> 2), 438 0x00000000, 439 (0x0400 << 16) | (0xc7a0 >> 2), 440 0x00000000, 441 (0x0400 << 16) | (0xc7a4 >> 2), 442 0x00000000, 443 (0x0400 << 16) | (0xc7a8 >> 2), 444 0x00000000, 445 (0x0400 << 16) | (0xc7ac >> 2), 446 0x00000000, 447 (0x0400 << 16) | (0xc7b0 >> 2), 448 0x00000000, 449 (0x0400 << 16) | (0xc7b4 >> 2), 450 0x00000000, 451 (0x0e00 << 16) | (0x9100 >> 2), 452 0x00000000, 453 (0x0e00 << 16) | (0x3c010 >> 2), 454 0x00000000, 455 (0x0e00 << 16) | (0x92a8 >> 2), 456 0x00000000, 457 (0x0e00 << 16) | (0x92ac >> 2), 458 0x00000000, 459 (0x0e00 << 16) | (0x92b4 >> 2), 460 0x00000000, 461 (0x0e00 << 16) | (0x92b8 >> 2), 462 0x00000000, 463 (0x0e00 << 16) | (0x92bc >> 2), 464 0x00000000, 465 (0x0e00 << 16) | (0x92c0 >> 2), 466 0x00000000, 467 (0x0e00 << 16) | (0x92c4 >> 2), 468 0x00000000, 469 (0x0e00 << 16) | (0x92c8 >> 2), 470 0x00000000, 471 (0x0e00 << 16) | (0x92cc >> 2), 472 0x00000000, 473 (0x0e00 << 16) | (0x92d0 >> 2), 474 0x00000000, 475 (0x0e00 << 16) | (0x8c00 >> 2), 476 0x00000000, 477 (0x0e00 << 16) | (0x8c04 >> 2), 478 0x00000000, 479 (0x0e00 << 16) | (0x8c20 >> 2), 480 0x00000000, 481 (0x0e00 << 16) | (0x8c38 >> 2), 482 0x00000000, 483 (0x0e00 << 16) | (0x8c3c >> 2), 484 0x00000000, 485 (0x0e00 << 16) | (0xae00 >> 2), 486 0x00000000, 487 (0x0e00 << 16) | (0x9604 >> 2), 488 0x00000000, 489 (0x0e00 << 16) | (0xac08 >> 2), 490 0x00000000, 491 (0x0e00 << 16) | (0xac0c >> 2), 492 0x00000000, 493 (0x0e00 << 16) | (0xac10 >> 2), 494 0x00000000, 495 (0x0e00 << 16) | (0xac14 >> 2), 496 0x00000000, 497 (0x0e00 << 16) | (0xac58 >> 2), 498 0x00000000, 499 (0x0e00 << 16) | (0xac68 >> 2), 500 0x00000000, 501 (0x0e00 << 16) | (0xac6c >> 2), 502 0x00000000, 503 (0x0e00 << 16) | (0xac70 >> 2), 504 0x00000000, 505 (0x0e00 << 16) | (0xac74 >> 2), 506 0x00000000, 507 (0x0e00 << 16) | (0xac78 >> 2), 508 0x00000000, 509 (0x0e00 << 16) | (0xac7c >> 2), 510 0x00000000, 511 (0x0e00 << 16) | (0xac80 >> 2), 512 0x00000000, 513 (0x0e00 << 16) | (0xac84 >> 2), 514 0x00000000, 515 (0x0e00 << 16) | (0xac88 >> 2), 516 0x00000000, 517 (0x0e00 << 16) | (0xac8c >> 2), 518 0x00000000, 519 (0x0e00 << 16) | (0x970c >> 2), 520 0x00000000, 521 (0x0e00 << 16) | (0x9714 >> 2), 522 0x00000000, 523 (0x0e00 << 16) | (0x9718 >> 2), 524 0x00000000, 525 (0x0e00 << 16) | (0x971c >> 2), 526 0x00000000, 527 (0x0e00 << 16) | (0x31068 >> 2), 528 0x00000000, 529 (0x4e00 << 16) | (0x31068 >> 2), 530 0x00000000, 531 (0x5e00 << 16) | (0x31068 >> 2), 532 0x00000000, 533 (0x6e00 << 16) | (0x31068 >> 2), 534 0x00000000, 535 (0x7e00 << 16) | (0x31068 >> 2), 536 0x00000000, 537 (0x8e00 << 16) | (0x31068 >> 2), 538 0x00000000, 539 (0x9e00 << 16) | (0x31068 >> 2), 540 0x00000000, 541 (0xae00 << 16) | (0x31068 >> 2), 542 0x00000000, 543 (0xbe00 << 16) | (0x31068 >> 2), 544 0x00000000, 545 (0x0e00 << 16) | (0xcd10 >> 2), 546 0x00000000, 547 (0x0e00 << 16) | (0xcd14 >> 2), 548 0x00000000, 549 (0x0e00 << 16) | (0x88b0 >> 2), 550 0x00000000, 551 (0x0e00 << 16) | (0x88b4 >> 2), 552 0x00000000, 553 (0x0e00 << 16) | (0x88b8 >> 2), 554 0x00000000, 555 (0x0e00 << 16) | (0x88bc >> 2), 556 0x00000000, 557 (0x0400 << 16) | (0x89c0 >> 2), 558 0x00000000, 559 (0x0e00 << 16) | (0x88c4 >> 2), 560 0x00000000, 561 (0x0e00 << 16) | (0x88c8 >> 2), 562 0x00000000, 563 (0x0e00 << 16) | (0x88d0 >> 2), 564 0x00000000, 565 (0x0e00 << 16) | (0x88d4 >> 2), 566 0x00000000, 567 (0x0e00 << 16) | (0x88d8 >> 2), 568 0x00000000, 569 (0x0e00 << 16) | (0x8980 >> 2), 570 0x00000000, 571 (0x0e00 << 16) | (0x30938 >> 2), 572 0x00000000, 573 (0x0e00 << 16) | (0x3093c >> 2), 574 0x00000000, 575 (0x0e00 << 16) | (0x30940 >> 2), 576 0x00000000, 577 (0x0e00 << 16) | (0x89a0 >> 2), 578 0x00000000, 579 (0x0e00 << 16) | (0x30900 >> 2), 580 0x00000000, 581 (0x0e00 << 16) | (0x30904 >> 2), 582 0x00000000, 583 (0x0e00 << 16) | (0x89b4 >> 2), 584 0x00000000, 585 (0x0e00 << 16) | (0x3c210 >> 2), 586 0x00000000, 587 (0x0e00 << 16) | (0x3c214 >> 2), 588 0x00000000, 589 (0x0e00 << 16) | (0x3c218 >> 2), 590 0x00000000, 591 (0x0e00 << 16) | (0x8904 >> 2), 592 0x00000000, 593 0x5, 594 (0x0e00 << 16) | (0x8c28 >> 2), 595 (0x0e00 << 16) | (0x8c2c >> 2), 596 (0x0e00 << 16) | (0x8c30 >> 2), 597 (0x0e00 << 16) | (0x8c34 >> 2), 598 (0x0e00 << 16) | (0x9600 >> 2), 599 }; 600 601 static const u32 kalindi_rlc_save_restore_register_list[] = 602 { 603 (0x0e00 << 16) | (0xc12c >> 2), 604 0x00000000, 605 (0x0e00 << 16) | (0xc140 >> 2), 606 0x00000000, 607 (0x0e00 << 16) | (0xc150 >> 2), 608 0x00000000, 609 (0x0e00 << 16) | (0xc15c >> 2), 610 0x00000000, 611 (0x0e00 << 16) | (0xc168 >> 2), 612 0x00000000, 613 (0x0e00 << 16) | (0xc170 >> 2), 614 0x00000000, 615 (0x0e00 << 16) | (0xc204 >> 2), 616 0x00000000, 617 (0x0e00 << 16) | (0xc2b4 >> 2), 618 0x00000000, 619 (0x0e00 << 16) | (0xc2b8 >> 2), 620 0x00000000, 621 (0x0e00 << 16) | (0xc2bc >> 2), 622 0x00000000, 623 (0x0e00 << 16) | (0xc2c0 >> 2), 624 0x00000000, 625 (0x0e00 << 16) | (0x8228 >> 2), 626 0x00000000, 627 (0x0e00 << 16) | (0x829c >> 2), 628 0x00000000, 629 (0x0e00 << 16) | (0x869c >> 2), 630 0x00000000, 631 (0x0600 << 16) | (0x98f4 >> 2), 632 0x00000000, 633 (0x0e00 << 16) | (0x98f8 >> 2), 634 0x00000000, 635 (0x0e00 << 16) | (0x9900 >> 2), 636 0x00000000, 637 (0x0e00 << 16) | (0xc260 >> 2), 638 0x00000000, 639 (0x0e00 << 16) | (0x90e8 >> 2), 640 0x00000000, 641 (0x0e00 << 16) | (0x3c000 >> 2), 642 0x00000000, 643 (0x0e00 << 16) | (0x3c00c >> 2), 644 0x00000000, 645 (0x0e00 << 16) | (0x8c1c >> 2), 646 0x00000000, 647 (0x0e00 << 16) | (0x9700 >> 2), 648 0x00000000, 649 (0x0e00 << 16) | (0xcd20 >> 2), 650 0x00000000, 651 (0x4e00 << 16) | (0xcd20 >> 2), 652 0x00000000, 653 (0x5e00 << 16) | (0xcd20 >> 2), 654 0x00000000, 655 (0x6e00 << 16) | (0xcd20 >> 2), 656 0x00000000, 657 (0x7e00 << 16) | (0xcd20 >> 2), 658 0x00000000, 659 (0x0e00 << 16) | (0x89bc >> 2), 660 0x00000000, 661 (0x0e00 << 16) | (0x8900 >> 2), 662 0x00000000, 663 0x3, 664 (0x0e00 << 16) | (0xc130 >> 2), 665 0x00000000, 666 (0x0e00 << 16) | (0xc134 >> 2), 667 0x00000000, 668 (0x0e00 << 16) | (0xc1fc >> 2), 669 0x00000000, 670 (0x0e00 << 16) | (0xc208 >> 2), 671 0x00000000, 672 (0x0e00 << 16) | (0xc264 >> 2), 673 0x00000000, 674 (0x0e00 << 16) | (0xc268 >> 2), 675 0x00000000, 676 (0x0e00 << 16) | (0xc26c >> 2), 677 0x00000000, 678 (0x0e00 << 16) | (0xc270 >> 2), 679 0x00000000, 680 (0x0e00 << 16) | (0xc274 >> 2), 681 0x00000000, 682 (0x0e00 << 16) | (0xc28c >> 2), 683 0x00000000, 684 (0x0e00 << 16) | (0xc290 >> 2), 685 0x00000000, 686 (0x0e00 << 16) | (0xc294 >> 2), 687 0x00000000, 688 (0x0e00 << 16) | (0xc298 >> 2), 689 0x00000000, 690 (0x0e00 << 16) | (0xc2a0 >> 2), 691 0x00000000, 692 (0x0e00 << 16) | (0xc2a4 >> 2), 693 0x00000000, 694 (0x0e00 << 16) | (0xc2a8 >> 2), 695 0x00000000, 696 (0x0e00 << 16) | (0xc2ac >> 2), 697 0x00000000, 698 (0x0e00 << 16) | (0x301d0 >> 2), 699 0x00000000, 700 (0x0e00 << 16) | (0x30238 >> 2), 701 0x00000000, 702 (0x0e00 << 16) | (0x30250 >> 2), 703 0x00000000, 704 (0x0e00 << 16) | (0x30254 >> 2), 705 0x00000000, 706 (0x0e00 << 16) | (0x30258 >> 2), 707 0x00000000, 708 (0x0e00 << 16) | (0x3025c >> 2), 709 0x00000000, 710 (0x4e00 << 16) | (0xc900 >> 2), 711 0x00000000, 712 (0x5e00 << 16) | (0xc900 >> 2), 713 0x00000000, 714 (0x6e00 << 16) | (0xc900 >> 2), 715 0x00000000, 716 (0x7e00 << 16) | (0xc900 >> 2), 717 0x00000000, 718 (0x4e00 << 16) | (0xc904 >> 2), 719 0x00000000, 720 (0x5e00 << 16) | (0xc904 >> 2), 721 0x00000000, 722 (0x6e00 << 16) | (0xc904 >> 2), 723 0x00000000, 724 (0x7e00 << 16) | (0xc904 >> 2), 725 0x00000000, 726 (0x4e00 << 16) | (0xc908 >> 2), 727 0x00000000, 728 (0x5e00 << 16) | (0xc908 >> 2), 729 0x00000000, 730 (0x6e00 << 16) | (0xc908 >> 2), 731 0x00000000, 732 (0x7e00 << 16) | (0xc908 >> 2), 733 0x00000000, 734 (0x4e00 << 16) | (0xc90c >> 2), 735 0x00000000, 736 (0x5e00 << 16) | (0xc90c >> 2), 737 0x00000000, 738 (0x6e00 << 16) | (0xc90c >> 2), 739 0x00000000, 740 (0x7e00 << 16) | (0xc90c >> 2), 741 0x00000000, 742 (0x4e00 << 16) | (0xc910 >> 2), 743 0x00000000, 744 (0x5e00 << 16) | (0xc910 >> 2), 745 0x00000000, 746 (0x6e00 << 16) | (0xc910 >> 2), 747 0x00000000, 748 (0x7e00 << 16) | (0xc910 >> 2), 749 0x00000000, 750 (0x0e00 << 16) | (0xc99c >> 2), 751 0x00000000, 752 (0x0e00 << 16) | (0x9834 >> 2), 753 0x00000000, 754 (0x0000 << 16) | (0x30f00 >> 2), 755 0x00000000, 756 (0x0000 << 16) | (0x30f04 >> 2), 757 0x00000000, 758 (0x0000 << 16) | (0x30f08 >> 2), 759 0x00000000, 760 (0x0000 << 16) | (0x30f0c >> 2), 761 0x00000000, 762 (0x0600 << 16) | (0x9b7c >> 2), 763 0x00000000, 764 (0x0e00 << 16) | (0x8a14 >> 2), 765 0x00000000, 766 (0x0e00 << 16) | (0x8a18 >> 2), 767 0x00000000, 768 (0x0600 << 16) | (0x30a00 >> 2), 769 0x00000000, 770 (0x0e00 << 16) | (0x8bf0 >> 2), 771 0x00000000, 772 (0x0e00 << 16) | (0x8bcc >> 2), 773 0x00000000, 774 (0x0e00 << 16) | (0x8b24 >> 2), 775 0x00000000, 776 (0x0e00 << 16) | (0x30a04 >> 2), 777 0x00000000, 778 (0x0600 << 16) | (0x30a10 >> 2), 779 0x00000000, 780 (0x0600 << 16) | (0x30a14 >> 2), 781 0x00000000, 782 (0x0600 << 16) | (0x30a18 >> 2), 783 0x00000000, 784 (0x0600 << 16) | (0x30a2c >> 2), 785 0x00000000, 786 (0x0e00 << 16) | (0xc700 >> 2), 787 0x00000000, 788 (0x0e00 << 16) | (0xc704 >> 2), 789 0x00000000, 790 (0x0e00 << 16) | (0xc708 >> 2), 791 0x00000000, 792 (0x0e00 << 16) | (0xc768 >> 2), 793 0x00000000, 794 (0x0400 << 16) | (0xc770 >> 2), 795 0x00000000, 796 (0x0400 << 16) | (0xc774 >> 2), 797 0x00000000, 798 (0x0400 << 16) | (0xc798 >> 2), 799 0x00000000, 800 (0x0400 << 16) | (0xc79c >> 2), 801 0x00000000, 802 (0x0e00 << 16) | (0x9100 >> 2), 803 0x00000000, 804 (0x0e00 << 16) | (0x3c010 >> 2), 805 0x00000000, 806 (0x0e00 << 16) | (0x8c00 >> 2), 807 0x00000000, 808 (0x0e00 << 16) | (0x8c04 >> 2), 809 0x00000000, 810 (0x0e00 << 16) | (0x8c20 >> 2), 811 0x00000000, 812 (0x0e00 << 16) | (0x8c38 >> 2), 813 0x00000000, 814 (0x0e00 << 16) | (0x8c3c >> 2), 815 0x00000000, 816 (0x0e00 << 16) | (0xae00 >> 2), 817 0x00000000, 818 (0x0e00 << 16) | (0x9604 >> 2), 819 0x00000000, 820 (0x0e00 << 16) | (0xac08 >> 2), 821 0x00000000, 822 (0x0e00 << 16) | (0xac0c >> 2), 823 0x00000000, 824 (0x0e00 << 16) | (0xac10 >> 2), 825 0x00000000, 826 (0x0e00 << 16) | (0xac14 >> 2), 827 0x00000000, 828 (0x0e00 << 16) | (0xac58 >> 2), 829 0x00000000, 830 (0x0e00 << 16) | (0xac68 >> 2), 831 0x00000000, 832 (0x0e00 << 16) | (0xac6c >> 2), 833 0x00000000, 834 (0x0e00 << 16) | (0xac70 >> 2), 835 0x00000000, 836 (0x0e00 << 16) | (0xac74 >> 2), 837 0x00000000, 838 (0x0e00 << 16) | (0xac78 >> 2), 839 0x00000000, 840 (0x0e00 << 16) | (0xac7c >> 2), 841 0x00000000, 842 (0x0e00 << 16) | (0xac80 >> 2), 843 0x00000000, 844 (0x0e00 << 16) | (0xac84 >> 2), 845 0x00000000, 846 (0x0e00 << 16) | (0xac88 >> 2), 847 0x00000000, 848 (0x0e00 << 16) | (0xac8c >> 2), 849 0x00000000, 850 (0x0e00 << 16) | (0x970c >> 2), 851 0x00000000, 852 (0x0e00 << 16) | (0x9714 >> 2), 853 0x00000000, 854 (0x0e00 << 16) | (0x9718 >> 2), 855 0x00000000, 856 (0x0e00 << 16) | (0x971c >> 2), 857 0x00000000, 858 (0x0e00 << 16) | (0x31068 >> 2), 859 0x00000000, 860 (0x4e00 << 16) | (0x31068 >> 2), 861 0x00000000, 862 (0x5e00 << 16) | (0x31068 >> 2), 863 0x00000000, 864 (0x6e00 << 16) | (0x31068 >> 2), 865 0x00000000, 866 (0x7e00 << 16) | (0x31068 >> 2), 867 0x00000000, 868 (0x0e00 << 16) | (0xcd10 >> 2), 869 0x00000000, 870 (0x0e00 << 16) | (0xcd14 >> 2), 871 0x00000000, 872 (0x0e00 << 16) | (0x88b0 >> 2), 873 0x00000000, 874 (0x0e00 << 16) | (0x88b4 >> 2), 875 0x00000000, 876 (0x0e00 << 16) | (0x88b8 >> 2), 877 0x00000000, 878 (0x0e00 << 16) | (0x88bc >> 2), 879 0x00000000, 880 (0x0400 << 16) | (0x89c0 >> 2), 881 0x00000000, 882 (0x0e00 << 16) | (0x88c4 >> 2), 883 0x00000000, 884 (0x0e00 << 16) | (0x88c8 >> 2), 885 0x00000000, 886 (0x0e00 << 16) | (0x88d0 >> 2), 887 0x00000000, 888 (0x0e00 << 16) | (0x88d4 >> 2), 889 0x00000000, 890 (0x0e00 << 16) | (0x88d8 >> 2), 891 0x00000000, 892 (0x0e00 << 16) | (0x8980 >> 2), 893 0x00000000, 894 (0x0e00 << 16) | (0x30938 >> 2), 895 0x00000000, 896 (0x0e00 << 16) | (0x3093c >> 2), 897 0x00000000, 898 (0x0e00 << 16) | (0x30940 >> 2), 899 0x00000000, 900 (0x0e00 << 16) | (0x89a0 >> 2), 901 0x00000000, 902 (0x0e00 << 16) | (0x30900 >> 2), 903 0x00000000, 904 (0x0e00 << 16) | (0x30904 >> 2), 905 0x00000000, 906 (0x0e00 << 16) | (0x89b4 >> 2), 907 0x00000000, 908 (0x0e00 << 16) | (0x3e1fc >> 2), 909 0x00000000, 910 (0x0e00 << 16) | (0x3c210 >> 2), 911 0x00000000, 912 (0x0e00 << 16) | (0x3c214 >> 2), 913 0x00000000, 914 (0x0e00 << 16) | (0x3c218 >> 2), 915 0x00000000, 916 (0x0e00 << 16) | (0x8904 >> 2), 917 0x00000000, 918 0x5, 919 (0x0e00 << 16) | (0x8c28 >> 2), 920 (0x0e00 << 16) | (0x8c2c >> 2), 921 (0x0e00 << 16) | (0x8c30 >> 2), 922 (0x0e00 << 16) | (0x8c34 >> 2), 923 (0x0e00 << 16) | (0x9600 >> 2), 924 }; 925 926 static const u32 bonaire_golden_spm_registers[] = 927 { 928 0x30800, 0xe0ffffff, 0xe0000000 929 }; 930 931 static const u32 bonaire_golden_common_registers[] = 932 { 933 0xc770, 0xffffffff, 0x00000800, 934 0xc774, 0xffffffff, 0x00000800, 935 0xc798, 0xffffffff, 0x00007fbf, 936 0xc79c, 0xffffffff, 0x00007faf 937 }; 938 939 static const u32 bonaire_golden_registers[] = 940 { 941 0x3354, 0x00000333, 0x00000333, 942 0x3350, 0x000c0fc0, 0x00040200, 943 0x9a10, 0x00010000, 0x00058208, 944 0x3c000, 0xffff1fff, 0x00140000, 945 0x3c200, 0xfdfc0fff, 0x00000100, 946 0x3c234, 0x40000000, 0x40000200, 947 0x9830, 0xffffffff, 0x00000000, 948 0x9834, 0xf00fffff, 0x00000400, 949 0x9838, 0x0002021c, 0x00020200, 950 0xc78, 0x00000080, 0x00000000, 951 0x5bb0, 0x000000f0, 0x00000070, 952 0x5bc0, 0xf0311fff, 0x80300000, 953 0x98f8, 0x73773777, 0x12010001, 954 0x350c, 0x00810000, 0x408af000, 955 0x7030, 0x31000111, 0x00000011, 956 0x2f48, 0x73773777, 0x12010001, 957 0x220c, 0x00007fb6, 0x0021a1b1, 958 0x2210, 0x00007fb6, 0x002021b1, 959 0x2180, 0x00007fb6, 0x00002191, 960 0x2218, 0x00007fb6, 0x002121b1, 961 0x221c, 0x00007fb6, 0x002021b1, 962 0x21dc, 0x00007fb6, 0x00002191, 963 0x21e0, 0x00007fb6, 0x00002191, 964 0x3628, 0x0000003f, 0x0000000a, 965 0x362c, 0x0000003f, 0x0000000a, 966 0x2ae4, 0x00073ffe, 0x000022a2, 967 0x240c, 0x000007ff, 0x00000000, 968 0x8a14, 0xf000003f, 0x00000007, 969 0x8bf0, 0x00002001, 0x00000001, 970 0x8b24, 0xffffffff, 0x00ffffff, 971 0x30a04, 0x0000ff0f, 0x00000000, 972 0x28a4c, 0x07ffffff, 0x06000000, 973 0x4d8, 0x00000fff, 0x00000100, 974 0x3e78, 0x00000001, 0x00000002, 975 0x9100, 0x03000000, 0x0362c688, 976 0x8c00, 0x000000ff, 0x00000001, 977 0xe40, 0x00001fff, 0x00001fff, 978 0x9060, 0x0000007f, 0x00000020, 979 0x9508, 0x00010000, 0x00010000, 980 0xac14, 0x000003ff, 0x000000f3, 981 0xac0c, 0xffffffff, 0x00001032 982 }; 983 984 static const u32 bonaire_mgcg_cgcg_init[] = 985 { 986 0xc420, 0xffffffff, 0xfffffffc, 987 0x30800, 0xffffffff, 0xe0000000, 988 0x3c2a0, 0xffffffff, 0x00000100, 989 0x3c208, 0xffffffff, 0x00000100, 990 0x3c2c0, 0xffffffff, 0xc0000100, 991 0x3c2c8, 0xffffffff, 0xc0000100, 992 0x3c2c4, 0xffffffff, 0xc0000100, 993 0x55e4, 0xffffffff, 0x00600100, 994 0x3c280, 0xffffffff, 0x00000100, 995 0x3c214, 0xffffffff, 0x06000100, 996 0x3c220, 0xffffffff, 0x00000100, 997 0x3c218, 0xffffffff, 0x06000100, 998 0x3c204, 0xffffffff, 0x00000100, 999 0x3c2e0, 0xffffffff, 0x00000100, 1000 0x3c224, 0xffffffff, 0x00000100, 1001 0x3c200, 0xffffffff, 0x00000100, 1002 0x3c230, 0xffffffff, 0x00000100, 1003 0x3c234, 0xffffffff, 0x00000100, 1004 0x3c250, 0xffffffff, 0x00000100, 1005 0x3c254, 0xffffffff, 0x00000100, 1006 0x3c258, 0xffffffff, 0x00000100, 1007 0x3c25c, 0xffffffff, 0x00000100, 1008 0x3c260, 0xffffffff, 0x00000100, 1009 0x3c27c, 0xffffffff, 0x00000100, 1010 0x3c278, 0xffffffff, 0x00000100, 1011 0x3c210, 0xffffffff, 0x06000100, 1012 0x3c290, 0xffffffff, 0x00000100, 1013 0x3c274, 0xffffffff, 0x00000100, 1014 0x3c2b4, 0xffffffff, 0x00000100, 1015 0x3c2b0, 0xffffffff, 0x00000100, 1016 0x3c270, 0xffffffff, 0x00000100, 1017 0x30800, 0xffffffff, 0xe0000000, 1018 0x3c020, 0xffffffff, 0x00010000, 1019 0x3c024, 0xffffffff, 0x00030002, 1020 0x3c028, 0xffffffff, 0x00040007, 1021 0x3c02c, 0xffffffff, 0x00060005, 1022 0x3c030, 0xffffffff, 0x00090008, 1023 0x3c034, 0xffffffff, 0x00010000, 1024 0x3c038, 0xffffffff, 0x00030002, 1025 0x3c03c, 0xffffffff, 0x00040007, 1026 0x3c040, 0xffffffff, 0x00060005, 1027 0x3c044, 0xffffffff, 0x00090008, 1028 0x3c048, 0xffffffff, 0x00010000, 1029 0x3c04c, 0xffffffff, 0x00030002, 1030 0x3c050, 0xffffffff, 0x00040007, 1031 0x3c054, 0xffffffff, 0x00060005, 1032 0x3c058, 0xffffffff, 0x00090008, 1033 0x3c05c, 0xffffffff, 0x00010000, 1034 0x3c060, 0xffffffff, 0x00030002, 1035 0x3c064, 0xffffffff, 0x00040007, 1036 0x3c068, 0xffffffff, 0x00060005, 1037 0x3c06c, 0xffffffff, 0x00090008, 1038 0x3c070, 0xffffffff, 0x00010000, 1039 0x3c074, 0xffffffff, 0x00030002, 1040 0x3c078, 0xffffffff, 0x00040007, 1041 0x3c07c, 0xffffffff, 0x00060005, 1042 0x3c080, 0xffffffff, 0x00090008, 1043 0x3c084, 0xffffffff, 0x00010000, 1044 0x3c088, 0xffffffff, 0x00030002, 1045 0x3c08c, 0xffffffff, 0x00040007, 1046 0x3c090, 0xffffffff, 0x00060005, 1047 0x3c094, 0xffffffff, 0x00090008, 1048 0x3c098, 0xffffffff, 0x00010000, 1049 0x3c09c, 0xffffffff, 0x00030002, 1050 0x3c0a0, 0xffffffff, 0x00040007, 1051 0x3c0a4, 0xffffffff, 0x00060005, 1052 0x3c0a8, 0xffffffff, 0x00090008, 1053 0x3c000, 0xffffffff, 0x96e00200, 1054 0x8708, 0xffffffff, 0x00900100, 1055 0xc424, 0xffffffff, 0x0020003f, 1056 0x38, 0xffffffff, 0x0140001c, 1057 0x3c, 0x000f0000, 0x000f0000, 1058 0x220, 0xffffffff, 0xC060000C, 1059 0x224, 0xc0000fff, 0x00000100, 1060 0xf90, 0xffffffff, 0x00000100, 1061 0xf98, 0x00000101, 0x00000000, 1062 0x20a8, 0xffffffff, 0x00000104, 1063 0x55e4, 0xff000fff, 0x00000100, 1064 0x30cc, 0xc0000fff, 0x00000104, 1065 0xc1e4, 0x00000001, 0x00000001, 1066 0xd00c, 0xff000ff0, 0x00000100, 1067 0xd80c, 0xff000ff0, 0x00000100 1068 }; 1069 1070 static const u32 spectre_golden_spm_registers[] = 1071 { 1072 0x30800, 0xe0ffffff, 0xe0000000 1073 }; 1074 1075 static const u32 spectre_golden_common_registers[] = 1076 { 1077 0xc770, 0xffffffff, 0x00000800, 1078 0xc774, 0xffffffff, 0x00000800, 1079 0xc798, 0xffffffff, 0x00007fbf, 1080 0xc79c, 0xffffffff, 0x00007faf 1081 }; 1082 1083 static const u32 spectre_golden_registers[] = 1084 { 1085 0x3c000, 0xffff1fff, 0x96940200, 1086 0x3c00c, 0xffff0001, 0xff000000, 1087 0x3c200, 0xfffc0fff, 0x00000100, 1088 0x6ed8, 0x00010101, 0x00010000, 1089 0x9834, 0xf00fffff, 0x00000400, 1090 0x9838, 0xfffffffc, 0x00020200, 1091 0x5bb0, 0x000000f0, 0x00000070, 1092 0x5bc0, 0xf0311fff, 0x80300000, 1093 0x98f8, 0x73773777, 0x12010001, 1094 0x9b7c, 0x00ff0000, 0x00fc0000, 1095 0x2f48, 0x73773777, 0x12010001, 1096 0x8a14, 0xf000003f, 0x00000007, 1097 0x8b24, 0xffffffff, 0x00ffffff, 1098 0x28350, 0x3f3f3fff, 0x00000082, 1099 0x28354, 0x0000003f, 0x00000000, 1100 0x3e78, 0x00000001, 0x00000002, 1101 0x913c, 0xffff03df, 0x00000004, 1102 0xc768, 0x00000008, 0x00000008, 1103 0x8c00, 0x000008ff, 0x00000800, 1104 0x9508, 0x00010000, 0x00010000, 1105 0xac0c, 0xffffffff, 0x54763210, 1106 0x214f8, 0x01ff01ff, 0x00000002, 1107 0x21498, 0x007ff800, 0x00200000, 1108 0x2015c, 0xffffffff, 0x00000f40, 1109 0x30934, 0xffffffff, 0x00000001 1110 }; 1111 1112 static const u32 spectre_mgcg_cgcg_init[] = 1113 { 1114 0xc420, 0xffffffff, 0xfffffffc, 1115 0x30800, 0xffffffff, 0xe0000000, 1116 0x3c2a0, 0xffffffff, 0x00000100, 1117 0x3c208, 0xffffffff, 0x00000100, 1118 0x3c2c0, 0xffffffff, 0x00000100, 1119 0x3c2c8, 0xffffffff, 0x00000100, 1120 0x3c2c4, 0xffffffff, 0x00000100, 1121 0x55e4, 0xffffffff, 0x00600100, 1122 0x3c280, 0xffffffff, 0x00000100, 1123 0x3c214, 0xffffffff, 0x06000100, 1124 0x3c220, 0xffffffff, 0x00000100, 1125 0x3c218, 0xffffffff, 0x06000100, 1126 0x3c204, 0xffffffff, 0x00000100, 1127 0x3c2e0, 0xffffffff, 0x00000100, 1128 0x3c224, 0xffffffff, 0x00000100, 1129 0x3c200, 0xffffffff, 0x00000100, 1130 0x3c230, 0xffffffff, 0x00000100, 1131 0x3c234, 0xffffffff, 0x00000100, 1132 0x3c250, 0xffffffff, 0x00000100, 1133 0x3c254, 0xffffffff, 0x00000100, 1134 0x3c258, 0xffffffff, 0x00000100, 1135 0x3c25c, 0xffffffff, 0x00000100, 1136 0x3c260, 0xffffffff, 0x00000100, 1137 0x3c27c, 0xffffffff, 0x00000100, 1138 0x3c278, 0xffffffff, 0x00000100, 1139 0x3c210, 0xffffffff, 0x06000100, 1140 0x3c290, 0xffffffff, 0x00000100, 1141 0x3c274, 0xffffffff, 0x00000100, 1142 0x3c2b4, 0xffffffff, 0x00000100, 1143 0x3c2b0, 0xffffffff, 0x00000100, 1144 0x3c270, 0xffffffff, 0x00000100, 1145 0x30800, 0xffffffff, 0xe0000000, 1146 0x3c020, 0xffffffff, 0x00010000, 1147 0x3c024, 0xffffffff, 0x00030002, 1148 0x3c028, 0xffffffff, 0x00040007, 1149 0x3c02c, 0xffffffff, 0x00060005, 1150 0x3c030, 0xffffffff, 0x00090008, 1151 0x3c034, 0xffffffff, 0x00010000, 1152 0x3c038, 0xffffffff, 0x00030002, 1153 0x3c03c, 0xffffffff, 0x00040007, 1154 0x3c040, 0xffffffff, 0x00060005, 1155 0x3c044, 0xffffffff, 0x00090008, 1156 0x3c048, 0xffffffff, 0x00010000, 1157 0x3c04c, 0xffffffff, 0x00030002, 1158 0x3c050, 0xffffffff, 0x00040007, 1159 0x3c054, 0xffffffff, 0x00060005, 1160 0x3c058, 0xffffffff, 0x00090008, 1161 0x3c05c, 0xffffffff, 0x00010000, 1162 0x3c060, 0xffffffff, 0x00030002, 1163 0x3c064, 0xffffffff, 0x00040007, 1164 0x3c068, 0xffffffff, 0x00060005, 1165 0x3c06c, 0xffffffff, 0x00090008, 1166 0x3c070, 0xffffffff, 0x00010000, 1167 0x3c074, 0xffffffff, 0x00030002, 1168 0x3c078, 0xffffffff, 0x00040007, 1169 0x3c07c, 0xffffffff, 0x00060005, 1170 0x3c080, 0xffffffff, 0x00090008, 1171 0x3c084, 0xffffffff, 0x00010000, 1172 0x3c088, 0xffffffff, 0x00030002, 1173 0x3c08c, 0xffffffff, 0x00040007, 1174 0x3c090, 0xffffffff, 0x00060005, 1175 0x3c094, 0xffffffff, 0x00090008, 1176 0x3c098, 0xffffffff, 0x00010000, 1177 0x3c09c, 0xffffffff, 0x00030002, 1178 0x3c0a0, 0xffffffff, 0x00040007, 1179 0x3c0a4, 0xffffffff, 0x00060005, 1180 0x3c0a8, 0xffffffff, 0x00090008, 1181 0x3c0ac, 0xffffffff, 0x00010000, 1182 0x3c0b0, 0xffffffff, 0x00030002, 1183 0x3c0b4, 0xffffffff, 0x00040007, 1184 0x3c0b8, 0xffffffff, 0x00060005, 1185 0x3c0bc, 0xffffffff, 0x00090008, 1186 0x3c000, 0xffffffff, 0x96e00200, 1187 0x8708, 0xffffffff, 0x00900100, 1188 0xc424, 0xffffffff, 0x0020003f, 1189 0x38, 0xffffffff, 0x0140001c, 1190 0x3c, 0x000f0000, 0x000f0000, 1191 0x220, 0xffffffff, 0xC060000C, 1192 0x224, 0xc0000fff, 0x00000100, 1193 0xf90, 0xffffffff, 0x00000100, 1194 0xf98, 0x00000101, 0x00000000, 1195 0x20a8, 0xffffffff, 0x00000104, 1196 0x55e4, 0xff000fff, 0x00000100, 1197 0x30cc, 0xc0000fff, 0x00000104, 1198 0xc1e4, 0x00000001, 0x00000001, 1199 0xd00c, 0xff000ff0, 0x00000100, 1200 0xd80c, 0xff000ff0, 0x00000100 1201 }; 1202 1203 static const u32 kalindi_golden_spm_registers[] = 1204 { 1205 0x30800, 0xe0ffffff, 0xe0000000 1206 }; 1207 1208 static const u32 kalindi_golden_common_registers[] = 1209 { 1210 0xc770, 0xffffffff, 0x00000800, 1211 0xc774, 0xffffffff, 0x00000800, 1212 0xc798, 0xffffffff, 0x00007fbf, 1213 0xc79c, 0xffffffff, 0x00007faf 1214 }; 1215 1216 static const u32 kalindi_golden_registers[] = 1217 { 1218 0x3c000, 0xffffdfff, 0x6e944040, 1219 0x55e4, 0xff607fff, 0xfc000100, 1220 0x3c220, 0xff000fff, 0x00000100, 1221 0x3c224, 0xff000fff, 0x00000100, 1222 0x3c200, 0xfffc0fff, 0x00000100, 1223 0x6ed8, 0x00010101, 0x00010000, 1224 0x9830, 0xffffffff, 0x00000000, 1225 0x9834, 0xf00fffff, 0x00000400, 1226 0x5bb0, 0x000000f0, 0x00000070, 1227 0x5bc0, 0xf0311fff, 0x80300000, 1228 0x98f8, 0x73773777, 0x12010001, 1229 0x98fc, 0xffffffff, 0x00000010, 1230 0x9b7c, 0x00ff0000, 0x00fc0000, 1231 0x8030, 0x00001f0f, 0x0000100a, 1232 0x2f48, 0x73773777, 0x12010001, 1233 0x2408, 0x000fffff, 0x000c007f, 1234 0x8a14, 0xf000003f, 0x00000007, 1235 0x8b24, 0x3fff3fff, 0x00ffcfff, 1236 0x30a04, 0x0000ff0f, 0x00000000, 1237 0x28a4c, 0x07ffffff, 0x06000000, 1238 0x4d8, 0x00000fff, 0x00000100, 1239 0x3e78, 0x00000001, 0x00000002, 1240 0xc768, 0x00000008, 0x00000008, 1241 0x8c00, 0x000000ff, 0x00000003, 1242 0x214f8, 0x01ff01ff, 0x00000002, 1243 0x21498, 0x007ff800, 0x00200000, 1244 0x2015c, 0xffffffff, 0x00000f40, 1245 0x88c4, 0x001f3ae3, 0x00000082, 1246 0x88d4, 0x0000001f, 0x00000010, 1247 0x30934, 0xffffffff, 0x00000000 1248 }; 1249 1250 static const u32 kalindi_mgcg_cgcg_init[] = 1251 { 1252 0xc420, 0xffffffff, 0xfffffffc, 1253 0x30800, 0xffffffff, 0xe0000000, 1254 0x3c2a0, 0xffffffff, 0x00000100, 1255 0x3c208, 0xffffffff, 0x00000100, 1256 0x3c2c0, 0xffffffff, 0x00000100, 1257 0x3c2c8, 0xffffffff, 0x00000100, 1258 0x3c2c4, 0xffffffff, 0x00000100, 1259 0x55e4, 0xffffffff, 0x00600100, 1260 0x3c280, 0xffffffff, 0x00000100, 1261 0x3c214, 0xffffffff, 0x06000100, 1262 0x3c220, 0xffffffff, 0x00000100, 1263 0x3c218, 0xffffffff, 0x06000100, 1264 0x3c204, 0xffffffff, 0x00000100, 1265 0x3c2e0, 0xffffffff, 0x00000100, 1266 0x3c224, 0xffffffff, 0x00000100, 1267 0x3c200, 0xffffffff, 0x00000100, 1268 0x3c230, 0xffffffff, 0x00000100, 1269 0x3c234, 0xffffffff, 0x00000100, 1270 0x3c250, 0xffffffff, 0x00000100, 1271 0x3c254, 0xffffffff, 0x00000100, 1272 0x3c258, 0xffffffff, 0x00000100, 1273 0x3c25c, 0xffffffff, 0x00000100, 1274 0x3c260, 0xffffffff, 0x00000100, 1275 0x3c27c, 0xffffffff, 0x00000100, 1276 0x3c278, 0xffffffff, 0x00000100, 1277 0x3c210, 0xffffffff, 0x06000100, 1278 0x3c290, 0xffffffff, 0x00000100, 1279 0x3c274, 0xffffffff, 0x00000100, 1280 0x3c2b4, 0xffffffff, 0x00000100, 1281 0x3c2b0, 0xffffffff, 0x00000100, 1282 0x3c270, 0xffffffff, 0x00000100, 1283 0x30800, 0xffffffff, 0xe0000000, 1284 0x3c020, 0xffffffff, 0x00010000, 1285 0x3c024, 0xffffffff, 0x00030002, 1286 0x3c028, 0xffffffff, 0x00040007, 1287 0x3c02c, 0xffffffff, 0x00060005, 1288 0x3c030, 0xffffffff, 0x00090008, 1289 0x3c034, 0xffffffff, 0x00010000, 1290 0x3c038, 0xffffffff, 0x00030002, 1291 0x3c03c, 0xffffffff, 0x00040007, 1292 0x3c040, 0xffffffff, 0x00060005, 1293 0x3c044, 0xffffffff, 0x00090008, 1294 0x3c000, 0xffffffff, 0x96e00200, 1295 0x8708, 0xffffffff, 0x00900100, 1296 0xc424, 0xffffffff, 0x0020003f, 1297 0x38, 0xffffffff, 0x0140001c, 1298 0x3c, 0x000f0000, 0x000f0000, 1299 0x220, 0xffffffff, 0xC060000C, 1300 0x224, 0xc0000fff, 0x00000100, 1301 0x20a8, 0xffffffff, 0x00000104, 1302 0x55e4, 0xff000fff, 0x00000100, 1303 0x30cc, 0xc0000fff, 0x00000104, 1304 0xc1e4, 0x00000001, 0x00000001, 1305 0xd00c, 0xff000ff0, 0x00000100, 1306 0xd80c, 0xff000ff0, 0x00000100 1307 }; 1308 1309 static const u32 hawaii_golden_spm_registers[] = 1310 { 1311 0x30800, 0xe0ffffff, 0xe0000000 1312 }; 1313 1314 static const u32 hawaii_golden_common_registers[] = 1315 { 1316 0x30800, 0xffffffff, 0xe0000000, 1317 0x28350, 0xffffffff, 0x3a00161a, 1318 0x28354, 0xffffffff, 0x0000002e, 1319 0x9a10, 0xffffffff, 0x00018208, 1320 0x98f8, 0xffffffff, 0x12011003 1321 }; 1322 1323 static const u32 hawaii_golden_registers[] = 1324 { 1325 0x3354, 0x00000333, 0x00000333, 1326 0x9a10, 0x00010000, 0x00058208, 1327 0x9830, 0xffffffff, 0x00000000, 1328 0x9834, 0xf00fffff, 0x00000400, 1329 0x9838, 0x0002021c, 0x00020200, 1330 0xc78, 0x00000080, 0x00000000, 1331 0x5bb0, 0x000000f0, 0x00000070, 1332 0x5bc0, 0xf0311fff, 0x80300000, 1333 0x350c, 0x00810000, 0x408af000, 1334 0x7030, 0x31000111, 0x00000011, 1335 0x2f48, 0x73773777, 0x12010001, 1336 0x2120, 0x0000007f, 0x0000001b, 1337 0x21dc, 0x00007fb6, 0x00002191, 1338 0x3628, 0x0000003f, 0x0000000a, 1339 0x362c, 0x0000003f, 0x0000000a, 1340 0x2ae4, 0x00073ffe, 0x000022a2, 1341 0x240c, 0x000007ff, 0x00000000, 1342 0x8bf0, 0x00002001, 0x00000001, 1343 0x8b24, 0xffffffff, 0x00ffffff, 1344 0x30a04, 0x0000ff0f, 0x00000000, 1345 0x28a4c, 0x07ffffff, 0x06000000, 1346 0x3e78, 0x00000001, 0x00000002, 1347 0xc768, 0x00000008, 0x00000008, 1348 0xc770, 0x00000f00, 0x00000800, 1349 0xc774, 0x00000f00, 0x00000800, 1350 0xc798, 0x00ffffff, 0x00ff7fbf, 1351 0xc79c, 0x00ffffff, 0x00ff7faf, 1352 0x8c00, 0x000000ff, 0x00000800, 1353 0xe40, 0x00001fff, 0x00001fff, 1354 0x9060, 0x0000007f, 0x00000020, 1355 0x9508, 0x00010000, 0x00010000, 1356 0xae00, 0x00100000, 0x000ff07c, 1357 0xac14, 0x000003ff, 0x0000000f, 1358 0xac10, 0xffffffff, 0x7564fdec, 1359 0xac0c, 0xffffffff, 0x3120b9a8, 1360 0xac08, 0x20000000, 0x0f9c0000 1361 }; 1362 1363 static const u32 hawaii_mgcg_cgcg_init[] = 1364 { 1365 0xc420, 0xffffffff, 0xfffffffd, 1366 0x30800, 0xffffffff, 0xe0000000, 1367 0x3c2a0, 0xffffffff, 0x00000100, 1368 0x3c208, 0xffffffff, 0x00000100, 1369 0x3c2c0, 0xffffffff, 0x00000100, 1370 0x3c2c8, 0xffffffff, 0x00000100, 1371 0x3c2c4, 0xffffffff, 0x00000100, 1372 0x55e4, 0xffffffff, 0x00200100, 1373 0x3c280, 0xffffffff, 0x00000100, 1374 0x3c214, 0xffffffff, 0x06000100, 1375 0x3c220, 0xffffffff, 0x00000100, 1376 0x3c218, 0xffffffff, 0x06000100, 1377 0x3c204, 0xffffffff, 0x00000100, 1378 0x3c2e0, 0xffffffff, 0x00000100, 1379 0x3c224, 0xffffffff, 0x00000100, 1380 0x3c200, 0xffffffff, 0x00000100, 1381 0x3c230, 0xffffffff, 0x00000100, 1382 0x3c234, 0xffffffff, 0x00000100, 1383 0x3c250, 0xffffffff, 0x00000100, 1384 0x3c254, 0xffffffff, 0x00000100, 1385 0x3c258, 0xffffffff, 0x00000100, 1386 0x3c25c, 0xffffffff, 0x00000100, 1387 0x3c260, 0xffffffff, 0x00000100, 1388 0x3c27c, 0xffffffff, 0x00000100, 1389 0x3c278, 0xffffffff, 0x00000100, 1390 0x3c210, 0xffffffff, 0x06000100, 1391 0x3c290, 0xffffffff, 0x00000100, 1392 0x3c274, 0xffffffff, 0x00000100, 1393 0x3c2b4, 0xffffffff, 0x00000100, 1394 0x3c2b0, 0xffffffff, 0x00000100, 1395 0x3c270, 0xffffffff, 0x00000100, 1396 0x30800, 0xffffffff, 0xe0000000, 1397 0x3c020, 0xffffffff, 0x00010000, 1398 0x3c024, 0xffffffff, 0x00030002, 1399 0x3c028, 0xffffffff, 0x00040007, 1400 0x3c02c, 0xffffffff, 0x00060005, 1401 0x3c030, 0xffffffff, 0x00090008, 1402 0x3c034, 0xffffffff, 0x00010000, 1403 0x3c038, 0xffffffff, 0x00030002, 1404 0x3c03c, 0xffffffff, 0x00040007, 1405 0x3c040, 0xffffffff, 0x00060005, 1406 0x3c044, 0xffffffff, 0x00090008, 1407 0x3c048, 0xffffffff, 0x00010000, 1408 0x3c04c, 0xffffffff, 0x00030002, 1409 0x3c050, 0xffffffff, 0x00040007, 1410 0x3c054, 0xffffffff, 0x00060005, 1411 0x3c058, 0xffffffff, 0x00090008, 1412 0x3c05c, 0xffffffff, 0x00010000, 1413 0x3c060, 0xffffffff, 0x00030002, 1414 0x3c064, 0xffffffff, 0x00040007, 1415 0x3c068, 0xffffffff, 0x00060005, 1416 0x3c06c, 0xffffffff, 0x00090008, 1417 0x3c070, 0xffffffff, 0x00010000, 1418 0x3c074, 0xffffffff, 0x00030002, 1419 0x3c078, 0xffffffff, 0x00040007, 1420 0x3c07c, 0xffffffff, 0x00060005, 1421 0x3c080, 0xffffffff, 0x00090008, 1422 0x3c084, 0xffffffff, 0x00010000, 1423 0x3c088, 0xffffffff, 0x00030002, 1424 0x3c08c, 0xffffffff, 0x00040007, 1425 0x3c090, 0xffffffff, 0x00060005, 1426 0x3c094, 0xffffffff, 0x00090008, 1427 0x3c098, 0xffffffff, 0x00010000, 1428 0x3c09c, 0xffffffff, 0x00030002, 1429 0x3c0a0, 0xffffffff, 0x00040007, 1430 0x3c0a4, 0xffffffff, 0x00060005, 1431 0x3c0a8, 0xffffffff, 0x00090008, 1432 0x3c0ac, 0xffffffff, 0x00010000, 1433 0x3c0b0, 0xffffffff, 0x00030002, 1434 0x3c0b4, 0xffffffff, 0x00040007, 1435 0x3c0b8, 0xffffffff, 0x00060005, 1436 0x3c0bc, 0xffffffff, 0x00090008, 1437 0x3c0c0, 0xffffffff, 0x00010000, 1438 0x3c0c4, 0xffffffff, 0x00030002, 1439 0x3c0c8, 0xffffffff, 0x00040007, 1440 0x3c0cc, 0xffffffff, 0x00060005, 1441 0x3c0d0, 0xffffffff, 0x00090008, 1442 0x3c0d4, 0xffffffff, 0x00010000, 1443 0x3c0d8, 0xffffffff, 0x00030002, 1444 0x3c0dc, 0xffffffff, 0x00040007, 1445 0x3c0e0, 0xffffffff, 0x00060005, 1446 0x3c0e4, 0xffffffff, 0x00090008, 1447 0x3c0e8, 0xffffffff, 0x00010000, 1448 0x3c0ec, 0xffffffff, 0x00030002, 1449 0x3c0f0, 0xffffffff, 0x00040007, 1450 0x3c0f4, 0xffffffff, 0x00060005, 1451 0x3c0f8, 0xffffffff, 0x00090008, 1452 0xc318, 0xffffffff, 0x00020200, 1453 0x3350, 0xffffffff, 0x00000200, 1454 0x15c0, 0xffffffff, 0x00000400, 1455 0x55e8, 0xffffffff, 0x00000000, 1456 0x2f50, 0xffffffff, 0x00000902, 1457 0x3c000, 0xffffffff, 0x96940200, 1458 0x8708, 0xffffffff, 0x00900100, 1459 0xc424, 0xffffffff, 0x0020003f, 1460 0x38, 0xffffffff, 0x0140001c, 1461 0x3c, 0x000f0000, 0x000f0000, 1462 0x220, 0xffffffff, 0xc060000c, 1463 0x224, 0xc0000fff, 0x00000100, 1464 0xf90, 0xffffffff, 0x00000100, 1465 0xf98, 0x00000101, 0x00000000, 1466 0x20a8, 0xffffffff, 0x00000104, 1467 0x55e4, 0xff000fff, 0x00000100, 1468 0x30cc, 0xc0000fff, 0x00000104, 1469 0xc1e4, 0x00000001, 0x00000001, 1470 0xd00c, 0xff000ff0, 0x00000100, 1471 0xd80c, 0xff000ff0, 0x00000100 1472 }; 1473 1474 static void cik_init_golden_registers(struct radeon_device *rdev) 1475 { 1476 switch (rdev->family) { 1477 case CHIP_BONAIRE: 1478 radeon_program_register_sequence(rdev, 1479 bonaire_mgcg_cgcg_init, 1480 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init)); 1481 radeon_program_register_sequence(rdev, 1482 bonaire_golden_registers, 1483 (const u32)ARRAY_SIZE(bonaire_golden_registers)); 1484 radeon_program_register_sequence(rdev, 1485 bonaire_golden_common_registers, 1486 (const u32)ARRAY_SIZE(bonaire_golden_common_registers)); 1487 radeon_program_register_sequence(rdev, 1488 bonaire_golden_spm_registers, 1489 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers)); 1490 break; 1491 case CHIP_KABINI: 1492 radeon_program_register_sequence(rdev, 1493 kalindi_mgcg_cgcg_init, 1494 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init)); 1495 radeon_program_register_sequence(rdev, 1496 kalindi_golden_registers, 1497 (const u32)ARRAY_SIZE(kalindi_golden_registers)); 1498 radeon_program_register_sequence(rdev, 1499 kalindi_golden_common_registers, 1500 (const u32)ARRAY_SIZE(kalindi_golden_common_registers)); 1501 radeon_program_register_sequence(rdev, 1502 kalindi_golden_spm_registers, 1503 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers)); 1504 break; 1505 case CHIP_KAVERI: 1506 radeon_program_register_sequence(rdev, 1507 spectre_mgcg_cgcg_init, 1508 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init)); 1509 radeon_program_register_sequence(rdev, 1510 spectre_golden_registers, 1511 (const u32)ARRAY_SIZE(spectre_golden_registers)); 1512 radeon_program_register_sequence(rdev, 1513 spectre_golden_common_registers, 1514 (const u32)ARRAY_SIZE(spectre_golden_common_registers)); 1515 radeon_program_register_sequence(rdev, 1516 spectre_golden_spm_registers, 1517 (const u32)ARRAY_SIZE(spectre_golden_spm_registers)); 1518 break; 1519 case CHIP_HAWAII: 1520 radeon_program_register_sequence(rdev, 1521 hawaii_mgcg_cgcg_init, 1522 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init)); 1523 radeon_program_register_sequence(rdev, 1524 hawaii_golden_registers, 1525 (const u32)ARRAY_SIZE(hawaii_golden_registers)); 1526 radeon_program_register_sequence(rdev, 1527 hawaii_golden_common_registers, 1528 (const u32)ARRAY_SIZE(hawaii_golden_common_registers)); 1529 radeon_program_register_sequence(rdev, 1530 hawaii_golden_spm_registers, 1531 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers)); 1532 break; 1533 default: 1534 break; 1535 } 1536 } 1537 1538 /** 1539 * cik_get_xclk - get the xclk 1540 * 1541 * @rdev: radeon_device pointer 1542 * 1543 * Returns the reference clock used by the gfx engine 1544 * (CIK). 1545 */ 1546 u32 cik_get_xclk(struct radeon_device *rdev) 1547 { 1548 u32 reference_clock = rdev->clock.spll.reference_freq; 1549 1550 if (rdev->flags & RADEON_IS_IGP) { 1551 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK) 1552 return reference_clock / 2; 1553 } else { 1554 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE) 1555 return reference_clock / 4; 1556 } 1557 return reference_clock; 1558 } 1559 1560 /** 1561 * cik_mm_rdoorbell - read a doorbell dword 1562 * 1563 * @rdev: radeon_device pointer 1564 * @index: doorbell index 1565 * 1566 * Returns the value in the doorbell aperture at the 1567 * requested doorbell index (CIK). 1568 */ 1569 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) 1570 { 1571 if (index < rdev->doorbell.num_doorbells) { 1572 return readl(rdev->doorbell.ptr + index); 1573 } else { 1574 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 1575 return 0; 1576 } 1577 } 1578 1579 /** 1580 * cik_mm_wdoorbell - write a doorbell dword 1581 * 1582 * @rdev: radeon_device pointer 1583 * @index: doorbell index 1584 * @v: value to write 1585 * 1586 * Writes @v to the doorbell aperture at the 1587 * requested doorbell index (CIK). 1588 */ 1589 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) 1590 { 1591 if (index < rdev->doorbell.num_doorbells) { 1592 writel(v, rdev->doorbell.ptr + index); 1593 } else { 1594 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 1595 } 1596 } 1597 1598 #define BONAIRE_IO_MC_REGS_SIZE 36 1599 1600 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] = 1601 { 1602 {0x00000070, 0x04400000}, 1603 {0x00000071, 0x80c01803}, 1604 {0x00000072, 0x00004004}, 1605 {0x00000073, 0x00000100}, 1606 {0x00000074, 0x00ff0000}, 1607 {0x00000075, 0x34000000}, 1608 {0x00000076, 0x08000014}, 1609 {0x00000077, 0x00cc08ec}, 1610 {0x00000078, 0x00000400}, 1611 {0x00000079, 0x00000000}, 1612 {0x0000007a, 0x04090000}, 1613 {0x0000007c, 0x00000000}, 1614 {0x0000007e, 0x4408a8e8}, 1615 {0x0000007f, 0x00000304}, 1616 {0x00000080, 0x00000000}, 1617 {0x00000082, 0x00000001}, 1618 {0x00000083, 0x00000002}, 1619 {0x00000084, 0xf3e4f400}, 1620 {0x00000085, 0x052024e3}, 1621 {0x00000087, 0x00000000}, 1622 {0x00000088, 0x01000000}, 1623 {0x0000008a, 0x1c0a0000}, 1624 {0x0000008b, 0xff010000}, 1625 {0x0000008d, 0xffffefff}, 1626 {0x0000008e, 0xfff3efff}, 1627 {0x0000008f, 0xfff3efbf}, 1628 {0x00000092, 0xf7ffffff}, 1629 {0x00000093, 0xffffff7f}, 1630 {0x00000095, 0x00101101}, 1631 {0x00000096, 0x00000fff}, 1632 {0x00000097, 0x00116fff}, 1633 {0x00000098, 0x60010000}, 1634 {0x00000099, 0x10010000}, 1635 {0x0000009a, 0x00006000}, 1636 {0x0000009b, 0x00001000}, 1637 {0x0000009f, 0x00b48000} 1638 }; 1639 1640 #define HAWAII_IO_MC_REGS_SIZE 22 1641 1642 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] = 1643 { 1644 {0x0000007d, 0x40000000}, 1645 {0x0000007e, 0x40180304}, 1646 {0x0000007f, 0x0000ff00}, 1647 {0x00000081, 0x00000000}, 1648 {0x00000083, 0x00000800}, 1649 {0x00000086, 0x00000000}, 1650 {0x00000087, 0x00000100}, 1651 {0x00000088, 0x00020100}, 1652 {0x00000089, 0x00000000}, 1653 {0x0000008b, 0x00040000}, 1654 {0x0000008c, 0x00000100}, 1655 {0x0000008e, 0xff010000}, 1656 {0x00000090, 0xffffefff}, 1657 {0x00000091, 0xfff3efff}, 1658 {0x00000092, 0xfff3efbf}, 1659 {0x00000093, 0xf7ffffff}, 1660 {0x00000094, 0xffffff7f}, 1661 {0x00000095, 0x00000fff}, 1662 {0x00000096, 0x00116fff}, 1663 {0x00000097, 0x60010000}, 1664 {0x00000098, 0x10010000}, 1665 {0x0000009f, 0x00c79000} 1666 }; 1667 1668 1669 /** 1670 * cik_srbm_select - select specific register instances 1671 * 1672 * @rdev: radeon_device pointer 1673 * @me: selected ME (micro engine) 1674 * @pipe: pipe 1675 * @queue: queue 1676 * @vmid: VMID 1677 * 1678 * Switches the currently active registers instances. Some 1679 * registers are instanced per VMID, others are instanced per 1680 * me/pipe/queue combination. 1681 */ 1682 static void cik_srbm_select(struct radeon_device *rdev, 1683 u32 me, u32 pipe, u32 queue, u32 vmid) 1684 { 1685 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) | 1686 MEID(me & 0x3) | 1687 VMID(vmid & 0xf) | 1688 QUEUEID(queue & 0x7)); 1689 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl); 1690 } 1691 1692 /* ucode loading */ 1693 /** 1694 * ci_mc_load_microcode - load MC ucode into the hw 1695 * 1696 * @rdev: radeon_device pointer 1697 * 1698 * Load the GDDR MC ucode into the hw (CIK). 1699 * Returns 0 on success, error on failure. 1700 */ 1701 int ci_mc_load_microcode(struct radeon_device *rdev) 1702 { 1703 const __be32 *fw_data; 1704 u32 running, blackout = 0; 1705 u32 *io_mc_regs; 1706 int i, ucode_size, regs_size; 1707 1708 if (!rdev->mc_fw) 1709 return -EINVAL; 1710 1711 switch (rdev->family) { 1712 case CHIP_BONAIRE: 1713 io_mc_regs = (u32 *)&bonaire_io_mc_regs; 1714 ucode_size = CIK_MC_UCODE_SIZE; 1715 regs_size = BONAIRE_IO_MC_REGS_SIZE; 1716 break; 1717 case CHIP_HAWAII: 1718 io_mc_regs = (u32 *)&hawaii_io_mc_regs; 1719 ucode_size = HAWAII_MC_UCODE_SIZE; 1720 regs_size = HAWAII_IO_MC_REGS_SIZE; 1721 break; 1722 default: 1723 return -EINVAL; 1724 } 1725 1726 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1727 1728 if (running == 0) { 1729 if (running) { 1730 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1731 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1732 } 1733 1734 /* reset the engine and set to writable */ 1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1736 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1737 1738 /* load mc io regs */ 1739 for (i = 0; i < regs_size; i++) { 1740 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1741 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1742 } 1743 /* load the MC ucode */ 1744 fw_data = (const __be32 *)rdev->mc_fw->data; 1745 for (i = 0; i < ucode_size; i++) 1746 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1747 1748 /* put the engine back into the active state */ 1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1751 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1752 1753 /* wait for training to complete */ 1754 for (i = 0; i < rdev->usec_timeout; i++) { 1755 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1756 break; 1757 udelay(1); 1758 } 1759 for (i = 0; i < rdev->usec_timeout; i++) { 1760 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1761 break; 1762 udelay(1); 1763 } 1764 1765 if (running) 1766 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1767 } 1768 1769 return 0; 1770 } 1771 1772 /** 1773 * cik_init_microcode - load ucode images from disk 1774 * 1775 * @rdev: radeon_device pointer 1776 * 1777 * Use the firmware interface to load the ucode images into 1778 * the driver (not loaded into hw). 1779 * Returns 0 on success, error on failure. 1780 */ 1781 static int cik_init_microcode(struct radeon_device *rdev) 1782 { 1783 const char *chip_name; 1784 size_t pfp_req_size, me_req_size, ce_req_size, 1785 mec_req_size, rlc_req_size, mc_req_size = 0, 1786 sdma_req_size, smc_req_size = 0; 1787 char fw_name[30]; 1788 int err; 1789 1790 DRM_DEBUG("\n"); 1791 1792 switch (rdev->family) { 1793 case CHIP_BONAIRE: 1794 chip_name = "BONAIRE"; 1795 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1796 me_req_size = CIK_ME_UCODE_SIZE * 4; 1797 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1798 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1799 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1800 mc_req_size = CIK_MC_UCODE_SIZE * 4; 1801 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1802 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4); 1803 break; 1804 case CHIP_HAWAII: 1805 chip_name = "HAWAII"; 1806 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1807 me_req_size = CIK_ME_UCODE_SIZE * 4; 1808 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1809 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1810 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4; 1811 mc_req_size = HAWAII_MC_UCODE_SIZE * 4; 1812 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1813 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4); 1814 break; 1815 case CHIP_KAVERI: 1816 chip_name = "KAVERI"; 1817 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1818 me_req_size = CIK_ME_UCODE_SIZE * 4; 1819 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1820 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1821 rlc_req_size = KV_RLC_UCODE_SIZE * 4; 1822 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1823 break; 1824 case CHIP_KABINI: 1825 chip_name = "KABINI"; 1826 pfp_req_size = CIK_PFP_UCODE_SIZE * 4; 1827 me_req_size = CIK_ME_UCODE_SIZE * 4; 1828 ce_req_size = CIK_CE_UCODE_SIZE * 4; 1829 mec_req_size = CIK_MEC_UCODE_SIZE * 4; 1830 rlc_req_size = KB_RLC_UCODE_SIZE * 4; 1831 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4; 1832 break; 1833 default: BUG(); 1834 } 1835 1836 DRM_INFO("Loading %s Microcode\n", chip_name); 1837 1838 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 1839 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1840 if (err) 1841 goto out; 1842 if (rdev->pfp_fw->size != pfp_req_size) { 1843 printk(KERN_ERR 1844 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 1845 rdev->pfp_fw->size, fw_name); 1846 err = -EINVAL; 1847 goto out; 1848 } 1849 1850 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 1851 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 1852 if (err) 1853 goto out; 1854 if (rdev->me_fw->size != me_req_size) { 1855 printk(KERN_ERR 1856 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 1857 rdev->me_fw->size, fw_name); 1858 err = -EINVAL; 1859 } 1860 1861 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 1862 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 1863 if (err) 1864 goto out; 1865 if (rdev->ce_fw->size != ce_req_size) { 1866 printk(KERN_ERR 1867 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 1868 rdev->ce_fw->size, fw_name); 1869 err = -EINVAL; 1870 } 1871 1872 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name); 1873 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); 1874 if (err) 1875 goto out; 1876 if (rdev->mec_fw->size != mec_req_size) { 1877 printk(KERN_ERR 1878 "cik_cp: Bogus length %zu in firmware \"%s\"\n", 1879 rdev->mec_fw->size, fw_name); 1880 err = -EINVAL; 1881 } 1882 1883 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); 1884 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 1885 if (err) 1886 goto out; 1887 if (rdev->rlc_fw->size != rlc_req_size) { 1888 printk(KERN_ERR 1889 "cik_rlc: Bogus length %zu in firmware \"%s\"\n", 1890 rdev->rlc_fw->size, fw_name); 1891 err = -EINVAL; 1892 } 1893 1894 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name); 1895 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); 1896 if (err) 1897 goto out; 1898 if (rdev->sdma_fw->size != sdma_req_size) { 1899 printk(KERN_ERR 1900 "cik_sdma: Bogus length %zu in firmware \"%s\"\n", 1901 rdev->sdma_fw->size, fw_name); 1902 err = -EINVAL; 1903 } 1904 1905 /* No SMC, MC ucode on APUs */ 1906 if (!(rdev->flags & RADEON_IS_IGP)) { 1907 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 1908 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 1909 if (err) 1910 goto out; 1911 if (rdev->mc_fw->size != mc_req_size) { 1912 printk(KERN_ERR 1913 "cik_mc: Bogus length %zu in firmware \"%s\"\n", 1914 rdev->mc_fw->size, fw_name); 1915 err = -EINVAL; 1916 } 1917 1918 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 1919 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 1920 if (err) { 1921 printk(KERN_ERR 1922 "smc: error loading firmware \"%s\"\n", 1923 fw_name); 1924 release_firmware(rdev->smc_fw); 1925 rdev->smc_fw = NULL; 1926 err = 0; 1927 } else if (rdev->smc_fw->size != smc_req_size) { 1928 printk(KERN_ERR 1929 "cik_smc: Bogus length %zu in firmware \"%s\"\n", 1930 rdev->smc_fw->size, fw_name); 1931 err = -EINVAL; 1932 } 1933 } 1934 1935 out: 1936 if (err) { 1937 if (err != -EINVAL) 1938 printk(KERN_ERR 1939 "cik_cp: Failed to load firmware \"%s\"\n", 1940 fw_name); 1941 release_firmware(rdev->pfp_fw); 1942 rdev->pfp_fw = NULL; 1943 release_firmware(rdev->me_fw); 1944 rdev->me_fw = NULL; 1945 release_firmware(rdev->ce_fw); 1946 rdev->ce_fw = NULL; 1947 release_firmware(rdev->rlc_fw); 1948 rdev->rlc_fw = NULL; 1949 release_firmware(rdev->mc_fw); 1950 rdev->mc_fw = NULL; 1951 release_firmware(rdev->smc_fw); 1952 rdev->smc_fw = NULL; 1953 } 1954 return err; 1955 } 1956 1957 /* 1958 * Core functions 1959 */ 1960 /** 1961 * cik_tiling_mode_table_init - init the hw tiling table 1962 * 1963 * @rdev: radeon_device pointer 1964 * 1965 * Starting with SI, the tiling setup is done globally in a 1966 * set of 32 tiling modes. Rather than selecting each set of 1967 * parameters per surface as on older asics, we just select 1968 * which index in the tiling table we want to use, and the 1969 * surface uses those parameters (CIK). 1970 */ 1971 static void cik_tiling_mode_table_init(struct radeon_device *rdev) 1972 { 1973 const u32 num_tile_mode_states = 32; 1974 const u32 num_secondary_tile_mode_states = 16; 1975 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 1976 u32 num_pipe_configs; 1977 u32 num_rbs = rdev->config.cik.max_backends_per_se * 1978 rdev->config.cik.max_shader_engines; 1979 1980 switch (rdev->config.cik.mem_row_size_in_kb) { 1981 case 1: 1982 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 1983 break; 1984 case 2: 1985 default: 1986 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 1987 break; 1988 case 4: 1989 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 1990 break; 1991 } 1992 1993 num_pipe_configs = rdev->config.cik.max_tile_pipes; 1994 if (num_pipe_configs > 8) 1995 num_pipe_configs = 16; 1996 1997 if (num_pipe_configs == 16) { 1998 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 1999 switch (reg_offset) { 2000 case 0: 2001 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2002 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2003 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2004 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2005 break; 2006 case 1: 2007 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2008 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2009 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2010 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2011 break; 2012 case 2: 2013 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2014 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2015 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2016 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2017 break; 2018 case 3: 2019 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2020 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2021 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2022 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2023 break; 2024 case 4: 2025 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2026 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2027 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2028 TILE_SPLIT(split_equal_to_row_size)); 2029 break; 2030 case 5: 2031 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2032 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2033 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2034 break; 2035 case 6: 2036 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2037 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2038 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2039 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2040 break; 2041 case 7: 2042 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2043 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2044 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2045 TILE_SPLIT(split_equal_to_row_size)); 2046 break; 2047 case 8: 2048 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2049 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2050 break; 2051 case 9: 2052 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2053 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2054 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2055 break; 2056 case 10: 2057 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2058 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2059 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2060 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2061 break; 2062 case 11: 2063 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2064 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2065 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2066 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2067 break; 2068 case 12: 2069 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2070 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2071 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2072 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2073 break; 2074 case 13: 2075 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2076 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2078 break; 2079 case 14: 2080 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2081 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2082 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2083 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2084 break; 2085 case 16: 2086 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2087 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2088 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2089 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2090 break; 2091 case 17: 2092 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2093 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2094 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2095 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2096 break; 2097 case 27: 2098 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2099 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2100 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2101 break; 2102 case 28: 2103 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2104 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2105 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2106 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2107 break; 2108 case 29: 2109 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2110 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2111 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) | 2112 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2113 break; 2114 case 30: 2115 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2116 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2117 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2118 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2119 break; 2120 default: 2121 gb_tile_moden = 0; 2122 break; 2123 } 2124 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2125 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2126 } 2127 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2128 switch (reg_offset) { 2129 case 0: 2130 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2131 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2132 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2133 NUM_BANKS(ADDR_SURF_16_BANK)); 2134 break; 2135 case 1: 2136 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2137 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2138 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2139 NUM_BANKS(ADDR_SURF_16_BANK)); 2140 break; 2141 case 2: 2142 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2143 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2144 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2145 NUM_BANKS(ADDR_SURF_16_BANK)); 2146 break; 2147 case 3: 2148 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2149 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2150 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2151 NUM_BANKS(ADDR_SURF_16_BANK)); 2152 break; 2153 case 4: 2154 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2155 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2156 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2157 NUM_BANKS(ADDR_SURF_8_BANK)); 2158 break; 2159 case 5: 2160 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2161 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2162 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2163 NUM_BANKS(ADDR_SURF_4_BANK)); 2164 break; 2165 case 6: 2166 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2167 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2168 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2169 NUM_BANKS(ADDR_SURF_2_BANK)); 2170 break; 2171 case 8: 2172 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2173 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2174 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2175 NUM_BANKS(ADDR_SURF_16_BANK)); 2176 break; 2177 case 9: 2178 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2179 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2180 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2181 NUM_BANKS(ADDR_SURF_16_BANK)); 2182 break; 2183 case 10: 2184 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2185 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2186 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2187 NUM_BANKS(ADDR_SURF_16_BANK)); 2188 break; 2189 case 11: 2190 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2191 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2192 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2193 NUM_BANKS(ADDR_SURF_8_BANK)); 2194 break; 2195 case 12: 2196 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2197 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2198 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2199 NUM_BANKS(ADDR_SURF_4_BANK)); 2200 break; 2201 case 13: 2202 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2203 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2204 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2205 NUM_BANKS(ADDR_SURF_2_BANK)); 2206 break; 2207 case 14: 2208 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2209 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2210 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2211 NUM_BANKS(ADDR_SURF_2_BANK)); 2212 break; 2213 default: 2214 gb_tile_moden = 0; 2215 break; 2216 } 2217 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2218 } 2219 } else if (num_pipe_configs == 8) { 2220 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2221 switch (reg_offset) { 2222 case 0: 2223 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2224 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2225 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2226 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2227 break; 2228 case 1: 2229 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2230 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2231 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2232 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2233 break; 2234 case 2: 2235 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2236 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2237 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2238 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2239 break; 2240 case 3: 2241 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2242 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2243 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2244 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2245 break; 2246 case 4: 2247 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2248 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2249 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2250 TILE_SPLIT(split_equal_to_row_size)); 2251 break; 2252 case 5: 2253 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2255 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2256 break; 2257 case 6: 2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2261 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2262 break; 2263 case 7: 2264 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2265 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2266 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2267 TILE_SPLIT(split_equal_to_row_size)); 2268 break; 2269 case 8: 2270 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2271 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2272 break; 2273 case 9: 2274 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2275 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2276 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2277 break; 2278 case 10: 2279 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2280 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2282 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2283 break; 2284 case 11: 2285 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2286 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2287 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2288 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2289 break; 2290 case 12: 2291 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2292 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2293 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2294 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2295 break; 2296 case 13: 2297 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2298 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2299 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2300 break; 2301 case 14: 2302 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2303 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2304 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2305 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2306 break; 2307 case 16: 2308 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2309 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2310 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2311 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2312 break; 2313 case 17: 2314 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2315 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2316 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2317 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2318 break; 2319 case 27: 2320 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2321 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2322 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2323 break; 2324 case 28: 2325 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2326 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2327 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2328 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2329 break; 2330 case 29: 2331 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2332 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2333 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2334 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2335 break; 2336 case 30: 2337 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2338 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2339 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2340 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2341 break; 2342 default: 2343 gb_tile_moden = 0; 2344 break; 2345 } 2346 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2347 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2348 } 2349 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2350 switch (reg_offset) { 2351 case 0: 2352 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2353 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2354 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2355 NUM_BANKS(ADDR_SURF_16_BANK)); 2356 break; 2357 case 1: 2358 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2359 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2360 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2361 NUM_BANKS(ADDR_SURF_16_BANK)); 2362 break; 2363 case 2: 2364 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2365 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2366 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2367 NUM_BANKS(ADDR_SURF_16_BANK)); 2368 break; 2369 case 3: 2370 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2371 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2372 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2373 NUM_BANKS(ADDR_SURF_16_BANK)); 2374 break; 2375 case 4: 2376 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2379 NUM_BANKS(ADDR_SURF_8_BANK)); 2380 break; 2381 case 5: 2382 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2383 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2384 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2385 NUM_BANKS(ADDR_SURF_4_BANK)); 2386 break; 2387 case 6: 2388 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2389 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2390 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2391 NUM_BANKS(ADDR_SURF_2_BANK)); 2392 break; 2393 case 8: 2394 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2395 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2396 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2397 NUM_BANKS(ADDR_SURF_16_BANK)); 2398 break; 2399 case 9: 2400 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2401 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2402 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2403 NUM_BANKS(ADDR_SURF_16_BANK)); 2404 break; 2405 case 10: 2406 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2409 NUM_BANKS(ADDR_SURF_16_BANK)); 2410 break; 2411 case 11: 2412 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2413 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2414 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2415 NUM_BANKS(ADDR_SURF_16_BANK)); 2416 break; 2417 case 12: 2418 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2419 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2420 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2421 NUM_BANKS(ADDR_SURF_8_BANK)); 2422 break; 2423 case 13: 2424 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2425 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2426 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2427 NUM_BANKS(ADDR_SURF_4_BANK)); 2428 break; 2429 case 14: 2430 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2431 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2432 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2433 NUM_BANKS(ADDR_SURF_2_BANK)); 2434 break; 2435 default: 2436 gb_tile_moden = 0; 2437 break; 2438 } 2439 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2440 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2441 } 2442 } else if (num_pipe_configs == 4) { 2443 if (num_rbs == 4) { 2444 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2445 switch (reg_offset) { 2446 case 0: 2447 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2448 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2449 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2450 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2451 break; 2452 case 1: 2453 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2454 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2455 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2456 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2457 break; 2458 case 2: 2459 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2460 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2461 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2462 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2463 break; 2464 case 3: 2465 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2466 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2467 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2468 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2469 break; 2470 case 4: 2471 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2472 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2473 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2474 TILE_SPLIT(split_equal_to_row_size)); 2475 break; 2476 case 5: 2477 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2478 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2480 break; 2481 case 6: 2482 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2483 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2484 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2485 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2486 break; 2487 case 7: 2488 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2490 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2491 TILE_SPLIT(split_equal_to_row_size)); 2492 break; 2493 case 8: 2494 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2495 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2496 break; 2497 case 9: 2498 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2499 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2500 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2501 break; 2502 case 10: 2503 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2504 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2505 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2506 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2507 break; 2508 case 11: 2509 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2511 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2512 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2513 break; 2514 case 12: 2515 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2516 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2517 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2518 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2519 break; 2520 case 13: 2521 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2522 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2523 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2524 break; 2525 case 14: 2526 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2528 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2530 break; 2531 case 16: 2532 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2533 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2534 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2535 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2536 break; 2537 case 17: 2538 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2539 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2540 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2541 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2542 break; 2543 case 27: 2544 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2545 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2546 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2547 break; 2548 case 28: 2549 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2550 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2551 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2553 break; 2554 case 29: 2555 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2556 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2557 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2558 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2559 break; 2560 case 30: 2561 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2562 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2563 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2564 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2565 break; 2566 default: 2567 gb_tile_moden = 0; 2568 break; 2569 } 2570 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2571 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2572 } 2573 } else if (num_rbs < 4) { 2574 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2575 switch (reg_offset) { 2576 case 0: 2577 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2579 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2580 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2581 break; 2582 case 1: 2583 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2584 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2585 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2587 break; 2588 case 2: 2589 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2590 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2591 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2592 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2593 break; 2594 case 3: 2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2597 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2599 break; 2600 case 4: 2601 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2603 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2604 TILE_SPLIT(split_equal_to_row_size)); 2605 break; 2606 case 5: 2607 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2608 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2609 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2610 break; 2611 case 6: 2612 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2613 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2614 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2616 break; 2617 case 7: 2618 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2621 TILE_SPLIT(split_equal_to_row_size)); 2622 break; 2623 case 8: 2624 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2625 PIPE_CONFIG(ADDR_SURF_P4_8x16)); 2626 break; 2627 case 9: 2628 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2629 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2630 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING)); 2631 break; 2632 case 10: 2633 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2634 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2635 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2637 break; 2638 case 11: 2639 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2640 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2641 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2643 break; 2644 case 12: 2645 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2646 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2647 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2648 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2649 break; 2650 case 13: 2651 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2652 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2654 break; 2655 case 14: 2656 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2657 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2658 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2659 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2660 break; 2661 case 16: 2662 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2663 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2666 break; 2667 case 17: 2668 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2669 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2672 break; 2673 case 27: 2674 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2676 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING)); 2677 break; 2678 case 28: 2679 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2680 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2681 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2682 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2683 break; 2684 case 29: 2685 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2686 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2687 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2688 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2689 break; 2690 case 30: 2691 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2692 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2693 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2694 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2695 break; 2696 default: 2697 gb_tile_moden = 0; 2698 break; 2699 } 2700 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2701 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2702 } 2703 } 2704 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2705 switch (reg_offset) { 2706 case 0: 2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2710 NUM_BANKS(ADDR_SURF_16_BANK)); 2711 break; 2712 case 1: 2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2716 NUM_BANKS(ADDR_SURF_16_BANK)); 2717 break; 2718 case 2: 2719 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2722 NUM_BANKS(ADDR_SURF_16_BANK)); 2723 break; 2724 case 3: 2725 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2728 NUM_BANKS(ADDR_SURF_16_BANK)); 2729 break; 2730 case 4: 2731 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2734 NUM_BANKS(ADDR_SURF_16_BANK)); 2735 break; 2736 case 5: 2737 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2740 NUM_BANKS(ADDR_SURF_8_BANK)); 2741 break; 2742 case 6: 2743 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2746 NUM_BANKS(ADDR_SURF_4_BANK)); 2747 break; 2748 case 8: 2749 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2752 NUM_BANKS(ADDR_SURF_16_BANK)); 2753 break; 2754 case 9: 2755 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2758 NUM_BANKS(ADDR_SURF_16_BANK)); 2759 break; 2760 case 10: 2761 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2764 NUM_BANKS(ADDR_SURF_16_BANK)); 2765 break; 2766 case 11: 2767 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2768 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2769 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2770 NUM_BANKS(ADDR_SURF_16_BANK)); 2771 break; 2772 case 12: 2773 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2774 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2775 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2776 NUM_BANKS(ADDR_SURF_16_BANK)); 2777 break; 2778 case 13: 2779 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2782 NUM_BANKS(ADDR_SURF_8_BANK)); 2783 break; 2784 case 14: 2785 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2786 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2787 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2788 NUM_BANKS(ADDR_SURF_4_BANK)); 2789 break; 2790 default: 2791 gb_tile_moden = 0; 2792 break; 2793 } 2794 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 2795 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2796 } 2797 } else if (num_pipe_configs == 2) { 2798 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2799 switch (reg_offset) { 2800 case 0: 2801 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2803 PIPE_CONFIG(ADDR_SURF_P2) | 2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B)); 2805 break; 2806 case 1: 2807 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2808 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2809 PIPE_CONFIG(ADDR_SURF_P2) | 2810 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B)); 2811 break; 2812 case 2: 2813 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2814 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2815 PIPE_CONFIG(ADDR_SURF_P2) | 2816 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2817 break; 2818 case 3: 2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2821 PIPE_CONFIG(ADDR_SURF_P2) | 2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B)); 2823 break; 2824 case 4: 2825 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2827 PIPE_CONFIG(ADDR_SURF_P2) | 2828 TILE_SPLIT(split_equal_to_row_size)); 2829 break; 2830 case 5: 2831 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2832 PIPE_CONFIG(ADDR_SURF_P2) | 2833 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2834 break; 2835 case 6: 2836 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2837 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2838 PIPE_CONFIG(ADDR_SURF_P2) | 2839 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B)); 2840 break; 2841 case 7: 2842 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2843 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) | 2844 PIPE_CONFIG(ADDR_SURF_P2) | 2845 TILE_SPLIT(split_equal_to_row_size)); 2846 break; 2847 case 8: 2848 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2849 PIPE_CONFIG(ADDR_SURF_P2); 2850 break; 2851 case 9: 2852 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2853 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2854 PIPE_CONFIG(ADDR_SURF_P2)); 2855 break; 2856 case 10: 2857 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2858 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2859 PIPE_CONFIG(ADDR_SURF_P2) | 2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2861 break; 2862 case 11: 2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2864 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2865 PIPE_CONFIG(ADDR_SURF_P2) | 2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2867 break; 2868 case 12: 2869 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2870 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2871 PIPE_CONFIG(ADDR_SURF_P2) | 2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2873 break; 2874 case 13: 2875 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2876 PIPE_CONFIG(ADDR_SURF_P2) | 2877 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING)); 2878 break; 2879 case 14: 2880 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2881 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2882 PIPE_CONFIG(ADDR_SURF_P2) | 2883 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2884 break; 2885 case 16: 2886 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2887 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2888 PIPE_CONFIG(ADDR_SURF_P2) | 2889 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2890 break; 2891 case 17: 2892 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2893 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2894 PIPE_CONFIG(ADDR_SURF_P2) | 2895 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2896 break; 2897 case 27: 2898 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2899 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2900 PIPE_CONFIG(ADDR_SURF_P2)); 2901 break; 2902 case 28: 2903 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2904 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2905 PIPE_CONFIG(ADDR_SURF_P2) | 2906 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2907 break; 2908 case 29: 2909 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2910 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2911 PIPE_CONFIG(ADDR_SURF_P2) | 2912 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2913 break; 2914 case 30: 2915 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) | 2916 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2917 PIPE_CONFIG(ADDR_SURF_P2) | 2918 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2919 break; 2920 default: 2921 gb_tile_moden = 0; 2922 break; 2923 } 2924 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; 2925 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2926 } 2927 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) { 2928 switch (reg_offset) { 2929 case 0: 2930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2933 NUM_BANKS(ADDR_SURF_16_BANK)); 2934 break; 2935 case 1: 2936 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2939 NUM_BANKS(ADDR_SURF_16_BANK)); 2940 break; 2941 case 2: 2942 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2945 NUM_BANKS(ADDR_SURF_16_BANK)); 2946 break; 2947 case 3: 2948 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2951 NUM_BANKS(ADDR_SURF_16_BANK)); 2952 break; 2953 case 4: 2954 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2957 NUM_BANKS(ADDR_SURF_16_BANK)); 2958 break; 2959 case 5: 2960 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2963 NUM_BANKS(ADDR_SURF_16_BANK)); 2964 break; 2965 case 6: 2966 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2969 NUM_BANKS(ADDR_SURF_8_BANK)); 2970 break; 2971 case 8: 2972 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2975 NUM_BANKS(ADDR_SURF_16_BANK)); 2976 break; 2977 case 9: 2978 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2981 NUM_BANKS(ADDR_SURF_16_BANK)); 2982 break; 2983 case 10: 2984 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2987 NUM_BANKS(ADDR_SURF_16_BANK)); 2988 break; 2989 case 11: 2990 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2991 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2992 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2993 NUM_BANKS(ADDR_SURF_16_BANK)); 2994 break; 2995 case 12: 2996 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2997 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2998 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2999 NUM_BANKS(ADDR_SURF_16_BANK)); 3000 break; 3001 case 13: 3002 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3003 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3004 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3005 NUM_BANKS(ADDR_SURF_16_BANK)); 3006 break; 3007 case 14: 3008 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3011 NUM_BANKS(ADDR_SURF_8_BANK)); 3012 break; 3013 default: 3014 gb_tile_moden = 0; 3015 break; 3016 } 3017 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; 3018 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden); 3019 } 3020 } else 3021 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs); 3022 } 3023 3024 /** 3025 * cik_select_se_sh - select which SE, SH to address 3026 * 3027 * @rdev: radeon_device pointer 3028 * @se_num: shader engine to address 3029 * @sh_num: sh block to address 3030 * 3031 * Select which SE, SH combinations to address. Certain 3032 * registers are instanced per SE or SH. 0xffffffff means 3033 * broadcast to all SEs or SHs (CIK). 3034 */ 3035 static void cik_select_se_sh(struct radeon_device *rdev, 3036 u32 se_num, u32 sh_num) 3037 { 3038 u32 data = INSTANCE_BROADCAST_WRITES; 3039 3040 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 3041 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 3042 else if (se_num == 0xffffffff) 3043 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 3044 else if (sh_num == 0xffffffff) 3045 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 3046 else 3047 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 3048 WREG32(GRBM_GFX_INDEX, data); 3049 } 3050 3051 /** 3052 * cik_create_bitmask - create a bitmask 3053 * 3054 * @bit_width: length of the mask 3055 * 3056 * create a variable length bit mask (CIK). 3057 * Returns the bitmask. 3058 */ 3059 static u32 cik_create_bitmask(u32 bit_width) 3060 { 3061 u32 i, mask = 0; 3062 3063 for (i = 0; i < bit_width; i++) { 3064 mask <<= 1; 3065 mask |= 1; 3066 } 3067 return mask; 3068 } 3069 3070 /** 3071 * cik_get_rb_disabled - computes the mask of disabled RBs 3072 * 3073 * @rdev: radeon_device pointer 3074 * @max_rb_num: max RBs (render backends) for the asic 3075 * @se_num: number of SEs (shader engines) for the asic 3076 * @sh_per_se: number of SH blocks per SE for the asic 3077 * 3078 * Calculates the bitmask of disabled RBs (CIK). 3079 * Returns the disabled RB bitmask. 3080 */ 3081 static u32 cik_get_rb_disabled(struct radeon_device *rdev, 3082 u32 max_rb_num_per_se, 3083 u32 sh_per_se) 3084 { 3085 u32 data, mask; 3086 3087 data = RREG32(CC_RB_BACKEND_DISABLE); 3088 if (data & 1) 3089 data &= BACKEND_DISABLE_MASK; 3090 else 3091 data = 0; 3092 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 3093 3094 data >>= BACKEND_DISABLE_SHIFT; 3095 3096 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se); 3097 3098 return data & mask; 3099 } 3100 3101 /** 3102 * cik_setup_rb - setup the RBs on the asic 3103 * 3104 * @rdev: radeon_device pointer 3105 * @se_num: number of SEs (shader engines) for the asic 3106 * @sh_per_se: number of SH blocks per SE for the asic 3107 * @max_rb_num: max RBs (render backends) for the asic 3108 * 3109 * Configures per-SE/SH RB registers (CIK). 3110 */ 3111 static void cik_setup_rb(struct radeon_device *rdev, 3112 u32 se_num, u32 sh_per_se, 3113 u32 max_rb_num_per_se) 3114 { 3115 int i, j; 3116 u32 data, mask; 3117 u32 disabled_rbs = 0; 3118 u32 enabled_rbs = 0; 3119 3120 for (i = 0; i < se_num; i++) { 3121 for (j = 0; j < sh_per_se; j++) { 3122 cik_select_se_sh(rdev, i, j); 3123 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3124 if (rdev->family == CHIP_HAWAII) 3125 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH); 3126 else 3127 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH); 3128 } 3129 } 3130 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3131 3132 mask = 1; 3133 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3134 if (!(disabled_rbs & mask)) 3135 enabled_rbs |= mask; 3136 mask <<= 1; 3137 } 3138 3139 rdev->config.cik.backend_enable_mask = enabled_rbs; 3140 3141 for (i = 0; i < se_num; i++) { 3142 cik_select_se_sh(rdev, i, 0xffffffff); 3143 data = 0; 3144 for (j = 0; j < sh_per_se; j++) { 3145 switch (enabled_rbs & 3) { 3146 case 0: 3147 if (j == 0) 3148 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3); 3149 else 3150 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0); 3151 break; 3152 case 1: 3153 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3154 break; 3155 case 2: 3156 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3157 break; 3158 case 3: 3159 default: 3160 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3161 break; 3162 } 3163 enabled_rbs >>= 2; 3164 } 3165 WREG32(PA_SC_RASTER_CONFIG, data); 3166 } 3167 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3168 } 3169 3170 /** 3171 * cik_gpu_init - setup the 3D engine 3172 * 3173 * @rdev: radeon_device pointer 3174 * 3175 * Configures the 3D engine and tiling configuration 3176 * registers so that the 3D engine is usable. 3177 */ 3178 static void cik_gpu_init(struct radeon_device *rdev) 3179 { 3180 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG); 3181 u32 mc_shared_chmap, mc_arb_ramcfg; 3182 u32 hdp_host_path_cntl; 3183 u32 tmp; 3184 int i, j; 3185 3186 switch (rdev->family) { 3187 case CHIP_BONAIRE: 3188 rdev->config.cik.max_shader_engines = 2; 3189 rdev->config.cik.max_tile_pipes = 4; 3190 rdev->config.cik.max_cu_per_sh = 7; 3191 rdev->config.cik.max_sh_per_se = 1; 3192 rdev->config.cik.max_backends_per_se = 2; 3193 rdev->config.cik.max_texture_channel_caches = 4; 3194 rdev->config.cik.max_gprs = 256; 3195 rdev->config.cik.max_gs_threads = 32; 3196 rdev->config.cik.max_hw_contexts = 8; 3197 3198 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3199 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3200 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3201 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3202 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3203 break; 3204 case CHIP_HAWAII: 3205 rdev->config.cik.max_shader_engines = 4; 3206 rdev->config.cik.max_tile_pipes = 16; 3207 rdev->config.cik.max_cu_per_sh = 11; 3208 rdev->config.cik.max_sh_per_se = 1; 3209 rdev->config.cik.max_backends_per_se = 4; 3210 rdev->config.cik.max_texture_channel_caches = 16; 3211 rdev->config.cik.max_gprs = 256; 3212 rdev->config.cik.max_gs_threads = 32; 3213 rdev->config.cik.max_hw_contexts = 8; 3214 3215 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3216 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3217 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3218 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3219 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN; 3220 break; 3221 case CHIP_KAVERI: 3222 rdev->config.cik.max_shader_engines = 1; 3223 rdev->config.cik.max_tile_pipes = 4; 3224 if ((rdev->pdev->device == 0x1304) || 3225 (rdev->pdev->device == 0x1305) || 3226 (rdev->pdev->device == 0x130C) || 3227 (rdev->pdev->device == 0x130F) || 3228 (rdev->pdev->device == 0x1310) || 3229 (rdev->pdev->device == 0x1311) || 3230 (rdev->pdev->device == 0x131C)) { 3231 rdev->config.cik.max_cu_per_sh = 8; 3232 rdev->config.cik.max_backends_per_se = 2; 3233 } else if ((rdev->pdev->device == 0x1309) || 3234 (rdev->pdev->device == 0x130A) || 3235 (rdev->pdev->device == 0x130D) || 3236 (rdev->pdev->device == 0x1313) || 3237 (rdev->pdev->device == 0x131D)) { 3238 rdev->config.cik.max_cu_per_sh = 6; 3239 rdev->config.cik.max_backends_per_se = 2; 3240 } else if ((rdev->pdev->device == 0x1306) || 3241 (rdev->pdev->device == 0x1307) || 3242 (rdev->pdev->device == 0x130B) || 3243 (rdev->pdev->device == 0x130E) || 3244 (rdev->pdev->device == 0x1315) || 3245 (rdev->pdev->device == 0x131B)) { 3246 rdev->config.cik.max_cu_per_sh = 4; 3247 rdev->config.cik.max_backends_per_se = 1; 3248 } else { 3249 rdev->config.cik.max_cu_per_sh = 3; 3250 rdev->config.cik.max_backends_per_se = 1; 3251 } 3252 rdev->config.cik.max_sh_per_se = 1; 3253 rdev->config.cik.max_texture_channel_caches = 4; 3254 rdev->config.cik.max_gprs = 256; 3255 rdev->config.cik.max_gs_threads = 16; 3256 rdev->config.cik.max_hw_contexts = 8; 3257 3258 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3259 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3260 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3261 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3262 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3263 break; 3264 case CHIP_KABINI: 3265 default: 3266 rdev->config.cik.max_shader_engines = 1; 3267 rdev->config.cik.max_tile_pipes = 2; 3268 rdev->config.cik.max_cu_per_sh = 2; 3269 rdev->config.cik.max_sh_per_se = 1; 3270 rdev->config.cik.max_backends_per_se = 1; 3271 rdev->config.cik.max_texture_channel_caches = 2; 3272 rdev->config.cik.max_gprs = 256; 3273 rdev->config.cik.max_gs_threads = 16; 3274 rdev->config.cik.max_hw_contexts = 8; 3275 3276 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; 3277 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; 3278 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; 3279 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; 3280 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN; 3281 break; 3282 } 3283 3284 /* Initialize HDP */ 3285 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3286 WREG32((0x2c14 + j), 0x00000000); 3287 WREG32((0x2c18 + j), 0x00000000); 3288 WREG32((0x2c1c + j), 0x00000000); 3289 WREG32((0x2c20 + j), 0x00000000); 3290 WREG32((0x2c24 + j), 0x00000000); 3291 } 3292 3293 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3294 3295 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3296 3297 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3298 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3299 3300 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; 3301 rdev->config.cik.mem_max_burst_length_bytes = 256; 3302 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3303 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3304 if (rdev->config.cik.mem_row_size_in_kb > 4) 3305 rdev->config.cik.mem_row_size_in_kb = 4; 3306 /* XXX use MC settings? */ 3307 rdev->config.cik.shader_engine_tile_size = 32; 3308 rdev->config.cik.num_gpus = 1; 3309 rdev->config.cik.multi_gpu_tile_size = 64; 3310 3311 /* fix up row size */ 3312 gb_addr_config &= ~ROW_SIZE_MASK; 3313 switch (rdev->config.cik.mem_row_size_in_kb) { 3314 case 1: 3315 default: 3316 gb_addr_config |= ROW_SIZE(0); 3317 break; 3318 case 2: 3319 gb_addr_config |= ROW_SIZE(1); 3320 break; 3321 case 4: 3322 gb_addr_config |= ROW_SIZE(2); 3323 break; 3324 } 3325 3326 /* setup tiling info dword. gb_addr_config is not adequate since it does 3327 * not have bank info, so create a custom tiling dword. 3328 * bits 3:0 num_pipes 3329 * bits 7:4 num_banks 3330 * bits 11:8 group_size 3331 * bits 15:12 row_size 3332 */ 3333 rdev->config.cik.tile_config = 0; 3334 switch (rdev->config.cik.num_tile_pipes) { 3335 case 1: 3336 rdev->config.cik.tile_config |= (0 << 0); 3337 break; 3338 case 2: 3339 rdev->config.cik.tile_config |= (1 << 0); 3340 break; 3341 case 4: 3342 rdev->config.cik.tile_config |= (2 << 0); 3343 break; 3344 case 8: 3345 default: 3346 /* XXX what about 12? */ 3347 rdev->config.cik.tile_config |= (3 << 0); 3348 break; 3349 } 3350 rdev->config.cik.tile_config |= 3351 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 3352 rdev->config.cik.tile_config |= 3353 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3354 rdev->config.cik.tile_config |= 3355 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3356 3357 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3358 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3359 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3360 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70); 3361 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70); 3362 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3363 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3364 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3365 3366 cik_tiling_mode_table_init(rdev); 3367 3368 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, 3369 rdev->config.cik.max_sh_per_se, 3370 rdev->config.cik.max_backends_per_se); 3371 3372 /* set HW defaults for 3D engine */ 3373 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3374 3375 WREG32(SX_DEBUG_1, 0x20); 3376 3377 WREG32(TA_CNTL_AUX, 0x00010000); 3378 3379 tmp = RREG32(SPI_CONFIG_CNTL); 3380 tmp |= 0x03000000; 3381 WREG32(SPI_CONFIG_CNTL, tmp); 3382 3383 WREG32(SQ_CONFIG, 1); 3384 3385 WREG32(DB_DEBUG, 0); 3386 3387 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff; 3388 tmp |= 0x00000400; 3389 WREG32(DB_DEBUG2, tmp); 3390 3391 tmp = RREG32(DB_DEBUG3) & ~0x0002021c; 3392 tmp |= 0x00020200; 3393 WREG32(DB_DEBUG3, tmp); 3394 3395 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000; 3396 tmp |= 0x00018208; 3397 WREG32(CB_HW_CONTROL, tmp); 3398 3399 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3400 3401 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | 3402 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | 3403 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | 3404 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); 3405 3406 WREG32(VGT_NUM_INSTANCES, 1); 3407 3408 WREG32(CP_PERFMON_CNTL, 0); 3409 3410 WREG32(SQ_CONFIG, 0); 3411 3412 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3413 FORCE_EOV_MAX_REZ_CNT(255))); 3414 3415 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3416 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3417 3418 WREG32(VGT_GS_VERTEX_REUSE, 16); 3419 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3420 3421 tmp = RREG32(HDP_MISC_CNTL); 3422 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3423 WREG32(HDP_MISC_CNTL, tmp); 3424 3425 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3426 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3427 3428 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3429 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER); 3430 3431 udelay(50); 3432 } 3433 3434 /* 3435 * GPU scratch registers helpers function. 3436 */ 3437 /** 3438 * cik_scratch_init - setup driver info for CP scratch regs 3439 * 3440 * @rdev: radeon_device pointer 3441 * 3442 * Set up the number and offset of the CP scratch registers. 3443 * NOTE: use of CP scratch registers is a legacy inferface and 3444 * is not used by default on newer asics (r6xx+). On newer asics, 3445 * memory buffers are used for fences rather than scratch regs. 3446 */ 3447 static void cik_scratch_init(struct radeon_device *rdev) 3448 { 3449 int i; 3450 3451 rdev->scratch.num_reg = 7; 3452 rdev->scratch.reg_base = SCRATCH_REG0; 3453 for (i = 0; i < rdev->scratch.num_reg; i++) { 3454 rdev->scratch.free[i] = true; 3455 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3456 } 3457 } 3458 3459 /** 3460 * cik_ring_test - basic gfx ring test 3461 * 3462 * @rdev: radeon_device pointer 3463 * @ring: radeon_ring structure holding ring information 3464 * 3465 * Allocate a scratch register and write to it using the gfx ring (CIK). 3466 * Provides a basic gfx ring test to verify that the ring is working. 3467 * Used by cik_cp_gfx_resume(); 3468 * Returns 0 on success, error on failure. 3469 */ 3470 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) 3471 { 3472 uint32_t scratch; 3473 uint32_t tmp = 0; 3474 unsigned i; 3475 int r; 3476 3477 r = radeon_scratch_get(rdev, &scratch); 3478 if (r) { 3479 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r); 3480 return r; 3481 } 3482 WREG32(scratch, 0xCAFEDEAD); 3483 r = radeon_ring_lock(rdev, ring, 3); 3484 if (r) { 3485 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r); 3486 radeon_scratch_free(rdev, scratch); 3487 return r; 3488 } 3489 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3490 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2)); 3491 radeon_ring_write(ring, 0xDEADBEEF); 3492 radeon_ring_unlock_commit(rdev, ring); 3493 3494 for (i = 0; i < rdev->usec_timeout; i++) { 3495 tmp = RREG32(scratch); 3496 if (tmp == 0xDEADBEEF) 3497 break; 3498 DRM_UDELAY(1); 3499 } 3500 if (i < rdev->usec_timeout) { 3501 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 3502 } else { 3503 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n", 3504 ring->idx, scratch, tmp); 3505 r = -EINVAL; 3506 } 3507 radeon_scratch_free(rdev, scratch); 3508 return r; 3509 } 3510 3511 /** 3512 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp 3513 * 3514 * @rdev: radeon_device pointer 3515 * @ridx: radeon ring index 3516 * 3517 * Emits an hdp flush on the cp. 3518 */ 3519 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, 3520 int ridx) 3521 { 3522 struct radeon_ring *ring = &rdev->ring[ridx]; 3523 u32 ref_and_mask; 3524 3525 switch (ring->idx) { 3526 case CAYMAN_RING_TYPE_CP1_INDEX: 3527 case CAYMAN_RING_TYPE_CP2_INDEX: 3528 default: 3529 switch (ring->me) { 3530 case 0: 3531 ref_and_mask = CP2 << ring->pipe; 3532 break; 3533 case 1: 3534 ref_and_mask = CP6 << ring->pipe; 3535 break; 3536 default: 3537 return; 3538 } 3539 break; 3540 case RADEON_RING_TYPE_GFX_INDEX: 3541 ref_and_mask = CP0; 3542 break; 3543 } 3544 3545 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 3546 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 3547 WAIT_REG_MEM_FUNCTION(3) | /* == */ 3548 WAIT_REG_MEM_ENGINE(1))); /* pfp */ 3549 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2); 3550 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2); 3551 radeon_ring_write(ring, ref_and_mask); 3552 radeon_ring_write(ring, ref_and_mask); 3553 radeon_ring_write(ring, 0x20); /* poll interval */ 3554 } 3555 3556 /** 3557 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring 3558 * 3559 * @rdev: radeon_device pointer 3560 * @fence: radeon fence object 3561 * 3562 * Emits a fence sequnce number on the gfx ring and flushes 3563 * GPU caches. 3564 */ 3565 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, 3566 struct radeon_fence *fence) 3567 { 3568 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3569 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3570 3571 /* EVENT_WRITE_EOP - flush caches, send int */ 3572 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3573 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3574 EOP_TC_ACTION_EN | 3575 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3576 EVENT_INDEX(5))); 3577 radeon_ring_write(ring, addr & 0xfffffffc); 3578 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2)); 3579 radeon_ring_write(ring, fence->seq); 3580 radeon_ring_write(ring, 0); 3581 /* HDP flush */ 3582 cik_hdp_flush_cp_ring_emit(rdev, fence->ring); 3583 } 3584 3585 /** 3586 * cik_fence_compute_ring_emit - emit a fence on the compute ring 3587 * 3588 * @rdev: radeon_device pointer 3589 * @fence: radeon fence object 3590 * 3591 * Emits a fence sequnce number on the compute ring and flushes 3592 * GPU caches. 3593 */ 3594 void cik_fence_compute_ring_emit(struct radeon_device *rdev, 3595 struct radeon_fence *fence) 3596 { 3597 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3598 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3599 3600 /* RELEASE_MEM - flush caches, send int */ 3601 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 3602 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN | 3603 EOP_TC_ACTION_EN | 3604 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3605 EVENT_INDEX(5))); 3606 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2)); 3607 radeon_ring_write(ring, addr & 0xfffffffc); 3608 radeon_ring_write(ring, upper_32_bits(addr)); 3609 radeon_ring_write(ring, fence->seq); 3610 radeon_ring_write(ring, 0); 3611 /* HDP flush */ 3612 cik_hdp_flush_cp_ring_emit(rdev, fence->ring); 3613 } 3614 3615 bool cik_semaphore_ring_emit(struct radeon_device *rdev, 3616 struct radeon_ring *ring, 3617 struct radeon_semaphore *semaphore, 3618 bool emit_wait) 3619 { 3620 uint64_t addr = semaphore->gpu_addr; 3621 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; 3622 3623 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1)); 3624 radeon_ring_write(ring, addr & 0xffffffff); 3625 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel); 3626 3627 return true; 3628 } 3629 3630 /** 3631 * cik_copy_cpdma - copy pages using the CP DMA engine 3632 * 3633 * @rdev: radeon_device pointer 3634 * @src_offset: src GPU address 3635 * @dst_offset: dst GPU address 3636 * @num_gpu_pages: number of GPU pages to xfer 3637 * @fence: radeon fence object 3638 * 3639 * Copy GPU paging using the CP DMA engine (CIK+). 3640 * Used by the radeon ttm implementation to move pages if 3641 * registered as the asic copy callback. 3642 */ 3643 int cik_copy_cpdma(struct radeon_device *rdev, 3644 uint64_t src_offset, uint64_t dst_offset, 3645 unsigned num_gpu_pages, 3646 struct radeon_fence **fence) 3647 { 3648 struct radeon_semaphore *sem = NULL; 3649 int ring_index = rdev->asic->copy.blit_ring_index; 3650 struct radeon_ring *ring = &rdev->ring[ring_index]; 3651 u32 size_in_bytes, cur_size_in_bytes, control; 3652 int i, num_loops; 3653 int r = 0; 3654 3655 r = radeon_semaphore_create(rdev, &sem); 3656 if (r) { 3657 DRM_ERROR("radeon: moving bo (%d).\n", r); 3658 return r; 3659 } 3660 3661 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 3662 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); 3663 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); 3664 if (r) { 3665 DRM_ERROR("radeon: moving bo (%d).\n", r); 3666 radeon_semaphore_free(rdev, &sem, NULL); 3667 return r; 3668 } 3669 3670 radeon_semaphore_sync_to(sem, *fence); 3671 radeon_semaphore_sync_rings(rdev, sem, ring->idx); 3672 3673 for (i = 0; i < num_loops; i++) { 3674 cur_size_in_bytes = size_in_bytes; 3675 if (cur_size_in_bytes > 0x1fffff) 3676 cur_size_in_bytes = 0x1fffff; 3677 size_in_bytes -= cur_size_in_bytes; 3678 control = 0; 3679 if (size_in_bytes == 0) 3680 control |= PACKET3_DMA_DATA_CP_SYNC; 3681 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 3682 radeon_ring_write(ring, control); 3683 radeon_ring_write(ring, lower_32_bits(src_offset)); 3684 radeon_ring_write(ring, upper_32_bits(src_offset)); 3685 radeon_ring_write(ring, lower_32_bits(dst_offset)); 3686 radeon_ring_write(ring, upper_32_bits(dst_offset)); 3687 radeon_ring_write(ring, cur_size_in_bytes); 3688 src_offset += cur_size_in_bytes; 3689 dst_offset += cur_size_in_bytes; 3690 } 3691 3692 r = radeon_fence_emit(rdev, fence, ring->idx); 3693 if (r) { 3694 radeon_ring_unlock_undo(rdev, ring); 3695 return r; 3696 } 3697 3698 radeon_ring_unlock_commit(rdev, ring); 3699 radeon_semaphore_free(rdev, &sem, *fence); 3700 3701 return r; 3702 } 3703 3704 /* 3705 * IB stuff 3706 */ 3707 /** 3708 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring 3709 * 3710 * @rdev: radeon_device pointer 3711 * @ib: radeon indirect buffer object 3712 * 3713 * Emits an DE (drawing engine) or CE (constant engine) IB 3714 * on the gfx ring. IBs are usually generated by userspace 3715 * acceleration drivers and submitted to the kernel for 3716 * sheduling on the ring. This function schedules the IB 3717 * on the gfx ring for execution by the GPU. 3718 */ 3719 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3720 { 3721 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3722 u32 header, control = INDIRECT_BUFFER_VALID; 3723 3724 if (ib->is_const_ib) { 3725 /* set switch buffer packet before const IB */ 3726 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 3727 radeon_ring_write(ring, 0); 3728 3729 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 3730 } else { 3731 u32 next_rptr; 3732 if (ring->rptr_save_reg) { 3733 next_rptr = ring->wptr + 3 + 4; 3734 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 3735 radeon_ring_write(ring, ((ring->rptr_save_reg - 3736 PACKET3_SET_UCONFIG_REG_START) >> 2)); 3737 radeon_ring_write(ring, next_rptr); 3738 } else if (rdev->wb.enabled) { 3739 next_rptr = ring->wptr + 5 + 4; 3740 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3741 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1)); 3742 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3743 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); 3744 radeon_ring_write(ring, next_rptr); 3745 } 3746 3747 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 3748 } 3749 3750 control |= ib->length_dw | 3751 (ib->vm ? (ib->vm->id << 24) : 0); 3752 3753 radeon_ring_write(ring, header); 3754 radeon_ring_write(ring, 3755 #ifdef __BIG_ENDIAN 3756 (2 << 0) | 3757 #endif 3758 (ib->gpu_addr & 0xFFFFFFFC)); 3759 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 3760 radeon_ring_write(ring, control); 3761 } 3762 3763 /** 3764 * cik_ib_test - basic gfx ring IB test 3765 * 3766 * @rdev: radeon_device pointer 3767 * @ring: radeon_ring structure holding ring information 3768 * 3769 * Allocate an IB and execute it on the gfx ring (CIK). 3770 * Provides a basic gfx ring test to verify that IBs are working. 3771 * Returns 0 on success, error on failure. 3772 */ 3773 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) 3774 { 3775 struct radeon_ib ib; 3776 uint32_t scratch; 3777 uint32_t tmp = 0; 3778 unsigned i; 3779 int r; 3780 3781 r = radeon_scratch_get(rdev, &scratch); 3782 if (r) { 3783 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r); 3784 return r; 3785 } 3786 WREG32(scratch, 0xCAFEDEAD); 3787 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); 3788 if (r) { 3789 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 3790 radeon_scratch_free(rdev, scratch); 3791 return r; 3792 } 3793 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); 3794 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2); 3795 ib.ptr[2] = 0xDEADBEEF; 3796 ib.length_dw = 3; 3797 r = radeon_ib_schedule(rdev, &ib, NULL); 3798 if (r) { 3799 radeon_scratch_free(rdev, scratch); 3800 radeon_ib_free(rdev, &ib); 3801 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r); 3802 return r; 3803 } 3804 r = radeon_fence_wait(ib.fence, false); 3805 if (r) { 3806 DRM_ERROR("radeon: fence wait failed (%d).\n", r); 3807 radeon_scratch_free(rdev, scratch); 3808 radeon_ib_free(rdev, &ib); 3809 return r; 3810 } 3811 for (i = 0; i < rdev->usec_timeout; i++) { 3812 tmp = RREG32(scratch); 3813 if (tmp == 0xDEADBEEF) 3814 break; 3815 DRM_UDELAY(1); 3816 } 3817 if (i < rdev->usec_timeout) { 3818 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i); 3819 } else { 3820 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 3821 scratch, tmp); 3822 r = -EINVAL; 3823 } 3824 radeon_scratch_free(rdev, scratch); 3825 radeon_ib_free(rdev, &ib); 3826 return r; 3827 } 3828 3829 /* 3830 * CP. 3831 * On CIK, gfx and compute now have independant command processors. 3832 * 3833 * GFX 3834 * Gfx consists of a single ring and can process both gfx jobs and 3835 * compute jobs. The gfx CP consists of three microengines (ME): 3836 * PFP - Pre-Fetch Parser 3837 * ME - Micro Engine 3838 * CE - Constant Engine 3839 * The PFP and ME make up what is considered the Drawing Engine (DE). 3840 * The CE is an asynchronous engine used for updating buffer desciptors 3841 * used by the DE so that they can be loaded into cache in parallel 3842 * while the DE is processing state update packets. 3843 * 3844 * Compute 3845 * The compute CP consists of two microengines (ME): 3846 * MEC1 - Compute MicroEngine 1 3847 * MEC2 - Compute MicroEngine 2 3848 * Each MEC supports 4 compute pipes and each pipe supports 8 queues. 3849 * The queues are exposed to userspace and are programmed directly 3850 * by the compute runtime. 3851 */ 3852 /** 3853 * cik_cp_gfx_enable - enable/disable the gfx CP MEs 3854 * 3855 * @rdev: radeon_device pointer 3856 * @enable: enable or disable the MEs 3857 * 3858 * Halts or unhalts the gfx MEs. 3859 */ 3860 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) 3861 { 3862 if (enable) 3863 WREG32(CP_ME_CNTL, 0); 3864 else { 3865 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 3866 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 3867 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 3868 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3869 } 3870 udelay(50); 3871 } 3872 3873 /** 3874 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode 3875 * 3876 * @rdev: radeon_device pointer 3877 * 3878 * Loads the gfx PFP, ME, and CE ucode. 3879 * Returns 0 for success, -EINVAL if the ucode is not available. 3880 */ 3881 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) 3882 { 3883 const __be32 *fw_data; 3884 int i; 3885 3886 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 3887 return -EINVAL; 3888 3889 cik_cp_gfx_enable(rdev, false); 3890 3891 /* PFP */ 3892 fw_data = (const __be32 *)rdev->pfp_fw->data; 3893 WREG32(CP_PFP_UCODE_ADDR, 0); 3894 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++) 3895 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3896 WREG32(CP_PFP_UCODE_ADDR, 0); 3897 3898 /* CE */ 3899 fw_data = (const __be32 *)rdev->ce_fw->data; 3900 WREG32(CP_CE_UCODE_ADDR, 0); 3901 for (i = 0; i < CIK_CE_UCODE_SIZE; i++) 3902 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 3903 WREG32(CP_CE_UCODE_ADDR, 0); 3904 3905 /* ME */ 3906 fw_data = (const __be32 *)rdev->me_fw->data; 3907 WREG32(CP_ME_RAM_WADDR, 0); 3908 for (i = 0; i < CIK_ME_UCODE_SIZE; i++) 3909 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3910 WREG32(CP_ME_RAM_WADDR, 0); 3911 3912 WREG32(CP_PFP_UCODE_ADDR, 0); 3913 WREG32(CP_CE_UCODE_ADDR, 0); 3914 WREG32(CP_ME_RAM_WADDR, 0); 3915 WREG32(CP_ME_RAM_RADDR, 0); 3916 return 0; 3917 } 3918 3919 /** 3920 * cik_cp_gfx_start - start the gfx ring 3921 * 3922 * @rdev: radeon_device pointer 3923 * 3924 * Enables the ring and loads the clear state context and other 3925 * packets required to init the ring. 3926 * Returns 0 for success, error for failure. 3927 */ 3928 static int cik_cp_gfx_start(struct radeon_device *rdev) 3929 { 3930 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3931 int r, i; 3932 3933 /* init the CP */ 3934 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); 3935 WREG32(CP_ENDIAN_SWAP, 0); 3936 WREG32(CP_DEVICE_ID, 1); 3937 3938 cik_cp_gfx_enable(rdev, true); 3939 3940 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); 3941 if (r) { 3942 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3943 return r; 3944 } 3945 3946 /* init the CE partitions. CE only used for gfx on CIK */ 3947 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3948 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3949 radeon_ring_write(ring, 0xc000); 3950 radeon_ring_write(ring, 0xc000); 3951 3952 /* setup clear context state */ 3953 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3954 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3955 3956 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3957 radeon_ring_write(ring, 0x80000000); 3958 radeon_ring_write(ring, 0x80000000); 3959 3960 for (i = 0; i < cik_default_size; i++) 3961 radeon_ring_write(ring, cik_default_state[i]); 3962 3963 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3964 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3965 3966 /* set clear context state */ 3967 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3968 radeon_ring_write(ring, 0); 3969 3970 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 3971 radeon_ring_write(ring, 0x00000316); 3972 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3973 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 3974 3975 radeon_ring_unlock_commit(rdev, ring); 3976 3977 return 0; 3978 } 3979 3980 /** 3981 * cik_cp_gfx_fini - stop the gfx ring 3982 * 3983 * @rdev: radeon_device pointer 3984 * 3985 * Stop the gfx ring and tear down the driver ring 3986 * info. 3987 */ 3988 static void cik_cp_gfx_fini(struct radeon_device *rdev) 3989 { 3990 cik_cp_gfx_enable(rdev, false); 3991 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 3992 } 3993 3994 /** 3995 * cik_cp_gfx_resume - setup the gfx ring buffer registers 3996 * 3997 * @rdev: radeon_device pointer 3998 * 3999 * Program the location and size of the gfx ring buffer 4000 * and test it to make sure it's working. 4001 * Returns 0 for success, error for failure. 4002 */ 4003 static int cik_cp_gfx_resume(struct radeon_device *rdev) 4004 { 4005 struct radeon_ring *ring; 4006 u32 tmp; 4007 u32 rb_bufsz; 4008 u64 rb_addr; 4009 int r; 4010 4011 WREG32(CP_SEM_WAIT_TIMER, 0x0); 4012 if (rdev->family != CHIP_HAWAII) 4013 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 4014 4015 /* Set the write pointer delay */ 4016 WREG32(CP_RB_WPTR_DELAY, 0); 4017 4018 /* set the RB to use vmid 0 */ 4019 WREG32(CP_RB_VMID, 0); 4020 4021 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 4022 4023 /* ring 0 - compute and gfx */ 4024 /* Set ring buffer size */ 4025 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4026 rb_bufsz = order_base_2(ring->ring_size / 8); 4027 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 4028 #ifdef __BIG_ENDIAN 4029 tmp |= BUF_SWAP_32BIT; 4030 #endif 4031 WREG32(CP_RB0_CNTL, tmp); 4032 4033 /* Initialize the ring buffer's read and write pointers */ 4034 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 4035 ring->wptr = 0; 4036 WREG32(CP_RB0_WPTR, ring->wptr); 4037 4038 /* set the wb address wether it's enabled or not */ 4039 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 4040 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 4041 4042 /* scratch register shadowing is no longer supported */ 4043 WREG32(SCRATCH_UMSK, 0); 4044 4045 if (!rdev->wb.enabled) 4046 tmp |= RB_NO_UPDATE; 4047 4048 mdelay(1); 4049 WREG32(CP_RB0_CNTL, tmp); 4050 4051 rb_addr = ring->gpu_addr >> 8; 4052 WREG32(CP_RB0_BASE, rb_addr); 4053 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4054 4055 /* start the ring */ 4056 cik_cp_gfx_start(rdev); 4057 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 4058 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 4059 if (r) { 4060 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 4061 return r; 4062 } 4063 4064 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 4065 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 4066 4067 return 0; 4068 } 4069 4070 u32 cik_gfx_get_rptr(struct radeon_device *rdev, 4071 struct radeon_ring *ring) 4072 { 4073 u32 rptr; 4074 4075 if (rdev->wb.enabled) 4076 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4077 else 4078 rptr = RREG32(CP_RB0_RPTR); 4079 4080 return rptr; 4081 } 4082 4083 u32 cik_gfx_get_wptr(struct radeon_device *rdev, 4084 struct radeon_ring *ring) 4085 { 4086 u32 wptr; 4087 4088 wptr = RREG32(CP_RB0_WPTR); 4089 4090 return wptr; 4091 } 4092 4093 void cik_gfx_set_wptr(struct radeon_device *rdev, 4094 struct radeon_ring *ring) 4095 { 4096 WREG32(CP_RB0_WPTR, ring->wptr); 4097 (void)RREG32(CP_RB0_WPTR); 4098 } 4099 4100 u32 cik_compute_get_rptr(struct radeon_device *rdev, 4101 struct radeon_ring *ring) 4102 { 4103 u32 rptr; 4104 4105 if (rdev->wb.enabled) { 4106 rptr = rdev->wb.wb[ring->rptr_offs/4]; 4107 } else { 4108 mutex_lock(&rdev->srbm_mutex); 4109 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4110 rptr = RREG32(CP_HQD_PQ_RPTR); 4111 cik_srbm_select(rdev, 0, 0, 0, 0); 4112 mutex_unlock(&rdev->srbm_mutex); 4113 } 4114 4115 return rptr; 4116 } 4117 4118 u32 cik_compute_get_wptr(struct radeon_device *rdev, 4119 struct radeon_ring *ring) 4120 { 4121 u32 wptr; 4122 4123 if (rdev->wb.enabled) { 4124 /* XXX check if swapping is necessary on BE */ 4125 wptr = rdev->wb.wb[ring->wptr_offs/4]; 4126 } else { 4127 mutex_lock(&rdev->srbm_mutex); 4128 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); 4129 wptr = RREG32(CP_HQD_PQ_WPTR); 4130 cik_srbm_select(rdev, 0, 0, 0, 0); 4131 mutex_unlock(&rdev->srbm_mutex); 4132 } 4133 4134 return wptr; 4135 } 4136 4137 void cik_compute_set_wptr(struct radeon_device *rdev, 4138 struct radeon_ring *ring) 4139 { 4140 /* XXX check if swapping is necessary on BE */ 4141 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; 4142 WDOORBELL32(ring->doorbell_index, ring->wptr); 4143 } 4144 4145 /** 4146 * cik_cp_compute_enable - enable/disable the compute CP MEs 4147 * 4148 * @rdev: radeon_device pointer 4149 * @enable: enable or disable the MEs 4150 * 4151 * Halts or unhalts the compute MEs. 4152 */ 4153 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) 4154 { 4155 if (enable) 4156 WREG32(CP_MEC_CNTL, 0); 4157 else { 4158 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT)); 4159 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 4160 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 4161 } 4162 udelay(50); 4163 } 4164 4165 /** 4166 * cik_cp_compute_load_microcode - load the compute CP ME ucode 4167 * 4168 * @rdev: radeon_device pointer 4169 * 4170 * Loads the compute MEC1&2 ucode. 4171 * Returns 0 for success, -EINVAL if the ucode is not available. 4172 */ 4173 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) 4174 { 4175 const __be32 *fw_data; 4176 int i; 4177 4178 if (!rdev->mec_fw) 4179 return -EINVAL; 4180 4181 cik_cp_compute_enable(rdev, false); 4182 4183 /* MEC1 */ 4184 fw_data = (const __be32 *)rdev->mec_fw->data; 4185 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4186 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4187 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++)); 4188 WREG32(CP_MEC_ME1_UCODE_ADDR, 0); 4189 4190 if (rdev->family == CHIP_KAVERI) { 4191 /* MEC2 */ 4192 fw_data = (const __be32 *)rdev->mec_fw->data; 4193 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4194 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++) 4195 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++)); 4196 WREG32(CP_MEC_ME2_UCODE_ADDR, 0); 4197 } 4198 4199 return 0; 4200 } 4201 4202 /** 4203 * cik_cp_compute_start - start the compute queues 4204 * 4205 * @rdev: radeon_device pointer 4206 * 4207 * Enable the compute queues. 4208 * Returns 0 for success, error for failure. 4209 */ 4210 static int cik_cp_compute_start(struct radeon_device *rdev) 4211 { 4212 cik_cp_compute_enable(rdev, true); 4213 4214 return 0; 4215 } 4216 4217 /** 4218 * cik_cp_compute_fini - stop the compute queues 4219 * 4220 * @rdev: radeon_device pointer 4221 * 4222 * Stop the compute queues and tear down the driver queue 4223 * info. 4224 */ 4225 static void cik_cp_compute_fini(struct radeon_device *rdev) 4226 { 4227 int i, idx, r; 4228 4229 cik_cp_compute_enable(rdev, false); 4230 4231 for (i = 0; i < 2; i++) { 4232 if (i == 0) 4233 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4234 else 4235 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4236 4237 if (rdev->ring[idx].mqd_obj) { 4238 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4239 if (unlikely(r != 0)) 4240 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); 4241 4242 radeon_bo_unpin(rdev->ring[idx].mqd_obj); 4243 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4244 4245 radeon_bo_unref(&rdev->ring[idx].mqd_obj); 4246 rdev->ring[idx].mqd_obj = NULL; 4247 } 4248 } 4249 } 4250 4251 static void cik_mec_fini(struct radeon_device *rdev) 4252 { 4253 int r; 4254 4255 if (rdev->mec.hpd_eop_obj) { 4256 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4257 if (unlikely(r != 0)) 4258 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); 4259 radeon_bo_unpin(rdev->mec.hpd_eop_obj); 4260 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4261 4262 radeon_bo_unref(&rdev->mec.hpd_eop_obj); 4263 rdev->mec.hpd_eop_obj = NULL; 4264 } 4265 } 4266 4267 #define MEC_HPD_SIZE 2048 4268 4269 static int cik_mec_init(struct radeon_device *rdev) 4270 { 4271 int r; 4272 u32 *hpd; 4273 4274 /* 4275 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total 4276 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total 4277 */ 4278 if (rdev->family == CHIP_KAVERI) 4279 rdev->mec.num_mec = 2; 4280 else 4281 rdev->mec.num_mec = 1; 4282 rdev->mec.num_pipe = 4; 4283 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; 4284 4285 if (rdev->mec.hpd_eop_obj == NULL) { 4286 r = radeon_bo_create(rdev, 4287 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, 4288 PAGE_SIZE, true, 4289 RADEON_GEM_DOMAIN_GTT, NULL, 4290 &rdev->mec.hpd_eop_obj); 4291 if (r) { 4292 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); 4293 return r; 4294 } 4295 } 4296 4297 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); 4298 if (unlikely(r != 0)) { 4299 cik_mec_fini(rdev); 4300 return r; 4301 } 4302 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, 4303 &rdev->mec.hpd_eop_gpu_addr); 4304 if (r) { 4305 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); 4306 cik_mec_fini(rdev); 4307 return r; 4308 } 4309 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); 4310 if (r) { 4311 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); 4312 cik_mec_fini(rdev); 4313 return r; 4314 } 4315 4316 /* clear memory. Not sure if this is required or not */ 4317 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); 4318 4319 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); 4320 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); 4321 4322 return 0; 4323 } 4324 4325 struct hqd_registers 4326 { 4327 u32 cp_mqd_base_addr; 4328 u32 cp_mqd_base_addr_hi; 4329 u32 cp_hqd_active; 4330 u32 cp_hqd_vmid; 4331 u32 cp_hqd_persistent_state; 4332 u32 cp_hqd_pipe_priority; 4333 u32 cp_hqd_queue_priority; 4334 u32 cp_hqd_quantum; 4335 u32 cp_hqd_pq_base; 4336 u32 cp_hqd_pq_base_hi; 4337 u32 cp_hqd_pq_rptr; 4338 u32 cp_hqd_pq_rptr_report_addr; 4339 u32 cp_hqd_pq_rptr_report_addr_hi; 4340 u32 cp_hqd_pq_wptr_poll_addr; 4341 u32 cp_hqd_pq_wptr_poll_addr_hi; 4342 u32 cp_hqd_pq_doorbell_control; 4343 u32 cp_hqd_pq_wptr; 4344 u32 cp_hqd_pq_control; 4345 u32 cp_hqd_ib_base_addr; 4346 u32 cp_hqd_ib_base_addr_hi; 4347 u32 cp_hqd_ib_rptr; 4348 u32 cp_hqd_ib_control; 4349 u32 cp_hqd_iq_timer; 4350 u32 cp_hqd_iq_rptr; 4351 u32 cp_hqd_dequeue_request; 4352 u32 cp_hqd_dma_offload; 4353 u32 cp_hqd_sema_cmd; 4354 u32 cp_hqd_msg_type; 4355 u32 cp_hqd_atomic0_preop_lo; 4356 u32 cp_hqd_atomic0_preop_hi; 4357 u32 cp_hqd_atomic1_preop_lo; 4358 u32 cp_hqd_atomic1_preop_hi; 4359 u32 cp_hqd_hq_scheduler0; 4360 u32 cp_hqd_hq_scheduler1; 4361 u32 cp_mqd_control; 4362 }; 4363 4364 struct bonaire_mqd 4365 { 4366 u32 header; 4367 u32 dispatch_initiator; 4368 u32 dimensions[3]; 4369 u32 start_idx[3]; 4370 u32 num_threads[3]; 4371 u32 pipeline_stat_enable; 4372 u32 perf_counter_enable; 4373 u32 pgm[2]; 4374 u32 tba[2]; 4375 u32 tma[2]; 4376 u32 pgm_rsrc[2]; 4377 u32 vmid; 4378 u32 resource_limits; 4379 u32 static_thread_mgmt01[2]; 4380 u32 tmp_ring_size; 4381 u32 static_thread_mgmt23[2]; 4382 u32 restart[3]; 4383 u32 thread_trace_enable; 4384 u32 reserved1; 4385 u32 user_data[16]; 4386 u32 vgtcs_invoke_count[2]; 4387 struct hqd_registers queue_state; 4388 u32 dequeue_cntr; 4389 u32 interrupt_queue[64]; 4390 }; 4391 4392 /** 4393 * cik_cp_compute_resume - setup the compute queue registers 4394 * 4395 * @rdev: radeon_device pointer 4396 * 4397 * Program the compute queues and test them to make sure they 4398 * are working. 4399 * Returns 0 for success, error for failure. 4400 */ 4401 static int cik_cp_compute_resume(struct radeon_device *rdev) 4402 { 4403 int r, i, idx; 4404 u32 tmp; 4405 bool use_doorbell = true; 4406 u64 hqd_gpu_addr; 4407 u64 mqd_gpu_addr; 4408 u64 eop_gpu_addr; 4409 u64 wb_gpu_addr; 4410 u32 *buf; 4411 struct bonaire_mqd *mqd; 4412 4413 r = cik_cp_compute_start(rdev); 4414 if (r) 4415 return r; 4416 4417 /* fix up chicken bits */ 4418 tmp = RREG32(CP_CPF_DEBUG); 4419 tmp |= (1 << 23); 4420 WREG32(CP_CPF_DEBUG, tmp); 4421 4422 /* init the pipes */ 4423 mutex_lock(&rdev->srbm_mutex); 4424 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) { 4425 int me = (i < 4) ? 1 : 2; 4426 int pipe = (i < 4) ? i : (i - 4); 4427 4428 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2); 4429 4430 cik_srbm_select(rdev, me, pipe, 0, 0); 4431 4432 /* write the EOP addr */ 4433 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8); 4434 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8); 4435 4436 /* set the VMID assigned */ 4437 WREG32(CP_HPD_EOP_VMID, 0); 4438 4439 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4440 tmp = RREG32(CP_HPD_EOP_CONTROL); 4441 tmp &= ~EOP_SIZE_MASK; 4442 tmp |= order_base_2(MEC_HPD_SIZE / 8); 4443 WREG32(CP_HPD_EOP_CONTROL, tmp); 4444 } 4445 cik_srbm_select(rdev, 0, 0, 0, 0); 4446 mutex_unlock(&rdev->srbm_mutex); 4447 4448 /* init the queues. Just two for now. */ 4449 for (i = 0; i < 2; i++) { 4450 if (i == 0) 4451 idx = CAYMAN_RING_TYPE_CP1_INDEX; 4452 else 4453 idx = CAYMAN_RING_TYPE_CP2_INDEX; 4454 4455 if (rdev->ring[idx].mqd_obj == NULL) { 4456 r = radeon_bo_create(rdev, 4457 sizeof(struct bonaire_mqd), 4458 PAGE_SIZE, true, 4459 RADEON_GEM_DOMAIN_GTT, NULL, 4460 &rdev->ring[idx].mqd_obj); 4461 if (r) { 4462 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); 4463 return r; 4464 } 4465 } 4466 4467 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); 4468 if (unlikely(r != 0)) { 4469 cik_cp_compute_fini(rdev); 4470 return r; 4471 } 4472 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, 4473 &mqd_gpu_addr); 4474 if (r) { 4475 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); 4476 cik_cp_compute_fini(rdev); 4477 return r; 4478 } 4479 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); 4480 if (r) { 4481 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); 4482 cik_cp_compute_fini(rdev); 4483 return r; 4484 } 4485 4486 /* init the mqd struct */ 4487 memset(buf, 0, sizeof(struct bonaire_mqd)); 4488 4489 mqd = (struct bonaire_mqd *)buf; 4490 mqd->header = 0xC0310800; 4491 mqd->static_thread_mgmt01[0] = 0xffffffff; 4492 mqd->static_thread_mgmt01[1] = 0xffffffff; 4493 mqd->static_thread_mgmt23[0] = 0xffffffff; 4494 mqd->static_thread_mgmt23[1] = 0xffffffff; 4495 4496 mutex_lock(&rdev->srbm_mutex); 4497 cik_srbm_select(rdev, rdev->ring[idx].me, 4498 rdev->ring[idx].pipe, 4499 rdev->ring[idx].queue, 0); 4500 4501 /* disable wptr polling */ 4502 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL); 4503 tmp &= ~WPTR_POLL_EN; 4504 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp); 4505 4506 /* enable doorbell? */ 4507 mqd->queue_state.cp_hqd_pq_doorbell_control = 4508 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4509 if (use_doorbell) 4510 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4511 else 4512 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN; 4513 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4514 mqd->queue_state.cp_hqd_pq_doorbell_control); 4515 4516 /* disable the queue if it's active */ 4517 mqd->queue_state.cp_hqd_dequeue_request = 0; 4518 mqd->queue_state.cp_hqd_pq_rptr = 0; 4519 mqd->queue_state.cp_hqd_pq_wptr= 0; 4520 if (RREG32(CP_HQD_ACTIVE) & 1) { 4521 WREG32(CP_HQD_DEQUEUE_REQUEST, 1); 4522 for (i = 0; i < rdev->usec_timeout; i++) { 4523 if (!(RREG32(CP_HQD_ACTIVE) & 1)) 4524 break; 4525 udelay(1); 4526 } 4527 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request); 4528 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr); 4529 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4530 } 4531 4532 /* set the pointer to the MQD */ 4533 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc; 4534 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr); 4535 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr); 4536 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi); 4537 /* set MQD vmid to 0 */ 4538 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL); 4539 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK; 4540 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control); 4541 4542 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4543 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; 4544 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr; 4545 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4546 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base); 4547 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi); 4548 4549 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4550 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL); 4551 mqd->queue_state.cp_hqd_pq_control &= 4552 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK); 4553 4554 mqd->queue_state.cp_hqd_pq_control |= 4555 order_base_2(rdev->ring[idx].ring_size / 8); 4556 mqd->queue_state.cp_hqd_pq_control |= 4557 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8); 4558 #ifdef __BIG_ENDIAN 4559 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT; 4560 #endif 4561 mqd->queue_state.cp_hqd_pq_control &= 4562 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE); 4563 mqd->queue_state.cp_hqd_pq_control |= 4564 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */ 4565 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control); 4566 4567 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */ 4568 if (i == 0) 4569 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; 4570 else 4571 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; 4572 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc; 4573 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4574 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr); 4575 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI, 4576 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi); 4577 4578 /* set the wb address wether it's enabled or not */ 4579 if (i == 0) 4580 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; 4581 else 4582 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; 4583 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc; 4584 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi = 4585 upper_32_bits(wb_gpu_addr) & 0xffff; 4586 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR, 4587 mqd->queue_state.cp_hqd_pq_rptr_report_addr); 4588 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4589 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi); 4590 4591 /* enable the doorbell if requested */ 4592 if (use_doorbell) { 4593 mqd->queue_state.cp_hqd_pq_doorbell_control = 4594 RREG32(CP_HQD_PQ_DOORBELL_CONTROL); 4595 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK; 4596 mqd->queue_state.cp_hqd_pq_doorbell_control |= 4597 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); 4598 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN; 4599 mqd->queue_state.cp_hqd_pq_doorbell_control &= 4600 ~(DOORBELL_SOURCE | DOORBELL_HIT); 4601 4602 } else { 4603 mqd->queue_state.cp_hqd_pq_doorbell_control = 0; 4604 } 4605 WREG32(CP_HQD_PQ_DOORBELL_CONTROL, 4606 mqd->queue_state.cp_hqd_pq_doorbell_control); 4607 4608 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4609 rdev->ring[idx].wptr = 0; 4610 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; 4611 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr); 4612 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR); 4613 4614 /* set the vmid for the queue */ 4615 mqd->queue_state.cp_hqd_vmid = 0; 4616 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid); 4617 4618 /* activate the queue */ 4619 mqd->queue_state.cp_hqd_active = 1; 4620 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active); 4621 4622 cik_srbm_select(rdev, 0, 0, 0, 0); 4623 mutex_unlock(&rdev->srbm_mutex); 4624 4625 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); 4626 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); 4627 4628 rdev->ring[idx].ready = true; 4629 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); 4630 if (r) 4631 rdev->ring[idx].ready = false; 4632 } 4633 4634 return 0; 4635 } 4636 4637 static void cik_cp_enable(struct radeon_device *rdev, bool enable) 4638 { 4639 cik_cp_gfx_enable(rdev, enable); 4640 cik_cp_compute_enable(rdev, enable); 4641 } 4642 4643 static int cik_cp_load_microcode(struct radeon_device *rdev) 4644 { 4645 int r; 4646 4647 r = cik_cp_gfx_load_microcode(rdev); 4648 if (r) 4649 return r; 4650 r = cik_cp_compute_load_microcode(rdev); 4651 if (r) 4652 return r; 4653 4654 return 0; 4655 } 4656 4657 static void cik_cp_fini(struct radeon_device *rdev) 4658 { 4659 cik_cp_gfx_fini(rdev); 4660 cik_cp_compute_fini(rdev); 4661 } 4662 4663 static int cik_cp_resume(struct radeon_device *rdev) 4664 { 4665 int r; 4666 4667 cik_enable_gui_idle_interrupt(rdev, false); 4668 4669 r = cik_cp_load_microcode(rdev); 4670 if (r) 4671 return r; 4672 4673 r = cik_cp_gfx_resume(rdev); 4674 if (r) 4675 return r; 4676 r = cik_cp_compute_resume(rdev); 4677 if (r) 4678 return r; 4679 4680 cik_enable_gui_idle_interrupt(rdev, true); 4681 4682 return 0; 4683 } 4684 4685 static void cik_print_gpu_status_regs(struct radeon_device *rdev) 4686 { 4687 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 4688 RREG32(GRBM_STATUS)); 4689 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 4690 RREG32(GRBM_STATUS2)); 4691 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 4692 RREG32(GRBM_STATUS_SE0)); 4693 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 4694 RREG32(GRBM_STATUS_SE1)); 4695 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", 4696 RREG32(GRBM_STATUS_SE2)); 4697 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", 4698 RREG32(GRBM_STATUS_SE3)); 4699 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 4700 RREG32(SRBM_STATUS)); 4701 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", 4702 RREG32(SRBM_STATUS2)); 4703 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", 4704 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET)); 4705 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", 4706 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET)); 4707 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); 4708 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", 4709 RREG32(CP_STALLED_STAT1)); 4710 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", 4711 RREG32(CP_STALLED_STAT2)); 4712 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", 4713 RREG32(CP_STALLED_STAT3)); 4714 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", 4715 RREG32(CP_CPF_BUSY_STAT)); 4716 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", 4717 RREG32(CP_CPF_STALLED_STAT1)); 4718 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); 4719 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); 4720 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", 4721 RREG32(CP_CPC_STALLED_STAT1)); 4722 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); 4723 } 4724 4725 /** 4726 * cik_gpu_check_soft_reset - check which blocks are busy 4727 * 4728 * @rdev: radeon_device pointer 4729 * 4730 * Check which blocks are busy and return the relevant reset 4731 * mask to be used by cik_gpu_soft_reset(). 4732 * Returns a mask of the blocks to be reset. 4733 */ 4734 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) 4735 { 4736 u32 reset_mask = 0; 4737 u32 tmp; 4738 4739 /* GRBM_STATUS */ 4740 tmp = RREG32(GRBM_STATUS); 4741 if (tmp & (PA_BUSY | SC_BUSY | 4742 BCI_BUSY | SX_BUSY | 4743 TA_BUSY | VGT_BUSY | 4744 DB_BUSY | CB_BUSY | 4745 GDS_BUSY | SPI_BUSY | 4746 IA_BUSY | IA_BUSY_NO_DMA)) 4747 reset_mask |= RADEON_RESET_GFX; 4748 4749 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY)) 4750 reset_mask |= RADEON_RESET_CP; 4751 4752 /* GRBM_STATUS2 */ 4753 tmp = RREG32(GRBM_STATUS2); 4754 if (tmp & RLC_BUSY) 4755 reset_mask |= RADEON_RESET_RLC; 4756 4757 /* SDMA0_STATUS_REG */ 4758 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET); 4759 if (!(tmp & SDMA_IDLE)) 4760 reset_mask |= RADEON_RESET_DMA; 4761 4762 /* SDMA1_STATUS_REG */ 4763 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET); 4764 if (!(tmp & SDMA_IDLE)) 4765 reset_mask |= RADEON_RESET_DMA1; 4766 4767 /* SRBM_STATUS2 */ 4768 tmp = RREG32(SRBM_STATUS2); 4769 if (tmp & SDMA_BUSY) 4770 reset_mask |= RADEON_RESET_DMA; 4771 4772 if (tmp & SDMA1_BUSY) 4773 reset_mask |= RADEON_RESET_DMA1; 4774 4775 /* SRBM_STATUS */ 4776 tmp = RREG32(SRBM_STATUS); 4777 4778 if (tmp & IH_BUSY) 4779 reset_mask |= RADEON_RESET_IH; 4780 4781 if (tmp & SEM_BUSY) 4782 reset_mask |= RADEON_RESET_SEM; 4783 4784 if (tmp & GRBM_RQ_PENDING) 4785 reset_mask |= RADEON_RESET_GRBM; 4786 4787 if (tmp & VMC_BUSY) 4788 reset_mask |= RADEON_RESET_VMC; 4789 4790 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 4791 MCC_BUSY | MCD_BUSY)) 4792 reset_mask |= RADEON_RESET_MC; 4793 4794 if (evergreen_is_display_hung(rdev)) 4795 reset_mask |= RADEON_RESET_DISPLAY; 4796 4797 /* Skip MC reset as it's mostly likely not hung, just busy */ 4798 if (reset_mask & RADEON_RESET_MC) { 4799 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 4800 reset_mask &= ~RADEON_RESET_MC; 4801 } 4802 4803 return reset_mask; 4804 } 4805 4806 /** 4807 * cik_gpu_soft_reset - soft reset GPU 4808 * 4809 * @rdev: radeon_device pointer 4810 * @reset_mask: mask of which blocks to reset 4811 * 4812 * Soft reset the blocks specified in @reset_mask. 4813 */ 4814 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 4815 { 4816 struct evergreen_mc_save save; 4817 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 4818 u32 tmp; 4819 4820 if (reset_mask == 0) 4821 return; 4822 4823 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 4824 4825 cik_print_gpu_status_regs(rdev); 4826 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 4827 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 4828 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 4829 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 4830 4831 /* disable CG/PG */ 4832 cik_fini_pg(rdev); 4833 cik_fini_cg(rdev); 4834 4835 /* stop the rlc */ 4836 cik_rlc_stop(rdev); 4837 4838 /* Disable GFX parsing/prefetching */ 4839 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 4840 4841 /* Disable MEC parsing/prefetching */ 4842 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 4843 4844 if (reset_mask & RADEON_RESET_DMA) { 4845 /* sdma0 */ 4846 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 4847 tmp |= SDMA_HALT; 4848 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 4849 } 4850 if (reset_mask & RADEON_RESET_DMA1) { 4851 /* sdma1 */ 4852 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 4853 tmp |= SDMA_HALT; 4854 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 4855 } 4856 4857 evergreen_mc_stop(rdev, &save); 4858 if (evergreen_mc_wait_for_idle(rdev)) { 4859 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 4860 } 4861 4862 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) 4863 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX; 4864 4865 if (reset_mask & RADEON_RESET_CP) { 4866 grbm_soft_reset |= SOFT_RESET_CP; 4867 4868 srbm_soft_reset |= SOFT_RESET_GRBM; 4869 } 4870 4871 if (reset_mask & RADEON_RESET_DMA) 4872 srbm_soft_reset |= SOFT_RESET_SDMA; 4873 4874 if (reset_mask & RADEON_RESET_DMA1) 4875 srbm_soft_reset |= SOFT_RESET_SDMA1; 4876 4877 if (reset_mask & RADEON_RESET_DISPLAY) 4878 srbm_soft_reset |= SOFT_RESET_DC; 4879 4880 if (reset_mask & RADEON_RESET_RLC) 4881 grbm_soft_reset |= SOFT_RESET_RLC; 4882 4883 if (reset_mask & RADEON_RESET_SEM) 4884 srbm_soft_reset |= SOFT_RESET_SEM; 4885 4886 if (reset_mask & RADEON_RESET_IH) 4887 srbm_soft_reset |= SOFT_RESET_IH; 4888 4889 if (reset_mask & RADEON_RESET_GRBM) 4890 srbm_soft_reset |= SOFT_RESET_GRBM; 4891 4892 if (reset_mask & RADEON_RESET_VMC) 4893 srbm_soft_reset |= SOFT_RESET_VMC; 4894 4895 if (!(rdev->flags & RADEON_IS_IGP)) { 4896 if (reset_mask & RADEON_RESET_MC) 4897 srbm_soft_reset |= SOFT_RESET_MC; 4898 } 4899 4900 if (grbm_soft_reset) { 4901 tmp = RREG32(GRBM_SOFT_RESET); 4902 tmp |= grbm_soft_reset; 4903 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4904 WREG32(GRBM_SOFT_RESET, tmp); 4905 tmp = RREG32(GRBM_SOFT_RESET); 4906 4907 udelay(50); 4908 4909 tmp &= ~grbm_soft_reset; 4910 WREG32(GRBM_SOFT_RESET, tmp); 4911 tmp = RREG32(GRBM_SOFT_RESET); 4912 } 4913 4914 if (srbm_soft_reset) { 4915 tmp = RREG32(SRBM_SOFT_RESET); 4916 tmp |= srbm_soft_reset; 4917 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 4918 WREG32(SRBM_SOFT_RESET, tmp); 4919 tmp = RREG32(SRBM_SOFT_RESET); 4920 4921 udelay(50); 4922 4923 tmp &= ~srbm_soft_reset; 4924 WREG32(SRBM_SOFT_RESET, tmp); 4925 tmp = RREG32(SRBM_SOFT_RESET); 4926 } 4927 4928 /* Wait a little for things to settle down */ 4929 udelay(50); 4930 4931 evergreen_mc_resume(rdev, &save); 4932 udelay(50); 4933 4934 cik_print_gpu_status_regs(rdev); 4935 } 4936 4937 struct kv_reset_save_regs { 4938 u32 gmcon_reng_execute; 4939 u32 gmcon_misc; 4940 u32 gmcon_misc3; 4941 }; 4942 4943 static void kv_save_regs_for_reset(struct radeon_device *rdev, 4944 struct kv_reset_save_regs *save) 4945 { 4946 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE); 4947 save->gmcon_misc = RREG32(GMCON_MISC); 4948 save->gmcon_misc3 = RREG32(GMCON_MISC3); 4949 4950 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP); 4951 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE | 4952 STCTRL_STUTTER_EN)); 4953 } 4954 4955 static void kv_restore_regs_for_reset(struct radeon_device *rdev, 4956 struct kv_reset_save_regs *save) 4957 { 4958 int i; 4959 4960 WREG32(GMCON_PGFSM_WRITE, 0); 4961 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff); 4962 4963 for (i = 0; i < 5; i++) 4964 WREG32(GMCON_PGFSM_WRITE, 0); 4965 4966 WREG32(GMCON_PGFSM_WRITE, 0); 4967 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff); 4968 4969 for (i = 0; i < 5; i++) 4970 WREG32(GMCON_PGFSM_WRITE, 0); 4971 4972 WREG32(GMCON_PGFSM_WRITE, 0x210000); 4973 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff); 4974 4975 for (i = 0; i < 5; i++) 4976 WREG32(GMCON_PGFSM_WRITE, 0); 4977 4978 WREG32(GMCON_PGFSM_WRITE, 0x21003); 4979 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff); 4980 4981 for (i = 0; i < 5; i++) 4982 WREG32(GMCON_PGFSM_WRITE, 0); 4983 4984 WREG32(GMCON_PGFSM_WRITE, 0x2b00); 4985 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff); 4986 4987 for (i = 0; i < 5; i++) 4988 WREG32(GMCON_PGFSM_WRITE, 0); 4989 4990 WREG32(GMCON_PGFSM_WRITE, 0); 4991 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff); 4992 4993 for (i = 0; i < 5; i++) 4994 WREG32(GMCON_PGFSM_WRITE, 0); 4995 4996 WREG32(GMCON_PGFSM_WRITE, 0x420000); 4997 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff); 4998 4999 for (i = 0; i < 5; i++) 5000 WREG32(GMCON_PGFSM_WRITE, 0); 5001 5002 WREG32(GMCON_PGFSM_WRITE, 0x120202); 5003 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff); 5004 5005 for (i = 0; i < 5; i++) 5006 WREG32(GMCON_PGFSM_WRITE, 0); 5007 5008 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36); 5009 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff); 5010 5011 for (i = 0; i < 5; i++) 5012 WREG32(GMCON_PGFSM_WRITE, 0); 5013 5014 WREG32(GMCON_PGFSM_WRITE, 0x373f3e); 5015 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff); 5016 5017 for (i = 0; i < 5; i++) 5018 WREG32(GMCON_PGFSM_WRITE, 0); 5019 5020 WREG32(GMCON_PGFSM_WRITE, 0x3e1332); 5021 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff); 5022 5023 WREG32(GMCON_MISC3, save->gmcon_misc3); 5024 WREG32(GMCON_MISC, save->gmcon_misc); 5025 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute); 5026 } 5027 5028 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) 5029 { 5030 struct evergreen_mc_save save; 5031 struct kv_reset_save_regs kv_save = { 0 }; 5032 u32 tmp, i; 5033 5034 dev_info(rdev->dev, "GPU pci config reset\n"); 5035 5036 /* disable dpm? */ 5037 5038 /* disable cg/pg */ 5039 cik_fini_pg(rdev); 5040 cik_fini_cg(rdev); 5041 5042 /* Disable GFX parsing/prefetching */ 5043 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 5044 5045 /* Disable MEC parsing/prefetching */ 5046 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT); 5047 5048 /* sdma0 */ 5049 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET); 5050 tmp |= SDMA_HALT; 5051 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp); 5052 /* sdma1 */ 5053 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET); 5054 tmp |= SDMA_HALT; 5055 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp); 5056 /* XXX other engines? */ 5057 5058 /* halt the rlc, disable cp internal ints */ 5059 cik_rlc_stop(rdev); 5060 5061 udelay(50); 5062 5063 /* disable mem access */ 5064 evergreen_mc_stop(rdev, &save); 5065 if (evergreen_mc_wait_for_idle(rdev)) { 5066 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 5067 } 5068 5069 if (rdev->flags & RADEON_IS_IGP) 5070 kv_save_regs_for_reset(rdev, &kv_save); 5071 5072 /* disable BM */ 5073 pci_clear_master(rdev->pdev); 5074 /* reset */ 5075 radeon_pci_config_reset(rdev); 5076 5077 udelay(100); 5078 5079 /* wait for asic to come out of reset */ 5080 for (i = 0; i < rdev->usec_timeout; i++) { 5081 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 5082 break; 5083 udelay(1); 5084 } 5085 5086 /* does asic init need to be run first??? */ 5087 if (rdev->flags & RADEON_IS_IGP) 5088 kv_restore_regs_for_reset(rdev, &kv_save); 5089 } 5090 5091 /** 5092 * cik_asic_reset - soft reset GPU 5093 * 5094 * @rdev: radeon_device pointer 5095 * 5096 * Look up which blocks are hung and attempt 5097 * to reset them. 5098 * Returns 0 for success. 5099 */ 5100 int cik_asic_reset(struct radeon_device *rdev) 5101 { 5102 u32 reset_mask; 5103 5104 reset_mask = cik_gpu_check_soft_reset(rdev); 5105 5106 if (reset_mask) 5107 r600_set_bios_scratch_engine_hung(rdev, true); 5108 5109 /* try soft reset */ 5110 cik_gpu_soft_reset(rdev, reset_mask); 5111 5112 reset_mask = cik_gpu_check_soft_reset(rdev); 5113 5114 /* try pci config reset */ 5115 if (reset_mask && radeon_hard_reset) 5116 cik_gpu_pci_config_reset(rdev); 5117 5118 reset_mask = cik_gpu_check_soft_reset(rdev); 5119 5120 if (!reset_mask) 5121 r600_set_bios_scratch_engine_hung(rdev, false); 5122 5123 return 0; 5124 } 5125 5126 /** 5127 * cik_gfx_is_lockup - check if the 3D engine is locked up 5128 * 5129 * @rdev: radeon_device pointer 5130 * @ring: radeon_ring structure holding ring information 5131 * 5132 * Check if the 3D engine is locked up (CIK). 5133 * Returns true if the engine is locked, false if not. 5134 */ 5135 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 5136 { 5137 u32 reset_mask = cik_gpu_check_soft_reset(rdev); 5138 5139 if (!(reset_mask & (RADEON_RESET_GFX | 5140 RADEON_RESET_COMPUTE | 5141 RADEON_RESET_CP))) { 5142 radeon_ring_lockup_update(rdev, ring); 5143 return false; 5144 } 5145 return radeon_ring_test_lockup(rdev, ring); 5146 } 5147 5148 /* MC */ 5149 /** 5150 * cik_mc_program - program the GPU memory controller 5151 * 5152 * @rdev: radeon_device pointer 5153 * 5154 * Set the location of vram, gart, and AGP in the GPU's 5155 * physical address space (CIK). 5156 */ 5157 static void cik_mc_program(struct radeon_device *rdev) 5158 { 5159 struct evergreen_mc_save save; 5160 u32 tmp; 5161 int i, j; 5162 5163 /* Initialize HDP */ 5164 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 5165 WREG32((0x2c14 + j), 0x00000000); 5166 WREG32((0x2c18 + j), 0x00000000); 5167 WREG32((0x2c1c + j), 0x00000000); 5168 WREG32((0x2c20 + j), 0x00000000); 5169 WREG32((0x2c24 + j), 0x00000000); 5170 } 5171 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 5172 5173 evergreen_mc_stop(rdev, &save); 5174 if (radeon_mc_wait_for_idle(rdev)) { 5175 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5176 } 5177 /* Lockout access through VGA aperture*/ 5178 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 5179 /* Update configuration */ 5180 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 5181 rdev->mc.vram_start >> 12); 5182 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 5183 rdev->mc.vram_end >> 12); 5184 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 5185 rdev->vram_scratch.gpu_addr >> 12); 5186 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 5187 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 5188 WREG32(MC_VM_FB_LOCATION, tmp); 5189 /* XXX double check these! */ 5190 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 5191 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 5192 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 5193 WREG32(MC_VM_AGP_BASE, 0); 5194 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 5195 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 5196 if (radeon_mc_wait_for_idle(rdev)) { 5197 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 5198 } 5199 evergreen_mc_resume(rdev, &save); 5200 /* we need to own VRAM, so turn off the VGA renderer here 5201 * to stop it overwriting our objects */ 5202 rv515_vga_render_disable(rdev); 5203 } 5204 5205 /** 5206 * cik_mc_init - initialize the memory controller driver params 5207 * 5208 * @rdev: radeon_device pointer 5209 * 5210 * Look up the amount of vram, vram width, and decide how to place 5211 * vram and gart within the GPU's physical address space (CIK). 5212 * Returns 0 for success. 5213 */ 5214 static int cik_mc_init(struct radeon_device *rdev) 5215 { 5216 u32 tmp; 5217 int chansize, numchan; 5218 5219 /* Get VRAM informations */ 5220 rdev->mc.vram_is_ddr = true; 5221 tmp = RREG32(MC_ARB_RAMCFG); 5222 if (tmp & CHANSIZE_MASK) { 5223 chansize = 64; 5224 } else { 5225 chansize = 32; 5226 } 5227 tmp = RREG32(MC_SHARED_CHMAP); 5228 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 5229 case 0: 5230 default: 5231 numchan = 1; 5232 break; 5233 case 1: 5234 numchan = 2; 5235 break; 5236 case 2: 5237 numchan = 4; 5238 break; 5239 case 3: 5240 numchan = 8; 5241 break; 5242 case 4: 5243 numchan = 3; 5244 break; 5245 case 5: 5246 numchan = 6; 5247 break; 5248 case 6: 5249 numchan = 10; 5250 break; 5251 case 7: 5252 numchan = 12; 5253 break; 5254 case 8: 5255 numchan = 16; 5256 break; 5257 } 5258 rdev->mc.vram_width = numchan * chansize; 5259 /* Could aper size report 0 ? */ 5260 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 5261 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 5262 /* size in MB on si */ 5263 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5264 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 5265 rdev->mc.visible_vram_size = rdev->mc.aper_size; 5266 si_vram_gtt_location(rdev, &rdev->mc); 5267 radeon_update_bandwidth_info(rdev); 5268 5269 return 0; 5270 } 5271 5272 /* 5273 * GART 5274 * VMID 0 is the physical GPU addresses as used by the kernel. 5275 * VMIDs 1-15 are used for userspace clients and are handled 5276 * by the radeon vm/hsa code. 5277 */ 5278 /** 5279 * cik_pcie_gart_tlb_flush - gart tlb flush callback 5280 * 5281 * @rdev: radeon_device pointer 5282 * 5283 * Flush the TLB for the VMID 0 page table (CIK). 5284 */ 5285 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) 5286 { 5287 /* flush hdp cache */ 5288 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0); 5289 5290 /* bits 0-15 are the VM contexts0-15 */ 5291 WREG32(VM_INVALIDATE_REQUEST, 0x1); 5292 } 5293 5294 /** 5295 * cik_pcie_gart_enable - gart enable 5296 * 5297 * @rdev: radeon_device pointer 5298 * 5299 * This sets up the TLBs, programs the page tables for VMID0, 5300 * sets up the hw for VMIDs 1-15 which are allocated on 5301 * demand, and sets up the global locations for the LDS, GDS, 5302 * and GPUVM for FSA64 clients (CIK). 5303 * Returns 0 for success, errors for failure. 5304 */ 5305 static int cik_pcie_gart_enable(struct radeon_device *rdev) 5306 { 5307 int r, i; 5308 5309 if (rdev->gart.robj == NULL) { 5310 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 5311 return -EINVAL; 5312 } 5313 r = radeon_gart_table_vram_pin(rdev); 5314 if (r) 5315 return r; 5316 radeon_gart_restore(rdev); 5317 /* Setup TLB control */ 5318 WREG32(MC_VM_MX_L1_TLB_CNTL, 5319 (0xA << 7) | 5320 ENABLE_L1_TLB | 5321 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5322 ENABLE_ADVANCED_DRIVER_MODEL | 5323 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5324 /* Setup L2 cache */ 5325 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 5326 ENABLE_L2_FRAGMENT_PROCESSING | 5327 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5328 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5329 EFFECTIVE_L2_QUEUE_SIZE(7) | 5330 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5331 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 5332 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5333 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5334 /* setup context0 */ 5335 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 5336 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 5337 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 5338 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 5339 (u32)(rdev->dummy_page.addr >> 12)); 5340 WREG32(VM_CONTEXT0_CNTL2, 0); 5341 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 5342 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 5343 5344 WREG32(0x15D4, 0); 5345 WREG32(0x15D8, 0); 5346 WREG32(0x15DC, 0); 5347 5348 /* empty context1-15 */ 5349 /* FIXME start with 4G, once using 2 level pt switch to full 5350 * vm size space 5351 */ 5352 /* set vm size, must be a multiple of 4 */ 5353 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 5354 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 5355 for (i = 1; i < 16; i++) { 5356 if (i < 8) 5357 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 5358 rdev->gart.table_addr >> 12); 5359 else 5360 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 5361 rdev->gart.table_addr >> 12); 5362 } 5363 5364 /* enable context1-15 */ 5365 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 5366 (u32)(rdev->dummy_page.addr >> 12)); 5367 WREG32(VM_CONTEXT1_CNTL2, 4); 5368 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 5369 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5370 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5371 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5372 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 5373 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 5374 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 5375 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 5376 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 5377 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 5378 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 5379 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 5380 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 5381 5382 if (rdev->family == CHIP_KAVERI) { 5383 u32 tmp = RREG32(CHUB_CONTROL); 5384 tmp &= ~BYPASS_VM; 5385 WREG32(CHUB_CONTROL, tmp); 5386 } 5387 5388 /* XXX SH_MEM regs */ 5389 /* where to put LDS, scratch, GPUVM in FSA64 space */ 5390 mutex_lock(&rdev->srbm_mutex); 5391 for (i = 0; i < 16; i++) { 5392 cik_srbm_select(rdev, 0, 0, 0, i); 5393 /* CP and shaders */ 5394 WREG32(SH_MEM_CONFIG, 0); 5395 WREG32(SH_MEM_APE1_BASE, 1); 5396 WREG32(SH_MEM_APE1_LIMIT, 0); 5397 WREG32(SH_MEM_BASES, 0); 5398 /* SDMA GFX */ 5399 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0); 5400 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0); 5401 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0); 5402 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0); 5403 /* XXX SDMA RLC - todo */ 5404 } 5405 cik_srbm_select(rdev, 0, 0, 0, 0); 5406 mutex_unlock(&rdev->srbm_mutex); 5407 5408 cik_pcie_gart_tlb_flush(rdev); 5409 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 5410 (unsigned)(rdev->mc.gtt_size >> 20), 5411 (unsigned long long)rdev->gart.table_addr); 5412 rdev->gart.ready = true; 5413 return 0; 5414 } 5415 5416 /** 5417 * cik_pcie_gart_disable - gart disable 5418 * 5419 * @rdev: radeon_device pointer 5420 * 5421 * This disables all VM page table (CIK). 5422 */ 5423 static void cik_pcie_gart_disable(struct radeon_device *rdev) 5424 { 5425 /* Disable all tables */ 5426 WREG32(VM_CONTEXT0_CNTL, 0); 5427 WREG32(VM_CONTEXT1_CNTL, 0); 5428 /* Setup TLB control */ 5429 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 5430 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 5431 /* Setup L2 cache */ 5432 WREG32(VM_L2_CNTL, 5433 ENABLE_L2_FRAGMENT_PROCESSING | 5434 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 5435 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 5436 EFFECTIVE_L2_QUEUE_SIZE(7) | 5437 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 5438 WREG32(VM_L2_CNTL2, 0); 5439 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 5440 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 5441 radeon_gart_table_vram_unpin(rdev); 5442 } 5443 5444 /** 5445 * cik_pcie_gart_fini - vm fini callback 5446 * 5447 * @rdev: radeon_device pointer 5448 * 5449 * Tears down the driver GART/VM setup (CIK). 5450 */ 5451 static void cik_pcie_gart_fini(struct radeon_device *rdev) 5452 { 5453 cik_pcie_gart_disable(rdev); 5454 radeon_gart_table_vram_free(rdev); 5455 radeon_gart_fini(rdev); 5456 } 5457 5458 /* vm parser */ 5459 /** 5460 * cik_ib_parse - vm ib_parse callback 5461 * 5462 * @rdev: radeon_device pointer 5463 * @ib: indirect buffer pointer 5464 * 5465 * CIK uses hw IB checking so this is a nop (CIK). 5466 */ 5467 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 5468 { 5469 return 0; 5470 } 5471 5472 /* 5473 * vm 5474 * VMID 0 is the physical GPU addresses as used by the kernel. 5475 * VMIDs 1-15 are used for userspace clients and are handled 5476 * by the radeon vm/hsa code. 5477 */ 5478 /** 5479 * cik_vm_init - cik vm init callback 5480 * 5481 * @rdev: radeon_device pointer 5482 * 5483 * Inits cik specific vm parameters (number of VMs, base of vram for 5484 * VMIDs 1-15) (CIK). 5485 * Returns 0 for success. 5486 */ 5487 int cik_vm_init(struct radeon_device *rdev) 5488 { 5489 /* number of VMs */ 5490 rdev->vm_manager.nvm = 16; 5491 /* base offset of vram pages */ 5492 if (rdev->flags & RADEON_IS_IGP) { 5493 u64 tmp = RREG32(MC_VM_FB_OFFSET); 5494 tmp <<= 22; 5495 rdev->vm_manager.vram_base_offset = tmp; 5496 } else 5497 rdev->vm_manager.vram_base_offset = 0; 5498 5499 return 0; 5500 } 5501 5502 /** 5503 * cik_vm_fini - cik vm fini callback 5504 * 5505 * @rdev: radeon_device pointer 5506 * 5507 * Tear down any asic specific VM setup (CIK). 5508 */ 5509 void cik_vm_fini(struct radeon_device *rdev) 5510 { 5511 } 5512 5513 /** 5514 * cik_vm_decode_fault - print human readable fault info 5515 * 5516 * @rdev: radeon_device pointer 5517 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 5518 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 5519 * 5520 * Print human readable fault information (CIK). 5521 */ 5522 static void cik_vm_decode_fault(struct radeon_device *rdev, 5523 u32 status, u32 addr, u32 mc_client) 5524 { 5525 u32 mc_id; 5526 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 5527 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 5528 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, 5529 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; 5530 5531 if (rdev->family == CHIP_HAWAII) 5532 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5533 else 5534 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 5535 5536 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", 5537 protections, vmid, addr, 5538 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 5539 block, mc_client, mc_id); 5540 } 5541 5542 /** 5543 * cik_vm_flush - cik vm flush using the CP 5544 * 5545 * @rdev: radeon_device pointer 5546 * 5547 * Update the page table base and flush the VM TLB 5548 * using the CP (CIK). 5549 */ 5550 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 5551 { 5552 struct radeon_ring *ring = &rdev->ring[ridx]; 5553 5554 if (vm == NULL) 5555 return; 5556 5557 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5558 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5559 WRITE_DATA_DST_SEL(0))); 5560 if (vm->id < 8) { 5561 radeon_ring_write(ring, 5562 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); 5563 } else { 5564 radeon_ring_write(ring, 5565 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); 5566 } 5567 radeon_ring_write(ring, 0); 5568 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 5569 5570 /* update SH_MEM_* regs */ 5571 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5572 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5573 WRITE_DATA_DST_SEL(0))); 5574 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5575 radeon_ring_write(ring, 0); 5576 radeon_ring_write(ring, VMID(vm->id)); 5577 5578 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); 5579 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5580 WRITE_DATA_DST_SEL(0))); 5581 radeon_ring_write(ring, SH_MEM_BASES >> 2); 5582 radeon_ring_write(ring, 0); 5583 5584 radeon_ring_write(ring, 0); /* SH_MEM_BASES */ 5585 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */ 5586 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */ 5587 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */ 5588 5589 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5590 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5591 WRITE_DATA_DST_SEL(0))); 5592 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); 5593 radeon_ring_write(ring, 0); 5594 radeon_ring_write(ring, VMID(0)); 5595 5596 /* HDP flush */ 5597 cik_hdp_flush_cp_ring_emit(rdev, ridx); 5598 5599 /* bits 0-15 are the VM contexts0-15 */ 5600 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5601 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5602 WRITE_DATA_DST_SEL(0))); 5603 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 5604 radeon_ring_write(ring, 0); 5605 radeon_ring_write(ring, 1 << vm->id); 5606 5607 /* compute doesn't have PFP */ 5608 if (ridx == RADEON_RING_TYPE_GFX_INDEX) { 5609 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5610 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5611 radeon_ring_write(ring, 0x0); 5612 } 5613 } 5614 5615 /* 5616 * RLC 5617 * The RLC is a multi-purpose microengine that handles a 5618 * variety of functions, the most important of which is 5619 * the interrupt controller. 5620 */ 5621 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, 5622 bool enable) 5623 { 5624 u32 tmp = RREG32(CP_INT_CNTL_RING0); 5625 5626 if (enable) 5627 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5628 else 5629 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5630 WREG32(CP_INT_CNTL_RING0, tmp); 5631 } 5632 5633 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) 5634 { 5635 u32 tmp; 5636 5637 tmp = RREG32(RLC_LB_CNTL); 5638 if (enable) 5639 tmp |= LOAD_BALANCE_ENABLE; 5640 else 5641 tmp &= ~LOAD_BALANCE_ENABLE; 5642 WREG32(RLC_LB_CNTL, tmp); 5643 } 5644 5645 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) 5646 { 5647 u32 i, j, k; 5648 u32 mask; 5649 5650 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 5651 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 5652 cik_select_se_sh(rdev, i, j); 5653 for (k = 0; k < rdev->usec_timeout; k++) { 5654 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0) 5655 break; 5656 udelay(1); 5657 } 5658 } 5659 } 5660 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5661 5662 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY; 5663 for (k = 0; k < rdev->usec_timeout; k++) { 5664 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 5665 break; 5666 udelay(1); 5667 } 5668 } 5669 5670 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) 5671 { 5672 u32 tmp; 5673 5674 tmp = RREG32(RLC_CNTL); 5675 if (tmp != rlc) 5676 WREG32(RLC_CNTL, rlc); 5677 } 5678 5679 static u32 cik_halt_rlc(struct radeon_device *rdev) 5680 { 5681 u32 data, orig; 5682 5683 orig = data = RREG32(RLC_CNTL); 5684 5685 if (data & RLC_ENABLE) { 5686 u32 i; 5687 5688 data &= ~RLC_ENABLE; 5689 WREG32(RLC_CNTL, data); 5690 5691 for (i = 0; i < rdev->usec_timeout; i++) { 5692 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0) 5693 break; 5694 udelay(1); 5695 } 5696 5697 cik_wait_for_rlc_serdes(rdev); 5698 } 5699 5700 return orig; 5701 } 5702 5703 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) 5704 { 5705 u32 tmp, i, mask; 5706 5707 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE); 5708 WREG32(RLC_GPR_REG2, tmp); 5709 5710 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS; 5711 for (i = 0; i < rdev->usec_timeout; i++) { 5712 if ((RREG32(RLC_GPM_STAT) & mask) == mask) 5713 break; 5714 udelay(1); 5715 } 5716 5717 for (i = 0; i < rdev->usec_timeout; i++) { 5718 if ((RREG32(RLC_GPR_REG2) & REQ) == 0) 5719 break; 5720 udelay(1); 5721 } 5722 } 5723 5724 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) 5725 { 5726 u32 tmp; 5727 5728 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE); 5729 WREG32(RLC_GPR_REG2, tmp); 5730 } 5731 5732 /** 5733 * cik_rlc_stop - stop the RLC ME 5734 * 5735 * @rdev: radeon_device pointer 5736 * 5737 * Halt the RLC ME (MicroEngine) (CIK). 5738 */ 5739 static void cik_rlc_stop(struct radeon_device *rdev) 5740 { 5741 WREG32(RLC_CNTL, 0); 5742 5743 cik_enable_gui_idle_interrupt(rdev, false); 5744 5745 cik_wait_for_rlc_serdes(rdev); 5746 } 5747 5748 /** 5749 * cik_rlc_start - start the RLC ME 5750 * 5751 * @rdev: radeon_device pointer 5752 * 5753 * Unhalt the RLC ME (MicroEngine) (CIK). 5754 */ 5755 static void cik_rlc_start(struct radeon_device *rdev) 5756 { 5757 WREG32(RLC_CNTL, RLC_ENABLE); 5758 5759 cik_enable_gui_idle_interrupt(rdev, true); 5760 5761 udelay(50); 5762 } 5763 5764 /** 5765 * cik_rlc_resume - setup the RLC hw 5766 * 5767 * @rdev: radeon_device pointer 5768 * 5769 * Initialize the RLC registers, load the ucode, 5770 * and start the RLC (CIK). 5771 * Returns 0 for success, -EINVAL if the ucode is not available. 5772 */ 5773 static int cik_rlc_resume(struct radeon_device *rdev) 5774 { 5775 u32 i, size, tmp; 5776 const __be32 *fw_data; 5777 5778 if (!rdev->rlc_fw) 5779 return -EINVAL; 5780 5781 switch (rdev->family) { 5782 case CHIP_BONAIRE: 5783 case CHIP_HAWAII: 5784 default: 5785 size = BONAIRE_RLC_UCODE_SIZE; 5786 break; 5787 case CHIP_KAVERI: 5788 size = KV_RLC_UCODE_SIZE; 5789 break; 5790 case CHIP_KABINI: 5791 size = KB_RLC_UCODE_SIZE; 5792 break; 5793 } 5794 5795 cik_rlc_stop(rdev); 5796 5797 /* disable CG */ 5798 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc; 5799 WREG32(RLC_CGCG_CGLS_CTRL, tmp); 5800 5801 si_rlc_reset(rdev); 5802 5803 cik_init_pg(rdev); 5804 5805 cik_init_cg(rdev); 5806 5807 WREG32(RLC_LB_CNTR_INIT, 0); 5808 WREG32(RLC_LB_CNTR_MAX, 0x00008000); 5809 5810 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5811 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 5812 WREG32(RLC_LB_PARAMS, 0x00600408); 5813 WREG32(RLC_LB_CNTL, 0x80000004); 5814 5815 WREG32(RLC_MC_CNTL, 0); 5816 WREG32(RLC_UCODE_CNTL, 0); 5817 5818 fw_data = (const __be32 *)rdev->rlc_fw->data; 5819 WREG32(RLC_GPM_UCODE_ADDR, 0); 5820 for (i = 0; i < size; i++) 5821 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++)); 5822 WREG32(RLC_GPM_UCODE_ADDR, 0); 5823 5824 /* XXX - find out what chips support lbpw */ 5825 cik_enable_lbpw(rdev, false); 5826 5827 if (rdev->family == CHIP_BONAIRE) 5828 WREG32(RLC_DRIVER_DMA_STATUS, 0); 5829 5830 cik_rlc_start(rdev); 5831 5832 return 0; 5833 } 5834 5835 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) 5836 { 5837 u32 data, orig, tmp, tmp2; 5838 5839 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 5840 5841 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 5842 cik_enable_gui_idle_interrupt(rdev, true); 5843 5844 tmp = cik_halt_rlc(rdev); 5845 5846 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5847 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 5848 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 5849 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE; 5850 WREG32(RLC_SERDES_WR_CTRL, tmp2); 5851 5852 cik_update_rlc(rdev, tmp); 5853 5854 data |= CGCG_EN | CGLS_EN; 5855 } else { 5856 cik_enable_gui_idle_interrupt(rdev, false); 5857 5858 RREG32(CB_CGTT_SCLK_CTRL); 5859 RREG32(CB_CGTT_SCLK_CTRL); 5860 RREG32(CB_CGTT_SCLK_CTRL); 5861 RREG32(CB_CGTT_SCLK_CTRL); 5862 5863 data &= ~(CGCG_EN | CGLS_EN); 5864 } 5865 5866 if (orig != data) 5867 WREG32(RLC_CGCG_CGLS_CTRL, data); 5868 5869 } 5870 5871 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) 5872 { 5873 u32 data, orig, tmp = 0; 5874 5875 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 5876 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { 5877 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 5878 orig = data = RREG32(CP_MEM_SLP_CNTL); 5879 data |= CP_MEM_LS_EN; 5880 if (orig != data) 5881 WREG32(CP_MEM_SLP_CNTL, data); 5882 } 5883 } 5884 5885 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5886 data &= 0xfffffffd; 5887 if (orig != data) 5888 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5889 5890 tmp = cik_halt_rlc(rdev); 5891 5892 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5893 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 5894 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 5895 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0; 5896 WREG32(RLC_SERDES_WR_CTRL, data); 5897 5898 cik_update_rlc(rdev, tmp); 5899 5900 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { 5901 orig = data = RREG32(CGTS_SM_CTRL_REG); 5902 data &= ~SM_MODE_MASK; 5903 data |= SM_MODE(0x2); 5904 data |= SM_MODE_ENABLE; 5905 data &= ~CGTS_OVERRIDE; 5906 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && 5907 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) 5908 data &= ~CGTS_LS_OVERRIDE; 5909 data &= ~ON_MONITOR_ADD_MASK; 5910 data |= ON_MONITOR_ADD_EN; 5911 data |= ON_MONITOR_ADD(0x96); 5912 if (orig != data) 5913 WREG32(CGTS_SM_CTRL_REG, data); 5914 } 5915 } else { 5916 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5917 data |= 0x00000002; 5918 if (orig != data) 5919 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5920 5921 data = RREG32(RLC_MEM_SLP_CNTL); 5922 if (data & RLC_MEM_LS_EN) { 5923 data &= ~RLC_MEM_LS_EN; 5924 WREG32(RLC_MEM_SLP_CNTL, data); 5925 } 5926 5927 data = RREG32(CP_MEM_SLP_CNTL); 5928 if (data & CP_MEM_LS_EN) { 5929 data &= ~CP_MEM_LS_EN; 5930 WREG32(CP_MEM_SLP_CNTL, data); 5931 } 5932 5933 orig = data = RREG32(CGTS_SM_CTRL_REG); 5934 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE; 5935 if (orig != data) 5936 WREG32(CGTS_SM_CTRL_REG, data); 5937 5938 tmp = cik_halt_rlc(rdev); 5939 5940 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5941 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 5942 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 5943 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1; 5944 WREG32(RLC_SERDES_WR_CTRL, data); 5945 5946 cik_update_rlc(rdev, tmp); 5947 } 5948 } 5949 5950 static const u32 mc_cg_registers[] = 5951 { 5952 MC_HUB_MISC_HUB_CG, 5953 MC_HUB_MISC_SIP_CG, 5954 MC_HUB_MISC_VM_CG, 5955 MC_XPB_CLK_GAT, 5956 ATC_MISC_CG, 5957 MC_CITF_MISC_WR_CG, 5958 MC_CITF_MISC_RD_CG, 5959 MC_CITF_MISC_VM_CG, 5960 VM_L2_CG, 5961 }; 5962 5963 static void cik_enable_mc_ls(struct radeon_device *rdev, 5964 bool enable) 5965 { 5966 int i; 5967 u32 orig, data; 5968 5969 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5970 orig = data = RREG32(mc_cg_registers[i]); 5971 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 5972 data |= MC_LS_ENABLE; 5973 else 5974 data &= ~MC_LS_ENABLE; 5975 if (data != orig) 5976 WREG32(mc_cg_registers[i], data); 5977 } 5978 } 5979 5980 static void cik_enable_mc_mgcg(struct radeon_device *rdev, 5981 bool enable) 5982 { 5983 int i; 5984 u32 orig, data; 5985 5986 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5987 orig = data = RREG32(mc_cg_registers[i]); 5988 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 5989 data |= MC_CG_ENABLE; 5990 else 5991 data &= ~MC_CG_ENABLE; 5992 if (data != orig) 5993 WREG32(mc_cg_registers[i], data); 5994 } 5995 } 5996 5997 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, 5998 bool enable) 5999 { 6000 u32 orig, data; 6001 6002 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 6003 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); 6004 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); 6005 } else { 6006 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); 6007 data |= 0xff000000; 6008 if (data != orig) 6009 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); 6010 6011 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); 6012 data |= 0xff000000; 6013 if (data != orig) 6014 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); 6015 } 6016 } 6017 6018 static void cik_enable_sdma_mgls(struct radeon_device *rdev, 6019 bool enable) 6020 { 6021 u32 orig, data; 6022 6023 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { 6024 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6025 data |= 0x100; 6026 if (orig != data) 6027 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6028 6029 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6030 data |= 0x100; 6031 if (orig != data) 6032 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6033 } else { 6034 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); 6035 data &= ~0x100; 6036 if (orig != data) 6037 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); 6038 6039 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); 6040 data &= ~0x100; 6041 if (orig != data) 6042 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); 6043 } 6044 } 6045 6046 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, 6047 bool enable) 6048 { 6049 u32 orig, data; 6050 6051 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 6052 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6053 data = 0xfff; 6054 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6055 6056 orig = data = RREG32(UVD_CGC_CTRL); 6057 data |= DCM; 6058 if (orig != data) 6059 WREG32(UVD_CGC_CTRL, data); 6060 } else { 6061 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 6062 data &= ~0xfff; 6063 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data); 6064 6065 orig = data = RREG32(UVD_CGC_CTRL); 6066 data &= ~DCM; 6067 if (orig != data) 6068 WREG32(UVD_CGC_CTRL, data); 6069 } 6070 } 6071 6072 static void cik_enable_bif_mgls(struct radeon_device *rdev, 6073 bool enable) 6074 { 6075 u32 orig, data; 6076 6077 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 6078 6079 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 6080 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 6081 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 6082 else 6083 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 6084 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 6085 6086 if (orig != data) 6087 WREG32_PCIE_PORT(PCIE_CNTL2, data); 6088 } 6089 6090 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, 6091 bool enable) 6092 { 6093 u32 orig, data; 6094 6095 orig = data = RREG32(HDP_HOST_PATH_CNTL); 6096 6097 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 6098 data &= ~CLOCK_GATING_DIS; 6099 else 6100 data |= CLOCK_GATING_DIS; 6101 6102 if (orig != data) 6103 WREG32(HDP_HOST_PATH_CNTL, data); 6104 } 6105 6106 static void cik_enable_hdp_ls(struct radeon_device *rdev, 6107 bool enable) 6108 { 6109 u32 orig, data; 6110 6111 orig = data = RREG32(HDP_MEM_POWER_LS); 6112 6113 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 6114 data |= HDP_LS_ENABLE; 6115 else 6116 data &= ~HDP_LS_ENABLE; 6117 6118 if (orig != data) 6119 WREG32(HDP_MEM_POWER_LS, data); 6120 } 6121 6122 void cik_update_cg(struct radeon_device *rdev, 6123 u32 block, bool enable) 6124 { 6125 6126 if (block & RADEON_CG_BLOCK_GFX) { 6127 cik_enable_gui_idle_interrupt(rdev, false); 6128 /* order matters! */ 6129 if (enable) { 6130 cik_enable_mgcg(rdev, true); 6131 cik_enable_cgcg(rdev, true); 6132 } else { 6133 cik_enable_cgcg(rdev, false); 6134 cik_enable_mgcg(rdev, false); 6135 } 6136 cik_enable_gui_idle_interrupt(rdev, true); 6137 } 6138 6139 if (block & RADEON_CG_BLOCK_MC) { 6140 if (!(rdev->flags & RADEON_IS_IGP)) { 6141 cik_enable_mc_mgcg(rdev, enable); 6142 cik_enable_mc_ls(rdev, enable); 6143 } 6144 } 6145 6146 if (block & RADEON_CG_BLOCK_SDMA) { 6147 cik_enable_sdma_mgcg(rdev, enable); 6148 cik_enable_sdma_mgls(rdev, enable); 6149 } 6150 6151 if (block & RADEON_CG_BLOCK_BIF) { 6152 cik_enable_bif_mgls(rdev, enable); 6153 } 6154 6155 if (block & RADEON_CG_BLOCK_UVD) { 6156 if (rdev->has_uvd) 6157 cik_enable_uvd_mgcg(rdev, enable); 6158 } 6159 6160 if (block & RADEON_CG_BLOCK_HDP) { 6161 cik_enable_hdp_mgcg(rdev, enable); 6162 cik_enable_hdp_ls(rdev, enable); 6163 } 6164 6165 if (block & RADEON_CG_BLOCK_VCE) { 6166 vce_v2_0_enable_mgcg(rdev, enable); 6167 } 6168 } 6169 6170 static void cik_init_cg(struct radeon_device *rdev) 6171 { 6172 6173 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); 6174 6175 if (rdev->has_uvd) 6176 si_init_uvd_internal_cg(rdev); 6177 6178 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6179 RADEON_CG_BLOCK_SDMA | 6180 RADEON_CG_BLOCK_BIF | 6181 RADEON_CG_BLOCK_UVD | 6182 RADEON_CG_BLOCK_HDP), true); 6183 } 6184 6185 static void cik_fini_cg(struct radeon_device *rdev) 6186 { 6187 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | 6188 RADEON_CG_BLOCK_SDMA | 6189 RADEON_CG_BLOCK_BIF | 6190 RADEON_CG_BLOCK_UVD | 6191 RADEON_CG_BLOCK_HDP), false); 6192 6193 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); 6194 } 6195 6196 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, 6197 bool enable) 6198 { 6199 u32 data, orig; 6200 6201 orig = data = RREG32(RLC_PG_CNTL); 6202 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6203 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6204 else 6205 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE; 6206 if (orig != data) 6207 WREG32(RLC_PG_CNTL, data); 6208 } 6209 6210 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, 6211 bool enable) 6212 { 6213 u32 data, orig; 6214 6215 orig = data = RREG32(RLC_PG_CNTL); 6216 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) 6217 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6218 else 6219 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE; 6220 if (orig != data) 6221 WREG32(RLC_PG_CNTL, data); 6222 } 6223 6224 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) 6225 { 6226 u32 data, orig; 6227 6228 orig = data = RREG32(RLC_PG_CNTL); 6229 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) 6230 data &= ~DISABLE_CP_PG; 6231 else 6232 data |= DISABLE_CP_PG; 6233 if (orig != data) 6234 WREG32(RLC_PG_CNTL, data); 6235 } 6236 6237 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) 6238 { 6239 u32 data, orig; 6240 6241 orig = data = RREG32(RLC_PG_CNTL); 6242 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) 6243 data &= ~DISABLE_GDS_PG; 6244 else 6245 data |= DISABLE_GDS_PG; 6246 if (orig != data) 6247 WREG32(RLC_PG_CNTL, data); 6248 } 6249 6250 #define CP_ME_TABLE_SIZE 96 6251 #define CP_ME_TABLE_OFFSET 2048 6252 #define CP_MEC_TABLE_OFFSET 4096 6253 6254 void cik_init_cp_pg_table(struct radeon_device *rdev) 6255 { 6256 const __be32 *fw_data; 6257 volatile u32 *dst_ptr; 6258 int me, i, max_me = 4; 6259 u32 bo_offset = 0; 6260 u32 table_offset; 6261 6262 if (rdev->family == CHIP_KAVERI) 6263 max_me = 5; 6264 6265 if (rdev->rlc.cp_table_ptr == NULL) 6266 return; 6267 6268 /* write the cp table buffer */ 6269 dst_ptr = rdev->rlc.cp_table_ptr; 6270 for (me = 0; me < max_me; me++) { 6271 if (me == 0) { 6272 fw_data = (const __be32 *)rdev->ce_fw->data; 6273 table_offset = CP_ME_TABLE_OFFSET; 6274 } else if (me == 1) { 6275 fw_data = (const __be32 *)rdev->pfp_fw->data; 6276 table_offset = CP_ME_TABLE_OFFSET; 6277 } else if (me == 2) { 6278 fw_data = (const __be32 *)rdev->me_fw->data; 6279 table_offset = CP_ME_TABLE_OFFSET; 6280 } else { 6281 fw_data = (const __be32 *)rdev->mec_fw->data; 6282 table_offset = CP_MEC_TABLE_OFFSET; 6283 } 6284 6285 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) { 6286 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i])); 6287 } 6288 bo_offset += CP_ME_TABLE_SIZE; 6289 } 6290 } 6291 6292 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, 6293 bool enable) 6294 { 6295 u32 data, orig; 6296 6297 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 6298 orig = data = RREG32(RLC_PG_CNTL); 6299 data |= GFX_PG_ENABLE; 6300 if (orig != data) 6301 WREG32(RLC_PG_CNTL, data); 6302 6303 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6304 data |= AUTO_PG_EN; 6305 if (orig != data) 6306 WREG32(RLC_AUTO_PG_CTRL, data); 6307 } else { 6308 orig = data = RREG32(RLC_PG_CNTL); 6309 data &= ~GFX_PG_ENABLE; 6310 if (orig != data) 6311 WREG32(RLC_PG_CNTL, data); 6312 6313 orig = data = RREG32(RLC_AUTO_PG_CTRL); 6314 data &= ~AUTO_PG_EN; 6315 if (orig != data) 6316 WREG32(RLC_AUTO_PG_CTRL, data); 6317 6318 data = RREG32(DB_RENDER_CONTROL); 6319 } 6320 } 6321 6322 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 6323 { 6324 u32 mask = 0, tmp, tmp1; 6325 int i; 6326 6327 cik_select_se_sh(rdev, se, sh); 6328 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 6329 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 6330 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); 6331 6332 tmp &= 0xffff0000; 6333 6334 tmp |= tmp1; 6335 tmp >>= 16; 6336 6337 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { 6338 mask <<= 1; 6339 mask |= 1; 6340 } 6341 6342 return (~tmp) & mask; 6343 } 6344 6345 static void cik_init_ao_cu_mask(struct radeon_device *rdev) 6346 { 6347 u32 i, j, k, active_cu_number = 0; 6348 u32 mask, counter, cu_bitmap; 6349 u32 tmp = 0; 6350 6351 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { 6352 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { 6353 mask = 1; 6354 cu_bitmap = 0; 6355 counter = 0; 6356 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { 6357 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { 6358 if (counter < 2) 6359 cu_bitmap |= mask; 6360 counter ++; 6361 } 6362 mask <<= 1; 6363 } 6364 6365 active_cu_number += counter; 6366 tmp |= (cu_bitmap << (i * 16 + j * 8)); 6367 } 6368 } 6369 6370 WREG32(RLC_PG_AO_CU_MASK, tmp); 6371 6372 tmp = RREG32(RLC_MAX_PG_CU); 6373 tmp &= ~MAX_PU_CU_MASK; 6374 tmp |= MAX_PU_CU(active_cu_number); 6375 WREG32(RLC_MAX_PG_CU, tmp); 6376 } 6377 6378 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, 6379 bool enable) 6380 { 6381 u32 data, orig; 6382 6383 orig = data = RREG32(RLC_PG_CNTL); 6384 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) 6385 data |= STATIC_PER_CU_PG_ENABLE; 6386 else 6387 data &= ~STATIC_PER_CU_PG_ENABLE; 6388 if (orig != data) 6389 WREG32(RLC_PG_CNTL, data); 6390 } 6391 6392 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, 6393 bool enable) 6394 { 6395 u32 data, orig; 6396 6397 orig = data = RREG32(RLC_PG_CNTL); 6398 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) 6399 data |= DYN_PER_CU_PG_ENABLE; 6400 else 6401 data &= ~DYN_PER_CU_PG_ENABLE; 6402 if (orig != data) 6403 WREG32(RLC_PG_CNTL, data); 6404 } 6405 6406 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90 6407 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D 6408 6409 static void cik_init_gfx_cgpg(struct radeon_device *rdev) 6410 { 6411 u32 data, orig; 6412 u32 i; 6413 6414 if (rdev->rlc.cs_data) { 6415 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6416 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); 6417 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); 6418 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); 6419 } else { 6420 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET); 6421 for (i = 0; i < 3; i++) 6422 WREG32(RLC_GPM_SCRATCH_DATA, 0); 6423 } 6424 if (rdev->rlc.reg_list) { 6425 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET); 6426 for (i = 0; i < rdev->rlc.reg_list_size; i++) 6427 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); 6428 } 6429 6430 orig = data = RREG32(RLC_PG_CNTL); 6431 data |= GFX_PG_SRC; 6432 if (orig != data) 6433 WREG32(RLC_PG_CNTL, data); 6434 6435 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 6436 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); 6437 6438 data = RREG32(CP_RB_WPTR_POLL_CNTL); 6439 data &= ~IDLE_POLL_COUNT_MASK; 6440 data |= IDLE_POLL_COUNT(0x60); 6441 WREG32(CP_RB_WPTR_POLL_CNTL, data); 6442 6443 data = 0x10101010; 6444 WREG32(RLC_PG_DELAY, data); 6445 6446 data = RREG32(RLC_PG_DELAY_2); 6447 data &= ~0xff; 6448 data |= 0x3; 6449 WREG32(RLC_PG_DELAY_2, data); 6450 6451 data = RREG32(RLC_AUTO_PG_CTRL); 6452 data &= ~GRBM_REG_SGIT_MASK; 6453 data |= GRBM_REG_SGIT(0x700); 6454 WREG32(RLC_AUTO_PG_CTRL, data); 6455 6456 } 6457 6458 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) 6459 { 6460 cik_enable_gfx_cgpg(rdev, enable); 6461 cik_enable_gfx_static_mgpg(rdev, enable); 6462 cik_enable_gfx_dynamic_mgpg(rdev, enable); 6463 } 6464 6465 u32 cik_get_csb_size(struct radeon_device *rdev) 6466 { 6467 u32 count = 0; 6468 const struct cs_section_def *sect = NULL; 6469 const struct cs_extent_def *ext = NULL; 6470 6471 if (rdev->rlc.cs_data == NULL) 6472 return 0; 6473 6474 /* begin clear state */ 6475 count += 2; 6476 /* context control state */ 6477 count += 3; 6478 6479 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6480 for (ext = sect->section; ext->extent != NULL; ++ext) { 6481 if (sect->id == SECT_CONTEXT) 6482 count += 2 + ext->reg_count; 6483 else 6484 return 0; 6485 } 6486 } 6487 /* pa_sc_raster_config/pa_sc_raster_config1 */ 6488 count += 4; 6489 /* end clear state */ 6490 count += 2; 6491 /* clear state */ 6492 count += 2; 6493 6494 return count; 6495 } 6496 6497 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 6498 { 6499 u32 count = 0, i; 6500 const struct cs_section_def *sect = NULL; 6501 const struct cs_extent_def *ext = NULL; 6502 6503 if (rdev->rlc.cs_data == NULL) 6504 return; 6505 if (buffer == NULL) 6506 return; 6507 6508 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 6509 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 6510 6511 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6512 buffer[count++] = cpu_to_le32(0x80000000); 6513 buffer[count++] = cpu_to_le32(0x80000000); 6514 6515 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 6516 for (ext = sect->section; ext->extent != NULL; ++ext) { 6517 if (sect->id == SECT_CONTEXT) { 6518 buffer[count++] = 6519 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 6520 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 6521 for (i = 0; i < ext->reg_count; i++) 6522 buffer[count++] = cpu_to_le32(ext->extent[i]); 6523 } else { 6524 return; 6525 } 6526 } 6527 } 6528 6529 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 6530 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 6531 switch (rdev->family) { 6532 case CHIP_BONAIRE: 6533 buffer[count++] = cpu_to_le32(0x16000012); 6534 buffer[count++] = cpu_to_le32(0x00000000); 6535 break; 6536 case CHIP_KAVERI: 6537 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 6538 buffer[count++] = cpu_to_le32(0x00000000); 6539 break; 6540 case CHIP_KABINI: 6541 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */ 6542 buffer[count++] = cpu_to_le32(0x00000000); 6543 break; 6544 case CHIP_HAWAII: 6545 buffer[count++] = cpu_to_le32(0x3a00161a); 6546 buffer[count++] = cpu_to_le32(0x0000002e); 6547 break; 6548 default: 6549 buffer[count++] = cpu_to_le32(0x00000000); 6550 buffer[count++] = cpu_to_le32(0x00000000); 6551 break; 6552 } 6553 6554 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 6555 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 6556 6557 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 6558 buffer[count++] = cpu_to_le32(0); 6559 } 6560 6561 static void cik_init_pg(struct radeon_device *rdev) 6562 { 6563 if (rdev->pg_flags) { 6564 cik_enable_sck_slowdown_on_pu(rdev, true); 6565 cik_enable_sck_slowdown_on_pd(rdev, true); 6566 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 6567 cik_init_gfx_cgpg(rdev); 6568 cik_enable_cp_pg(rdev, true); 6569 cik_enable_gds_pg(rdev, true); 6570 } 6571 cik_init_ao_cu_mask(rdev); 6572 cik_update_gfx_pg(rdev, true); 6573 } 6574 } 6575 6576 static void cik_fini_pg(struct radeon_device *rdev) 6577 { 6578 if (rdev->pg_flags) { 6579 cik_update_gfx_pg(rdev, false); 6580 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 6581 cik_enable_cp_pg(rdev, false); 6582 cik_enable_gds_pg(rdev, false); 6583 } 6584 } 6585 } 6586 6587 /* 6588 * Interrupts 6589 * Starting with r6xx, interrupts are handled via a ring buffer. 6590 * Ring buffers are areas of GPU accessible memory that the GPU 6591 * writes interrupt vectors into and the host reads vectors out of. 6592 * There is a rptr (read pointer) that determines where the 6593 * host is currently reading, and a wptr (write pointer) 6594 * which determines where the GPU has written. When the 6595 * pointers are equal, the ring is idle. When the GPU 6596 * writes vectors to the ring buffer, it increments the 6597 * wptr. When there is an interrupt, the host then starts 6598 * fetching commands and processing them until the pointers are 6599 * equal again at which point it updates the rptr. 6600 */ 6601 6602 /** 6603 * cik_enable_interrupts - Enable the interrupt ring buffer 6604 * 6605 * @rdev: radeon_device pointer 6606 * 6607 * Enable the interrupt ring buffer (CIK). 6608 */ 6609 static void cik_enable_interrupts(struct radeon_device *rdev) 6610 { 6611 u32 ih_cntl = RREG32(IH_CNTL); 6612 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 6613 6614 ih_cntl |= ENABLE_INTR; 6615 ih_rb_cntl |= IH_RB_ENABLE; 6616 WREG32(IH_CNTL, ih_cntl); 6617 WREG32(IH_RB_CNTL, ih_rb_cntl); 6618 rdev->ih.enabled = true; 6619 } 6620 6621 /** 6622 * cik_disable_interrupts - Disable the interrupt ring buffer 6623 * 6624 * @rdev: radeon_device pointer 6625 * 6626 * Disable the interrupt ring buffer (CIK). 6627 */ 6628 static void cik_disable_interrupts(struct radeon_device *rdev) 6629 { 6630 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 6631 u32 ih_cntl = RREG32(IH_CNTL); 6632 6633 ih_rb_cntl &= ~IH_RB_ENABLE; 6634 ih_cntl &= ~ENABLE_INTR; 6635 WREG32(IH_RB_CNTL, ih_rb_cntl); 6636 WREG32(IH_CNTL, ih_cntl); 6637 /* set rptr, wptr to 0 */ 6638 WREG32(IH_RB_RPTR, 0); 6639 WREG32(IH_RB_WPTR, 0); 6640 rdev->ih.enabled = false; 6641 rdev->ih.rptr = 0; 6642 } 6643 6644 /** 6645 * cik_disable_interrupt_state - Disable all interrupt sources 6646 * 6647 * @rdev: radeon_device pointer 6648 * 6649 * Clear all interrupt enable bits used by the driver (CIK). 6650 */ 6651 static void cik_disable_interrupt_state(struct radeon_device *rdev) 6652 { 6653 u32 tmp; 6654 6655 /* gfx ring */ 6656 tmp = RREG32(CP_INT_CNTL_RING0) & 6657 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6658 WREG32(CP_INT_CNTL_RING0, tmp); 6659 /* sdma */ 6660 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 6661 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp); 6662 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 6663 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp); 6664 /* compute queues */ 6665 WREG32(CP_ME1_PIPE0_INT_CNTL, 0); 6666 WREG32(CP_ME1_PIPE1_INT_CNTL, 0); 6667 WREG32(CP_ME1_PIPE2_INT_CNTL, 0); 6668 WREG32(CP_ME1_PIPE3_INT_CNTL, 0); 6669 WREG32(CP_ME2_PIPE0_INT_CNTL, 0); 6670 WREG32(CP_ME2_PIPE1_INT_CNTL, 0); 6671 WREG32(CP_ME2_PIPE2_INT_CNTL, 0); 6672 WREG32(CP_ME2_PIPE3_INT_CNTL, 0); 6673 /* grbm */ 6674 WREG32(GRBM_INT_CNTL, 0); 6675 /* vline/vblank, etc. */ 6676 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 6677 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 6678 if (rdev->num_crtc >= 4) { 6679 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 6680 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 6681 } 6682 if (rdev->num_crtc >= 6) { 6683 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 6684 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 6685 } 6686 6687 /* dac hotplug */ 6688 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 6689 6690 /* digital hotplug */ 6691 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6692 WREG32(DC_HPD1_INT_CONTROL, tmp); 6693 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6694 WREG32(DC_HPD2_INT_CONTROL, tmp); 6695 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6696 WREG32(DC_HPD3_INT_CONTROL, tmp); 6697 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6698 WREG32(DC_HPD4_INT_CONTROL, tmp); 6699 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6700 WREG32(DC_HPD5_INT_CONTROL, tmp); 6701 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 6702 WREG32(DC_HPD6_INT_CONTROL, tmp); 6703 6704 } 6705 6706 /** 6707 * cik_irq_init - init and enable the interrupt ring 6708 * 6709 * @rdev: radeon_device pointer 6710 * 6711 * Allocate a ring buffer for the interrupt controller, 6712 * enable the RLC, disable interrupts, enable the IH 6713 * ring buffer and enable it (CIK). 6714 * Called at device load and reume. 6715 * Returns 0 for success, errors for failure. 6716 */ 6717 static int cik_irq_init(struct radeon_device *rdev) 6718 { 6719 int ret = 0; 6720 int rb_bufsz; 6721 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 6722 6723 /* allocate ring */ 6724 ret = r600_ih_ring_alloc(rdev); 6725 if (ret) 6726 return ret; 6727 6728 /* disable irqs */ 6729 cik_disable_interrupts(rdev); 6730 6731 /* init rlc */ 6732 ret = cik_rlc_resume(rdev); 6733 if (ret) { 6734 r600_ih_ring_fini(rdev); 6735 return ret; 6736 } 6737 6738 /* setup interrupt control */ 6739 /* XXX this should actually be a bus address, not an MC address. same on older asics */ 6740 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 6741 interrupt_cntl = RREG32(INTERRUPT_CNTL); 6742 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 6743 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 6744 */ 6745 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 6746 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 6747 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 6748 WREG32(INTERRUPT_CNTL, interrupt_cntl); 6749 6750 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 6751 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 6752 6753 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 6754 IH_WPTR_OVERFLOW_CLEAR | 6755 (rb_bufsz << 1)); 6756 6757 if (rdev->wb.enabled) 6758 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 6759 6760 /* set the writeback address whether it's enabled or not */ 6761 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 6762 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 6763 6764 WREG32(IH_RB_CNTL, ih_rb_cntl); 6765 6766 /* set rptr, wptr to 0 */ 6767 WREG32(IH_RB_RPTR, 0); 6768 WREG32(IH_RB_WPTR, 0); 6769 6770 /* Default settings for IH_CNTL (disabled at first) */ 6771 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 6772 /* RPTR_REARM only works if msi's are enabled */ 6773 if (rdev->msi_enabled) 6774 ih_cntl |= RPTR_REARM; 6775 WREG32(IH_CNTL, ih_cntl); 6776 6777 /* force the active interrupt state to all disabled */ 6778 cik_disable_interrupt_state(rdev); 6779 6780 pci_set_master(rdev->pdev); 6781 6782 /* enable irqs */ 6783 cik_enable_interrupts(rdev); 6784 6785 return ret; 6786 } 6787 6788 /** 6789 * cik_irq_set - enable/disable interrupt sources 6790 * 6791 * @rdev: radeon_device pointer 6792 * 6793 * Enable interrupt sources on the GPU (vblanks, hpd, 6794 * etc.) (CIK). 6795 * Returns 0 for success, errors for failure. 6796 */ 6797 int cik_irq_set(struct radeon_device *rdev) 6798 { 6799 u32 cp_int_cntl; 6800 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3; 6801 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3; 6802 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 6803 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 6804 u32 grbm_int_cntl = 0; 6805 u32 dma_cntl, dma_cntl1; 6806 u32 thermal_int; 6807 6808 if (!rdev->irq.installed) { 6809 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 6810 return -EINVAL; 6811 } 6812 /* don't enable anything if the ih is disabled */ 6813 if (!rdev->ih.enabled) { 6814 cik_disable_interrupts(rdev); 6815 /* force the active interrupt state to all disabled */ 6816 cik_disable_interrupt_state(rdev); 6817 return 0; 6818 } 6819 6820 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 6821 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6822 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE; 6823 6824 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 6825 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 6826 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 6827 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 6828 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 6829 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 6830 6831 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 6832 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 6833 6834 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6835 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6836 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6837 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6838 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6839 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6840 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6841 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE; 6842 6843 if (rdev->flags & RADEON_IS_IGP) 6844 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) & 6845 ~(THERM_INTH_MASK | THERM_INTL_MASK); 6846 else 6847 thermal_int = RREG32_SMC(CG_THERMAL_INT) & 6848 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 6849 6850 /* enable CP interrupts on all rings */ 6851 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 6852 DRM_DEBUG("cik_irq_set: sw int gfx\n"); 6853 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 6854 } 6855 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 6856 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6857 DRM_DEBUG("si_irq_set: sw int cp1\n"); 6858 if (ring->me == 1) { 6859 switch (ring->pipe) { 6860 case 0: 6861 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 6862 break; 6863 case 1: 6864 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 6865 break; 6866 case 2: 6867 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 6868 break; 6869 case 3: 6870 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 6871 break; 6872 default: 6873 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 6874 break; 6875 } 6876 } else if (ring->me == 2) { 6877 switch (ring->pipe) { 6878 case 0: 6879 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 6880 break; 6881 case 1: 6882 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 6883 break; 6884 case 2: 6885 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 6886 break; 6887 case 3: 6888 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 6889 break; 6890 default: 6891 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe); 6892 break; 6893 } 6894 } else { 6895 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me); 6896 } 6897 } 6898 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 6899 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6900 DRM_DEBUG("si_irq_set: sw int cp2\n"); 6901 if (ring->me == 1) { 6902 switch (ring->pipe) { 6903 case 0: 6904 cp_m1p0 |= TIME_STAMP_INT_ENABLE; 6905 break; 6906 case 1: 6907 cp_m1p1 |= TIME_STAMP_INT_ENABLE; 6908 break; 6909 case 2: 6910 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 6911 break; 6912 case 3: 6913 cp_m1p2 |= TIME_STAMP_INT_ENABLE; 6914 break; 6915 default: 6916 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 6917 break; 6918 } 6919 } else if (ring->me == 2) { 6920 switch (ring->pipe) { 6921 case 0: 6922 cp_m2p0 |= TIME_STAMP_INT_ENABLE; 6923 break; 6924 case 1: 6925 cp_m2p1 |= TIME_STAMP_INT_ENABLE; 6926 break; 6927 case 2: 6928 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 6929 break; 6930 case 3: 6931 cp_m2p2 |= TIME_STAMP_INT_ENABLE; 6932 break; 6933 default: 6934 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe); 6935 break; 6936 } 6937 } else { 6938 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me); 6939 } 6940 } 6941 6942 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 6943 DRM_DEBUG("cik_irq_set: sw int dma\n"); 6944 dma_cntl |= TRAP_ENABLE; 6945 } 6946 6947 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 6948 DRM_DEBUG("cik_irq_set: sw int dma1\n"); 6949 dma_cntl1 |= TRAP_ENABLE; 6950 } 6951 6952 if (rdev->irq.crtc_vblank_int[0] || 6953 atomic_read(&rdev->irq.pflip[0])) { 6954 DRM_DEBUG("cik_irq_set: vblank 0\n"); 6955 crtc1 |= VBLANK_INTERRUPT_MASK; 6956 } 6957 if (rdev->irq.crtc_vblank_int[1] || 6958 atomic_read(&rdev->irq.pflip[1])) { 6959 DRM_DEBUG("cik_irq_set: vblank 1\n"); 6960 crtc2 |= VBLANK_INTERRUPT_MASK; 6961 } 6962 if (rdev->irq.crtc_vblank_int[2] || 6963 atomic_read(&rdev->irq.pflip[2])) { 6964 DRM_DEBUG("cik_irq_set: vblank 2\n"); 6965 crtc3 |= VBLANK_INTERRUPT_MASK; 6966 } 6967 if (rdev->irq.crtc_vblank_int[3] || 6968 atomic_read(&rdev->irq.pflip[3])) { 6969 DRM_DEBUG("cik_irq_set: vblank 3\n"); 6970 crtc4 |= VBLANK_INTERRUPT_MASK; 6971 } 6972 if (rdev->irq.crtc_vblank_int[4] || 6973 atomic_read(&rdev->irq.pflip[4])) { 6974 DRM_DEBUG("cik_irq_set: vblank 4\n"); 6975 crtc5 |= VBLANK_INTERRUPT_MASK; 6976 } 6977 if (rdev->irq.crtc_vblank_int[5] || 6978 atomic_read(&rdev->irq.pflip[5])) { 6979 DRM_DEBUG("cik_irq_set: vblank 5\n"); 6980 crtc6 |= VBLANK_INTERRUPT_MASK; 6981 } 6982 if (rdev->irq.hpd[0]) { 6983 DRM_DEBUG("cik_irq_set: hpd 1\n"); 6984 hpd1 |= DC_HPDx_INT_EN; 6985 } 6986 if (rdev->irq.hpd[1]) { 6987 DRM_DEBUG("cik_irq_set: hpd 2\n"); 6988 hpd2 |= DC_HPDx_INT_EN; 6989 } 6990 if (rdev->irq.hpd[2]) { 6991 DRM_DEBUG("cik_irq_set: hpd 3\n"); 6992 hpd3 |= DC_HPDx_INT_EN; 6993 } 6994 if (rdev->irq.hpd[3]) { 6995 DRM_DEBUG("cik_irq_set: hpd 4\n"); 6996 hpd4 |= DC_HPDx_INT_EN; 6997 } 6998 if (rdev->irq.hpd[4]) { 6999 DRM_DEBUG("cik_irq_set: hpd 5\n"); 7000 hpd5 |= DC_HPDx_INT_EN; 7001 } 7002 if (rdev->irq.hpd[5]) { 7003 DRM_DEBUG("cik_irq_set: hpd 6\n"); 7004 hpd6 |= DC_HPDx_INT_EN; 7005 } 7006 7007 if (rdev->irq.dpm_thermal) { 7008 DRM_DEBUG("dpm thermal\n"); 7009 if (rdev->flags & RADEON_IS_IGP) 7010 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK; 7011 else 7012 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 7013 } 7014 7015 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 7016 7017 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl); 7018 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1); 7019 7020 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0); 7021 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1); 7022 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2); 7023 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3); 7024 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0); 7025 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1); 7026 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2); 7027 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3); 7028 7029 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 7030 7031 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 7032 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 7033 if (rdev->num_crtc >= 4) { 7034 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 7035 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 7036 } 7037 if (rdev->num_crtc >= 6) { 7038 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 7039 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 7040 } 7041 7042 WREG32(DC_HPD1_INT_CONTROL, hpd1); 7043 WREG32(DC_HPD2_INT_CONTROL, hpd2); 7044 WREG32(DC_HPD3_INT_CONTROL, hpd3); 7045 WREG32(DC_HPD4_INT_CONTROL, hpd4); 7046 WREG32(DC_HPD5_INT_CONTROL, hpd5); 7047 WREG32(DC_HPD6_INT_CONTROL, hpd6); 7048 7049 if (rdev->flags & RADEON_IS_IGP) 7050 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int); 7051 else 7052 WREG32_SMC(CG_THERMAL_INT, thermal_int); 7053 7054 return 0; 7055 } 7056 7057 /** 7058 * cik_irq_ack - ack interrupt sources 7059 * 7060 * @rdev: radeon_device pointer 7061 * 7062 * Ack interrupt sources on the GPU (vblanks, hpd, 7063 * etc.) (CIK). Certain interrupts sources are sw 7064 * generated and do not require an explicit ack. 7065 */ 7066 static inline void cik_irq_ack(struct radeon_device *rdev) 7067 { 7068 u32 tmp; 7069 7070 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); 7071 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 7072 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 7073 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 7074 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 7075 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 7076 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); 7077 7078 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) 7079 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 7080 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) 7081 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 7082 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 7083 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 7084 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) 7085 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 7086 7087 if (rdev->num_crtc >= 4) { 7088 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 7089 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 7090 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 7091 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 7092 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 7093 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 7094 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 7095 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 7096 } 7097 7098 if (rdev->num_crtc >= 6) { 7099 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 7100 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 7101 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 7102 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 7103 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 7104 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 7105 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 7106 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 7107 } 7108 7109 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7110 tmp = RREG32(DC_HPD1_INT_CONTROL); 7111 tmp |= DC_HPDx_INT_ACK; 7112 WREG32(DC_HPD1_INT_CONTROL, tmp); 7113 } 7114 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7115 tmp = RREG32(DC_HPD2_INT_CONTROL); 7116 tmp |= DC_HPDx_INT_ACK; 7117 WREG32(DC_HPD2_INT_CONTROL, tmp); 7118 } 7119 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7120 tmp = RREG32(DC_HPD3_INT_CONTROL); 7121 tmp |= DC_HPDx_INT_ACK; 7122 WREG32(DC_HPD3_INT_CONTROL, tmp); 7123 } 7124 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7125 tmp = RREG32(DC_HPD4_INT_CONTROL); 7126 tmp |= DC_HPDx_INT_ACK; 7127 WREG32(DC_HPD4_INT_CONTROL, tmp); 7128 } 7129 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7130 tmp = RREG32(DC_HPD5_INT_CONTROL); 7131 tmp |= DC_HPDx_INT_ACK; 7132 WREG32(DC_HPD5_INT_CONTROL, tmp); 7133 } 7134 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7135 tmp = RREG32(DC_HPD5_INT_CONTROL); 7136 tmp |= DC_HPDx_INT_ACK; 7137 WREG32(DC_HPD6_INT_CONTROL, tmp); 7138 } 7139 } 7140 7141 /** 7142 * cik_irq_disable - disable interrupts 7143 * 7144 * @rdev: radeon_device pointer 7145 * 7146 * Disable interrupts on the hw (CIK). 7147 */ 7148 static void cik_irq_disable(struct radeon_device *rdev) 7149 { 7150 cik_disable_interrupts(rdev); 7151 /* Wait and acknowledge irq */ 7152 mdelay(1); 7153 cik_irq_ack(rdev); 7154 cik_disable_interrupt_state(rdev); 7155 } 7156 7157 /** 7158 * cik_irq_disable - disable interrupts for suspend 7159 * 7160 * @rdev: radeon_device pointer 7161 * 7162 * Disable interrupts and stop the RLC (CIK). 7163 * Used for suspend. 7164 */ 7165 static void cik_irq_suspend(struct radeon_device *rdev) 7166 { 7167 cik_irq_disable(rdev); 7168 cik_rlc_stop(rdev); 7169 } 7170 7171 /** 7172 * cik_irq_fini - tear down interrupt support 7173 * 7174 * @rdev: radeon_device pointer 7175 * 7176 * Disable interrupts on the hw and free the IH ring 7177 * buffer (CIK). 7178 * Used for driver unload. 7179 */ 7180 static void cik_irq_fini(struct radeon_device *rdev) 7181 { 7182 cik_irq_suspend(rdev); 7183 r600_ih_ring_fini(rdev); 7184 } 7185 7186 /** 7187 * cik_get_ih_wptr - get the IH ring buffer wptr 7188 * 7189 * @rdev: radeon_device pointer 7190 * 7191 * Get the IH ring buffer wptr from either the register 7192 * or the writeback memory buffer (CIK). Also check for 7193 * ring buffer overflow and deal with it. 7194 * Used by cik_irq_process(). 7195 * Returns the value of the wptr. 7196 */ 7197 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) 7198 { 7199 u32 wptr, tmp; 7200 7201 if (rdev->wb.enabled) 7202 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 7203 else 7204 wptr = RREG32(IH_RB_WPTR); 7205 7206 if (wptr & RB_OVERFLOW) { 7207 /* When a ring buffer overflow happen start parsing interrupt 7208 * from the last not overwritten vector (wptr + 16). Hopefully 7209 * this should allow us to catchup. 7210 */ 7211 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 7212 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 7213 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 7214 tmp = RREG32(IH_RB_CNTL); 7215 tmp |= IH_WPTR_OVERFLOW_CLEAR; 7216 WREG32(IH_RB_CNTL, tmp); 7217 } 7218 return (wptr & rdev->ih.ptr_mask); 7219 } 7220 7221 /* CIK IV Ring 7222 * Each IV ring entry is 128 bits: 7223 * [7:0] - interrupt source id 7224 * [31:8] - reserved 7225 * [59:32] - interrupt source data 7226 * [63:60] - reserved 7227 * [71:64] - RINGID 7228 * CP: 7229 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] 7230 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher 7231 * - for gfx, hw shader state (0=PS...5=LS, 6=CS) 7232 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes 7233 * PIPE_ID - ME0 0=3D 7234 * - ME1&2 compute dispatcher (4 pipes each) 7235 * SDMA: 7236 * INSTANCE_ID [1:0], QUEUE_ID[1:0] 7237 * INSTANCE_ID - 0 = sdma0, 1 = sdma1 7238 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 7239 * [79:72] - VMID 7240 * [95:80] - PASID 7241 * [127:96] - reserved 7242 */ 7243 /** 7244 * cik_irq_process - interrupt handler 7245 * 7246 * @rdev: radeon_device pointer 7247 * 7248 * Interrupt hander (CIK). Walk the IH ring, 7249 * ack interrupts and schedule work to handle 7250 * interrupt events. 7251 * Returns irq process return code. 7252 */ 7253 int cik_irq_process(struct radeon_device *rdev) 7254 { 7255 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7256 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7257 u32 wptr; 7258 u32 rptr; 7259 u32 src_id, src_data, ring_id; 7260 u8 me_id, pipe_id, queue_id; 7261 u32 ring_index; 7262 bool queue_hotplug = false; 7263 bool queue_reset = false; 7264 u32 addr, status, mc_client; 7265 bool queue_thermal = false; 7266 7267 if (!rdev->ih.enabled || rdev->shutdown) 7268 return IRQ_NONE; 7269 7270 wptr = cik_get_ih_wptr(rdev); 7271 7272 restart_ih: 7273 /* is somebody else already processing irqs? */ 7274 if (atomic_xchg(&rdev->ih.lock, 1)) 7275 return IRQ_NONE; 7276 7277 rptr = rdev->ih.rptr; 7278 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 7279 7280 /* Order reading of wptr vs. reading of IH ring data */ 7281 rmb(); 7282 7283 /* display interrupts */ 7284 cik_irq_ack(rdev); 7285 7286 while (rptr != wptr) { 7287 /* wptr/rptr are in bytes! */ 7288 ring_index = rptr / 4; 7289 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 7290 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 7291 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 7292 7293 switch (src_id) { 7294 case 1: /* D1 vblank/vline */ 7295 switch (src_data) { 7296 case 0: /* D1 vblank */ 7297 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) { 7298 if (rdev->irq.crtc_vblank_int[0]) { 7299 drm_handle_vblank(rdev->ddev, 0); 7300 rdev->pm.vblank_sync = true; 7301 wake_up(&rdev->irq.vblank_queue); 7302 } 7303 if (atomic_read(&rdev->irq.pflip[0])) 7304 radeon_crtc_handle_flip(rdev, 0); 7305 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 7306 DRM_DEBUG("IH: D1 vblank\n"); 7307 } 7308 break; 7309 case 1: /* D1 vline */ 7310 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) { 7311 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; 7312 DRM_DEBUG("IH: D1 vline\n"); 7313 } 7314 break; 7315 default: 7316 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7317 break; 7318 } 7319 break; 7320 case 2: /* D2 vblank/vline */ 7321 switch (src_data) { 7322 case 0: /* D2 vblank */ 7323 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 7324 if (rdev->irq.crtc_vblank_int[1]) { 7325 drm_handle_vblank(rdev->ddev, 1); 7326 rdev->pm.vblank_sync = true; 7327 wake_up(&rdev->irq.vblank_queue); 7328 } 7329 if (atomic_read(&rdev->irq.pflip[1])) 7330 radeon_crtc_handle_flip(rdev, 1); 7331 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 7332 DRM_DEBUG("IH: D2 vblank\n"); 7333 } 7334 break; 7335 case 1: /* D2 vline */ 7336 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 7337 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 7338 DRM_DEBUG("IH: D2 vline\n"); 7339 } 7340 break; 7341 default: 7342 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7343 break; 7344 } 7345 break; 7346 case 3: /* D3 vblank/vline */ 7347 switch (src_data) { 7348 case 0: /* D3 vblank */ 7349 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 7350 if (rdev->irq.crtc_vblank_int[2]) { 7351 drm_handle_vblank(rdev->ddev, 2); 7352 rdev->pm.vblank_sync = true; 7353 wake_up(&rdev->irq.vblank_queue); 7354 } 7355 if (atomic_read(&rdev->irq.pflip[2])) 7356 radeon_crtc_handle_flip(rdev, 2); 7357 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 7358 DRM_DEBUG("IH: D3 vblank\n"); 7359 } 7360 break; 7361 case 1: /* D3 vline */ 7362 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 7363 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 7364 DRM_DEBUG("IH: D3 vline\n"); 7365 } 7366 break; 7367 default: 7368 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7369 break; 7370 } 7371 break; 7372 case 4: /* D4 vblank/vline */ 7373 switch (src_data) { 7374 case 0: /* D4 vblank */ 7375 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 7376 if (rdev->irq.crtc_vblank_int[3]) { 7377 drm_handle_vblank(rdev->ddev, 3); 7378 rdev->pm.vblank_sync = true; 7379 wake_up(&rdev->irq.vblank_queue); 7380 } 7381 if (atomic_read(&rdev->irq.pflip[3])) 7382 radeon_crtc_handle_flip(rdev, 3); 7383 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 7384 DRM_DEBUG("IH: D4 vblank\n"); 7385 } 7386 break; 7387 case 1: /* D4 vline */ 7388 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 7389 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 7390 DRM_DEBUG("IH: D4 vline\n"); 7391 } 7392 break; 7393 default: 7394 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7395 break; 7396 } 7397 break; 7398 case 5: /* D5 vblank/vline */ 7399 switch (src_data) { 7400 case 0: /* D5 vblank */ 7401 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 7402 if (rdev->irq.crtc_vblank_int[4]) { 7403 drm_handle_vblank(rdev->ddev, 4); 7404 rdev->pm.vblank_sync = true; 7405 wake_up(&rdev->irq.vblank_queue); 7406 } 7407 if (atomic_read(&rdev->irq.pflip[4])) 7408 radeon_crtc_handle_flip(rdev, 4); 7409 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 7410 DRM_DEBUG("IH: D5 vblank\n"); 7411 } 7412 break; 7413 case 1: /* D5 vline */ 7414 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 7415 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 7416 DRM_DEBUG("IH: D5 vline\n"); 7417 } 7418 break; 7419 default: 7420 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7421 break; 7422 } 7423 break; 7424 case 6: /* D6 vblank/vline */ 7425 switch (src_data) { 7426 case 0: /* D6 vblank */ 7427 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 7428 if (rdev->irq.crtc_vblank_int[5]) { 7429 drm_handle_vblank(rdev->ddev, 5); 7430 rdev->pm.vblank_sync = true; 7431 wake_up(&rdev->irq.vblank_queue); 7432 } 7433 if (atomic_read(&rdev->irq.pflip[5])) 7434 radeon_crtc_handle_flip(rdev, 5); 7435 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 7436 DRM_DEBUG("IH: D6 vblank\n"); 7437 } 7438 break; 7439 case 1: /* D6 vline */ 7440 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 7441 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 7442 DRM_DEBUG("IH: D6 vline\n"); 7443 } 7444 break; 7445 default: 7446 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7447 break; 7448 } 7449 break; 7450 case 42: /* HPD hotplug */ 7451 switch (src_data) { 7452 case 0: 7453 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { 7454 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; 7455 queue_hotplug = true; 7456 DRM_DEBUG("IH: HPD1\n"); 7457 } 7458 break; 7459 case 1: 7460 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { 7461 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; 7462 queue_hotplug = true; 7463 DRM_DEBUG("IH: HPD2\n"); 7464 } 7465 break; 7466 case 2: 7467 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { 7468 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 7469 queue_hotplug = true; 7470 DRM_DEBUG("IH: HPD3\n"); 7471 } 7472 break; 7473 case 3: 7474 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { 7475 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 7476 queue_hotplug = true; 7477 DRM_DEBUG("IH: HPD4\n"); 7478 } 7479 break; 7480 case 4: 7481 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { 7482 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 7483 queue_hotplug = true; 7484 DRM_DEBUG("IH: HPD5\n"); 7485 } 7486 break; 7487 case 5: 7488 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { 7489 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 7490 queue_hotplug = true; 7491 DRM_DEBUG("IH: HPD6\n"); 7492 } 7493 break; 7494 default: 7495 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7496 break; 7497 } 7498 break; 7499 case 124: /* UVD */ 7500 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 7501 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 7502 break; 7503 case 146: 7504 case 147: 7505 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 7506 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 7507 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); 7508 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 7509 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 7510 addr); 7511 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 7512 status); 7513 cik_vm_decode_fault(rdev, status, addr, mc_client); 7514 /* reset addr and status */ 7515 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 7516 break; 7517 case 167: /* VCE */ 7518 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data); 7519 switch (src_data) { 7520 case 0: 7521 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); 7522 break; 7523 case 1: 7524 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); 7525 break; 7526 default: 7527 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 7528 break; 7529 } 7530 break; 7531 case 176: /* GFX RB CP_INT */ 7532 case 177: /* GFX IB CP_INT */ 7533 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 7534 break; 7535 case 181: /* CP EOP event */ 7536 DRM_DEBUG("IH: CP EOP\n"); 7537 /* XXX check the bitfield order! */ 7538 me_id = (ring_id & 0x60) >> 5; 7539 pipe_id = (ring_id & 0x18) >> 3; 7540 queue_id = (ring_id & 0x7) >> 0; 7541 switch (me_id) { 7542 case 0: 7543 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 7544 break; 7545 case 1: 7546 case 2: 7547 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id)) 7548 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 7549 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id)) 7550 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 7551 break; 7552 } 7553 break; 7554 case 184: /* CP Privileged reg access */ 7555 DRM_ERROR("Illegal register access in command stream\n"); 7556 /* XXX check the bitfield order! */ 7557 me_id = (ring_id & 0x60) >> 5; 7558 pipe_id = (ring_id & 0x18) >> 3; 7559 queue_id = (ring_id & 0x7) >> 0; 7560 switch (me_id) { 7561 case 0: 7562 /* This results in a full GPU reset, but all we need to do is soft 7563 * reset the CP for gfx 7564 */ 7565 queue_reset = true; 7566 break; 7567 case 1: 7568 /* XXX compute */ 7569 queue_reset = true; 7570 break; 7571 case 2: 7572 /* XXX compute */ 7573 queue_reset = true; 7574 break; 7575 } 7576 break; 7577 case 185: /* CP Privileged inst */ 7578 DRM_ERROR("Illegal instruction in command stream\n"); 7579 /* XXX check the bitfield order! */ 7580 me_id = (ring_id & 0x60) >> 5; 7581 pipe_id = (ring_id & 0x18) >> 3; 7582 queue_id = (ring_id & 0x7) >> 0; 7583 switch (me_id) { 7584 case 0: 7585 /* This results in a full GPU reset, but all we need to do is soft 7586 * reset the CP for gfx 7587 */ 7588 queue_reset = true; 7589 break; 7590 case 1: 7591 /* XXX compute */ 7592 queue_reset = true; 7593 break; 7594 case 2: 7595 /* XXX compute */ 7596 queue_reset = true; 7597 break; 7598 } 7599 break; 7600 case 224: /* SDMA trap event */ 7601 /* XXX check the bitfield order! */ 7602 me_id = (ring_id & 0x3) >> 0; 7603 queue_id = (ring_id & 0xc) >> 2; 7604 DRM_DEBUG("IH: SDMA trap\n"); 7605 switch (me_id) { 7606 case 0: 7607 switch (queue_id) { 7608 case 0: 7609 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 7610 break; 7611 case 1: 7612 /* XXX compute */ 7613 break; 7614 case 2: 7615 /* XXX compute */ 7616 break; 7617 } 7618 break; 7619 case 1: 7620 switch (queue_id) { 7621 case 0: 7622 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 7623 break; 7624 case 1: 7625 /* XXX compute */ 7626 break; 7627 case 2: 7628 /* XXX compute */ 7629 break; 7630 } 7631 break; 7632 } 7633 break; 7634 case 230: /* thermal low to high */ 7635 DRM_DEBUG("IH: thermal low to high\n"); 7636 rdev->pm.dpm.thermal.high_to_low = false; 7637 queue_thermal = true; 7638 break; 7639 case 231: /* thermal high to low */ 7640 DRM_DEBUG("IH: thermal high to low\n"); 7641 rdev->pm.dpm.thermal.high_to_low = true; 7642 queue_thermal = true; 7643 break; 7644 case 233: /* GUI IDLE */ 7645 DRM_DEBUG("IH: GUI idle\n"); 7646 break; 7647 case 241: /* SDMA Privileged inst */ 7648 case 247: /* SDMA Privileged inst */ 7649 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 7650 /* XXX check the bitfield order! */ 7651 me_id = (ring_id & 0x3) >> 0; 7652 queue_id = (ring_id & 0xc) >> 2; 7653 switch (me_id) { 7654 case 0: 7655 switch (queue_id) { 7656 case 0: 7657 queue_reset = true; 7658 break; 7659 case 1: 7660 /* XXX compute */ 7661 queue_reset = true; 7662 break; 7663 case 2: 7664 /* XXX compute */ 7665 queue_reset = true; 7666 break; 7667 } 7668 break; 7669 case 1: 7670 switch (queue_id) { 7671 case 0: 7672 queue_reset = true; 7673 break; 7674 case 1: 7675 /* XXX compute */ 7676 queue_reset = true; 7677 break; 7678 case 2: 7679 /* XXX compute */ 7680 queue_reset = true; 7681 break; 7682 } 7683 break; 7684 } 7685 break; 7686 default: 7687 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 7688 break; 7689 } 7690 7691 /* wptr/rptr are in bytes! */ 7692 rptr += 16; 7693 rptr &= rdev->ih.ptr_mask; 7694 } 7695 if (queue_hotplug) 7696 schedule_work(&rdev->hotplug_work); 7697 if (queue_reset) 7698 schedule_work(&rdev->reset_work); 7699 if (queue_thermal) 7700 schedule_work(&rdev->pm.dpm.thermal.work); 7701 rdev->ih.rptr = rptr; 7702 WREG32(IH_RB_RPTR, rdev->ih.rptr); 7703 atomic_set(&rdev->ih.lock, 0); 7704 7705 /* make sure wptr hasn't changed while processing */ 7706 wptr = cik_get_ih_wptr(rdev); 7707 if (wptr != rptr) 7708 goto restart_ih; 7709 7710 return IRQ_HANDLED; 7711 } 7712 7713 /* 7714 * startup/shutdown callbacks 7715 */ 7716 /** 7717 * cik_startup - program the asic to a functional state 7718 * 7719 * @rdev: radeon_device pointer 7720 * 7721 * Programs the asic to a functional state (CIK). 7722 * Called by cik_init() and cik_resume(). 7723 * Returns 0 for success, error for failure. 7724 */ 7725 static int cik_startup(struct radeon_device *rdev) 7726 { 7727 struct radeon_ring *ring; 7728 int r; 7729 7730 /* enable pcie gen2/3 link */ 7731 cik_pcie_gen3_enable(rdev); 7732 /* enable aspm */ 7733 cik_program_aspm(rdev); 7734 7735 /* scratch needs to be initialized before MC */ 7736 r = r600_vram_scratch_init(rdev); 7737 if (r) 7738 return r; 7739 7740 cik_mc_program(rdev); 7741 7742 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { 7743 r = ci_mc_load_microcode(rdev); 7744 if (r) { 7745 DRM_ERROR("Failed to load MC firmware!\n"); 7746 return r; 7747 } 7748 } 7749 7750 r = cik_pcie_gart_enable(rdev); 7751 if (r) 7752 return r; 7753 cik_gpu_init(rdev); 7754 7755 /* allocate rlc buffers */ 7756 if (rdev->flags & RADEON_IS_IGP) { 7757 if (rdev->family == CHIP_KAVERI) { 7758 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; 7759 rdev->rlc.reg_list_size = 7760 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list); 7761 } else { 7762 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; 7763 rdev->rlc.reg_list_size = 7764 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list); 7765 } 7766 } 7767 rdev->rlc.cs_data = ci_cs_data; 7768 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; 7769 r = sumo_rlc_init(rdev); 7770 if (r) { 7771 DRM_ERROR("Failed to init rlc BOs!\n"); 7772 return r; 7773 } 7774 7775 /* allocate wb buffer */ 7776 r = radeon_wb_init(rdev); 7777 if (r) 7778 return r; 7779 7780 /* allocate mec buffers */ 7781 r = cik_mec_init(rdev); 7782 if (r) { 7783 DRM_ERROR("Failed to init MEC BOs!\n"); 7784 return r; 7785 } 7786 7787 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 7788 if (r) { 7789 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 7790 return r; 7791 } 7792 7793 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 7794 if (r) { 7795 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 7796 return r; 7797 } 7798 7799 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 7800 if (r) { 7801 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 7802 return r; 7803 } 7804 7805 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 7806 if (r) { 7807 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 7808 return r; 7809 } 7810 7811 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 7812 if (r) { 7813 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 7814 return r; 7815 } 7816 7817 r = radeon_uvd_resume(rdev); 7818 if (!r) { 7819 r = uvd_v4_2_resume(rdev); 7820 if (!r) { 7821 r = radeon_fence_driver_start_ring(rdev, 7822 R600_RING_TYPE_UVD_INDEX); 7823 if (r) 7824 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 7825 } 7826 } 7827 if (r) 7828 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 7829 7830 r = radeon_vce_resume(rdev); 7831 if (!r) { 7832 r = vce_v2_0_resume(rdev); 7833 if (!r) 7834 r = radeon_fence_driver_start_ring(rdev, 7835 TN_RING_TYPE_VCE1_INDEX); 7836 if (!r) 7837 r = radeon_fence_driver_start_ring(rdev, 7838 TN_RING_TYPE_VCE2_INDEX); 7839 } 7840 if (r) { 7841 dev_err(rdev->dev, "VCE init error (%d).\n", r); 7842 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; 7843 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; 7844 } 7845 7846 /* Enable IRQ */ 7847 if (!rdev->irq.installed) { 7848 r = radeon_irq_kms_init(rdev); 7849 if (r) 7850 return r; 7851 } 7852 7853 r = cik_irq_init(rdev); 7854 if (r) { 7855 DRM_ERROR("radeon: IH init failed (%d).\n", r); 7856 radeon_irq_kms_fini(rdev); 7857 return r; 7858 } 7859 cik_irq_set(rdev); 7860 7861 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 7862 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 7863 PACKET3(PACKET3_NOP, 0x3FFF)); 7864 if (r) 7865 return r; 7866 7867 /* set up the compute queues */ 7868 /* type-2 packets are deprecated on MEC, use type-3 instead */ 7869 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 7870 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 7871 PACKET3(PACKET3_NOP, 0x3FFF)); 7872 if (r) 7873 return r; 7874 ring->me = 1; /* first MEC */ 7875 ring->pipe = 0; /* first pipe */ 7876 ring->queue = 0; /* first queue */ 7877 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET; 7878 7879 /* type-2 packets are deprecated on MEC, use type-3 instead */ 7880 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 7881 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 7882 PACKET3(PACKET3_NOP, 0x3FFF)); 7883 if (r) 7884 return r; 7885 /* dGPU only have 1 MEC */ 7886 ring->me = 1; /* first MEC */ 7887 ring->pipe = 0; /* first pipe */ 7888 ring->queue = 1; /* second queue */ 7889 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET; 7890 7891 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 7892 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 7893 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 7894 if (r) 7895 return r; 7896 7897 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 7898 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 7899 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); 7900 if (r) 7901 return r; 7902 7903 r = cik_cp_resume(rdev); 7904 if (r) 7905 return r; 7906 7907 r = cik_sdma_resume(rdev); 7908 if (r) 7909 return r; 7910 7911 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 7912 if (ring->ring_size) { 7913 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 7914 RADEON_CP_PACKET2); 7915 if (!r) 7916 r = uvd_v1_0_init(rdev); 7917 if (r) 7918 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 7919 } 7920 7921 r = -ENOENT; 7922 7923 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 7924 if (ring->ring_size) 7925 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 7926 VCE_CMD_NO_OP); 7927 7928 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 7929 if (ring->ring_size) 7930 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 7931 VCE_CMD_NO_OP); 7932 7933 if (!r) 7934 r = vce_v1_0_init(rdev); 7935 else if (r != -ENOENT) 7936 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r); 7937 7938 r = radeon_ib_pool_init(rdev); 7939 if (r) { 7940 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 7941 return r; 7942 } 7943 7944 r = radeon_vm_manager_init(rdev); 7945 if (r) { 7946 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 7947 return r; 7948 } 7949 7950 r = dce6_audio_init(rdev); 7951 if (r) 7952 return r; 7953 7954 return 0; 7955 } 7956 7957 /** 7958 * cik_resume - resume the asic to a functional state 7959 * 7960 * @rdev: radeon_device pointer 7961 * 7962 * Programs the asic to a functional state (CIK). 7963 * Called at resume. 7964 * Returns 0 for success, error for failure. 7965 */ 7966 int cik_resume(struct radeon_device *rdev) 7967 { 7968 int r; 7969 7970 /* post card */ 7971 atom_asic_init(rdev->mode_info.atom_context); 7972 7973 /* init golden registers */ 7974 cik_init_golden_registers(rdev); 7975 7976 if (rdev->pm.pm_method == PM_METHOD_DPM) 7977 radeon_pm_resume(rdev); 7978 7979 rdev->accel_working = true; 7980 r = cik_startup(rdev); 7981 if (r) { 7982 DRM_ERROR("cik startup failed on resume\n"); 7983 rdev->accel_working = false; 7984 return r; 7985 } 7986 7987 return r; 7988 7989 } 7990 7991 /** 7992 * cik_suspend - suspend the asic 7993 * 7994 * @rdev: radeon_device pointer 7995 * 7996 * Bring the chip into a state suitable for suspend (CIK). 7997 * Called at suspend. 7998 * Returns 0 for success. 7999 */ 8000 int cik_suspend(struct radeon_device *rdev) 8001 { 8002 radeon_pm_suspend(rdev); 8003 dce6_audio_fini(rdev); 8004 radeon_vm_manager_fini(rdev); 8005 cik_cp_enable(rdev, false); 8006 cik_sdma_enable(rdev, false); 8007 uvd_v1_0_fini(rdev); 8008 radeon_uvd_suspend(rdev); 8009 radeon_vce_suspend(rdev); 8010 cik_fini_pg(rdev); 8011 cik_fini_cg(rdev); 8012 cik_irq_suspend(rdev); 8013 radeon_wb_disable(rdev); 8014 cik_pcie_gart_disable(rdev); 8015 return 0; 8016 } 8017 8018 /* Plan is to move initialization in that function and use 8019 * helper function so that radeon_device_init pretty much 8020 * do nothing more than calling asic specific function. This 8021 * should also allow to remove a bunch of callback function 8022 * like vram_info. 8023 */ 8024 /** 8025 * cik_init - asic specific driver and hw init 8026 * 8027 * @rdev: radeon_device pointer 8028 * 8029 * Setup asic specific driver variables and program the hw 8030 * to a functional state (CIK). 8031 * Called at driver startup. 8032 * Returns 0 for success, errors for failure. 8033 */ 8034 int cik_init(struct radeon_device *rdev) 8035 { 8036 struct radeon_ring *ring; 8037 int r; 8038 8039 /* Read BIOS */ 8040 if (!radeon_get_bios(rdev)) { 8041 if (ASIC_IS_AVIVO(rdev)) 8042 return -EINVAL; 8043 } 8044 /* Must be an ATOMBIOS */ 8045 if (!rdev->is_atom_bios) { 8046 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 8047 return -EINVAL; 8048 } 8049 r = radeon_atombios_init(rdev); 8050 if (r) 8051 return r; 8052 8053 /* Post card if necessary */ 8054 if (!radeon_card_posted(rdev)) { 8055 if (!rdev->bios) { 8056 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 8057 return -EINVAL; 8058 } 8059 DRM_INFO("GPU not posted. posting now...\n"); 8060 atom_asic_init(rdev->mode_info.atom_context); 8061 } 8062 /* init golden registers */ 8063 cik_init_golden_registers(rdev); 8064 /* Initialize scratch registers */ 8065 cik_scratch_init(rdev); 8066 /* Initialize surface registers */ 8067 radeon_surface_init(rdev); 8068 /* Initialize clocks */ 8069 radeon_get_clock_info(rdev->ddev); 8070 8071 /* Fence driver */ 8072 r = radeon_fence_driver_init(rdev); 8073 if (r) 8074 return r; 8075 8076 /* initialize memory controller */ 8077 r = cik_mc_init(rdev); 8078 if (r) 8079 return r; 8080 /* Memory manager */ 8081 r = radeon_bo_init(rdev); 8082 if (r) 8083 return r; 8084 8085 if (rdev->flags & RADEON_IS_IGP) { 8086 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8087 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { 8088 r = cik_init_microcode(rdev); 8089 if (r) { 8090 DRM_ERROR("Failed to load firmware!\n"); 8091 return r; 8092 } 8093 } 8094 } else { 8095 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 8096 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || 8097 !rdev->mc_fw) { 8098 r = cik_init_microcode(rdev); 8099 if (r) { 8100 DRM_ERROR("Failed to load firmware!\n"); 8101 return r; 8102 } 8103 } 8104 } 8105 8106 /* Initialize power management */ 8107 radeon_pm_init(rdev); 8108 8109 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 8110 ring->ring_obj = NULL; 8111 r600_ring_init(rdev, ring, 1024 * 1024); 8112 8113 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 8114 ring->ring_obj = NULL; 8115 r600_ring_init(rdev, ring, 1024 * 1024); 8116 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8117 if (r) 8118 return r; 8119 8120 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 8121 ring->ring_obj = NULL; 8122 r600_ring_init(rdev, ring, 1024 * 1024); 8123 r = radeon_doorbell_get(rdev, &ring->doorbell_index); 8124 if (r) 8125 return r; 8126 8127 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 8128 ring->ring_obj = NULL; 8129 r600_ring_init(rdev, ring, 256 * 1024); 8130 8131 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 8132 ring->ring_obj = NULL; 8133 r600_ring_init(rdev, ring, 256 * 1024); 8134 8135 r = radeon_uvd_init(rdev); 8136 if (!r) { 8137 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 8138 ring->ring_obj = NULL; 8139 r600_ring_init(rdev, ring, 4096); 8140 } 8141 8142 r = radeon_vce_init(rdev); 8143 if (!r) { 8144 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; 8145 ring->ring_obj = NULL; 8146 r600_ring_init(rdev, ring, 4096); 8147 8148 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; 8149 ring->ring_obj = NULL; 8150 r600_ring_init(rdev, ring, 4096); 8151 } 8152 8153 rdev->ih.ring_obj = NULL; 8154 r600_ih_ring_init(rdev, 64 * 1024); 8155 8156 r = r600_pcie_gart_init(rdev); 8157 if (r) 8158 return r; 8159 8160 rdev->accel_working = true; 8161 r = cik_startup(rdev); 8162 if (r) { 8163 dev_err(rdev->dev, "disabling GPU acceleration\n"); 8164 cik_cp_fini(rdev); 8165 cik_sdma_fini(rdev); 8166 cik_irq_fini(rdev); 8167 sumo_rlc_fini(rdev); 8168 cik_mec_fini(rdev); 8169 radeon_wb_fini(rdev); 8170 radeon_ib_pool_fini(rdev); 8171 radeon_vm_manager_fini(rdev); 8172 radeon_irq_kms_fini(rdev); 8173 cik_pcie_gart_fini(rdev); 8174 rdev->accel_working = false; 8175 } 8176 8177 /* Don't start up if the MC ucode is missing. 8178 * The default clocks and voltages before the MC ucode 8179 * is loaded are not suffient for advanced operations. 8180 */ 8181 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 8182 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 8183 return -EINVAL; 8184 } 8185 8186 return 0; 8187 } 8188 8189 /** 8190 * cik_fini - asic specific driver and hw fini 8191 * 8192 * @rdev: radeon_device pointer 8193 * 8194 * Tear down the asic specific driver variables and program the hw 8195 * to an idle state (CIK). 8196 * Called at driver unload. 8197 */ 8198 void cik_fini(struct radeon_device *rdev) 8199 { 8200 radeon_pm_fini(rdev); 8201 cik_cp_fini(rdev); 8202 cik_sdma_fini(rdev); 8203 cik_fini_pg(rdev); 8204 cik_fini_cg(rdev); 8205 cik_irq_fini(rdev); 8206 sumo_rlc_fini(rdev); 8207 cik_mec_fini(rdev); 8208 radeon_wb_fini(rdev); 8209 radeon_vm_manager_fini(rdev); 8210 radeon_ib_pool_fini(rdev); 8211 radeon_irq_kms_fini(rdev); 8212 uvd_v1_0_fini(rdev); 8213 radeon_uvd_fini(rdev); 8214 radeon_vce_fini(rdev); 8215 cik_pcie_gart_fini(rdev); 8216 r600_vram_scratch_fini(rdev); 8217 radeon_gem_fini(rdev); 8218 radeon_fence_driver_fini(rdev); 8219 radeon_bo_fini(rdev); 8220 radeon_atombios_fini(rdev); 8221 kfree(rdev->bios); 8222 rdev->bios = NULL; 8223 } 8224 8225 void dce8_program_fmt(struct drm_encoder *encoder) 8226 { 8227 struct drm_device *dev = encoder->dev; 8228 struct radeon_device *rdev = dev->dev_private; 8229 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 8230 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 8231 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 8232 int bpc = 0; 8233 u32 tmp = 0; 8234 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 8235 8236 if (connector) { 8237 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 8238 bpc = radeon_get_monitor_bpc(connector); 8239 dither = radeon_connector->dither; 8240 } 8241 8242 /* LVDS/eDP FMT is set up by atom */ 8243 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 8244 return; 8245 8246 /* not needed for analog */ 8247 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 8248 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 8249 return; 8250 8251 if (bpc == 0) 8252 return; 8253 8254 switch (bpc) { 8255 case 6: 8256 if (dither == RADEON_FMT_DITHER_ENABLE) 8257 /* XXX sort out optimal dither settings */ 8258 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8259 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0)); 8260 else 8261 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0)); 8262 break; 8263 case 8: 8264 if (dither == RADEON_FMT_DITHER_ENABLE) 8265 /* XXX sort out optimal dither settings */ 8266 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8267 FMT_RGB_RANDOM_ENABLE | 8268 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1)); 8269 else 8270 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1)); 8271 break; 8272 case 10: 8273 if (dither == RADEON_FMT_DITHER_ENABLE) 8274 /* XXX sort out optimal dither settings */ 8275 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 8276 FMT_RGB_RANDOM_ENABLE | 8277 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2)); 8278 else 8279 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2)); 8280 break; 8281 default: 8282 /* not needed */ 8283 break; 8284 } 8285 8286 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 8287 } 8288 8289 /* display watermark setup */ 8290 /** 8291 * dce8_line_buffer_adjust - Set up the line buffer 8292 * 8293 * @rdev: radeon_device pointer 8294 * @radeon_crtc: the selected display controller 8295 * @mode: the current display mode on the selected display 8296 * controller 8297 * 8298 * Setup up the line buffer allocation for 8299 * the selected display controller (CIK). 8300 * Returns the line buffer size in pixels. 8301 */ 8302 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, 8303 struct radeon_crtc *radeon_crtc, 8304 struct drm_display_mode *mode) 8305 { 8306 u32 tmp, buffer_alloc, i; 8307 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 8308 /* 8309 * Line Buffer Setup 8310 * There are 6 line buffers, one for each display controllers. 8311 * There are 3 partitions per LB. Select the number of partitions 8312 * to enable based on the display width. For display widths larger 8313 * than 4096, you need use to use 2 display controllers and combine 8314 * them using the stereo blender. 8315 */ 8316 if (radeon_crtc->base.enabled && mode) { 8317 if (mode->crtc_hdisplay < 1920) { 8318 tmp = 1; 8319 buffer_alloc = 2; 8320 } else if (mode->crtc_hdisplay < 2560) { 8321 tmp = 2; 8322 buffer_alloc = 2; 8323 } else if (mode->crtc_hdisplay < 4096) { 8324 tmp = 0; 8325 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8326 } else { 8327 DRM_DEBUG_KMS("Mode too big for LB!\n"); 8328 tmp = 0; 8329 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; 8330 } 8331 } else { 8332 tmp = 1; 8333 buffer_alloc = 0; 8334 } 8335 8336 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset, 8337 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0)); 8338 8339 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 8340 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 8341 for (i = 0; i < rdev->usec_timeout; i++) { 8342 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 8343 DMIF_BUFFERS_ALLOCATED_COMPLETED) 8344 break; 8345 udelay(1); 8346 } 8347 8348 if (radeon_crtc->base.enabled && mode) { 8349 switch (tmp) { 8350 case 0: 8351 default: 8352 return 4096 * 2; 8353 case 1: 8354 return 1920 * 2; 8355 case 2: 8356 return 2560 * 2; 8357 } 8358 } 8359 8360 /* controller not enabled, so no lb used */ 8361 return 0; 8362 } 8363 8364 /** 8365 * cik_get_number_of_dram_channels - get the number of dram channels 8366 * 8367 * @rdev: radeon_device pointer 8368 * 8369 * Look up the number of video ram channels (CIK). 8370 * Used for display watermark bandwidth calculations 8371 * Returns the number of dram channels 8372 */ 8373 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) 8374 { 8375 u32 tmp = RREG32(MC_SHARED_CHMAP); 8376 8377 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 8378 case 0: 8379 default: 8380 return 1; 8381 case 1: 8382 return 2; 8383 case 2: 8384 return 4; 8385 case 3: 8386 return 8; 8387 case 4: 8388 return 3; 8389 case 5: 8390 return 6; 8391 case 6: 8392 return 10; 8393 case 7: 8394 return 12; 8395 case 8: 8396 return 16; 8397 } 8398 } 8399 8400 struct dce8_wm_params { 8401 u32 dram_channels; /* number of dram channels */ 8402 u32 yclk; /* bandwidth per dram data pin in kHz */ 8403 u32 sclk; /* engine clock in kHz */ 8404 u32 disp_clk; /* display clock in kHz */ 8405 u32 src_width; /* viewport width */ 8406 u32 active_time; /* active display time in ns */ 8407 u32 blank_time; /* blank time in ns */ 8408 bool interlaced; /* mode is interlaced */ 8409 fixed20_12 vsc; /* vertical scale ratio */ 8410 u32 num_heads; /* number of active crtcs */ 8411 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 8412 u32 lb_size; /* line buffer allocated to pipe */ 8413 u32 vtaps; /* vertical scaler taps */ 8414 }; 8415 8416 /** 8417 * dce8_dram_bandwidth - get the dram bandwidth 8418 * 8419 * @wm: watermark calculation data 8420 * 8421 * Calculate the raw dram bandwidth (CIK). 8422 * Used for display watermark bandwidth calculations 8423 * Returns the dram bandwidth in MBytes/s 8424 */ 8425 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm) 8426 { 8427 /* Calculate raw DRAM Bandwidth */ 8428 fixed20_12 dram_efficiency; /* 0.7 */ 8429 fixed20_12 yclk, dram_channels, bandwidth; 8430 fixed20_12 a; 8431 8432 a.full = dfixed_const(1000); 8433 yclk.full = dfixed_const(wm->yclk); 8434 yclk.full = dfixed_div(yclk, a); 8435 dram_channels.full = dfixed_const(wm->dram_channels * 4); 8436 a.full = dfixed_const(10); 8437 dram_efficiency.full = dfixed_const(7); 8438 dram_efficiency.full = dfixed_div(dram_efficiency, a); 8439 bandwidth.full = dfixed_mul(dram_channels, yclk); 8440 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 8441 8442 return dfixed_trunc(bandwidth); 8443 } 8444 8445 /** 8446 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display 8447 * 8448 * @wm: watermark calculation data 8449 * 8450 * Calculate the dram bandwidth used for display (CIK). 8451 * Used for display watermark bandwidth calculations 8452 * Returns the dram bandwidth for display in MBytes/s 8453 */ 8454 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm) 8455 { 8456 /* Calculate DRAM Bandwidth and the part allocated to display. */ 8457 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 8458 fixed20_12 yclk, dram_channels, bandwidth; 8459 fixed20_12 a; 8460 8461 a.full = dfixed_const(1000); 8462 yclk.full = dfixed_const(wm->yclk); 8463 yclk.full = dfixed_div(yclk, a); 8464 dram_channels.full = dfixed_const(wm->dram_channels * 4); 8465 a.full = dfixed_const(10); 8466 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 8467 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 8468 bandwidth.full = dfixed_mul(dram_channels, yclk); 8469 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 8470 8471 return dfixed_trunc(bandwidth); 8472 } 8473 8474 /** 8475 * dce8_data_return_bandwidth - get the data return bandwidth 8476 * 8477 * @wm: watermark calculation data 8478 * 8479 * Calculate the data return bandwidth used for display (CIK). 8480 * Used for display watermark bandwidth calculations 8481 * Returns the data return bandwidth in MBytes/s 8482 */ 8483 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm) 8484 { 8485 /* Calculate the display Data return Bandwidth */ 8486 fixed20_12 return_efficiency; /* 0.8 */ 8487 fixed20_12 sclk, bandwidth; 8488 fixed20_12 a; 8489 8490 a.full = dfixed_const(1000); 8491 sclk.full = dfixed_const(wm->sclk); 8492 sclk.full = dfixed_div(sclk, a); 8493 a.full = dfixed_const(10); 8494 return_efficiency.full = dfixed_const(8); 8495 return_efficiency.full = dfixed_div(return_efficiency, a); 8496 a.full = dfixed_const(32); 8497 bandwidth.full = dfixed_mul(a, sclk); 8498 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 8499 8500 return dfixed_trunc(bandwidth); 8501 } 8502 8503 /** 8504 * dce8_dmif_request_bandwidth - get the dmif bandwidth 8505 * 8506 * @wm: watermark calculation data 8507 * 8508 * Calculate the dmif bandwidth used for display (CIK). 8509 * Used for display watermark bandwidth calculations 8510 * Returns the dmif bandwidth in MBytes/s 8511 */ 8512 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm) 8513 { 8514 /* Calculate the DMIF Request Bandwidth */ 8515 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 8516 fixed20_12 disp_clk, bandwidth; 8517 fixed20_12 a, b; 8518 8519 a.full = dfixed_const(1000); 8520 disp_clk.full = dfixed_const(wm->disp_clk); 8521 disp_clk.full = dfixed_div(disp_clk, a); 8522 a.full = dfixed_const(32); 8523 b.full = dfixed_mul(a, disp_clk); 8524 8525 a.full = dfixed_const(10); 8526 disp_clk_request_efficiency.full = dfixed_const(8); 8527 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 8528 8529 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency); 8530 8531 return dfixed_trunc(bandwidth); 8532 } 8533 8534 /** 8535 * dce8_available_bandwidth - get the min available bandwidth 8536 * 8537 * @wm: watermark calculation data 8538 * 8539 * Calculate the min available bandwidth used for display (CIK). 8540 * Used for display watermark bandwidth calculations 8541 * Returns the min available bandwidth in MBytes/s 8542 */ 8543 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm) 8544 { 8545 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 8546 u32 dram_bandwidth = dce8_dram_bandwidth(wm); 8547 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm); 8548 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm); 8549 8550 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 8551 } 8552 8553 /** 8554 * dce8_average_bandwidth - get the average available bandwidth 8555 * 8556 * @wm: watermark calculation data 8557 * 8558 * Calculate the average available bandwidth used for display (CIK). 8559 * Used for display watermark bandwidth calculations 8560 * Returns the average available bandwidth in MBytes/s 8561 */ 8562 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm) 8563 { 8564 /* Calculate the display mode Average Bandwidth 8565 * DisplayMode should contain the source and destination dimensions, 8566 * timing, etc. 8567 */ 8568 fixed20_12 bpp; 8569 fixed20_12 line_time; 8570 fixed20_12 src_width; 8571 fixed20_12 bandwidth; 8572 fixed20_12 a; 8573 8574 a.full = dfixed_const(1000); 8575 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 8576 line_time.full = dfixed_div(line_time, a); 8577 bpp.full = dfixed_const(wm->bytes_per_pixel); 8578 src_width.full = dfixed_const(wm->src_width); 8579 bandwidth.full = dfixed_mul(src_width, bpp); 8580 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 8581 bandwidth.full = dfixed_div(bandwidth, line_time); 8582 8583 return dfixed_trunc(bandwidth); 8584 } 8585 8586 /** 8587 * dce8_latency_watermark - get the latency watermark 8588 * 8589 * @wm: watermark calculation data 8590 * 8591 * Calculate the latency watermark (CIK). 8592 * Used for display watermark bandwidth calculations 8593 * Returns the latency watermark in ns 8594 */ 8595 static u32 dce8_latency_watermark(struct dce8_wm_params *wm) 8596 { 8597 /* First calculate the latency in ns */ 8598 u32 mc_latency = 2000; /* 2000 ns. */ 8599 u32 available_bandwidth = dce8_available_bandwidth(wm); 8600 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 8601 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 8602 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 8603 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 8604 (wm->num_heads * cursor_line_pair_return_time); 8605 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 8606 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 8607 u32 tmp, dmif_size = 12288; 8608 fixed20_12 a, b, c; 8609 8610 if (wm->num_heads == 0) 8611 return 0; 8612 8613 a.full = dfixed_const(2); 8614 b.full = dfixed_const(1); 8615 if ((wm->vsc.full > a.full) || 8616 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 8617 (wm->vtaps >= 5) || 8618 ((wm->vsc.full >= a.full) && wm->interlaced)) 8619 max_src_lines_per_dst_line = 4; 8620 else 8621 max_src_lines_per_dst_line = 2; 8622 8623 a.full = dfixed_const(available_bandwidth); 8624 b.full = dfixed_const(wm->num_heads); 8625 a.full = dfixed_div(a, b); 8626 8627 b.full = dfixed_const(mc_latency + 512); 8628 c.full = dfixed_const(wm->disp_clk); 8629 b.full = dfixed_div(b, c); 8630 8631 c.full = dfixed_const(dmif_size); 8632 b.full = dfixed_div(c, b); 8633 8634 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 8635 8636 b.full = dfixed_const(1000); 8637 c.full = dfixed_const(wm->disp_clk); 8638 b.full = dfixed_div(c, b); 8639 c.full = dfixed_const(wm->bytes_per_pixel); 8640 b.full = dfixed_mul(b, c); 8641 8642 lb_fill_bw = min(tmp, dfixed_trunc(b)); 8643 8644 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 8645 b.full = dfixed_const(1000); 8646 c.full = dfixed_const(lb_fill_bw); 8647 b.full = dfixed_div(c, b); 8648 a.full = dfixed_div(a, b); 8649 line_fill_time = dfixed_trunc(a); 8650 8651 if (line_fill_time < wm->active_time) 8652 return latency; 8653 else 8654 return latency + (line_fill_time - wm->active_time); 8655 8656 } 8657 8658 /** 8659 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check 8660 * average and available dram bandwidth 8661 * 8662 * @wm: watermark calculation data 8663 * 8664 * Check if the display average bandwidth fits in the display 8665 * dram bandwidth (CIK). 8666 * Used for display watermark bandwidth calculations 8667 * Returns true if the display fits, false if not. 8668 */ 8669 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm) 8670 { 8671 if (dce8_average_bandwidth(wm) <= 8672 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads)) 8673 return true; 8674 else 8675 return false; 8676 } 8677 8678 /** 8679 * dce8_average_bandwidth_vs_available_bandwidth - check 8680 * average and available bandwidth 8681 * 8682 * @wm: watermark calculation data 8683 * 8684 * Check if the display average bandwidth fits in the display 8685 * available bandwidth (CIK). 8686 * Used for display watermark bandwidth calculations 8687 * Returns true if the display fits, false if not. 8688 */ 8689 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm) 8690 { 8691 if (dce8_average_bandwidth(wm) <= 8692 (dce8_available_bandwidth(wm) / wm->num_heads)) 8693 return true; 8694 else 8695 return false; 8696 } 8697 8698 /** 8699 * dce8_check_latency_hiding - check latency hiding 8700 * 8701 * @wm: watermark calculation data 8702 * 8703 * Check latency hiding (CIK). 8704 * Used for display watermark bandwidth calculations 8705 * Returns true if the display fits, false if not. 8706 */ 8707 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm) 8708 { 8709 u32 lb_partitions = wm->lb_size / wm->src_width; 8710 u32 line_time = wm->active_time + wm->blank_time; 8711 u32 latency_tolerant_lines; 8712 u32 latency_hiding; 8713 fixed20_12 a; 8714 8715 a.full = dfixed_const(1); 8716 if (wm->vsc.full > a.full) 8717 latency_tolerant_lines = 1; 8718 else { 8719 if (lb_partitions <= (wm->vtaps + 1)) 8720 latency_tolerant_lines = 1; 8721 else 8722 latency_tolerant_lines = 2; 8723 } 8724 8725 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 8726 8727 if (dce8_latency_watermark(wm) <= latency_hiding) 8728 return true; 8729 else 8730 return false; 8731 } 8732 8733 /** 8734 * dce8_program_watermarks - program display watermarks 8735 * 8736 * @rdev: radeon_device pointer 8737 * @radeon_crtc: the selected display controller 8738 * @lb_size: line buffer size 8739 * @num_heads: number of display controllers in use 8740 * 8741 * Calculate and program the display watermarks for the 8742 * selected display controller (CIK). 8743 */ 8744 static void dce8_program_watermarks(struct radeon_device *rdev, 8745 struct radeon_crtc *radeon_crtc, 8746 u32 lb_size, u32 num_heads) 8747 { 8748 struct drm_display_mode *mode = &radeon_crtc->base.mode; 8749 struct dce8_wm_params wm_low, wm_high; 8750 u32 pixel_period; 8751 u32 line_time = 0; 8752 u32 latency_watermark_a = 0, latency_watermark_b = 0; 8753 u32 tmp, wm_mask; 8754 8755 if (radeon_crtc->base.enabled && num_heads && mode) { 8756 pixel_period = 1000000 / (u32)mode->clock; 8757 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 8758 8759 /* watermark for high clocks */ 8760 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 8761 rdev->pm.dpm_enabled) { 8762 wm_high.yclk = 8763 radeon_dpm_get_mclk(rdev, false) * 10; 8764 wm_high.sclk = 8765 radeon_dpm_get_sclk(rdev, false) * 10; 8766 } else { 8767 wm_high.yclk = rdev->pm.current_mclk * 10; 8768 wm_high.sclk = rdev->pm.current_sclk * 10; 8769 } 8770 8771 wm_high.disp_clk = mode->clock; 8772 wm_high.src_width = mode->crtc_hdisplay; 8773 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 8774 wm_high.blank_time = line_time - wm_high.active_time; 8775 wm_high.interlaced = false; 8776 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 8777 wm_high.interlaced = true; 8778 wm_high.vsc = radeon_crtc->vsc; 8779 wm_high.vtaps = 1; 8780 if (radeon_crtc->rmx_type != RMX_OFF) 8781 wm_high.vtaps = 2; 8782 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 8783 wm_high.lb_size = lb_size; 8784 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); 8785 wm_high.num_heads = num_heads; 8786 8787 /* set for high clocks */ 8788 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535); 8789 8790 /* possibly force display priority to high */ 8791 /* should really do this at mode validation time... */ 8792 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 8793 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) || 8794 !dce8_check_latency_hiding(&wm_high) || 8795 (rdev->disp_priority == 2)) { 8796 DRM_DEBUG_KMS("force priority to high\n"); 8797 } 8798 8799 /* watermark for low clocks */ 8800 if ((rdev->pm.pm_method == PM_METHOD_DPM) && 8801 rdev->pm.dpm_enabled) { 8802 wm_low.yclk = 8803 radeon_dpm_get_mclk(rdev, true) * 10; 8804 wm_low.sclk = 8805 radeon_dpm_get_sclk(rdev, true) * 10; 8806 } else { 8807 wm_low.yclk = rdev->pm.current_mclk * 10; 8808 wm_low.sclk = rdev->pm.current_sclk * 10; 8809 } 8810 8811 wm_low.disp_clk = mode->clock; 8812 wm_low.src_width = mode->crtc_hdisplay; 8813 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 8814 wm_low.blank_time = line_time - wm_low.active_time; 8815 wm_low.interlaced = false; 8816 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 8817 wm_low.interlaced = true; 8818 wm_low.vsc = radeon_crtc->vsc; 8819 wm_low.vtaps = 1; 8820 if (radeon_crtc->rmx_type != RMX_OFF) 8821 wm_low.vtaps = 2; 8822 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 8823 wm_low.lb_size = lb_size; 8824 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); 8825 wm_low.num_heads = num_heads; 8826 8827 /* set for low clocks */ 8828 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535); 8829 8830 /* possibly force display priority to high */ 8831 /* should really do this at mode validation time... */ 8832 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 8833 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) || 8834 !dce8_check_latency_hiding(&wm_low) || 8835 (rdev->disp_priority == 2)) { 8836 DRM_DEBUG_KMS("force priority to high\n"); 8837 } 8838 } 8839 8840 /* select wm A */ 8841 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 8842 tmp = wm_mask; 8843 tmp &= ~LATENCY_WATERMARK_MASK(3); 8844 tmp |= LATENCY_WATERMARK_MASK(1); 8845 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 8846 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 8847 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 8848 LATENCY_HIGH_WATERMARK(line_time))); 8849 /* select wm B */ 8850 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset); 8851 tmp &= ~LATENCY_WATERMARK_MASK(3); 8852 tmp |= LATENCY_WATERMARK_MASK(2); 8853 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp); 8854 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 8855 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 8856 LATENCY_HIGH_WATERMARK(line_time))); 8857 /* restore original selection */ 8858 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask); 8859 8860 /* save values for DPM */ 8861 radeon_crtc->line_time = line_time; 8862 radeon_crtc->wm_high = latency_watermark_a; 8863 radeon_crtc->wm_low = latency_watermark_b; 8864 } 8865 8866 /** 8867 * dce8_bandwidth_update - program display watermarks 8868 * 8869 * @rdev: radeon_device pointer 8870 * 8871 * Calculate and program the display watermarks and line 8872 * buffer allocation (CIK). 8873 */ 8874 void dce8_bandwidth_update(struct radeon_device *rdev) 8875 { 8876 struct drm_display_mode *mode = NULL; 8877 u32 num_heads = 0, lb_size; 8878 int i; 8879 8880 radeon_update_display_priority(rdev); 8881 8882 for (i = 0; i < rdev->num_crtc; i++) { 8883 if (rdev->mode_info.crtcs[i]->base.enabled) 8884 num_heads++; 8885 } 8886 for (i = 0; i < rdev->num_crtc; i++) { 8887 mode = &rdev->mode_info.crtcs[i]->base.mode; 8888 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); 8889 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 8890 } 8891 } 8892 8893 /** 8894 * cik_get_gpu_clock_counter - return GPU clock counter snapshot 8895 * 8896 * @rdev: radeon_device pointer 8897 * 8898 * Fetches a GPU clock counter snapshot (SI). 8899 * Returns the 64 bit clock counter snapshot. 8900 */ 8901 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) 8902 { 8903 uint64_t clock; 8904 8905 mutex_lock(&rdev->gpu_clock_mutex); 8906 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 8907 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 8908 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 8909 mutex_unlock(&rdev->gpu_clock_mutex); 8910 return clock; 8911 } 8912 8913 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, 8914 u32 cntl_reg, u32 status_reg) 8915 { 8916 int r, i; 8917 struct atom_clock_dividers dividers; 8918 uint32_t tmp; 8919 8920 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 8921 clock, false, ÷rs); 8922 if (r) 8923 return r; 8924 8925 tmp = RREG32_SMC(cntl_reg); 8926 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK); 8927 tmp |= dividers.post_divider; 8928 WREG32_SMC(cntl_reg, tmp); 8929 8930 for (i = 0; i < 100; i++) { 8931 if (RREG32_SMC(status_reg) & DCLK_STATUS) 8932 break; 8933 mdelay(10); 8934 } 8935 if (i == 100) 8936 return -ETIMEDOUT; 8937 8938 return 0; 8939 } 8940 8941 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 8942 { 8943 int r = 0; 8944 8945 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 8946 if (r) 8947 return r; 8948 8949 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 8950 return r; 8951 } 8952 8953 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) 8954 { 8955 int r, i; 8956 struct atom_clock_dividers dividers; 8957 u32 tmp; 8958 8959 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, 8960 ecclk, false, ÷rs); 8961 if (r) 8962 return r; 8963 8964 for (i = 0; i < 100; i++) { 8965 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 8966 break; 8967 mdelay(10); 8968 } 8969 if (i == 100) 8970 return -ETIMEDOUT; 8971 8972 tmp = RREG32_SMC(CG_ECLK_CNTL); 8973 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK); 8974 tmp |= dividers.post_divider; 8975 WREG32_SMC(CG_ECLK_CNTL, tmp); 8976 8977 for (i = 0; i < 100; i++) { 8978 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS) 8979 break; 8980 mdelay(10); 8981 } 8982 if (i == 100) 8983 return -ETIMEDOUT; 8984 8985 return 0; 8986 } 8987 8988 static void cik_pcie_gen3_enable(struct radeon_device *rdev) 8989 { 8990 struct pci_dev *root = rdev->pdev->bus->self; 8991 int bridge_pos, gpu_pos; 8992 u32 speed_cntl, mask, current_data_rate; 8993 int ret, i; 8994 u16 tmp16; 8995 8996 if (radeon_pcie_gen2 == 0) 8997 return; 8998 8999 if (rdev->flags & RADEON_IS_IGP) 9000 return; 9001 9002 if (!(rdev->flags & RADEON_IS_PCIE)) 9003 return; 9004 9005 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 9006 if (ret != 0) 9007 return; 9008 9009 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 9010 return; 9011 9012 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9013 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 9014 LC_CURRENT_DATA_RATE_SHIFT; 9015 if (mask & DRM_PCIE_SPEED_80) { 9016 if (current_data_rate == 2) { 9017 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 9018 return; 9019 } 9020 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 9021 } else if (mask & DRM_PCIE_SPEED_50) { 9022 if (current_data_rate == 1) { 9023 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 9024 return; 9025 } 9026 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 9027 } 9028 9029 bridge_pos = pci_pcie_cap(root); 9030 if (!bridge_pos) 9031 return; 9032 9033 gpu_pos = pci_pcie_cap(rdev->pdev); 9034 if (!gpu_pos) 9035 return; 9036 9037 if (mask & DRM_PCIE_SPEED_80) { 9038 /* re-try equalization if gen3 is not already enabled */ 9039 if (current_data_rate != 2) { 9040 u16 bridge_cfg, gpu_cfg; 9041 u16 bridge_cfg2, gpu_cfg2; 9042 u32 max_lw, current_lw, tmp; 9043 9044 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9045 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9046 9047 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 9048 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9049 9050 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 9051 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9052 9053 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9054 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 9055 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 9056 9057 if (current_lw < max_lw) { 9058 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9059 if (tmp & LC_RENEGOTIATION_SUPPORT) { 9060 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 9061 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 9062 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 9063 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 9064 } 9065 } 9066 9067 for (i = 0; i < 10; i++) { 9068 /* check status */ 9069 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 9070 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 9071 break; 9072 9073 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 9074 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 9075 9076 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 9077 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 9078 9079 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9080 tmp |= LC_SET_QUIESCE; 9081 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9082 9083 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9084 tmp |= LC_REDO_EQ; 9085 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9086 9087 mdelay(100); 9088 9089 /* linkctl */ 9090 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 9091 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9092 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 9093 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 9094 9095 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 9096 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 9097 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 9098 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 9099 9100 /* linkctl2 */ 9101 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 9102 tmp16 &= ~((1 << 4) | (7 << 9)); 9103 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 9104 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 9105 9106 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9107 tmp16 &= ~((1 << 4) | (7 << 9)); 9108 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 9109 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9110 9111 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 9112 tmp &= ~LC_SET_QUIESCE; 9113 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 9114 } 9115 } 9116 } 9117 9118 /* set the link speed */ 9119 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 9120 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 9121 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9122 9123 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 9124 tmp16 &= ~0xf; 9125 if (mask & DRM_PCIE_SPEED_80) 9126 tmp16 |= 3; /* gen3 */ 9127 else if (mask & DRM_PCIE_SPEED_50) 9128 tmp16 |= 2; /* gen2 */ 9129 else 9130 tmp16 |= 1; /* gen1 */ 9131 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 9132 9133 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9134 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 9135 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 9136 9137 for (i = 0; i < rdev->usec_timeout; i++) { 9138 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 9139 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 9140 break; 9141 udelay(1); 9142 } 9143 } 9144 9145 static void cik_program_aspm(struct radeon_device *rdev) 9146 { 9147 u32 data, orig; 9148 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 9149 bool disable_clkreq = false; 9150 9151 if (radeon_aspm == 0) 9152 return; 9153 9154 /* XXX double check IGPs */ 9155 if (rdev->flags & RADEON_IS_IGP) 9156 return; 9157 9158 if (!(rdev->flags & RADEON_IS_PCIE)) 9159 return; 9160 9161 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9162 data &= ~LC_XMIT_N_FTS_MASK; 9163 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 9164 if (orig != data) 9165 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 9166 9167 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 9168 data |= LC_GO_TO_RECOVERY; 9169 if (orig != data) 9170 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 9171 9172 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL); 9173 data |= P_IGNORE_EDB_ERR; 9174 if (orig != data) 9175 WREG32_PCIE_PORT(PCIE_P_CNTL, data); 9176 9177 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9178 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 9179 data |= LC_PMI_TO_L1_DIS; 9180 if (!disable_l0s) 9181 data |= LC_L0S_INACTIVITY(7); 9182 9183 if (!disable_l1) { 9184 data |= LC_L1_INACTIVITY(7); 9185 data &= ~LC_PMI_TO_L1_DIS; 9186 if (orig != data) 9187 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9188 9189 if (!disable_plloff_in_l1) { 9190 bool clk_req_support; 9191 9192 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0); 9193 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9194 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9195 if (orig != data) 9196 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data); 9197 9198 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1); 9199 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9200 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9201 if (orig != data) 9202 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data); 9203 9204 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0); 9205 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 9206 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 9207 if (orig != data) 9208 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data); 9209 9210 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1); 9211 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 9212 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 9213 if (orig != data) 9214 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data); 9215 9216 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 9217 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 9218 data |= LC_DYN_LANES_PWR_STATE(3); 9219 if (orig != data) 9220 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 9221 9222 if (!disable_clkreq) { 9223 struct pci_dev *root = rdev->pdev->bus->self; 9224 u32 lnkcap; 9225 9226 clk_req_support = false; 9227 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 9228 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 9229 clk_req_support = true; 9230 } else { 9231 clk_req_support = false; 9232 } 9233 9234 if (clk_req_support) { 9235 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 9236 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 9237 if (orig != data) 9238 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 9239 9240 orig = data = RREG32_SMC(THM_CLK_CNTL); 9241 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 9242 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 9243 if (orig != data) 9244 WREG32_SMC(THM_CLK_CNTL, data); 9245 9246 orig = data = RREG32_SMC(MISC_CLK_CTRL); 9247 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 9248 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 9249 if (orig != data) 9250 WREG32_SMC(MISC_CLK_CTRL, data); 9251 9252 orig = data = RREG32_SMC(CG_CLKPIN_CNTL); 9253 data &= ~BCLK_AS_XCLK; 9254 if (orig != data) 9255 WREG32_SMC(CG_CLKPIN_CNTL, data); 9256 9257 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2); 9258 data &= ~FORCE_BIF_REFCLK_EN; 9259 if (orig != data) 9260 WREG32_SMC(CG_CLKPIN_CNTL_2, data); 9261 9262 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL); 9263 data &= ~MPLL_CLKOUT_SEL_MASK; 9264 data |= MPLL_CLKOUT_SEL(4); 9265 if (orig != data) 9266 WREG32_SMC(MPLL_BYPASSCLK_SEL, data); 9267 } 9268 } 9269 } else { 9270 if (orig != data) 9271 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9272 } 9273 9274 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2); 9275 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 9276 if (orig != data) 9277 WREG32_PCIE_PORT(PCIE_CNTL2, data); 9278 9279 if (!disable_l0s) { 9280 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 9281 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 9282 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1); 9283 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 9284 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 9285 data &= ~LC_L0S_INACTIVITY_MASK; 9286 if (orig != data) 9287 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 9288 } 9289 } 9290 } 9291 } 9292