1 /* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <drm/drmP.h> 27 #include "radeon.h" 28 #include "radeon_asic.h" 29 #include "radeon_audio.h" 30 #include <drm/radeon_drm.h> 31 #include "evergreend.h" 32 #include "atom.h" 33 #include "avivod.h" 34 #include "evergreen_reg.h" 35 #include "evergreen_blit_shaders.h" 36 #include "radeon_ucode.h" 37 38 /* 39 * Indirect registers accessor 40 */ 41 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg) 42 { 43 unsigned long flags; 44 u32 r; 45 46 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 47 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 48 r = RREG32(EVERGREEN_CG_IND_DATA); 49 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 50 return r; 51 } 52 53 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v) 54 { 55 unsigned long flags; 56 57 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 58 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 59 WREG32(EVERGREEN_CG_IND_DATA, (v)); 60 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 61 } 62 63 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg) 64 { 65 unsigned long flags; 66 u32 r; 67 68 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 69 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 70 r = RREG32(EVERGREEN_PIF_PHY0_DATA); 71 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 72 return r; 73 } 74 75 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v) 76 { 77 unsigned long flags; 78 79 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 80 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 81 WREG32(EVERGREEN_PIF_PHY0_DATA, (v)); 82 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 83 } 84 85 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg) 86 { 87 unsigned long flags; 88 u32 r; 89 90 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 91 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 92 r = RREG32(EVERGREEN_PIF_PHY1_DATA); 93 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 94 return r; 95 } 96 97 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v) 98 { 99 unsigned long flags; 100 101 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 102 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 103 WREG32(EVERGREEN_PIF_PHY1_DATA, (v)); 104 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 105 } 106 107 static const u32 crtc_offsets[6] = 108 { 109 EVERGREEN_CRTC0_REGISTER_OFFSET, 110 EVERGREEN_CRTC1_REGISTER_OFFSET, 111 EVERGREEN_CRTC2_REGISTER_OFFSET, 112 EVERGREEN_CRTC3_REGISTER_OFFSET, 113 EVERGREEN_CRTC4_REGISTER_OFFSET, 114 EVERGREEN_CRTC5_REGISTER_OFFSET 115 }; 116 117 #include "clearstate_evergreen.h" 118 119 static const u32 sumo_rlc_save_restore_register_list[] = 120 { 121 0x98fc, 122 0x9830, 123 0x9834, 124 0x9838, 125 0x9870, 126 0x9874, 127 0x8a14, 128 0x8b24, 129 0x8bcc, 130 0x8b10, 131 0x8d00, 132 0x8d04, 133 0x8c00, 134 0x8c04, 135 0x8c08, 136 0x8c0c, 137 0x8d8c, 138 0x8c20, 139 0x8c24, 140 0x8c28, 141 0x8c18, 142 0x8c1c, 143 0x8cf0, 144 0x8e2c, 145 0x8e38, 146 0x8c30, 147 0x9508, 148 0x9688, 149 0x9608, 150 0x960c, 151 0x9610, 152 0x9614, 153 0x88c4, 154 0x88d4, 155 0xa008, 156 0x900c, 157 0x9100, 158 0x913c, 159 0x98f8, 160 0x98f4, 161 0x9b7c, 162 0x3f8c, 163 0x8950, 164 0x8954, 165 0x8a18, 166 0x8b28, 167 0x9144, 168 0x9148, 169 0x914c, 170 0x3f90, 171 0x3f94, 172 0x915c, 173 0x9160, 174 0x9178, 175 0x917c, 176 0x9180, 177 0x918c, 178 0x9190, 179 0x9194, 180 0x9198, 181 0x919c, 182 0x91a8, 183 0x91ac, 184 0x91b0, 185 0x91b4, 186 0x91b8, 187 0x91c4, 188 0x91c8, 189 0x91cc, 190 0x91d0, 191 0x91d4, 192 0x91e0, 193 0x91e4, 194 0x91ec, 195 0x91f0, 196 0x91f4, 197 0x9200, 198 0x9204, 199 0x929c, 200 0x9150, 201 0x802c, 202 }; 203 204 static void evergreen_gpu_init(struct radeon_device *rdev); 205 void evergreen_fini(struct radeon_device *rdev); 206 void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 207 void evergreen_program_aspm(struct radeon_device *rdev); 208 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev, 209 int ring, u32 cp_int_cntl); 210 extern void cayman_vm_decode_fault(struct radeon_device *rdev, 211 u32 status, u32 addr); 212 void cik_init_cp_pg_table(struct radeon_device *rdev); 213 214 extern u32 si_get_csb_size(struct radeon_device *rdev); 215 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 216 extern u32 cik_get_csb_size(struct radeon_device *rdev); 217 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 218 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev); 219 220 static const u32 evergreen_golden_registers[] = 221 { 222 0x3f90, 0xffff0000, 0xff000000, 223 0x9148, 0xffff0000, 0xff000000, 224 0x3f94, 0xffff0000, 0xff000000, 225 0x914c, 0xffff0000, 0xff000000, 226 0x9b7c, 0xffffffff, 0x00000000, 227 0x8a14, 0xffffffff, 0x00000007, 228 0x8b10, 0xffffffff, 0x00000000, 229 0x960c, 0xffffffff, 0x54763210, 230 0x88c4, 0xffffffff, 0x000000c2, 231 0x88d4, 0xffffffff, 0x00000010, 232 0x8974, 0xffffffff, 0x00000000, 233 0xc78, 0x00000080, 0x00000080, 234 0x5eb4, 0xffffffff, 0x00000002, 235 0x5e78, 0xffffffff, 0x001000f0, 236 0x6104, 0x01000300, 0x00000000, 237 0x5bc0, 0x00300000, 0x00000000, 238 0x7030, 0xffffffff, 0x00000011, 239 0x7c30, 0xffffffff, 0x00000011, 240 0x10830, 0xffffffff, 0x00000011, 241 0x11430, 0xffffffff, 0x00000011, 242 0x12030, 0xffffffff, 0x00000011, 243 0x12c30, 0xffffffff, 0x00000011, 244 0xd02c, 0xffffffff, 0x08421000, 245 0x240c, 0xffffffff, 0x00000380, 246 0x8b24, 0xffffffff, 0x00ff0fff, 247 0x28a4c, 0x06000000, 0x06000000, 248 0x10c, 0x00000001, 0x00000001, 249 0x8d00, 0xffffffff, 0x100e4848, 250 0x8d04, 0xffffffff, 0x00164745, 251 0x8c00, 0xffffffff, 0xe4000003, 252 0x8c04, 0xffffffff, 0x40600060, 253 0x8c08, 0xffffffff, 0x001c001c, 254 0x8cf0, 0xffffffff, 0x08e00620, 255 0x8c20, 0xffffffff, 0x00800080, 256 0x8c24, 0xffffffff, 0x00800080, 257 0x8c18, 0xffffffff, 0x20202078, 258 0x8c1c, 0xffffffff, 0x00001010, 259 0x28350, 0xffffffff, 0x00000000, 260 0xa008, 0xffffffff, 0x00010000, 261 0x5c4, 0xffffffff, 0x00000001, 262 0x9508, 0xffffffff, 0x00000002, 263 0x913c, 0x0000000f, 0x0000000a 264 }; 265 266 static const u32 evergreen_golden_registers2[] = 267 { 268 0x2f4c, 0xffffffff, 0x00000000, 269 0x54f4, 0xffffffff, 0x00000000, 270 0x54f0, 0xffffffff, 0x00000000, 271 0x5498, 0xffffffff, 0x00000000, 272 0x549c, 0xffffffff, 0x00000000, 273 0x5494, 0xffffffff, 0x00000000, 274 0x53cc, 0xffffffff, 0x00000000, 275 0x53c8, 0xffffffff, 0x00000000, 276 0x53c4, 0xffffffff, 0x00000000, 277 0x53c0, 0xffffffff, 0x00000000, 278 0x53bc, 0xffffffff, 0x00000000, 279 0x53b8, 0xffffffff, 0x00000000, 280 0x53b4, 0xffffffff, 0x00000000, 281 0x53b0, 0xffffffff, 0x00000000 282 }; 283 284 static const u32 cypress_mgcg_init[] = 285 { 286 0x802c, 0xffffffff, 0xc0000000, 287 0x5448, 0xffffffff, 0x00000100, 288 0x55e4, 0xffffffff, 0x00000100, 289 0x160c, 0xffffffff, 0x00000100, 290 0x5644, 0xffffffff, 0x00000100, 291 0xc164, 0xffffffff, 0x00000100, 292 0x8a18, 0xffffffff, 0x00000100, 293 0x897c, 0xffffffff, 0x06000100, 294 0x8b28, 0xffffffff, 0x00000100, 295 0x9144, 0xffffffff, 0x00000100, 296 0x9a60, 0xffffffff, 0x00000100, 297 0x9868, 0xffffffff, 0x00000100, 298 0x8d58, 0xffffffff, 0x00000100, 299 0x9510, 0xffffffff, 0x00000100, 300 0x949c, 0xffffffff, 0x00000100, 301 0x9654, 0xffffffff, 0x00000100, 302 0x9030, 0xffffffff, 0x00000100, 303 0x9034, 0xffffffff, 0x00000100, 304 0x9038, 0xffffffff, 0x00000100, 305 0x903c, 0xffffffff, 0x00000100, 306 0x9040, 0xffffffff, 0x00000100, 307 0xa200, 0xffffffff, 0x00000100, 308 0xa204, 0xffffffff, 0x00000100, 309 0xa208, 0xffffffff, 0x00000100, 310 0xa20c, 0xffffffff, 0x00000100, 311 0x971c, 0xffffffff, 0x00000100, 312 0x977c, 0xffffffff, 0x00000100, 313 0x3f80, 0xffffffff, 0x00000100, 314 0xa210, 0xffffffff, 0x00000100, 315 0xa214, 0xffffffff, 0x00000100, 316 0x4d8, 0xffffffff, 0x00000100, 317 0x9784, 0xffffffff, 0x00000100, 318 0x9698, 0xffffffff, 0x00000100, 319 0x4d4, 0xffffffff, 0x00000200, 320 0x30cc, 0xffffffff, 0x00000100, 321 0xd0c0, 0xffffffff, 0xff000100, 322 0x802c, 0xffffffff, 0x40000000, 323 0x915c, 0xffffffff, 0x00010000, 324 0x9160, 0xffffffff, 0x00030002, 325 0x9178, 0xffffffff, 0x00070000, 326 0x917c, 0xffffffff, 0x00030002, 327 0x9180, 0xffffffff, 0x00050004, 328 0x918c, 0xffffffff, 0x00010006, 329 0x9190, 0xffffffff, 0x00090008, 330 0x9194, 0xffffffff, 0x00070000, 331 0x9198, 0xffffffff, 0x00030002, 332 0x919c, 0xffffffff, 0x00050004, 333 0x91a8, 0xffffffff, 0x00010006, 334 0x91ac, 0xffffffff, 0x00090008, 335 0x91b0, 0xffffffff, 0x00070000, 336 0x91b4, 0xffffffff, 0x00030002, 337 0x91b8, 0xffffffff, 0x00050004, 338 0x91c4, 0xffffffff, 0x00010006, 339 0x91c8, 0xffffffff, 0x00090008, 340 0x91cc, 0xffffffff, 0x00070000, 341 0x91d0, 0xffffffff, 0x00030002, 342 0x91d4, 0xffffffff, 0x00050004, 343 0x91e0, 0xffffffff, 0x00010006, 344 0x91e4, 0xffffffff, 0x00090008, 345 0x91e8, 0xffffffff, 0x00000000, 346 0x91ec, 0xffffffff, 0x00070000, 347 0x91f0, 0xffffffff, 0x00030002, 348 0x91f4, 0xffffffff, 0x00050004, 349 0x9200, 0xffffffff, 0x00010006, 350 0x9204, 0xffffffff, 0x00090008, 351 0x9208, 0xffffffff, 0x00070000, 352 0x920c, 0xffffffff, 0x00030002, 353 0x9210, 0xffffffff, 0x00050004, 354 0x921c, 0xffffffff, 0x00010006, 355 0x9220, 0xffffffff, 0x00090008, 356 0x9224, 0xffffffff, 0x00070000, 357 0x9228, 0xffffffff, 0x00030002, 358 0x922c, 0xffffffff, 0x00050004, 359 0x9238, 0xffffffff, 0x00010006, 360 0x923c, 0xffffffff, 0x00090008, 361 0x9240, 0xffffffff, 0x00070000, 362 0x9244, 0xffffffff, 0x00030002, 363 0x9248, 0xffffffff, 0x00050004, 364 0x9254, 0xffffffff, 0x00010006, 365 0x9258, 0xffffffff, 0x00090008, 366 0x925c, 0xffffffff, 0x00070000, 367 0x9260, 0xffffffff, 0x00030002, 368 0x9264, 0xffffffff, 0x00050004, 369 0x9270, 0xffffffff, 0x00010006, 370 0x9274, 0xffffffff, 0x00090008, 371 0x9278, 0xffffffff, 0x00070000, 372 0x927c, 0xffffffff, 0x00030002, 373 0x9280, 0xffffffff, 0x00050004, 374 0x928c, 0xffffffff, 0x00010006, 375 0x9290, 0xffffffff, 0x00090008, 376 0x9294, 0xffffffff, 0x00000000, 377 0x929c, 0xffffffff, 0x00000001, 378 0x802c, 0xffffffff, 0x40010000, 379 0x915c, 0xffffffff, 0x00010000, 380 0x9160, 0xffffffff, 0x00030002, 381 0x9178, 0xffffffff, 0x00070000, 382 0x917c, 0xffffffff, 0x00030002, 383 0x9180, 0xffffffff, 0x00050004, 384 0x918c, 0xffffffff, 0x00010006, 385 0x9190, 0xffffffff, 0x00090008, 386 0x9194, 0xffffffff, 0x00070000, 387 0x9198, 0xffffffff, 0x00030002, 388 0x919c, 0xffffffff, 0x00050004, 389 0x91a8, 0xffffffff, 0x00010006, 390 0x91ac, 0xffffffff, 0x00090008, 391 0x91b0, 0xffffffff, 0x00070000, 392 0x91b4, 0xffffffff, 0x00030002, 393 0x91b8, 0xffffffff, 0x00050004, 394 0x91c4, 0xffffffff, 0x00010006, 395 0x91c8, 0xffffffff, 0x00090008, 396 0x91cc, 0xffffffff, 0x00070000, 397 0x91d0, 0xffffffff, 0x00030002, 398 0x91d4, 0xffffffff, 0x00050004, 399 0x91e0, 0xffffffff, 0x00010006, 400 0x91e4, 0xffffffff, 0x00090008, 401 0x91e8, 0xffffffff, 0x00000000, 402 0x91ec, 0xffffffff, 0x00070000, 403 0x91f0, 0xffffffff, 0x00030002, 404 0x91f4, 0xffffffff, 0x00050004, 405 0x9200, 0xffffffff, 0x00010006, 406 0x9204, 0xffffffff, 0x00090008, 407 0x9208, 0xffffffff, 0x00070000, 408 0x920c, 0xffffffff, 0x00030002, 409 0x9210, 0xffffffff, 0x00050004, 410 0x921c, 0xffffffff, 0x00010006, 411 0x9220, 0xffffffff, 0x00090008, 412 0x9224, 0xffffffff, 0x00070000, 413 0x9228, 0xffffffff, 0x00030002, 414 0x922c, 0xffffffff, 0x00050004, 415 0x9238, 0xffffffff, 0x00010006, 416 0x923c, 0xffffffff, 0x00090008, 417 0x9240, 0xffffffff, 0x00070000, 418 0x9244, 0xffffffff, 0x00030002, 419 0x9248, 0xffffffff, 0x00050004, 420 0x9254, 0xffffffff, 0x00010006, 421 0x9258, 0xffffffff, 0x00090008, 422 0x925c, 0xffffffff, 0x00070000, 423 0x9260, 0xffffffff, 0x00030002, 424 0x9264, 0xffffffff, 0x00050004, 425 0x9270, 0xffffffff, 0x00010006, 426 0x9274, 0xffffffff, 0x00090008, 427 0x9278, 0xffffffff, 0x00070000, 428 0x927c, 0xffffffff, 0x00030002, 429 0x9280, 0xffffffff, 0x00050004, 430 0x928c, 0xffffffff, 0x00010006, 431 0x9290, 0xffffffff, 0x00090008, 432 0x9294, 0xffffffff, 0x00000000, 433 0x929c, 0xffffffff, 0x00000001, 434 0x802c, 0xffffffff, 0xc0000000 435 }; 436 437 static const u32 redwood_mgcg_init[] = 438 { 439 0x802c, 0xffffffff, 0xc0000000, 440 0x5448, 0xffffffff, 0x00000100, 441 0x55e4, 0xffffffff, 0x00000100, 442 0x160c, 0xffffffff, 0x00000100, 443 0x5644, 0xffffffff, 0x00000100, 444 0xc164, 0xffffffff, 0x00000100, 445 0x8a18, 0xffffffff, 0x00000100, 446 0x897c, 0xffffffff, 0x06000100, 447 0x8b28, 0xffffffff, 0x00000100, 448 0x9144, 0xffffffff, 0x00000100, 449 0x9a60, 0xffffffff, 0x00000100, 450 0x9868, 0xffffffff, 0x00000100, 451 0x8d58, 0xffffffff, 0x00000100, 452 0x9510, 0xffffffff, 0x00000100, 453 0x949c, 0xffffffff, 0x00000100, 454 0x9654, 0xffffffff, 0x00000100, 455 0x9030, 0xffffffff, 0x00000100, 456 0x9034, 0xffffffff, 0x00000100, 457 0x9038, 0xffffffff, 0x00000100, 458 0x903c, 0xffffffff, 0x00000100, 459 0x9040, 0xffffffff, 0x00000100, 460 0xa200, 0xffffffff, 0x00000100, 461 0xa204, 0xffffffff, 0x00000100, 462 0xa208, 0xffffffff, 0x00000100, 463 0xa20c, 0xffffffff, 0x00000100, 464 0x971c, 0xffffffff, 0x00000100, 465 0x977c, 0xffffffff, 0x00000100, 466 0x3f80, 0xffffffff, 0x00000100, 467 0xa210, 0xffffffff, 0x00000100, 468 0xa214, 0xffffffff, 0x00000100, 469 0x4d8, 0xffffffff, 0x00000100, 470 0x9784, 0xffffffff, 0x00000100, 471 0x9698, 0xffffffff, 0x00000100, 472 0x4d4, 0xffffffff, 0x00000200, 473 0x30cc, 0xffffffff, 0x00000100, 474 0xd0c0, 0xffffffff, 0xff000100, 475 0x802c, 0xffffffff, 0x40000000, 476 0x915c, 0xffffffff, 0x00010000, 477 0x9160, 0xffffffff, 0x00030002, 478 0x9178, 0xffffffff, 0x00070000, 479 0x917c, 0xffffffff, 0x00030002, 480 0x9180, 0xffffffff, 0x00050004, 481 0x918c, 0xffffffff, 0x00010006, 482 0x9190, 0xffffffff, 0x00090008, 483 0x9194, 0xffffffff, 0x00070000, 484 0x9198, 0xffffffff, 0x00030002, 485 0x919c, 0xffffffff, 0x00050004, 486 0x91a8, 0xffffffff, 0x00010006, 487 0x91ac, 0xffffffff, 0x00090008, 488 0x91b0, 0xffffffff, 0x00070000, 489 0x91b4, 0xffffffff, 0x00030002, 490 0x91b8, 0xffffffff, 0x00050004, 491 0x91c4, 0xffffffff, 0x00010006, 492 0x91c8, 0xffffffff, 0x00090008, 493 0x91cc, 0xffffffff, 0x00070000, 494 0x91d0, 0xffffffff, 0x00030002, 495 0x91d4, 0xffffffff, 0x00050004, 496 0x91e0, 0xffffffff, 0x00010006, 497 0x91e4, 0xffffffff, 0x00090008, 498 0x91e8, 0xffffffff, 0x00000000, 499 0x91ec, 0xffffffff, 0x00070000, 500 0x91f0, 0xffffffff, 0x00030002, 501 0x91f4, 0xffffffff, 0x00050004, 502 0x9200, 0xffffffff, 0x00010006, 503 0x9204, 0xffffffff, 0x00090008, 504 0x9294, 0xffffffff, 0x00000000, 505 0x929c, 0xffffffff, 0x00000001, 506 0x802c, 0xffffffff, 0xc0000000 507 }; 508 509 static const u32 cedar_golden_registers[] = 510 { 511 0x3f90, 0xffff0000, 0xff000000, 512 0x9148, 0xffff0000, 0xff000000, 513 0x3f94, 0xffff0000, 0xff000000, 514 0x914c, 0xffff0000, 0xff000000, 515 0x9b7c, 0xffffffff, 0x00000000, 516 0x8a14, 0xffffffff, 0x00000007, 517 0x8b10, 0xffffffff, 0x00000000, 518 0x960c, 0xffffffff, 0x54763210, 519 0x88c4, 0xffffffff, 0x000000c2, 520 0x88d4, 0xffffffff, 0x00000000, 521 0x8974, 0xffffffff, 0x00000000, 522 0xc78, 0x00000080, 0x00000080, 523 0x5eb4, 0xffffffff, 0x00000002, 524 0x5e78, 0xffffffff, 0x001000f0, 525 0x6104, 0x01000300, 0x00000000, 526 0x5bc0, 0x00300000, 0x00000000, 527 0x7030, 0xffffffff, 0x00000011, 528 0x7c30, 0xffffffff, 0x00000011, 529 0x10830, 0xffffffff, 0x00000011, 530 0x11430, 0xffffffff, 0x00000011, 531 0xd02c, 0xffffffff, 0x08421000, 532 0x240c, 0xffffffff, 0x00000380, 533 0x8b24, 0xffffffff, 0x00ff0fff, 534 0x28a4c, 0x06000000, 0x06000000, 535 0x10c, 0x00000001, 0x00000001, 536 0x8d00, 0xffffffff, 0x100e4848, 537 0x8d04, 0xffffffff, 0x00164745, 538 0x8c00, 0xffffffff, 0xe4000003, 539 0x8c04, 0xffffffff, 0x40600060, 540 0x8c08, 0xffffffff, 0x001c001c, 541 0x8cf0, 0xffffffff, 0x08e00410, 542 0x8c20, 0xffffffff, 0x00800080, 543 0x8c24, 0xffffffff, 0x00800080, 544 0x8c18, 0xffffffff, 0x20202078, 545 0x8c1c, 0xffffffff, 0x00001010, 546 0x28350, 0xffffffff, 0x00000000, 547 0xa008, 0xffffffff, 0x00010000, 548 0x5c4, 0xffffffff, 0x00000001, 549 0x9508, 0xffffffff, 0x00000002 550 }; 551 552 static const u32 cedar_mgcg_init[] = 553 { 554 0x802c, 0xffffffff, 0xc0000000, 555 0x5448, 0xffffffff, 0x00000100, 556 0x55e4, 0xffffffff, 0x00000100, 557 0x160c, 0xffffffff, 0x00000100, 558 0x5644, 0xffffffff, 0x00000100, 559 0xc164, 0xffffffff, 0x00000100, 560 0x8a18, 0xffffffff, 0x00000100, 561 0x897c, 0xffffffff, 0x06000100, 562 0x8b28, 0xffffffff, 0x00000100, 563 0x9144, 0xffffffff, 0x00000100, 564 0x9a60, 0xffffffff, 0x00000100, 565 0x9868, 0xffffffff, 0x00000100, 566 0x8d58, 0xffffffff, 0x00000100, 567 0x9510, 0xffffffff, 0x00000100, 568 0x949c, 0xffffffff, 0x00000100, 569 0x9654, 0xffffffff, 0x00000100, 570 0x9030, 0xffffffff, 0x00000100, 571 0x9034, 0xffffffff, 0x00000100, 572 0x9038, 0xffffffff, 0x00000100, 573 0x903c, 0xffffffff, 0x00000100, 574 0x9040, 0xffffffff, 0x00000100, 575 0xa200, 0xffffffff, 0x00000100, 576 0xa204, 0xffffffff, 0x00000100, 577 0xa208, 0xffffffff, 0x00000100, 578 0xa20c, 0xffffffff, 0x00000100, 579 0x971c, 0xffffffff, 0x00000100, 580 0x977c, 0xffffffff, 0x00000100, 581 0x3f80, 0xffffffff, 0x00000100, 582 0xa210, 0xffffffff, 0x00000100, 583 0xa214, 0xffffffff, 0x00000100, 584 0x4d8, 0xffffffff, 0x00000100, 585 0x9784, 0xffffffff, 0x00000100, 586 0x9698, 0xffffffff, 0x00000100, 587 0x4d4, 0xffffffff, 0x00000200, 588 0x30cc, 0xffffffff, 0x00000100, 589 0xd0c0, 0xffffffff, 0xff000100, 590 0x802c, 0xffffffff, 0x40000000, 591 0x915c, 0xffffffff, 0x00010000, 592 0x9178, 0xffffffff, 0x00050000, 593 0x917c, 0xffffffff, 0x00030002, 594 0x918c, 0xffffffff, 0x00010004, 595 0x9190, 0xffffffff, 0x00070006, 596 0x9194, 0xffffffff, 0x00050000, 597 0x9198, 0xffffffff, 0x00030002, 598 0x91a8, 0xffffffff, 0x00010004, 599 0x91ac, 0xffffffff, 0x00070006, 600 0x91e8, 0xffffffff, 0x00000000, 601 0x9294, 0xffffffff, 0x00000000, 602 0x929c, 0xffffffff, 0x00000001, 603 0x802c, 0xffffffff, 0xc0000000 604 }; 605 606 static const u32 juniper_mgcg_init[] = 607 { 608 0x802c, 0xffffffff, 0xc0000000, 609 0x5448, 0xffffffff, 0x00000100, 610 0x55e4, 0xffffffff, 0x00000100, 611 0x160c, 0xffffffff, 0x00000100, 612 0x5644, 0xffffffff, 0x00000100, 613 0xc164, 0xffffffff, 0x00000100, 614 0x8a18, 0xffffffff, 0x00000100, 615 0x897c, 0xffffffff, 0x06000100, 616 0x8b28, 0xffffffff, 0x00000100, 617 0x9144, 0xffffffff, 0x00000100, 618 0x9a60, 0xffffffff, 0x00000100, 619 0x9868, 0xffffffff, 0x00000100, 620 0x8d58, 0xffffffff, 0x00000100, 621 0x9510, 0xffffffff, 0x00000100, 622 0x949c, 0xffffffff, 0x00000100, 623 0x9654, 0xffffffff, 0x00000100, 624 0x9030, 0xffffffff, 0x00000100, 625 0x9034, 0xffffffff, 0x00000100, 626 0x9038, 0xffffffff, 0x00000100, 627 0x903c, 0xffffffff, 0x00000100, 628 0x9040, 0xffffffff, 0x00000100, 629 0xa200, 0xffffffff, 0x00000100, 630 0xa204, 0xffffffff, 0x00000100, 631 0xa208, 0xffffffff, 0x00000100, 632 0xa20c, 0xffffffff, 0x00000100, 633 0x971c, 0xffffffff, 0x00000100, 634 0xd0c0, 0xffffffff, 0xff000100, 635 0x802c, 0xffffffff, 0x40000000, 636 0x915c, 0xffffffff, 0x00010000, 637 0x9160, 0xffffffff, 0x00030002, 638 0x9178, 0xffffffff, 0x00070000, 639 0x917c, 0xffffffff, 0x00030002, 640 0x9180, 0xffffffff, 0x00050004, 641 0x918c, 0xffffffff, 0x00010006, 642 0x9190, 0xffffffff, 0x00090008, 643 0x9194, 0xffffffff, 0x00070000, 644 0x9198, 0xffffffff, 0x00030002, 645 0x919c, 0xffffffff, 0x00050004, 646 0x91a8, 0xffffffff, 0x00010006, 647 0x91ac, 0xffffffff, 0x00090008, 648 0x91b0, 0xffffffff, 0x00070000, 649 0x91b4, 0xffffffff, 0x00030002, 650 0x91b8, 0xffffffff, 0x00050004, 651 0x91c4, 0xffffffff, 0x00010006, 652 0x91c8, 0xffffffff, 0x00090008, 653 0x91cc, 0xffffffff, 0x00070000, 654 0x91d0, 0xffffffff, 0x00030002, 655 0x91d4, 0xffffffff, 0x00050004, 656 0x91e0, 0xffffffff, 0x00010006, 657 0x91e4, 0xffffffff, 0x00090008, 658 0x91e8, 0xffffffff, 0x00000000, 659 0x91ec, 0xffffffff, 0x00070000, 660 0x91f0, 0xffffffff, 0x00030002, 661 0x91f4, 0xffffffff, 0x00050004, 662 0x9200, 0xffffffff, 0x00010006, 663 0x9204, 0xffffffff, 0x00090008, 664 0x9208, 0xffffffff, 0x00070000, 665 0x920c, 0xffffffff, 0x00030002, 666 0x9210, 0xffffffff, 0x00050004, 667 0x921c, 0xffffffff, 0x00010006, 668 0x9220, 0xffffffff, 0x00090008, 669 0x9224, 0xffffffff, 0x00070000, 670 0x9228, 0xffffffff, 0x00030002, 671 0x922c, 0xffffffff, 0x00050004, 672 0x9238, 0xffffffff, 0x00010006, 673 0x923c, 0xffffffff, 0x00090008, 674 0x9240, 0xffffffff, 0x00070000, 675 0x9244, 0xffffffff, 0x00030002, 676 0x9248, 0xffffffff, 0x00050004, 677 0x9254, 0xffffffff, 0x00010006, 678 0x9258, 0xffffffff, 0x00090008, 679 0x925c, 0xffffffff, 0x00070000, 680 0x9260, 0xffffffff, 0x00030002, 681 0x9264, 0xffffffff, 0x00050004, 682 0x9270, 0xffffffff, 0x00010006, 683 0x9274, 0xffffffff, 0x00090008, 684 0x9278, 0xffffffff, 0x00070000, 685 0x927c, 0xffffffff, 0x00030002, 686 0x9280, 0xffffffff, 0x00050004, 687 0x928c, 0xffffffff, 0x00010006, 688 0x9290, 0xffffffff, 0x00090008, 689 0x9294, 0xffffffff, 0x00000000, 690 0x929c, 0xffffffff, 0x00000001, 691 0x802c, 0xffffffff, 0xc0000000, 692 0x977c, 0xffffffff, 0x00000100, 693 0x3f80, 0xffffffff, 0x00000100, 694 0xa210, 0xffffffff, 0x00000100, 695 0xa214, 0xffffffff, 0x00000100, 696 0x4d8, 0xffffffff, 0x00000100, 697 0x9784, 0xffffffff, 0x00000100, 698 0x9698, 0xffffffff, 0x00000100, 699 0x4d4, 0xffffffff, 0x00000200, 700 0x30cc, 0xffffffff, 0x00000100, 701 0x802c, 0xffffffff, 0xc0000000 702 }; 703 704 static const u32 supersumo_golden_registers[] = 705 { 706 0x5eb4, 0xffffffff, 0x00000002, 707 0x5c4, 0xffffffff, 0x00000001, 708 0x7030, 0xffffffff, 0x00000011, 709 0x7c30, 0xffffffff, 0x00000011, 710 0x6104, 0x01000300, 0x00000000, 711 0x5bc0, 0x00300000, 0x00000000, 712 0x8c04, 0xffffffff, 0x40600060, 713 0x8c08, 0xffffffff, 0x001c001c, 714 0x8c20, 0xffffffff, 0x00800080, 715 0x8c24, 0xffffffff, 0x00800080, 716 0x8c18, 0xffffffff, 0x20202078, 717 0x8c1c, 0xffffffff, 0x00001010, 718 0x918c, 0xffffffff, 0x00010006, 719 0x91a8, 0xffffffff, 0x00010006, 720 0x91c4, 0xffffffff, 0x00010006, 721 0x91e0, 0xffffffff, 0x00010006, 722 0x9200, 0xffffffff, 0x00010006, 723 0x9150, 0xffffffff, 0x6e944040, 724 0x917c, 0xffffffff, 0x00030002, 725 0x9180, 0xffffffff, 0x00050004, 726 0x9198, 0xffffffff, 0x00030002, 727 0x919c, 0xffffffff, 0x00050004, 728 0x91b4, 0xffffffff, 0x00030002, 729 0x91b8, 0xffffffff, 0x00050004, 730 0x91d0, 0xffffffff, 0x00030002, 731 0x91d4, 0xffffffff, 0x00050004, 732 0x91f0, 0xffffffff, 0x00030002, 733 0x91f4, 0xffffffff, 0x00050004, 734 0x915c, 0xffffffff, 0x00010000, 735 0x9160, 0xffffffff, 0x00030002, 736 0x3f90, 0xffff0000, 0xff000000, 737 0x9178, 0xffffffff, 0x00070000, 738 0x9194, 0xffffffff, 0x00070000, 739 0x91b0, 0xffffffff, 0x00070000, 740 0x91cc, 0xffffffff, 0x00070000, 741 0x91ec, 0xffffffff, 0x00070000, 742 0x9148, 0xffff0000, 0xff000000, 743 0x9190, 0xffffffff, 0x00090008, 744 0x91ac, 0xffffffff, 0x00090008, 745 0x91c8, 0xffffffff, 0x00090008, 746 0x91e4, 0xffffffff, 0x00090008, 747 0x9204, 0xffffffff, 0x00090008, 748 0x3f94, 0xffff0000, 0xff000000, 749 0x914c, 0xffff0000, 0xff000000, 750 0x929c, 0xffffffff, 0x00000001, 751 0x8a18, 0xffffffff, 0x00000100, 752 0x8b28, 0xffffffff, 0x00000100, 753 0x9144, 0xffffffff, 0x00000100, 754 0x5644, 0xffffffff, 0x00000100, 755 0x9b7c, 0xffffffff, 0x00000000, 756 0x8030, 0xffffffff, 0x0000100a, 757 0x8a14, 0xffffffff, 0x00000007, 758 0x8b24, 0xffffffff, 0x00ff0fff, 759 0x8b10, 0xffffffff, 0x00000000, 760 0x28a4c, 0x06000000, 0x06000000, 761 0x4d8, 0xffffffff, 0x00000100, 762 0x913c, 0xffff000f, 0x0100000a, 763 0x960c, 0xffffffff, 0x54763210, 764 0x88c4, 0xffffffff, 0x000000c2, 765 0x88d4, 0xffffffff, 0x00000010, 766 0x8974, 0xffffffff, 0x00000000, 767 0xc78, 0x00000080, 0x00000080, 768 0x5e78, 0xffffffff, 0x001000f0, 769 0xd02c, 0xffffffff, 0x08421000, 770 0xa008, 0xffffffff, 0x00010000, 771 0x8d00, 0xffffffff, 0x100e4848, 772 0x8d04, 0xffffffff, 0x00164745, 773 0x8c00, 0xffffffff, 0xe4000003, 774 0x8cf0, 0x1fffffff, 0x08e00620, 775 0x28350, 0xffffffff, 0x00000000, 776 0x9508, 0xffffffff, 0x00000002 777 }; 778 779 static const u32 sumo_golden_registers[] = 780 { 781 0x900c, 0x00ffffff, 0x0017071f, 782 0x8c18, 0xffffffff, 0x10101060, 783 0x8c1c, 0xffffffff, 0x00001010, 784 0x8c30, 0x0000000f, 0x00000005, 785 0x9688, 0x0000000f, 0x00000007 786 }; 787 788 static const u32 wrestler_golden_registers[] = 789 { 790 0x5eb4, 0xffffffff, 0x00000002, 791 0x5c4, 0xffffffff, 0x00000001, 792 0x7030, 0xffffffff, 0x00000011, 793 0x7c30, 0xffffffff, 0x00000011, 794 0x6104, 0x01000300, 0x00000000, 795 0x5bc0, 0x00300000, 0x00000000, 796 0x918c, 0xffffffff, 0x00010006, 797 0x91a8, 0xffffffff, 0x00010006, 798 0x9150, 0xffffffff, 0x6e944040, 799 0x917c, 0xffffffff, 0x00030002, 800 0x9198, 0xffffffff, 0x00030002, 801 0x915c, 0xffffffff, 0x00010000, 802 0x3f90, 0xffff0000, 0xff000000, 803 0x9178, 0xffffffff, 0x00070000, 804 0x9194, 0xffffffff, 0x00070000, 805 0x9148, 0xffff0000, 0xff000000, 806 0x9190, 0xffffffff, 0x00090008, 807 0x91ac, 0xffffffff, 0x00090008, 808 0x3f94, 0xffff0000, 0xff000000, 809 0x914c, 0xffff0000, 0xff000000, 810 0x929c, 0xffffffff, 0x00000001, 811 0x8a18, 0xffffffff, 0x00000100, 812 0x8b28, 0xffffffff, 0x00000100, 813 0x9144, 0xffffffff, 0x00000100, 814 0x9b7c, 0xffffffff, 0x00000000, 815 0x8030, 0xffffffff, 0x0000100a, 816 0x8a14, 0xffffffff, 0x00000001, 817 0x8b24, 0xffffffff, 0x00ff0fff, 818 0x8b10, 0xffffffff, 0x00000000, 819 0x28a4c, 0x06000000, 0x06000000, 820 0x4d8, 0xffffffff, 0x00000100, 821 0x913c, 0xffff000f, 0x0100000a, 822 0x960c, 0xffffffff, 0x54763210, 823 0x88c4, 0xffffffff, 0x000000c2, 824 0x88d4, 0xffffffff, 0x00000010, 825 0x8974, 0xffffffff, 0x00000000, 826 0xc78, 0x00000080, 0x00000080, 827 0x5e78, 0xffffffff, 0x001000f0, 828 0xd02c, 0xffffffff, 0x08421000, 829 0xa008, 0xffffffff, 0x00010000, 830 0x8d00, 0xffffffff, 0x100e4848, 831 0x8d04, 0xffffffff, 0x00164745, 832 0x8c00, 0xffffffff, 0xe4000003, 833 0x8cf0, 0x1fffffff, 0x08e00410, 834 0x28350, 0xffffffff, 0x00000000, 835 0x9508, 0xffffffff, 0x00000002, 836 0x900c, 0xffffffff, 0x0017071f, 837 0x8c18, 0xffffffff, 0x10101060, 838 0x8c1c, 0xffffffff, 0x00001010 839 }; 840 841 static const u32 barts_golden_registers[] = 842 { 843 0x5eb4, 0xffffffff, 0x00000002, 844 0x5e78, 0x8f311ff1, 0x001000f0, 845 0x3f90, 0xffff0000, 0xff000000, 846 0x9148, 0xffff0000, 0xff000000, 847 0x3f94, 0xffff0000, 0xff000000, 848 0x914c, 0xffff0000, 0xff000000, 849 0xc78, 0x00000080, 0x00000080, 850 0xbd4, 0x70073777, 0x00010001, 851 0xd02c, 0xbfffff1f, 0x08421000, 852 0xd0b8, 0x03773777, 0x02011003, 853 0x5bc0, 0x00200000, 0x50100000, 854 0x98f8, 0x33773777, 0x02011003, 855 0x98fc, 0xffffffff, 0x76543210, 856 0x7030, 0x31000311, 0x00000011, 857 0x2f48, 0x00000007, 0x02011003, 858 0x6b28, 0x00000010, 0x00000012, 859 0x7728, 0x00000010, 0x00000012, 860 0x10328, 0x00000010, 0x00000012, 861 0x10f28, 0x00000010, 0x00000012, 862 0x11b28, 0x00000010, 0x00000012, 863 0x12728, 0x00000010, 0x00000012, 864 0x240c, 0x000007ff, 0x00000380, 865 0x8a14, 0xf000001f, 0x00000007, 866 0x8b24, 0x3fff3fff, 0x00ff0fff, 867 0x8b10, 0x0000ff0f, 0x00000000, 868 0x28a4c, 0x07ffffff, 0x06000000, 869 0x10c, 0x00000001, 0x00010003, 870 0xa02c, 0xffffffff, 0x0000009b, 871 0x913c, 0x0000000f, 0x0100000a, 872 0x8d00, 0xffff7f7f, 0x100e4848, 873 0x8d04, 0x00ffffff, 0x00164745, 874 0x8c00, 0xfffc0003, 0xe4000003, 875 0x8c04, 0xf8ff00ff, 0x40600060, 876 0x8c08, 0x00ff00ff, 0x001c001c, 877 0x8cf0, 0x1fff1fff, 0x08e00620, 878 0x8c20, 0x0fff0fff, 0x00800080, 879 0x8c24, 0x0fff0fff, 0x00800080, 880 0x8c18, 0xffffffff, 0x20202078, 881 0x8c1c, 0x0000ffff, 0x00001010, 882 0x28350, 0x00000f01, 0x00000000, 883 0x9508, 0x3700001f, 0x00000002, 884 0x960c, 0xffffffff, 0x54763210, 885 0x88c4, 0x001f3ae3, 0x000000c2, 886 0x88d4, 0x0000001f, 0x00000010, 887 0x8974, 0xffffffff, 0x00000000 888 }; 889 890 static const u32 turks_golden_registers[] = 891 { 892 0x5eb4, 0xffffffff, 0x00000002, 893 0x5e78, 0x8f311ff1, 0x001000f0, 894 0x8c8, 0x00003000, 0x00001070, 895 0x8cc, 0x000fffff, 0x00040035, 896 0x3f90, 0xffff0000, 0xfff00000, 897 0x9148, 0xffff0000, 0xfff00000, 898 0x3f94, 0xffff0000, 0xfff00000, 899 0x914c, 0xffff0000, 0xfff00000, 900 0xc78, 0x00000080, 0x00000080, 901 0xbd4, 0x00073007, 0x00010002, 902 0xd02c, 0xbfffff1f, 0x08421000, 903 0xd0b8, 0x03773777, 0x02010002, 904 0x5bc0, 0x00200000, 0x50100000, 905 0x98f8, 0x33773777, 0x00010002, 906 0x98fc, 0xffffffff, 0x33221100, 907 0x7030, 0x31000311, 0x00000011, 908 0x2f48, 0x33773777, 0x00010002, 909 0x6b28, 0x00000010, 0x00000012, 910 0x7728, 0x00000010, 0x00000012, 911 0x10328, 0x00000010, 0x00000012, 912 0x10f28, 0x00000010, 0x00000012, 913 0x11b28, 0x00000010, 0x00000012, 914 0x12728, 0x00000010, 0x00000012, 915 0x240c, 0x000007ff, 0x00000380, 916 0x8a14, 0xf000001f, 0x00000007, 917 0x8b24, 0x3fff3fff, 0x00ff0fff, 918 0x8b10, 0x0000ff0f, 0x00000000, 919 0x28a4c, 0x07ffffff, 0x06000000, 920 0x10c, 0x00000001, 0x00010003, 921 0xa02c, 0xffffffff, 0x0000009b, 922 0x913c, 0x0000000f, 0x0100000a, 923 0x8d00, 0xffff7f7f, 0x100e4848, 924 0x8d04, 0x00ffffff, 0x00164745, 925 0x8c00, 0xfffc0003, 0xe4000003, 926 0x8c04, 0xf8ff00ff, 0x40600060, 927 0x8c08, 0x00ff00ff, 0x001c001c, 928 0x8cf0, 0x1fff1fff, 0x08e00410, 929 0x8c20, 0x0fff0fff, 0x00800080, 930 0x8c24, 0x0fff0fff, 0x00800080, 931 0x8c18, 0xffffffff, 0x20202078, 932 0x8c1c, 0x0000ffff, 0x00001010, 933 0x28350, 0x00000f01, 0x00000000, 934 0x9508, 0x3700001f, 0x00000002, 935 0x960c, 0xffffffff, 0x54763210, 936 0x88c4, 0x001f3ae3, 0x000000c2, 937 0x88d4, 0x0000001f, 0x00000010, 938 0x8974, 0xffffffff, 0x00000000 939 }; 940 941 static const u32 caicos_golden_registers[] = 942 { 943 0x5eb4, 0xffffffff, 0x00000002, 944 0x5e78, 0x8f311ff1, 0x001000f0, 945 0x8c8, 0x00003420, 0x00001450, 946 0x8cc, 0x000fffff, 0x00040035, 947 0x3f90, 0xffff0000, 0xfffc0000, 948 0x9148, 0xffff0000, 0xfffc0000, 949 0x3f94, 0xffff0000, 0xfffc0000, 950 0x914c, 0xffff0000, 0xfffc0000, 951 0xc78, 0x00000080, 0x00000080, 952 0xbd4, 0x00073007, 0x00010001, 953 0xd02c, 0xbfffff1f, 0x08421000, 954 0xd0b8, 0x03773777, 0x02010001, 955 0x5bc0, 0x00200000, 0x50100000, 956 0x98f8, 0x33773777, 0x02010001, 957 0x98fc, 0xffffffff, 0x33221100, 958 0x7030, 0x31000311, 0x00000011, 959 0x2f48, 0x33773777, 0x02010001, 960 0x6b28, 0x00000010, 0x00000012, 961 0x7728, 0x00000010, 0x00000012, 962 0x10328, 0x00000010, 0x00000012, 963 0x10f28, 0x00000010, 0x00000012, 964 0x11b28, 0x00000010, 0x00000012, 965 0x12728, 0x00000010, 0x00000012, 966 0x240c, 0x000007ff, 0x00000380, 967 0x8a14, 0xf000001f, 0x00000001, 968 0x8b24, 0x3fff3fff, 0x00ff0fff, 969 0x8b10, 0x0000ff0f, 0x00000000, 970 0x28a4c, 0x07ffffff, 0x06000000, 971 0x10c, 0x00000001, 0x00010003, 972 0xa02c, 0xffffffff, 0x0000009b, 973 0x913c, 0x0000000f, 0x0100000a, 974 0x8d00, 0xffff7f7f, 0x100e4848, 975 0x8d04, 0x00ffffff, 0x00164745, 976 0x8c00, 0xfffc0003, 0xe4000003, 977 0x8c04, 0xf8ff00ff, 0x40600060, 978 0x8c08, 0x00ff00ff, 0x001c001c, 979 0x8cf0, 0x1fff1fff, 0x08e00410, 980 0x8c20, 0x0fff0fff, 0x00800080, 981 0x8c24, 0x0fff0fff, 0x00800080, 982 0x8c18, 0xffffffff, 0x20202078, 983 0x8c1c, 0x0000ffff, 0x00001010, 984 0x28350, 0x00000f01, 0x00000000, 985 0x9508, 0x3700001f, 0x00000002, 986 0x960c, 0xffffffff, 0x54763210, 987 0x88c4, 0x001f3ae3, 0x000000c2, 988 0x88d4, 0x0000001f, 0x00000010, 989 0x8974, 0xffffffff, 0x00000000 990 }; 991 992 static void evergreen_init_golden_registers(struct radeon_device *rdev) 993 { 994 switch (rdev->family) { 995 case CHIP_CYPRESS: 996 case CHIP_HEMLOCK: 997 radeon_program_register_sequence(rdev, 998 evergreen_golden_registers, 999 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1000 radeon_program_register_sequence(rdev, 1001 evergreen_golden_registers2, 1002 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1003 radeon_program_register_sequence(rdev, 1004 cypress_mgcg_init, 1005 (const u32)ARRAY_SIZE(cypress_mgcg_init)); 1006 break; 1007 case CHIP_JUNIPER: 1008 radeon_program_register_sequence(rdev, 1009 evergreen_golden_registers, 1010 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1011 radeon_program_register_sequence(rdev, 1012 evergreen_golden_registers2, 1013 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1014 radeon_program_register_sequence(rdev, 1015 juniper_mgcg_init, 1016 (const u32)ARRAY_SIZE(juniper_mgcg_init)); 1017 break; 1018 case CHIP_REDWOOD: 1019 radeon_program_register_sequence(rdev, 1020 evergreen_golden_registers, 1021 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1022 radeon_program_register_sequence(rdev, 1023 evergreen_golden_registers2, 1024 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1025 radeon_program_register_sequence(rdev, 1026 redwood_mgcg_init, 1027 (const u32)ARRAY_SIZE(redwood_mgcg_init)); 1028 break; 1029 case CHIP_CEDAR: 1030 radeon_program_register_sequence(rdev, 1031 cedar_golden_registers, 1032 (const u32)ARRAY_SIZE(cedar_golden_registers)); 1033 radeon_program_register_sequence(rdev, 1034 evergreen_golden_registers2, 1035 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1036 radeon_program_register_sequence(rdev, 1037 cedar_mgcg_init, 1038 (const u32)ARRAY_SIZE(cedar_mgcg_init)); 1039 break; 1040 case CHIP_PALM: 1041 radeon_program_register_sequence(rdev, 1042 wrestler_golden_registers, 1043 (const u32)ARRAY_SIZE(wrestler_golden_registers)); 1044 break; 1045 case CHIP_SUMO: 1046 radeon_program_register_sequence(rdev, 1047 supersumo_golden_registers, 1048 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1049 break; 1050 case CHIP_SUMO2: 1051 radeon_program_register_sequence(rdev, 1052 supersumo_golden_registers, 1053 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1054 radeon_program_register_sequence(rdev, 1055 sumo_golden_registers, 1056 (const u32)ARRAY_SIZE(sumo_golden_registers)); 1057 break; 1058 case CHIP_BARTS: 1059 radeon_program_register_sequence(rdev, 1060 barts_golden_registers, 1061 (const u32)ARRAY_SIZE(barts_golden_registers)); 1062 break; 1063 case CHIP_TURKS: 1064 radeon_program_register_sequence(rdev, 1065 turks_golden_registers, 1066 (const u32)ARRAY_SIZE(turks_golden_registers)); 1067 break; 1068 case CHIP_CAICOS: 1069 radeon_program_register_sequence(rdev, 1070 caicos_golden_registers, 1071 (const u32)ARRAY_SIZE(caicos_golden_registers)); 1072 break; 1073 default: 1074 break; 1075 } 1076 } 1077 1078 /** 1079 * evergreen_get_allowed_info_register - fetch the register for the info ioctl 1080 * 1081 * @rdev: radeon_device pointer 1082 * @reg: register offset in bytes 1083 * @val: register value 1084 * 1085 * Returns 0 for success or -EINVAL for an invalid register 1086 * 1087 */ 1088 int evergreen_get_allowed_info_register(struct radeon_device *rdev, 1089 u32 reg, u32 *val) 1090 { 1091 switch (reg) { 1092 case GRBM_STATUS: 1093 case GRBM_STATUS_SE0: 1094 case GRBM_STATUS_SE1: 1095 case SRBM_STATUS: 1096 case SRBM_STATUS2: 1097 case DMA_STATUS_REG: 1098 case UVD_STATUS: 1099 *val = RREG32(reg); 1100 return 0; 1101 default: 1102 return -EINVAL; 1103 } 1104 } 1105 1106 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, 1107 unsigned *bankh, unsigned *mtaspect, 1108 unsigned *tile_split) 1109 { 1110 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 1111 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 1112 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 1113 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 1114 switch (*bankw) { 1115 default: 1116 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break; 1117 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break; 1118 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break; 1119 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break; 1120 } 1121 switch (*bankh) { 1122 default: 1123 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break; 1124 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break; 1125 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break; 1126 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break; 1127 } 1128 switch (*mtaspect) { 1129 default: 1130 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break; 1131 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break; 1132 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break; 1133 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break; 1134 } 1135 } 1136 1137 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock, 1138 u32 cntl_reg, u32 status_reg) 1139 { 1140 int r, i; 1141 struct atom_clock_dividers dividers; 1142 1143 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, 1144 clock, false, ÷rs); 1145 if (r) 1146 return r; 1147 1148 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK)); 1149 1150 for (i = 0; i < 100; i++) { 1151 if (RREG32(status_reg) & DCLK_STATUS) 1152 break; 1153 mdelay(10); 1154 } 1155 if (i == 100) 1156 return -ETIMEDOUT; 1157 1158 return 0; 1159 } 1160 1161 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1162 { 1163 int r = 0; 1164 u32 cg_scratch = RREG32(CG_SCRATCH1); 1165 1166 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 1167 if (r) 1168 goto done; 1169 cg_scratch &= 0xffff0000; 1170 cg_scratch |= vclk / 100; /* Mhz */ 1171 1172 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 1173 if (r) 1174 goto done; 1175 cg_scratch &= 0x0000ffff; 1176 cg_scratch |= (dclk / 100) << 16; /* Mhz */ 1177 1178 done: 1179 WREG32(CG_SCRATCH1, cg_scratch); 1180 1181 return r; 1182 } 1183 1184 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1185 { 1186 /* start off with something large */ 1187 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 1188 int r; 1189 1190 /* bypass vclk and dclk with bclk */ 1191 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1192 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 1193 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1194 1195 /* put PLL in bypass mode */ 1196 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 1197 1198 if (!vclk || !dclk) { 1199 /* keep the Bypass mode, put PLL to sleep */ 1200 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1201 return 0; 1202 } 1203 1204 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 1205 16384, 0x03FFFFFF, 0, 128, 5, 1206 &fb_div, &vclk_div, &dclk_div); 1207 if (r) 1208 return r; 1209 1210 /* set VCO_MODE to 1 */ 1211 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 1212 1213 /* toggle UPLL_SLEEP to 1 then back to 0 */ 1214 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1215 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 1216 1217 /* deassert UPLL_RESET */ 1218 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1219 1220 mdelay(1); 1221 1222 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1223 if (r) 1224 return r; 1225 1226 /* assert UPLL_RESET again */ 1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 1228 1229 /* disable spread spectrum. */ 1230 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 1231 1232 /* set feedback divider */ 1233 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 1234 1235 /* set ref divider to 0 */ 1236 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 1237 1238 if (fb_div < 307200) 1239 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 1240 else 1241 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 1242 1243 /* set PDIV_A and PDIV_B */ 1244 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1245 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 1246 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 1247 1248 /* give the PLL some time to settle */ 1249 mdelay(15); 1250 1251 /* deassert PLL_RESET */ 1252 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1253 1254 mdelay(15); 1255 1256 /* switch from bypass mode to normal mode */ 1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 1258 1259 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1260 if (r) 1261 return r; 1262 1263 /* switch VCLK and DCLK selection */ 1264 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1265 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 1266 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1267 1268 mdelay(100); 1269 1270 return 0; 1271 } 1272 1273 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 1274 { 1275 int readrq; 1276 u16 v; 1277 1278 readrq = pcie_get_readrq(rdev->pdev); 1279 v = ffs(readrq) - 8; 1280 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it 1281 * to avoid hangs or perfomance issues 1282 */ 1283 if ((v == 0) || (v == 6) || (v == 7)) 1284 pcie_set_readrq(rdev->pdev, 512); 1285 } 1286 1287 void dce4_program_fmt(struct drm_encoder *encoder) 1288 { 1289 struct drm_device *dev = encoder->dev; 1290 struct radeon_device *rdev = dev->dev_private; 1291 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 1292 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 1293 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 1294 int bpc = 0; 1295 u32 tmp = 0; 1296 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 1297 1298 if (connector) { 1299 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1300 bpc = radeon_get_monitor_bpc(connector); 1301 dither = radeon_connector->dither; 1302 } 1303 1304 /* LVDS/eDP FMT is set up by atom */ 1305 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 1306 return; 1307 1308 /* not needed for analog */ 1309 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 1310 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 1311 return; 1312 1313 if (bpc == 0) 1314 return; 1315 1316 switch (bpc) { 1317 case 6: 1318 if (dither == RADEON_FMT_DITHER_ENABLE) 1319 /* XXX sort out optimal dither settings */ 1320 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1321 FMT_SPATIAL_DITHER_EN); 1322 else 1323 tmp |= FMT_TRUNCATE_EN; 1324 break; 1325 case 8: 1326 if (dither == RADEON_FMT_DITHER_ENABLE) 1327 /* XXX sort out optimal dither settings */ 1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1329 FMT_RGB_RANDOM_ENABLE | 1330 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); 1331 else 1332 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); 1333 break; 1334 case 10: 1335 default: 1336 /* not needed */ 1337 break; 1338 } 1339 1340 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 1341 } 1342 1343 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) 1344 { 1345 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK) 1346 return true; 1347 else 1348 return false; 1349 } 1350 1351 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc) 1352 { 1353 u32 pos1, pos2; 1354 1355 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1356 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1357 1358 if (pos1 != pos2) 1359 return true; 1360 else 1361 return false; 1362 } 1363 1364 /** 1365 * dce4_wait_for_vblank - vblank wait asic callback. 1366 * 1367 * @rdev: radeon_device pointer 1368 * @crtc: crtc to wait for vblank on 1369 * 1370 * Wait for vblank on the requested crtc (evergreen+). 1371 */ 1372 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) 1373 { 1374 unsigned i = 0; 1375 1376 if (crtc >= rdev->num_crtc) 1377 return; 1378 1379 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN)) 1380 return; 1381 1382 /* depending on when we hit vblank, we may be close to active; if so, 1383 * wait for another frame. 1384 */ 1385 while (dce4_is_in_vblank(rdev, crtc)) { 1386 if (i++ % 100 == 0) { 1387 if (!dce4_is_counter_moving(rdev, crtc)) 1388 break; 1389 } 1390 } 1391 1392 while (!dce4_is_in_vblank(rdev, crtc)) { 1393 if (i++ % 100 == 0) { 1394 if (!dce4_is_counter_moving(rdev, crtc)) 1395 break; 1396 } 1397 } 1398 } 1399 1400 /** 1401 * evergreen_page_flip - pageflip callback. 1402 * 1403 * @rdev: radeon_device pointer 1404 * @crtc_id: crtc to cleanup pageflip on 1405 * @crtc_base: new address of the crtc (GPU MC address) 1406 * 1407 * Triggers the actual pageflip by updating the primary 1408 * surface base address (evergreen+). 1409 */ 1410 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, 1411 bool async) 1412 { 1413 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1414 1415 /* update the scanout addresses */ 1416 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, 1417 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0); 1418 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1419 upper_32_bits(crtc_base)); 1420 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1421 (u32)crtc_base); 1422 /* post the write */ 1423 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset); 1424 } 1425 1426 /** 1427 * evergreen_page_flip_pending - check if page flip is still pending 1428 * 1429 * @rdev: radeon_device pointer 1430 * @crtc_id: crtc to check 1431 * 1432 * Returns the current update pending status. 1433 */ 1434 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) 1435 { 1436 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1437 1438 /* Return current update_pending status: */ 1439 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & 1440 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); 1441 } 1442 1443 /* get temperature in millidegrees */ 1444 int evergreen_get_temp(struct radeon_device *rdev) 1445 { 1446 u32 temp, toffset; 1447 int actual_temp = 0; 1448 1449 if (rdev->family == CHIP_JUNIPER) { 1450 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 1451 TOFFSET_SHIFT; 1452 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 1453 TS0_ADC_DOUT_SHIFT; 1454 1455 if (toffset & 0x100) 1456 actual_temp = temp / 2 - (0x200 - toffset); 1457 else 1458 actual_temp = temp / 2 + toffset; 1459 1460 actual_temp = actual_temp * 1000; 1461 1462 } else { 1463 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 1464 ASIC_T_SHIFT; 1465 1466 if (temp & 0x400) 1467 actual_temp = -256; 1468 else if (temp & 0x200) 1469 actual_temp = 255; 1470 else if (temp & 0x100) { 1471 actual_temp = temp & 0x1ff; 1472 actual_temp |= ~0x1ff; 1473 } else 1474 actual_temp = temp & 0xff; 1475 1476 actual_temp = (actual_temp * 1000) / 2; 1477 } 1478 1479 return actual_temp; 1480 } 1481 1482 int sumo_get_temp(struct radeon_device *rdev) 1483 { 1484 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 1485 int actual_temp = temp - 49; 1486 1487 return actual_temp * 1000; 1488 } 1489 1490 /** 1491 * sumo_pm_init_profile - Initialize power profiles callback. 1492 * 1493 * @rdev: radeon_device pointer 1494 * 1495 * Initialize the power states used in profile mode 1496 * (sumo, trinity, SI). 1497 * Used for profile mode only. 1498 */ 1499 void sumo_pm_init_profile(struct radeon_device *rdev) 1500 { 1501 int idx; 1502 1503 /* default */ 1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1505 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1506 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0; 1508 1509 /* low,mid sh/mh */ 1510 if (rdev->flags & RADEON_IS_MOBILITY) 1511 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1512 else 1513 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1514 1515 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1516 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1517 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1518 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1519 1520 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1521 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1522 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1523 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1524 1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1526 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1527 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1528 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0; 1529 1530 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1531 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1532 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1533 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0; 1534 1535 /* high sh/mh */ 1536 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1537 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1538 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1539 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1540 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 1541 rdev->pm.power_state[idx].num_clock_modes - 1; 1542 1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1544 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1546 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 1547 rdev->pm.power_state[idx].num_clock_modes - 1; 1548 } 1549 1550 /** 1551 * btc_pm_init_profile - Initialize power profiles callback. 1552 * 1553 * @rdev: radeon_device pointer 1554 * 1555 * Initialize the power states used in profile mode 1556 * (BTC, cayman). 1557 * Used for profile mode only. 1558 */ 1559 void btc_pm_init_profile(struct radeon_device *rdev) 1560 { 1561 int idx; 1562 1563 /* default */ 1564 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1565 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1566 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1567 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2; 1568 /* starting with BTC, there is one state that is used for both 1569 * MH and SH. Difference is that we always use the high clock index for 1570 * mclk. 1571 */ 1572 if (rdev->flags & RADEON_IS_MOBILITY) 1573 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1574 else 1575 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1576 /* low sh */ 1577 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1578 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1579 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1580 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1581 /* mid sh */ 1582 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1583 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1584 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1585 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1; 1586 /* high sh */ 1587 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1588 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1589 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1590 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2; 1591 /* low mh */ 1592 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1593 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1594 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1595 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1596 /* mid mh */ 1597 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1598 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1599 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1600 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1; 1601 /* high mh */ 1602 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1603 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1604 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1605 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2; 1606 } 1607 1608 /** 1609 * evergreen_pm_misc - set additional pm hw parameters callback. 1610 * 1611 * @rdev: radeon_device pointer 1612 * 1613 * Set non-clock parameters associated with a power state 1614 * (voltage, etc.) (evergreen+). 1615 */ 1616 void evergreen_pm_misc(struct radeon_device *rdev) 1617 { 1618 int req_ps_idx = rdev->pm.requested_power_state_index; 1619 int req_cm_idx = rdev->pm.requested_clock_mode_index; 1620 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 1621 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 1622 1623 if (voltage->type == VOLTAGE_SW) { 1624 /* 0xff0x are flags rather then an actual voltage */ 1625 if ((voltage->voltage & 0xff00) == 0xff00) 1626 return; 1627 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 1628 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 1629 rdev->pm.current_vddc = voltage->voltage; 1630 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 1631 } 1632 1633 /* starting with BTC, there is one state that is used for both 1634 * MH and SH. Difference is that we always use the high clock index for 1635 * mclk and vddci. 1636 */ 1637 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) && 1638 (rdev->family >= CHIP_BARTS) && 1639 rdev->pm.active_crtc_count && 1640 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) || 1641 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX))) 1642 voltage = &rdev->pm.power_state[req_ps_idx]. 1643 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; 1644 1645 /* 0xff0x are flags rather then an actual voltage */ 1646 if ((voltage->vddci & 0xff00) == 0xff00) 1647 return; 1648 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 1649 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 1650 rdev->pm.current_vddci = voltage->vddci; 1651 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 1652 } 1653 } 1654 } 1655 1656 /** 1657 * evergreen_pm_prepare - pre-power state change callback. 1658 * 1659 * @rdev: radeon_device pointer 1660 * 1661 * Prepare for a power state change (evergreen+). 1662 */ 1663 void evergreen_pm_prepare(struct radeon_device *rdev) 1664 { 1665 struct drm_device *ddev = rdev->ddev; 1666 struct drm_crtc *crtc; 1667 struct radeon_crtc *radeon_crtc; 1668 u32 tmp; 1669 1670 /* disable any active CRTCs */ 1671 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1672 radeon_crtc = to_radeon_crtc(crtc); 1673 if (radeon_crtc->enabled) { 1674 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1675 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1676 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1677 } 1678 } 1679 } 1680 1681 /** 1682 * evergreen_pm_finish - post-power state change callback. 1683 * 1684 * @rdev: radeon_device pointer 1685 * 1686 * Clean up after a power state change (evergreen+). 1687 */ 1688 void evergreen_pm_finish(struct radeon_device *rdev) 1689 { 1690 struct drm_device *ddev = rdev->ddev; 1691 struct drm_crtc *crtc; 1692 struct radeon_crtc *radeon_crtc; 1693 u32 tmp; 1694 1695 /* enable any active CRTCs */ 1696 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1697 radeon_crtc = to_radeon_crtc(crtc); 1698 if (radeon_crtc->enabled) { 1699 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1700 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1701 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1702 } 1703 } 1704 } 1705 1706 /** 1707 * evergreen_hpd_sense - hpd sense callback. 1708 * 1709 * @rdev: radeon_device pointer 1710 * @hpd: hpd (hotplug detect) pin 1711 * 1712 * Checks if a digital monitor is connected (evergreen+). 1713 * Returns true if connected, false if not connected. 1714 */ 1715 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 1716 { 1717 bool connected = false; 1718 1719 switch (hpd) { 1720 case RADEON_HPD_1: 1721 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE) 1722 connected = true; 1723 break; 1724 case RADEON_HPD_2: 1725 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE) 1726 connected = true; 1727 break; 1728 case RADEON_HPD_3: 1729 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE) 1730 connected = true; 1731 break; 1732 case RADEON_HPD_4: 1733 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE) 1734 connected = true; 1735 break; 1736 case RADEON_HPD_5: 1737 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE) 1738 connected = true; 1739 break; 1740 case RADEON_HPD_6: 1741 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) 1742 connected = true; 1743 break; 1744 default: 1745 break; 1746 } 1747 1748 return connected; 1749 } 1750 1751 /** 1752 * evergreen_hpd_set_polarity - hpd set polarity callback. 1753 * 1754 * @rdev: radeon_device pointer 1755 * @hpd: hpd (hotplug detect) pin 1756 * 1757 * Set the polarity of the hpd pin (evergreen+). 1758 */ 1759 void evergreen_hpd_set_polarity(struct radeon_device *rdev, 1760 enum radeon_hpd_id hpd) 1761 { 1762 u32 tmp; 1763 bool connected = evergreen_hpd_sense(rdev, hpd); 1764 1765 switch (hpd) { 1766 case RADEON_HPD_1: 1767 tmp = RREG32(DC_HPD1_INT_CONTROL); 1768 if (connected) 1769 tmp &= ~DC_HPDx_INT_POLARITY; 1770 else 1771 tmp |= DC_HPDx_INT_POLARITY; 1772 WREG32(DC_HPD1_INT_CONTROL, tmp); 1773 break; 1774 case RADEON_HPD_2: 1775 tmp = RREG32(DC_HPD2_INT_CONTROL); 1776 if (connected) 1777 tmp &= ~DC_HPDx_INT_POLARITY; 1778 else 1779 tmp |= DC_HPDx_INT_POLARITY; 1780 WREG32(DC_HPD2_INT_CONTROL, tmp); 1781 break; 1782 case RADEON_HPD_3: 1783 tmp = RREG32(DC_HPD3_INT_CONTROL); 1784 if (connected) 1785 tmp &= ~DC_HPDx_INT_POLARITY; 1786 else 1787 tmp |= DC_HPDx_INT_POLARITY; 1788 WREG32(DC_HPD3_INT_CONTROL, tmp); 1789 break; 1790 case RADEON_HPD_4: 1791 tmp = RREG32(DC_HPD4_INT_CONTROL); 1792 if (connected) 1793 tmp &= ~DC_HPDx_INT_POLARITY; 1794 else 1795 tmp |= DC_HPDx_INT_POLARITY; 1796 WREG32(DC_HPD4_INT_CONTROL, tmp); 1797 break; 1798 case RADEON_HPD_5: 1799 tmp = RREG32(DC_HPD5_INT_CONTROL); 1800 if (connected) 1801 tmp &= ~DC_HPDx_INT_POLARITY; 1802 else 1803 tmp |= DC_HPDx_INT_POLARITY; 1804 WREG32(DC_HPD5_INT_CONTROL, tmp); 1805 break; 1806 case RADEON_HPD_6: 1807 tmp = RREG32(DC_HPD6_INT_CONTROL); 1808 if (connected) 1809 tmp &= ~DC_HPDx_INT_POLARITY; 1810 else 1811 tmp |= DC_HPDx_INT_POLARITY; 1812 WREG32(DC_HPD6_INT_CONTROL, tmp); 1813 break; 1814 default: 1815 break; 1816 } 1817 } 1818 1819 /** 1820 * evergreen_hpd_init - hpd setup callback. 1821 * 1822 * @rdev: radeon_device pointer 1823 * 1824 * Setup the hpd pins used by the card (evergreen+). 1825 * Enable the pin, set the polarity, and enable the hpd interrupts. 1826 */ 1827 void evergreen_hpd_init(struct radeon_device *rdev) 1828 { 1829 struct drm_device *dev = rdev->ddev; 1830 struct drm_connector *connector; 1831 unsigned enabled = 0; 1832 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 1833 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 1834 1835 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1836 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1837 1838 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || 1839 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { 1840 /* don't try to enable hpd on eDP or LVDS avoid breaking the 1841 * aux dp channel on imac and help (but not completely fix) 1842 * https://bugzilla.redhat.com/show_bug.cgi?id=726143 1843 * also avoid interrupt storms during dpms. 1844 */ 1845 continue; 1846 } 1847 switch (radeon_connector->hpd.hpd) { 1848 case RADEON_HPD_1: 1849 WREG32(DC_HPD1_CONTROL, tmp); 1850 break; 1851 case RADEON_HPD_2: 1852 WREG32(DC_HPD2_CONTROL, tmp); 1853 break; 1854 case RADEON_HPD_3: 1855 WREG32(DC_HPD3_CONTROL, tmp); 1856 break; 1857 case RADEON_HPD_4: 1858 WREG32(DC_HPD4_CONTROL, tmp); 1859 break; 1860 case RADEON_HPD_5: 1861 WREG32(DC_HPD5_CONTROL, tmp); 1862 break; 1863 case RADEON_HPD_6: 1864 WREG32(DC_HPD6_CONTROL, tmp); 1865 break; 1866 default: 1867 break; 1868 } 1869 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd); 1870 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE) 1871 enabled |= 1 << radeon_connector->hpd.hpd; 1872 } 1873 radeon_irq_kms_enable_hpd(rdev, enabled); 1874 } 1875 1876 /** 1877 * evergreen_hpd_fini - hpd tear down callback. 1878 * 1879 * @rdev: radeon_device pointer 1880 * 1881 * Tear down the hpd pins used by the card (evergreen+). 1882 * Disable the hpd interrupts. 1883 */ 1884 void evergreen_hpd_fini(struct radeon_device *rdev) 1885 { 1886 struct drm_device *dev = rdev->ddev; 1887 struct drm_connector *connector; 1888 unsigned disabled = 0; 1889 1890 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1891 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1892 switch (radeon_connector->hpd.hpd) { 1893 case RADEON_HPD_1: 1894 WREG32(DC_HPD1_CONTROL, 0); 1895 break; 1896 case RADEON_HPD_2: 1897 WREG32(DC_HPD2_CONTROL, 0); 1898 break; 1899 case RADEON_HPD_3: 1900 WREG32(DC_HPD3_CONTROL, 0); 1901 break; 1902 case RADEON_HPD_4: 1903 WREG32(DC_HPD4_CONTROL, 0); 1904 break; 1905 case RADEON_HPD_5: 1906 WREG32(DC_HPD5_CONTROL, 0); 1907 break; 1908 case RADEON_HPD_6: 1909 WREG32(DC_HPD6_CONTROL, 0); 1910 break; 1911 default: 1912 break; 1913 } 1914 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE) 1915 disabled |= 1 << radeon_connector->hpd.hpd; 1916 } 1917 radeon_irq_kms_disable_hpd(rdev, disabled); 1918 } 1919 1920 /* watermark setup */ 1921 1922 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 1923 struct radeon_crtc *radeon_crtc, 1924 struct drm_display_mode *mode, 1925 struct drm_display_mode *other_mode) 1926 { 1927 u32 tmp, buffer_alloc, i; 1928 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1929 /* 1930 * Line Buffer Setup 1931 * There are 3 line buffers, each one shared by 2 display controllers. 1932 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1933 * the display controllers. The paritioning is done via one of four 1934 * preset allocations specified in bits 2:0: 1935 * first display controller 1936 * 0 - first half of lb (3840 * 2) 1937 * 1 - first 3/4 of lb (5760 * 2) 1938 * 2 - whole lb (7680 * 2), other crtc must be disabled 1939 * 3 - first 1/4 of lb (1920 * 2) 1940 * second display controller 1941 * 4 - second half of lb (3840 * 2) 1942 * 5 - second 3/4 of lb (5760 * 2) 1943 * 6 - whole lb (7680 * 2), other crtc must be disabled 1944 * 7 - last 1/4 of lb (1920 * 2) 1945 */ 1946 /* this can get tricky if we have two large displays on a paired group 1947 * of crtcs. Ideally for multiple large displays we'd assign them to 1948 * non-linked crtcs for maximum line buffer allocation. 1949 */ 1950 if (radeon_crtc->base.enabled && mode) { 1951 if (other_mode) { 1952 tmp = 0; /* 1/2 */ 1953 buffer_alloc = 1; 1954 } else { 1955 tmp = 2; /* whole */ 1956 buffer_alloc = 2; 1957 } 1958 } else { 1959 tmp = 0; 1960 buffer_alloc = 0; 1961 } 1962 1963 /* second controller of the pair uses second half of the lb */ 1964 if (radeon_crtc->crtc_id % 2) 1965 tmp += 4; 1966 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 1967 1968 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { 1969 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1970 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1971 for (i = 0; i < rdev->usec_timeout; i++) { 1972 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1973 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1974 break; 1975 udelay(1); 1976 } 1977 } 1978 1979 if (radeon_crtc->base.enabled && mode) { 1980 switch (tmp) { 1981 case 0: 1982 case 4: 1983 default: 1984 if (ASIC_IS_DCE5(rdev)) 1985 return 4096 * 2; 1986 else 1987 return 3840 * 2; 1988 case 1: 1989 case 5: 1990 if (ASIC_IS_DCE5(rdev)) 1991 return 6144 * 2; 1992 else 1993 return 5760 * 2; 1994 case 2: 1995 case 6: 1996 if (ASIC_IS_DCE5(rdev)) 1997 return 8192 * 2; 1998 else 1999 return 7680 * 2; 2000 case 3: 2001 case 7: 2002 if (ASIC_IS_DCE5(rdev)) 2003 return 2048 * 2; 2004 else 2005 return 1920 * 2; 2006 } 2007 } 2008 2009 /* controller not enabled, so no lb used */ 2010 return 0; 2011 } 2012 2013 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 2014 { 2015 u32 tmp = RREG32(MC_SHARED_CHMAP); 2016 2017 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 2018 case 0: 2019 default: 2020 return 1; 2021 case 1: 2022 return 2; 2023 case 2: 2024 return 4; 2025 case 3: 2026 return 8; 2027 } 2028 } 2029 2030 struct evergreen_wm_params { 2031 u32 dram_channels; /* number of dram channels */ 2032 u32 yclk; /* bandwidth per dram data pin in kHz */ 2033 u32 sclk; /* engine clock in kHz */ 2034 u32 disp_clk; /* display clock in kHz */ 2035 u32 src_width; /* viewport width */ 2036 u32 active_time; /* active display time in ns */ 2037 u32 blank_time; /* blank time in ns */ 2038 bool interlaced; /* mode is interlaced */ 2039 fixed20_12 vsc; /* vertical scale ratio */ 2040 u32 num_heads; /* number of active crtcs */ 2041 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 2042 u32 lb_size; /* line buffer allocated to pipe */ 2043 u32 vtaps; /* vertical scaler taps */ 2044 }; 2045 2046 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 2047 { 2048 /* Calculate DRAM Bandwidth and the part allocated to display. */ 2049 fixed20_12 dram_efficiency; /* 0.7 */ 2050 fixed20_12 yclk, dram_channels, bandwidth; 2051 fixed20_12 a; 2052 2053 a.full = dfixed_const(1000); 2054 yclk.full = dfixed_const(wm->yclk); 2055 yclk.full = dfixed_div(yclk, a); 2056 dram_channels.full = dfixed_const(wm->dram_channels * 4); 2057 a.full = dfixed_const(10); 2058 dram_efficiency.full = dfixed_const(7); 2059 dram_efficiency.full = dfixed_div(dram_efficiency, a); 2060 bandwidth.full = dfixed_mul(dram_channels, yclk); 2061 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 2062 2063 return dfixed_trunc(bandwidth); 2064 } 2065 2066 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2067 { 2068 /* Calculate DRAM Bandwidth and the part allocated to display. */ 2069 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 2070 fixed20_12 yclk, dram_channels, bandwidth; 2071 fixed20_12 a; 2072 2073 a.full = dfixed_const(1000); 2074 yclk.full = dfixed_const(wm->yclk); 2075 yclk.full = dfixed_div(yclk, a); 2076 dram_channels.full = dfixed_const(wm->dram_channels * 4); 2077 a.full = dfixed_const(10); 2078 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 2079 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 2080 bandwidth.full = dfixed_mul(dram_channels, yclk); 2081 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 2082 2083 return dfixed_trunc(bandwidth); 2084 } 2085 2086 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 2087 { 2088 /* Calculate the display Data return Bandwidth */ 2089 fixed20_12 return_efficiency; /* 0.8 */ 2090 fixed20_12 sclk, bandwidth; 2091 fixed20_12 a; 2092 2093 a.full = dfixed_const(1000); 2094 sclk.full = dfixed_const(wm->sclk); 2095 sclk.full = dfixed_div(sclk, a); 2096 a.full = dfixed_const(10); 2097 return_efficiency.full = dfixed_const(8); 2098 return_efficiency.full = dfixed_div(return_efficiency, a); 2099 a.full = dfixed_const(32); 2100 bandwidth.full = dfixed_mul(a, sclk); 2101 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 2102 2103 return dfixed_trunc(bandwidth); 2104 } 2105 2106 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 2107 { 2108 /* Calculate the DMIF Request Bandwidth */ 2109 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 2110 fixed20_12 disp_clk, bandwidth; 2111 fixed20_12 a; 2112 2113 a.full = dfixed_const(1000); 2114 disp_clk.full = dfixed_const(wm->disp_clk); 2115 disp_clk.full = dfixed_div(disp_clk, a); 2116 a.full = dfixed_const(10); 2117 disp_clk_request_efficiency.full = dfixed_const(8); 2118 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 2119 a.full = dfixed_const(32); 2120 bandwidth.full = dfixed_mul(a, disp_clk); 2121 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 2122 2123 return dfixed_trunc(bandwidth); 2124 } 2125 2126 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 2127 { 2128 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 2129 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 2130 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 2131 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 2132 2133 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 2134 } 2135 2136 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 2137 { 2138 /* Calculate the display mode Average Bandwidth 2139 * DisplayMode should contain the source and destination dimensions, 2140 * timing, etc. 2141 */ 2142 fixed20_12 bpp; 2143 fixed20_12 line_time; 2144 fixed20_12 src_width; 2145 fixed20_12 bandwidth; 2146 fixed20_12 a; 2147 2148 a.full = dfixed_const(1000); 2149 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 2150 line_time.full = dfixed_div(line_time, a); 2151 bpp.full = dfixed_const(wm->bytes_per_pixel); 2152 src_width.full = dfixed_const(wm->src_width); 2153 bandwidth.full = dfixed_mul(src_width, bpp); 2154 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 2155 bandwidth.full = dfixed_div(bandwidth, line_time); 2156 2157 return dfixed_trunc(bandwidth); 2158 } 2159 2160 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 2161 { 2162 /* First calcualte the latency in ns */ 2163 u32 mc_latency = 2000; /* 2000 ns. */ 2164 u32 available_bandwidth = evergreen_available_bandwidth(wm); 2165 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 2166 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 2167 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 2168 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 2169 (wm->num_heads * cursor_line_pair_return_time); 2170 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 2171 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 2172 fixed20_12 a, b, c; 2173 2174 if (wm->num_heads == 0) 2175 return 0; 2176 2177 a.full = dfixed_const(2); 2178 b.full = dfixed_const(1); 2179 if ((wm->vsc.full > a.full) || 2180 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 2181 (wm->vtaps >= 5) || 2182 ((wm->vsc.full >= a.full) && wm->interlaced)) 2183 max_src_lines_per_dst_line = 4; 2184 else 2185 max_src_lines_per_dst_line = 2; 2186 2187 a.full = dfixed_const(available_bandwidth); 2188 b.full = dfixed_const(wm->num_heads); 2189 a.full = dfixed_div(a, b); 2190 2191 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000); 2192 2193 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 2194 b.full = dfixed_const(1000); 2195 c.full = dfixed_const(lb_fill_bw); 2196 b.full = dfixed_div(c, b); 2197 a.full = dfixed_div(a, b); 2198 line_fill_time = dfixed_trunc(a); 2199 2200 if (line_fill_time < wm->active_time) 2201 return latency; 2202 else 2203 return latency + (line_fill_time - wm->active_time); 2204 2205 } 2206 2207 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2208 { 2209 if (evergreen_average_bandwidth(wm) <= 2210 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 2211 return true; 2212 else 2213 return false; 2214 }; 2215 2216 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 2217 { 2218 if (evergreen_average_bandwidth(wm) <= 2219 (evergreen_available_bandwidth(wm) / wm->num_heads)) 2220 return true; 2221 else 2222 return false; 2223 }; 2224 2225 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 2226 { 2227 u32 lb_partitions = wm->lb_size / wm->src_width; 2228 u32 line_time = wm->active_time + wm->blank_time; 2229 u32 latency_tolerant_lines; 2230 u32 latency_hiding; 2231 fixed20_12 a; 2232 2233 a.full = dfixed_const(1); 2234 if (wm->vsc.full > a.full) 2235 latency_tolerant_lines = 1; 2236 else { 2237 if (lb_partitions <= (wm->vtaps + 1)) 2238 latency_tolerant_lines = 1; 2239 else 2240 latency_tolerant_lines = 2; 2241 } 2242 2243 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2244 2245 if (evergreen_latency_watermark(wm) <= latency_hiding) 2246 return true; 2247 else 2248 return false; 2249 } 2250 2251 static void evergreen_program_watermarks(struct radeon_device *rdev, 2252 struct radeon_crtc *radeon_crtc, 2253 u32 lb_size, u32 num_heads) 2254 { 2255 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2256 struct evergreen_wm_params wm_low, wm_high; 2257 u32 dram_channels; 2258 u32 active_time; 2259 u32 line_time = 0; 2260 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2261 u32 priority_a_mark = 0, priority_b_mark = 0; 2262 u32 priority_a_cnt = PRIORITY_OFF; 2263 u32 priority_b_cnt = PRIORITY_OFF; 2264 u32 pipe_offset = radeon_crtc->crtc_id * 16; 2265 u32 tmp, arb_control3; 2266 fixed20_12 a, b, c; 2267 2268 if (radeon_crtc->base.enabled && num_heads && mode) { 2269 active_time = 1000000UL * (u32)mode->crtc_hdisplay / (u32)mode->clock; 2270 line_time = min((u32) (1000000UL * (u32)mode->crtc_htotal / (u32)mode->clock), (u32)65535); 2271 priority_a_cnt = 0; 2272 priority_b_cnt = 0; 2273 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2274 2275 /* watermark for high clocks */ 2276 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2277 wm_high.yclk = 2278 radeon_dpm_get_mclk(rdev, false) * 10; 2279 wm_high.sclk = 2280 radeon_dpm_get_sclk(rdev, false) * 10; 2281 } else { 2282 wm_high.yclk = rdev->pm.current_mclk * 10; 2283 wm_high.sclk = rdev->pm.current_sclk * 10; 2284 } 2285 2286 wm_high.disp_clk = mode->clock; 2287 wm_high.src_width = mode->crtc_hdisplay; 2288 wm_high.active_time = active_time; 2289 wm_high.blank_time = line_time - wm_high.active_time; 2290 wm_high.interlaced = false; 2291 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2292 wm_high.interlaced = true; 2293 wm_high.vsc = radeon_crtc->vsc; 2294 wm_high.vtaps = 1; 2295 if (radeon_crtc->rmx_type != RMX_OFF) 2296 wm_high.vtaps = 2; 2297 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2298 wm_high.lb_size = lb_size; 2299 wm_high.dram_channels = dram_channels; 2300 wm_high.num_heads = num_heads; 2301 2302 /* watermark for low clocks */ 2303 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2304 wm_low.yclk = 2305 radeon_dpm_get_mclk(rdev, true) * 10; 2306 wm_low.sclk = 2307 radeon_dpm_get_sclk(rdev, true) * 10; 2308 } else { 2309 wm_low.yclk = rdev->pm.current_mclk * 10; 2310 wm_low.sclk = rdev->pm.current_sclk * 10; 2311 } 2312 2313 wm_low.disp_clk = mode->clock; 2314 wm_low.src_width = mode->crtc_hdisplay; 2315 wm_low.active_time = active_time; 2316 wm_low.blank_time = line_time - wm_low.active_time; 2317 wm_low.interlaced = false; 2318 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2319 wm_low.interlaced = true; 2320 wm_low.vsc = radeon_crtc->vsc; 2321 wm_low.vtaps = 1; 2322 if (radeon_crtc->rmx_type != RMX_OFF) 2323 wm_low.vtaps = 2; 2324 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2325 wm_low.lb_size = lb_size; 2326 wm_low.dram_channels = dram_channels; 2327 wm_low.num_heads = num_heads; 2328 2329 /* set for high clocks */ 2330 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); 2331 /* set for low clocks */ 2332 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); 2333 2334 /* possibly force display priority to high */ 2335 /* should really do this at mode validation time... */ 2336 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2337 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || 2338 !evergreen_check_latency_hiding(&wm_high) || 2339 (rdev->disp_priority == 2)) { 2340 DRM_DEBUG_KMS("force priority a to high\n"); 2341 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2342 } 2343 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2344 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || 2345 !evergreen_check_latency_hiding(&wm_low) || 2346 (rdev->disp_priority == 2)) { 2347 DRM_DEBUG_KMS("force priority b to high\n"); 2348 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2349 } 2350 2351 a.full = dfixed_const(1000); 2352 b.full = dfixed_const(mode->clock); 2353 b.full = dfixed_div(b, a); 2354 c.full = dfixed_const(latency_watermark_a); 2355 c.full = dfixed_mul(c, b); 2356 c.full = dfixed_mul(c, radeon_crtc->hsc); 2357 c.full = dfixed_div(c, a); 2358 a.full = dfixed_const(16); 2359 c.full = dfixed_div(c, a); 2360 priority_a_mark = dfixed_trunc(c); 2361 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2362 2363 a.full = dfixed_const(1000); 2364 b.full = dfixed_const(mode->clock); 2365 b.full = dfixed_div(b, a); 2366 c.full = dfixed_const(latency_watermark_b); 2367 c.full = dfixed_mul(c, b); 2368 c.full = dfixed_mul(c, radeon_crtc->hsc); 2369 c.full = dfixed_div(c, a); 2370 a.full = dfixed_const(16); 2371 c.full = dfixed_div(c, a); 2372 priority_b_mark = dfixed_trunc(c); 2373 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2374 2375 /* Save number of lines the linebuffer leads before the scanout */ 2376 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); 2377 } 2378 2379 /* select wm A */ 2380 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2381 tmp = arb_control3; 2382 tmp &= ~LATENCY_WATERMARK_MASK(3); 2383 tmp |= LATENCY_WATERMARK_MASK(1); 2384 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2385 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2386 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2387 LATENCY_HIGH_WATERMARK(line_time))); 2388 /* select wm B */ 2389 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2390 tmp &= ~LATENCY_WATERMARK_MASK(3); 2391 tmp |= LATENCY_WATERMARK_MASK(2); 2392 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2393 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2394 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2395 LATENCY_HIGH_WATERMARK(line_time))); 2396 /* restore original selection */ 2397 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 2398 2399 /* write the priority marks */ 2400 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2401 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2402 2403 /* save values for DPM */ 2404 radeon_crtc->line_time = line_time; 2405 radeon_crtc->wm_high = latency_watermark_a; 2406 radeon_crtc->wm_low = latency_watermark_b; 2407 } 2408 2409 /** 2410 * evergreen_bandwidth_update - update display watermarks callback. 2411 * 2412 * @rdev: radeon_device pointer 2413 * 2414 * Update the display watermarks based on the requested mode(s) 2415 * (evergreen+). 2416 */ 2417 void evergreen_bandwidth_update(struct radeon_device *rdev) 2418 { 2419 struct drm_display_mode *mode0 = NULL; 2420 struct drm_display_mode *mode1 = NULL; 2421 u32 num_heads = 0, lb_size; 2422 int i; 2423 2424 if (!rdev->mode_info.mode_config_initialized) 2425 return; 2426 2427 radeon_update_display_priority(rdev); 2428 2429 for (i = 0; i < rdev->num_crtc; i++) { 2430 if (rdev->mode_info.crtcs[i]->base.enabled) 2431 num_heads++; 2432 } 2433 for (i = 0; i < rdev->num_crtc; i += 2) { 2434 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2435 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2436 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2437 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2438 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2439 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2440 } 2441 } 2442 2443 /** 2444 * evergreen_mc_wait_for_idle - wait for MC idle callback. 2445 * 2446 * @rdev: radeon_device pointer 2447 * 2448 * Wait for the MC (memory controller) to be idle. 2449 * (evergreen+). 2450 * Returns 0 if the MC is idle, -1 if not. 2451 */ 2452 int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 2453 { 2454 unsigned i; 2455 u32 tmp; 2456 2457 for (i = 0; i < rdev->usec_timeout; i++) { 2458 /* read MC_STATUS */ 2459 tmp = RREG32(SRBM_STATUS) & 0x1F00; 2460 if (!tmp) 2461 return 0; 2462 udelay(1); 2463 } 2464 return -1; 2465 } 2466 2467 /* 2468 * GART 2469 */ 2470 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 2471 { 2472 unsigned i; 2473 u32 tmp; 2474 2475 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 2476 2477 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 2478 for (i = 0; i < rdev->usec_timeout; i++) { 2479 /* read MC_STATUS */ 2480 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 2481 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 2482 if (tmp == 2) { 2483 pr_warn("[drm] r600 flush TLB failed\n"); 2484 return; 2485 } 2486 if (tmp) { 2487 return; 2488 } 2489 udelay(1); 2490 } 2491 } 2492 2493 static int evergreen_pcie_gart_enable(struct radeon_device *rdev) 2494 { 2495 u32 tmp; 2496 int r; 2497 2498 if (rdev->gart.robj == NULL) { 2499 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 2500 return -EINVAL; 2501 } 2502 r = radeon_gart_table_vram_pin(rdev); 2503 if (r) 2504 return r; 2505 /* Setup L2 cache */ 2506 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2507 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2508 EFFECTIVE_L2_QUEUE_SIZE(7)); 2509 WREG32(VM_L2_CNTL2, 0); 2510 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2511 /* Setup TLB control */ 2512 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2513 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2514 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2515 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2516 if (rdev->flags & RADEON_IS_IGP) { 2517 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 2518 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 2519 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 2520 } else { 2521 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2522 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2523 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2524 if ((rdev->family == CHIP_JUNIPER) || 2525 (rdev->family == CHIP_CYPRESS) || 2526 (rdev->family == CHIP_HEMLOCK) || 2527 (rdev->family == CHIP_BARTS)) 2528 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp); 2529 } 2530 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2531 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2532 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2533 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2534 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 2535 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 2536 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 2537 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 2538 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 2539 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 2540 (u32)(rdev->dummy_page.addr >> 12)); 2541 WREG32(VM_CONTEXT1_CNTL, 0); 2542 2543 evergreen_pcie_gart_tlb_flush(rdev); 2544 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 2545 (unsigned)(rdev->mc.gtt_size >> 20), 2546 (unsigned long long)rdev->gart.table_addr); 2547 rdev->gart.ready = true; 2548 return 0; 2549 } 2550 2551 static void evergreen_pcie_gart_disable(struct radeon_device *rdev) 2552 { 2553 u32 tmp; 2554 2555 /* Disable all tables */ 2556 WREG32(VM_CONTEXT0_CNTL, 0); 2557 WREG32(VM_CONTEXT1_CNTL, 0); 2558 2559 /* Setup L2 cache */ 2560 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 2561 EFFECTIVE_L2_QUEUE_SIZE(7)); 2562 WREG32(VM_L2_CNTL2, 0); 2563 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2564 /* Setup TLB control */ 2565 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2566 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2567 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2568 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2569 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2570 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2571 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2572 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2573 radeon_gart_table_vram_unpin(rdev); 2574 } 2575 2576 static void evergreen_pcie_gart_fini(struct radeon_device *rdev) 2577 { 2578 evergreen_pcie_gart_disable(rdev); 2579 radeon_gart_table_vram_free(rdev); 2580 radeon_gart_fini(rdev); 2581 } 2582 2583 2584 static void evergreen_agp_enable(struct radeon_device *rdev) 2585 { 2586 u32 tmp; 2587 2588 /* Setup L2 cache */ 2589 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2590 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2591 EFFECTIVE_L2_QUEUE_SIZE(7)); 2592 WREG32(VM_L2_CNTL2, 0); 2593 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2594 /* Setup TLB control */ 2595 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2596 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2597 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2598 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2599 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2600 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2601 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2602 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2603 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2604 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2605 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2606 WREG32(VM_CONTEXT0_CNTL, 0); 2607 WREG32(VM_CONTEXT1_CNTL, 0); 2608 } 2609 2610 static const unsigned ni_dig_offsets[] = 2611 { 2612 NI_DIG0_REGISTER_OFFSET, 2613 NI_DIG1_REGISTER_OFFSET, 2614 NI_DIG2_REGISTER_OFFSET, 2615 NI_DIG3_REGISTER_OFFSET, 2616 NI_DIG4_REGISTER_OFFSET, 2617 NI_DIG5_REGISTER_OFFSET 2618 }; 2619 2620 static const unsigned ni_tx_offsets[] = 2621 { 2622 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1, 2623 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1, 2624 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1, 2625 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1, 2626 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1, 2627 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1 2628 }; 2629 2630 static const unsigned evergreen_dp_offsets[] = 2631 { 2632 EVERGREEN_DP0_REGISTER_OFFSET, 2633 EVERGREEN_DP1_REGISTER_OFFSET, 2634 EVERGREEN_DP2_REGISTER_OFFSET, 2635 EVERGREEN_DP3_REGISTER_OFFSET, 2636 EVERGREEN_DP4_REGISTER_OFFSET, 2637 EVERGREEN_DP5_REGISTER_OFFSET 2638 }; 2639 2640 2641 /* 2642 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc 2643 * We go from crtc to connector and it is not relible since it 2644 * should be an opposite direction .If crtc is enable then 2645 * find the dig_fe which selects this crtc and insure that it enable. 2646 * if such dig_fe is found then find dig_be which selects found dig_be and 2647 * insure that it enable and in DP_SST mode. 2648 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing 2649 * from dp symbols clocks . 2650 */ 2651 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev, 2652 unsigned crtc_id, unsigned *ret_dig_fe) 2653 { 2654 unsigned i; 2655 unsigned dig_fe; 2656 unsigned dig_be; 2657 unsigned dig_en_be; 2658 unsigned uniphy_pll; 2659 unsigned digs_fe_selected; 2660 unsigned dig_be_mode; 2661 unsigned dig_fe_mask; 2662 bool is_enabled = false; 2663 bool found_crtc = false; 2664 2665 /* loop through all running dig_fe to find selected crtc */ 2666 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2667 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]); 2668 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON && 2669 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) { 2670 /* found running pipe */ 2671 found_crtc = true; 2672 dig_fe_mask = 1 << i; 2673 dig_fe = i; 2674 break; 2675 } 2676 } 2677 2678 if (found_crtc) { 2679 /* loop through all running dig_be to find selected dig_fe */ 2680 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2681 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]); 2682 /* if dig_fe_selected by dig_be? */ 2683 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be); 2684 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be); 2685 if (dig_fe_mask & digs_fe_selected && 2686 /* if dig_be in sst mode? */ 2687 dig_be_mode == NI_DIG_BE_DPSST) { 2688 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL + 2689 ni_dig_offsets[i]); 2690 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 + 2691 ni_tx_offsets[i]); 2692 /* dig_be enable and tx is running */ 2693 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE && 2694 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON && 2695 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) { 2696 is_enabled = true; 2697 *ret_dig_fe = dig_fe; 2698 break; 2699 } 2700 } 2701 } 2702 } 2703 2704 return is_enabled; 2705 } 2706 2707 /* 2708 * Blank dig when in dp sst mode 2709 * Dig ignores crtc timing 2710 */ 2711 static void evergreen_blank_dp_output(struct radeon_device *rdev, 2712 unsigned dig_fe) 2713 { 2714 unsigned stream_ctrl; 2715 unsigned fifo_ctrl; 2716 unsigned counter = 0; 2717 2718 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) { 2719 DRM_ERROR("invalid dig_fe %d\n", dig_fe); 2720 return; 2721 } 2722 2723 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2724 evergreen_dp_offsets[dig_fe]); 2725 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) { 2726 DRM_ERROR("dig %d , should be enable\n", dig_fe); 2727 return; 2728 } 2729 2730 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE; 2731 WREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2732 evergreen_dp_offsets[dig_fe], stream_ctrl); 2733 2734 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2735 evergreen_dp_offsets[dig_fe]); 2736 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) { 2737 msleep(1); 2738 counter++; 2739 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2740 evergreen_dp_offsets[dig_fe]); 2741 } 2742 if (counter >= 32 ) 2743 DRM_ERROR("counter exceeds %d\n", counter); 2744 2745 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]); 2746 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET; 2747 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl); 2748 2749 } 2750 2751 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 2752 { 2753 u32 crtc_enabled, tmp, frame_count, blackout; 2754 int i, j; 2755 unsigned dig_fe; 2756 2757 if (!ASIC_IS_NODCE(rdev)) { 2758 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 2759 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 2760 2761 /* disable VGA render */ 2762 WREG32(VGA_RENDER_CONTROL, 0); 2763 } 2764 /* blank the display controllers */ 2765 for (i = 0; i < rdev->num_crtc; i++) { 2766 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; 2767 if (crtc_enabled) { 2768 save->crtc_enabled[i] = true; 2769 if (ASIC_IS_DCE6(rdev)) { 2770 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2771 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) { 2772 radeon_wait_for_vblank(rdev, i); 2773 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2774 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 2775 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2776 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2777 } 2778 } else { 2779 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2780 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) { 2781 radeon_wait_for_vblank(rdev, i); 2782 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2783 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2784 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2785 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2786 } 2787 } 2788 /* wait for the next frame */ 2789 frame_count = radeon_get_vblank_counter(rdev, i); 2790 for (j = 0; j < rdev->usec_timeout; j++) { 2791 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2792 break; 2793 udelay(1); 2794 } 2795 /*we should disable dig if it drives dp sst*/ 2796 /*but we are in radeon_device_init and the topology is unknown*/ 2797 /*and it is available after radeon_modeset_init*/ 2798 /*the following method radeon_atom_encoder_dpms_dig*/ 2799 /*does the job if we initialize it properly*/ 2800 /*for now we do it this manually*/ 2801 /**/ 2802 if (ASIC_IS_DCE5(rdev) && 2803 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe)) 2804 evergreen_blank_dp_output(rdev, dig_fe); 2805 /*we could remove 6 lines below*/ 2806 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ 2807 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2808 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2809 tmp &= ~EVERGREEN_CRTC_MASTER_EN; 2810 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2811 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2812 save->crtc_enabled[i] = false; 2813 /* ***** */ 2814 } else { 2815 save->crtc_enabled[i] = false; 2816 } 2817 } 2818 2819 radeon_mc_wait_for_idle(rdev); 2820 2821 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 2822 if ((blackout & BLACKOUT_MODE_MASK) != 1) { 2823 /* Block CPU access */ 2824 WREG32(BIF_FB_EN, 0); 2825 /* blackout the MC */ 2826 blackout &= ~BLACKOUT_MODE_MASK; 2827 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 2828 } 2829 /* wait for the MC to settle */ 2830 udelay(100); 2831 2832 /* lock double buffered regs */ 2833 for (i = 0; i < rdev->num_crtc; i++) { 2834 if (save->crtc_enabled[i]) { 2835 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2836 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) { 2837 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 2838 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2839 } 2840 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2841 if (!(tmp & 1)) { 2842 tmp |= 1; 2843 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2844 } 2845 } 2846 } 2847 } 2848 2849 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 2850 { 2851 u32 tmp, frame_count; 2852 int i, j; 2853 2854 /* update crtc base addresses */ 2855 for (i = 0; i < rdev->num_crtc; i++) { 2856 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2857 upper_32_bits(rdev->mc.vram_start)); 2858 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2859 upper_32_bits(rdev->mc.vram_start)); 2860 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], 2861 (u32)rdev->mc.vram_start); 2862 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], 2863 (u32)rdev->mc.vram_start); 2864 } 2865 2866 if (!ASIC_IS_NODCE(rdev)) { 2867 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 2868 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 2869 } 2870 2871 /* unlock regs and wait for update */ 2872 for (i = 0; i < rdev->num_crtc; i++) { 2873 if (save->crtc_enabled[i]) { 2874 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); 2875 if ((tmp & 0x7) != 0) { 2876 tmp &= ~0x7; 2877 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); 2878 } 2879 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2880 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) { 2881 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 2882 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2883 } 2884 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2885 if (tmp & 1) { 2886 tmp &= ~1; 2887 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2888 } 2889 for (j = 0; j < rdev->usec_timeout; j++) { 2890 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2891 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0) 2892 break; 2893 udelay(1); 2894 } 2895 } 2896 } 2897 2898 /* unblackout the MC */ 2899 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL); 2900 tmp &= ~BLACKOUT_MODE_MASK; 2901 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp); 2902 /* allow CPU access */ 2903 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2904 2905 for (i = 0; i < rdev->num_crtc; i++) { 2906 if (save->crtc_enabled[i]) { 2907 if (ASIC_IS_DCE6(rdev)) { 2908 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2909 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN; 2910 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2911 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2912 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2913 } else { 2914 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2915 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2916 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2917 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2918 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2919 } 2920 /* wait for the next frame */ 2921 frame_count = radeon_get_vblank_counter(rdev, i); 2922 for (j = 0; j < rdev->usec_timeout; j++) { 2923 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2924 break; 2925 udelay(1); 2926 } 2927 } 2928 } 2929 if (!ASIC_IS_NODCE(rdev)) { 2930 /* Unlock vga access */ 2931 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 2932 mdelay(1); 2933 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 2934 } 2935 } 2936 2937 void evergreen_mc_program(struct radeon_device *rdev) 2938 { 2939 struct evergreen_mc_save save; 2940 u32 tmp; 2941 int i, j; 2942 2943 /* Initialize HDP */ 2944 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2945 WREG32((0x2c14 + j), 0x00000000); 2946 WREG32((0x2c18 + j), 0x00000000); 2947 WREG32((0x2c1c + j), 0x00000000); 2948 WREG32((0x2c20 + j), 0x00000000); 2949 WREG32((0x2c24 + j), 0x00000000); 2950 } 2951 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 2952 2953 evergreen_mc_stop(rdev, &save); 2954 if (evergreen_mc_wait_for_idle(rdev)) { 2955 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2956 } 2957 /* Lockout access through VGA aperture*/ 2958 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 2959 /* Update configuration */ 2960 if (rdev->flags & RADEON_IS_AGP) { 2961 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 2962 /* VRAM before AGP */ 2963 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2964 rdev->mc.vram_start >> 12); 2965 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2966 rdev->mc.gtt_end >> 12); 2967 } else { 2968 /* VRAM after AGP */ 2969 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2970 rdev->mc.gtt_start >> 12); 2971 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2972 rdev->mc.vram_end >> 12); 2973 } 2974 } else { 2975 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2976 rdev->mc.vram_start >> 12); 2977 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2978 rdev->mc.vram_end >> 12); 2979 } 2980 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12); 2981 /* llano/ontario only */ 2982 if ((rdev->family == CHIP_PALM) || 2983 (rdev->family == CHIP_SUMO) || 2984 (rdev->family == CHIP_SUMO2)) { 2985 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 2986 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 2987 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 2988 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 2989 } 2990 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 2991 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 2992 WREG32(MC_VM_FB_LOCATION, tmp); 2993 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 2994 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 2995 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 2996 if (rdev->flags & RADEON_IS_AGP) { 2997 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 2998 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 2999 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 3000 } else { 3001 WREG32(MC_VM_AGP_BASE, 0); 3002 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 3003 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 3004 } 3005 if (evergreen_mc_wait_for_idle(rdev)) { 3006 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3007 } 3008 evergreen_mc_resume(rdev, &save); 3009 /* we need to own VRAM, so turn off the VGA renderer here 3010 * to stop it overwriting our objects */ 3011 rv515_vga_render_disable(rdev); 3012 } 3013 3014 /* 3015 * CP. 3016 */ 3017 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3018 { 3019 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3020 u32 next_rptr; 3021 3022 /* set to DX10/11 mode */ 3023 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 3024 radeon_ring_write(ring, 1); 3025 3026 if (ring->rptr_save_reg) { 3027 next_rptr = ring->wptr + 3 + 4; 3028 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3029 radeon_ring_write(ring, ((ring->rptr_save_reg - 3030 PACKET3_SET_CONFIG_REG_START) >> 2)); 3031 radeon_ring_write(ring, next_rptr); 3032 } else if (rdev->wb.enabled) { 3033 next_rptr = ring->wptr + 5 + 4; 3034 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3)); 3035 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3036 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18)); 3037 radeon_ring_write(ring, next_rptr); 3038 radeon_ring_write(ring, 0); 3039 } 3040 3041 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 3042 radeon_ring_write(ring, 3043 #ifdef __BIG_ENDIAN 3044 (2 << 0) | 3045 #endif 3046 (ib->gpu_addr & 0xFFFFFFFC)); 3047 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 3048 radeon_ring_write(ring, ib->length_dw); 3049 } 3050 3051 3052 static int evergreen_cp_load_microcode(struct radeon_device *rdev) 3053 { 3054 const __be32 *fw_data; 3055 int i; 3056 3057 if (!rdev->me_fw || !rdev->pfp_fw) 3058 return -EINVAL; 3059 3060 r700_cp_stop(rdev); 3061 WREG32(CP_RB_CNTL, 3062 #ifdef __BIG_ENDIAN 3063 BUF_SWAP_32BIT | 3064 #endif 3065 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 3066 3067 fw_data = (const __be32 *)rdev->pfp_fw->data; 3068 WREG32(CP_PFP_UCODE_ADDR, 0); 3069 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 3070 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3071 WREG32(CP_PFP_UCODE_ADDR, 0); 3072 3073 fw_data = (const __be32 *)rdev->me_fw->data; 3074 WREG32(CP_ME_RAM_WADDR, 0); 3075 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 3076 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3077 3078 WREG32(CP_PFP_UCODE_ADDR, 0); 3079 WREG32(CP_ME_RAM_WADDR, 0); 3080 WREG32(CP_ME_RAM_RADDR, 0); 3081 return 0; 3082 } 3083 3084 static int evergreen_cp_start(struct radeon_device *rdev) 3085 { 3086 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3087 int r, i; 3088 uint32_t cp_me; 3089 3090 r = radeon_ring_lock(rdev, ring, 7); 3091 if (r) { 3092 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3093 return r; 3094 } 3095 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3096 radeon_ring_write(ring, 0x1); 3097 radeon_ring_write(ring, 0x0); 3098 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1); 3099 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3100 radeon_ring_write(ring, 0); 3101 radeon_ring_write(ring, 0); 3102 radeon_ring_unlock_commit(rdev, ring, false); 3103 3104 cp_me = 0xff; 3105 WREG32(CP_ME_CNTL, cp_me); 3106 3107 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19); 3108 if (r) { 3109 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3110 return r; 3111 } 3112 3113 /* setup clear context state */ 3114 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3115 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3116 3117 for (i = 0; i < evergreen_default_size; i++) 3118 radeon_ring_write(ring, evergreen_default_state[i]); 3119 3120 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3121 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3122 3123 /* set clear context state */ 3124 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3125 radeon_ring_write(ring, 0); 3126 3127 /* SQ_VTX_BASE_VTX_LOC */ 3128 radeon_ring_write(ring, 0xc0026f00); 3129 radeon_ring_write(ring, 0x00000000); 3130 radeon_ring_write(ring, 0x00000000); 3131 radeon_ring_write(ring, 0x00000000); 3132 3133 /* Clear consts */ 3134 radeon_ring_write(ring, 0xc0036f00); 3135 radeon_ring_write(ring, 0x00000bc4); 3136 radeon_ring_write(ring, 0xffffffff); 3137 radeon_ring_write(ring, 0xffffffff); 3138 radeon_ring_write(ring, 0xffffffff); 3139 3140 radeon_ring_write(ring, 0xc0026900); 3141 radeon_ring_write(ring, 0x00000316); 3142 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3143 radeon_ring_write(ring, 0x00000010); /* */ 3144 3145 radeon_ring_unlock_commit(rdev, ring, false); 3146 3147 return 0; 3148 } 3149 3150 static int evergreen_cp_resume(struct radeon_device *rdev) 3151 { 3152 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3153 u32 tmp; 3154 u32 rb_bufsz; 3155 int r; 3156 3157 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 3158 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 3159 SOFT_RESET_PA | 3160 SOFT_RESET_SH | 3161 SOFT_RESET_VGT | 3162 SOFT_RESET_SPI | 3163 SOFT_RESET_SX)); 3164 RREG32(GRBM_SOFT_RESET); 3165 mdelay(15); 3166 WREG32(GRBM_SOFT_RESET, 0); 3167 RREG32(GRBM_SOFT_RESET); 3168 3169 /* Set ring buffer size */ 3170 rb_bufsz = order_base_2(ring->ring_size / 8); 3171 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3172 #ifdef __BIG_ENDIAN 3173 tmp |= BUF_SWAP_32BIT; 3174 #endif 3175 WREG32(CP_RB_CNTL, tmp); 3176 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3177 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3178 3179 /* Set the write pointer delay */ 3180 WREG32(CP_RB_WPTR_DELAY, 0); 3181 3182 /* Initialize the ring buffer's read and write pointers */ 3183 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 3184 WREG32(CP_RB_RPTR_WR, 0); 3185 ring->wptr = 0; 3186 WREG32(CP_RB_WPTR, ring->wptr); 3187 3188 /* set the wb address whether it's enabled or not */ 3189 WREG32(CP_RB_RPTR_ADDR, 3190 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 3191 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3192 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3193 3194 if (rdev->wb.enabled) 3195 WREG32(SCRATCH_UMSK, 0xff); 3196 else { 3197 tmp |= RB_NO_UPDATE; 3198 WREG32(SCRATCH_UMSK, 0); 3199 } 3200 3201 mdelay(1); 3202 WREG32(CP_RB_CNTL, tmp); 3203 3204 WREG32(CP_RB_BASE, ring->gpu_addr >> 8); 3205 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 3206 3207 evergreen_cp_start(rdev); 3208 ring->ready = true; 3209 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); 3210 if (r) { 3211 ring->ready = false; 3212 return r; 3213 } 3214 return 0; 3215 } 3216 3217 /* 3218 * Core functions 3219 */ 3220 static void evergreen_gpu_init(struct radeon_device *rdev) 3221 { 3222 u32 gb_addr_config; 3223 u32 mc_shared_chmap, mc_arb_ramcfg; 3224 u32 sx_debug_1; 3225 u32 smx_dc_ctl0; 3226 u32 sq_config; 3227 u32 sq_lds_resource_mgmt; 3228 u32 sq_gpr_resource_mgmt_1; 3229 u32 sq_gpr_resource_mgmt_2; 3230 u32 sq_gpr_resource_mgmt_3; 3231 u32 sq_thread_resource_mgmt; 3232 u32 sq_thread_resource_mgmt_2; 3233 u32 sq_stack_resource_mgmt_1; 3234 u32 sq_stack_resource_mgmt_2; 3235 u32 sq_stack_resource_mgmt_3; 3236 u32 vgt_cache_invalidation; 3237 u32 hdp_host_path_cntl, tmp; 3238 u32 disabled_rb_mask; 3239 int i, j, ps_thread_count; 3240 3241 switch (rdev->family) { 3242 case CHIP_CYPRESS: 3243 case CHIP_HEMLOCK: 3244 rdev->config.evergreen.num_ses = 2; 3245 rdev->config.evergreen.max_pipes = 4; 3246 rdev->config.evergreen.max_tile_pipes = 8; 3247 rdev->config.evergreen.max_simds = 10; 3248 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3249 rdev->config.evergreen.max_gprs = 256; 3250 rdev->config.evergreen.max_threads = 248; 3251 rdev->config.evergreen.max_gs_threads = 32; 3252 rdev->config.evergreen.max_stack_entries = 512; 3253 rdev->config.evergreen.sx_num_of_sets = 4; 3254 rdev->config.evergreen.sx_max_export_size = 256; 3255 rdev->config.evergreen.sx_max_export_pos_size = 64; 3256 rdev->config.evergreen.sx_max_export_smx_size = 192; 3257 rdev->config.evergreen.max_hw_contexts = 8; 3258 rdev->config.evergreen.sq_num_cf_insts = 2; 3259 3260 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3261 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3262 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3263 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN; 3264 break; 3265 case CHIP_JUNIPER: 3266 rdev->config.evergreen.num_ses = 1; 3267 rdev->config.evergreen.max_pipes = 4; 3268 rdev->config.evergreen.max_tile_pipes = 4; 3269 rdev->config.evergreen.max_simds = 10; 3270 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3271 rdev->config.evergreen.max_gprs = 256; 3272 rdev->config.evergreen.max_threads = 248; 3273 rdev->config.evergreen.max_gs_threads = 32; 3274 rdev->config.evergreen.max_stack_entries = 512; 3275 rdev->config.evergreen.sx_num_of_sets = 4; 3276 rdev->config.evergreen.sx_max_export_size = 256; 3277 rdev->config.evergreen.sx_max_export_pos_size = 64; 3278 rdev->config.evergreen.sx_max_export_smx_size = 192; 3279 rdev->config.evergreen.max_hw_contexts = 8; 3280 rdev->config.evergreen.sq_num_cf_insts = 2; 3281 3282 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3283 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3284 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3285 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN; 3286 break; 3287 case CHIP_REDWOOD: 3288 rdev->config.evergreen.num_ses = 1; 3289 rdev->config.evergreen.max_pipes = 4; 3290 rdev->config.evergreen.max_tile_pipes = 4; 3291 rdev->config.evergreen.max_simds = 5; 3292 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3293 rdev->config.evergreen.max_gprs = 256; 3294 rdev->config.evergreen.max_threads = 248; 3295 rdev->config.evergreen.max_gs_threads = 32; 3296 rdev->config.evergreen.max_stack_entries = 256; 3297 rdev->config.evergreen.sx_num_of_sets = 4; 3298 rdev->config.evergreen.sx_max_export_size = 256; 3299 rdev->config.evergreen.sx_max_export_pos_size = 64; 3300 rdev->config.evergreen.sx_max_export_smx_size = 192; 3301 rdev->config.evergreen.max_hw_contexts = 8; 3302 rdev->config.evergreen.sq_num_cf_insts = 2; 3303 3304 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3305 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3306 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3307 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN; 3308 break; 3309 case CHIP_CEDAR: 3310 default: 3311 rdev->config.evergreen.num_ses = 1; 3312 rdev->config.evergreen.max_pipes = 2; 3313 rdev->config.evergreen.max_tile_pipes = 2; 3314 rdev->config.evergreen.max_simds = 2; 3315 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3316 rdev->config.evergreen.max_gprs = 256; 3317 rdev->config.evergreen.max_threads = 192; 3318 rdev->config.evergreen.max_gs_threads = 16; 3319 rdev->config.evergreen.max_stack_entries = 256; 3320 rdev->config.evergreen.sx_num_of_sets = 4; 3321 rdev->config.evergreen.sx_max_export_size = 128; 3322 rdev->config.evergreen.sx_max_export_pos_size = 32; 3323 rdev->config.evergreen.sx_max_export_smx_size = 96; 3324 rdev->config.evergreen.max_hw_contexts = 4; 3325 rdev->config.evergreen.sq_num_cf_insts = 1; 3326 3327 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3328 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3329 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3330 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3331 break; 3332 case CHIP_PALM: 3333 rdev->config.evergreen.num_ses = 1; 3334 rdev->config.evergreen.max_pipes = 2; 3335 rdev->config.evergreen.max_tile_pipes = 2; 3336 rdev->config.evergreen.max_simds = 2; 3337 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3338 rdev->config.evergreen.max_gprs = 256; 3339 rdev->config.evergreen.max_threads = 192; 3340 rdev->config.evergreen.max_gs_threads = 16; 3341 rdev->config.evergreen.max_stack_entries = 256; 3342 rdev->config.evergreen.sx_num_of_sets = 4; 3343 rdev->config.evergreen.sx_max_export_size = 128; 3344 rdev->config.evergreen.sx_max_export_pos_size = 32; 3345 rdev->config.evergreen.sx_max_export_smx_size = 96; 3346 rdev->config.evergreen.max_hw_contexts = 4; 3347 rdev->config.evergreen.sq_num_cf_insts = 1; 3348 3349 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3350 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3351 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3352 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3353 break; 3354 case CHIP_SUMO: 3355 rdev->config.evergreen.num_ses = 1; 3356 rdev->config.evergreen.max_pipes = 4; 3357 rdev->config.evergreen.max_tile_pipes = 4; 3358 if (rdev->pdev->device == 0x9648) 3359 rdev->config.evergreen.max_simds = 3; 3360 else if ((rdev->pdev->device == 0x9647) || 3361 (rdev->pdev->device == 0x964a)) 3362 rdev->config.evergreen.max_simds = 4; 3363 else 3364 rdev->config.evergreen.max_simds = 5; 3365 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3366 rdev->config.evergreen.max_gprs = 256; 3367 rdev->config.evergreen.max_threads = 248; 3368 rdev->config.evergreen.max_gs_threads = 32; 3369 rdev->config.evergreen.max_stack_entries = 256; 3370 rdev->config.evergreen.sx_num_of_sets = 4; 3371 rdev->config.evergreen.sx_max_export_size = 256; 3372 rdev->config.evergreen.sx_max_export_pos_size = 64; 3373 rdev->config.evergreen.sx_max_export_smx_size = 192; 3374 rdev->config.evergreen.max_hw_contexts = 8; 3375 rdev->config.evergreen.sq_num_cf_insts = 2; 3376 3377 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3378 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3379 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3380 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN; 3381 break; 3382 case CHIP_SUMO2: 3383 rdev->config.evergreen.num_ses = 1; 3384 rdev->config.evergreen.max_pipes = 4; 3385 rdev->config.evergreen.max_tile_pipes = 4; 3386 rdev->config.evergreen.max_simds = 2; 3387 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3388 rdev->config.evergreen.max_gprs = 256; 3389 rdev->config.evergreen.max_threads = 248; 3390 rdev->config.evergreen.max_gs_threads = 32; 3391 rdev->config.evergreen.max_stack_entries = 512; 3392 rdev->config.evergreen.sx_num_of_sets = 4; 3393 rdev->config.evergreen.sx_max_export_size = 256; 3394 rdev->config.evergreen.sx_max_export_pos_size = 64; 3395 rdev->config.evergreen.sx_max_export_smx_size = 192; 3396 rdev->config.evergreen.max_hw_contexts = 4; 3397 rdev->config.evergreen.sq_num_cf_insts = 2; 3398 3399 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3400 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3401 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3402 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN; 3403 break; 3404 case CHIP_BARTS: 3405 rdev->config.evergreen.num_ses = 2; 3406 rdev->config.evergreen.max_pipes = 4; 3407 rdev->config.evergreen.max_tile_pipes = 8; 3408 rdev->config.evergreen.max_simds = 7; 3409 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3410 rdev->config.evergreen.max_gprs = 256; 3411 rdev->config.evergreen.max_threads = 248; 3412 rdev->config.evergreen.max_gs_threads = 32; 3413 rdev->config.evergreen.max_stack_entries = 512; 3414 rdev->config.evergreen.sx_num_of_sets = 4; 3415 rdev->config.evergreen.sx_max_export_size = 256; 3416 rdev->config.evergreen.sx_max_export_pos_size = 64; 3417 rdev->config.evergreen.sx_max_export_smx_size = 192; 3418 rdev->config.evergreen.max_hw_contexts = 8; 3419 rdev->config.evergreen.sq_num_cf_insts = 2; 3420 3421 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3422 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3423 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3424 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN; 3425 break; 3426 case CHIP_TURKS: 3427 rdev->config.evergreen.num_ses = 1; 3428 rdev->config.evergreen.max_pipes = 4; 3429 rdev->config.evergreen.max_tile_pipes = 4; 3430 rdev->config.evergreen.max_simds = 6; 3431 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3432 rdev->config.evergreen.max_gprs = 256; 3433 rdev->config.evergreen.max_threads = 248; 3434 rdev->config.evergreen.max_gs_threads = 32; 3435 rdev->config.evergreen.max_stack_entries = 256; 3436 rdev->config.evergreen.sx_num_of_sets = 4; 3437 rdev->config.evergreen.sx_max_export_size = 256; 3438 rdev->config.evergreen.sx_max_export_pos_size = 64; 3439 rdev->config.evergreen.sx_max_export_smx_size = 192; 3440 rdev->config.evergreen.max_hw_contexts = 8; 3441 rdev->config.evergreen.sq_num_cf_insts = 2; 3442 3443 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3444 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3445 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3446 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN; 3447 break; 3448 case CHIP_CAICOS: 3449 rdev->config.evergreen.num_ses = 1; 3450 rdev->config.evergreen.max_pipes = 2; 3451 rdev->config.evergreen.max_tile_pipes = 2; 3452 rdev->config.evergreen.max_simds = 2; 3453 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3454 rdev->config.evergreen.max_gprs = 256; 3455 rdev->config.evergreen.max_threads = 192; 3456 rdev->config.evergreen.max_gs_threads = 16; 3457 rdev->config.evergreen.max_stack_entries = 256; 3458 rdev->config.evergreen.sx_num_of_sets = 4; 3459 rdev->config.evergreen.sx_max_export_size = 128; 3460 rdev->config.evergreen.sx_max_export_pos_size = 32; 3461 rdev->config.evergreen.sx_max_export_smx_size = 96; 3462 rdev->config.evergreen.max_hw_contexts = 4; 3463 rdev->config.evergreen.sq_num_cf_insts = 1; 3464 3465 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3466 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3467 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3468 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN; 3469 break; 3470 } 3471 3472 /* Initialize HDP */ 3473 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3474 WREG32((0x2c14 + j), 0x00000000); 3475 WREG32((0x2c18 + j), 0x00000000); 3476 WREG32((0x2c1c + j), 0x00000000); 3477 WREG32((0x2c20 + j), 0x00000000); 3478 WREG32((0x2c24 + j), 0x00000000); 3479 } 3480 3481 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3482 WREG32(SRBM_INT_CNTL, 0x1); 3483 WREG32(SRBM_INT_ACK, 0x1); 3484 3485 evergreen_fix_pci_max_read_req_size(rdev); 3486 3487 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3488 if ((rdev->family == CHIP_PALM) || 3489 (rdev->family == CHIP_SUMO) || 3490 (rdev->family == CHIP_SUMO2)) 3491 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 3492 else 3493 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3494 3495 /* setup tiling info dword. gb_addr_config is not adequate since it does 3496 * not have bank info, so create a custom tiling dword. 3497 * bits 3:0 num_pipes 3498 * bits 7:4 num_banks 3499 * bits 11:8 group_size 3500 * bits 15:12 row_size 3501 */ 3502 rdev->config.evergreen.tile_config = 0; 3503 switch (rdev->config.evergreen.max_tile_pipes) { 3504 case 1: 3505 default: 3506 rdev->config.evergreen.tile_config |= (0 << 0); 3507 break; 3508 case 2: 3509 rdev->config.evergreen.tile_config |= (1 << 0); 3510 break; 3511 case 4: 3512 rdev->config.evergreen.tile_config |= (2 << 0); 3513 break; 3514 case 8: 3515 rdev->config.evergreen.tile_config |= (3 << 0); 3516 break; 3517 } 3518 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 3519 if (rdev->flags & RADEON_IS_IGP) 3520 rdev->config.evergreen.tile_config |= 1 << 4; 3521 else { 3522 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3523 case 0: /* four banks */ 3524 rdev->config.evergreen.tile_config |= 0 << 4; 3525 break; 3526 case 1: /* eight banks */ 3527 rdev->config.evergreen.tile_config |= 1 << 4; 3528 break; 3529 case 2: /* sixteen banks */ 3530 default: 3531 rdev->config.evergreen.tile_config |= 2 << 4; 3532 break; 3533 } 3534 } 3535 rdev->config.evergreen.tile_config |= 0 << 8; 3536 rdev->config.evergreen.tile_config |= 3537 ((gb_addr_config & 0x30000000) >> 28) << 12; 3538 3539 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { 3540 u32 efuse_straps_4; 3541 u32 efuse_straps_3; 3542 3543 efuse_straps_4 = RREG32_RCU(0x204); 3544 efuse_straps_3 = RREG32_RCU(0x203); 3545 tmp = (((efuse_straps_4 & 0xf) << 4) | 3546 ((efuse_straps_3 & 0xf0000000) >> 28)); 3547 } else { 3548 tmp = 0; 3549 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) { 3550 u32 rb_disable_bitmap; 3551 3552 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3553 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3554 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 3555 tmp <<= 4; 3556 tmp |= rb_disable_bitmap; 3557 } 3558 } 3559 /* enabled rb are just the one not disabled :) */ 3560 disabled_rb_mask = tmp; 3561 tmp = 0; 3562 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3563 tmp |= (1 << i); 3564 /* if all the backends are disabled, fix it up here */ 3565 if ((disabled_rb_mask & tmp) == tmp) { 3566 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3567 disabled_rb_mask &= ~(1 << i); 3568 } 3569 3570 for (i = 0; i < rdev->config.evergreen.num_ses; i++) { 3571 u32 simd_disable_bitmap; 3572 3573 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3574 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3575 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 3576 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; 3577 tmp <<= 16; 3578 tmp |= simd_disable_bitmap; 3579 } 3580 rdev->config.evergreen.active_simds = hweight32(~tmp); 3581 3582 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3583 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3584 3585 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3586 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3587 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3588 WREG32(DMA_TILING_CONFIG, gb_addr_config); 3589 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3590 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3591 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3592 3593 if ((rdev->config.evergreen.max_backends == 1) && 3594 (rdev->flags & RADEON_IS_IGP)) { 3595 if ((disabled_rb_mask & 3) == 1) { 3596 /* RB0 disabled, RB1 enabled */ 3597 tmp = 0x11111111; 3598 } else { 3599 /* RB1 disabled, RB0 enabled */ 3600 tmp = 0x00000000; 3601 } 3602 } else { 3603 tmp = gb_addr_config & NUM_PIPES_MASK; 3604 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends, 3605 EVERGREEN_MAX_BACKENDS, disabled_rb_mask); 3606 } 3607 WREG32(GB_BACKEND_MAP, tmp); 3608 3609 WREG32(CGTS_SYS_TCC_DISABLE, 0); 3610 WREG32(CGTS_TCC_DISABLE, 0); 3611 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 3612 WREG32(CGTS_USER_TCC_DISABLE, 0); 3613 3614 /* set HW defaults for 3D engine */ 3615 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3616 ROQ_IB2_START(0x2b))); 3617 3618 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 3619 3620 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 3621 SYNC_GRADIENT | 3622 SYNC_WALKER | 3623 SYNC_ALIGNER)); 3624 3625 sx_debug_1 = RREG32(SX_DEBUG_1); 3626 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 3627 WREG32(SX_DEBUG_1, sx_debug_1); 3628 3629 3630 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 3631 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 3632 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 3633 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 3634 3635 if (rdev->family <= CHIP_SUMO2) 3636 WREG32(SMX_SAR_CTL0, 0x00010000); 3637 3638 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 3639 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 3640 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 3641 3642 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 3643 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 3644 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 3645 3646 WREG32(VGT_NUM_INSTANCES, 1); 3647 WREG32(SPI_CONFIG_CNTL, 0); 3648 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3649 WREG32(CP_PERFMON_CNTL, 0); 3650 3651 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 3652 FETCH_FIFO_HIWATER(0x4) | 3653 DONE_FIFO_HIWATER(0xe0) | 3654 ALU_UPDATE_FIFO_HIWATER(0x8))); 3655 3656 sq_config = RREG32(SQ_CONFIG); 3657 sq_config &= ~(PS_PRIO(3) | 3658 VS_PRIO(3) | 3659 GS_PRIO(3) | 3660 ES_PRIO(3)); 3661 sq_config |= (VC_ENABLE | 3662 EXPORT_SRC_C | 3663 PS_PRIO(0) | 3664 VS_PRIO(1) | 3665 GS_PRIO(2) | 3666 ES_PRIO(3)); 3667 3668 switch (rdev->family) { 3669 case CHIP_CEDAR: 3670 case CHIP_PALM: 3671 case CHIP_SUMO: 3672 case CHIP_SUMO2: 3673 case CHIP_CAICOS: 3674 /* no vertex cache */ 3675 sq_config &= ~VC_ENABLE; 3676 break; 3677 default: 3678 break; 3679 } 3680 3681 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 3682 3683 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 3684 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 3685 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 3686 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3687 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3688 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3689 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3690 3691 switch (rdev->family) { 3692 case CHIP_CEDAR: 3693 case CHIP_PALM: 3694 case CHIP_SUMO: 3695 case CHIP_SUMO2: 3696 ps_thread_count = 96; 3697 break; 3698 default: 3699 ps_thread_count = 128; 3700 break; 3701 } 3702 3703 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 3704 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3705 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3706 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3707 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3708 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3709 3710 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3711 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3712 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3713 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3714 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3715 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3716 3717 WREG32(SQ_CONFIG, sq_config); 3718 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 3719 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 3720 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 3721 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 3722 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 3723 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 3724 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 3725 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 3726 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 3727 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 3728 3729 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3730 FORCE_EOV_MAX_REZ_CNT(255))); 3731 3732 switch (rdev->family) { 3733 case CHIP_CEDAR: 3734 case CHIP_PALM: 3735 case CHIP_SUMO: 3736 case CHIP_SUMO2: 3737 case CHIP_CAICOS: 3738 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 3739 break; 3740 default: 3741 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 3742 break; 3743 } 3744 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 3745 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 3746 3747 WREG32(VGT_GS_VERTEX_REUSE, 16); 3748 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 3749 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3750 3751 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 3752 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 3753 3754 WREG32(CB_PERF_CTR0_SEL_0, 0); 3755 WREG32(CB_PERF_CTR0_SEL_1, 0); 3756 WREG32(CB_PERF_CTR1_SEL_0, 0); 3757 WREG32(CB_PERF_CTR1_SEL_1, 0); 3758 WREG32(CB_PERF_CTR2_SEL_0, 0); 3759 WREG32(CB_PERF_CTR2_SEL_1, 0); 3760 WREG32(CB_PERF_CTR3_SEL_0, 0); 3761 WREG32(CB_PERF_CTR3_SEL_1, 0); 3762 3763 /* clear render buffer base addresses */ 3764 WREG32(CB_COLOR0_BASE, 0); 3765 WREG32(CB_COLOR1_BASE, 0); 3766 WREG32(CB_COLOR2_BASE, 0); 3767 WREG32(CB_COLOR3_BASE, 0); 3768 WREG32(CB_COLOR4_BASE, 0); 3769 WREG32(CB_COLOR5_BASE, 0); 3770 WREG32(CB_COLOR6_BASE, 0); 3771 WREG32(CB_COLOR7_BASE, 0); 3772 WREG32(CB_COLOR8_BASE, 0); 3773 WREG32(CB_COLOR9_BASE, 0); 3774 WREG32(CB_COLOR10_BASE, 0); 3775 WREG32(CB_COLOR11_BASE, 0); 3776 3777 /* set the shader const cache sizes to 0 */ 3778 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 3779 WREG32(i, 0); 3780 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 3781 WREG32(i, 0); 3782 3783 tmp = RREG32(HDP_MISC_CNTL); 3784 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3785 WREG32(HDP_MISC_CNTL, tmp); 3786 3787 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3788 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3789 3790 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3791 3792 udelay(50); 3793 3794 } 3795 3796 int evergreen_mc_init(struct radeon_device *rdev) 3797 { 3798 u32 tmp; 3799 int chansize, numchan; 3800 3801 /* Get VRAM informations */ 3802 rdev->mc.vram_is_ddr = true; 3803 if ((rdev->family == CHIP_PALM) || 3804 (rdev->family == CHIP_SUMO) || 3805 (rdev->family == CHIP_SUMO2)) 3806 tmp = RREG32(FUS_MC_ARB_RAMCFG); 3807 else 3808 tmp = RREG32(MC_ARB_RAMCFG); 3809 if (tmp & CHANSIZE_OVERRIDE) { 3810 chansize = 16; 3811 } else if (tmp & CHANSIZE_MASK) { 3812 chansize = 64; 3813 } else { 3814 chansize = 32; 3815 } 3816 tmp = RREG32(MC_SHARED_CHMAP); 3817 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3818 case 0: 3819 default: 3820 numchan = 1; 3821 break; 3822 case 1: 3823 numchan = 2; 3824 break; 3825 case 2: 3826 numchan = 4; 3827 break; 3828 case 3: 3829 numchan = 8; 3830 break; 3831 } 3832 rdev->mc.vram_width = numchan * chansize; 3833 /* Could aper size report 0 ? */ 3834 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 3835 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 3836 /* Setup GPU memory space */ 3837 if ((rdev->family == CHIP_PALM) || 3838 (rdev->family == CHIP_SUMO) || 3839 (rdev->family == CHIP_SUMO2)) { 3840 /* size in bytes on fusion */ 3841 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 3842 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 3843 } else { 3844 /* size in MB on evergreen/cayman/tn */ 3845 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3846 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3847 } 3848 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3849 r700_vram_gtt_location(rdev, &rdev->mc); 3850 radeon_update_bandwidth_info(rdev); 3851 3852 return 0; 3853 } 3854 3855 void evergreen_print_gpu_status_regs(struct radeon_device *rdev) 3856 { 3857 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", 3858 RREG32(GRBM_STATUS)); 3859 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", 3860 RREG32(GRBM_STATUS_SE0)); 3861 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", 3862 RREG32(GRBM_STATUS_SE1)); 3863 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", 3864 RREG32(SRBM_STATUS)); 3865 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", 3866 RREG32(SRBM_STATUS2)); 3867 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", 3868 RREG32(CP_STALLED_STAT1)); 3869 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", 3870 RREG32(CP_STALLED_STAT2)); 3871 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", 3872 RREG32(CP_BUSY_STAT)); 3873 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", 3874 RREG32(CP_STAT)); 3875 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", 3876 RREG32(DMA_STATUS_REG)); 3877 if (rdev->family >= CHIP_CAYMAN) { 3878 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", 3879 RREG32(DMA_STATUS_REG + 0x800)); 3880 } 3881 } 3882 3883 bool evergreen_is_display_hung(struct radeon_device *rdev) 3884 { 3885 u32 crtc_hung = 0; 3886 u32 crtc_status[6]; 3887 u32 i, j, tmp; 3888 3889 for (i = 0; i < rdev->num_crtc; i++) { 3890 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { 3891 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3892 crtc_hung |= (1 << i); 3893 } 3894 } 3895 3896 for (j = 0; j < 10; j++) { 3897 for (i = 0; i < rdev->num_crtc; i++) { 3898 if (crtc_hung & (1 << i)) { 3899 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3900 if (tmp != crtc_status[i]) 3901 crtc_hung &= ~(1 << i); 3902 } 3903 } 3904 if (crtc_hung == 0) 3905 return false; 3906 udelay(100); 3907 } 3908 3909 return true; 3910 } 3911 3912 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) 3913 { 3914 u32 reset_mask = 0; 3915 u32 tmp; 3916 3917 /* GRBM_STATUS */ 3918 tmp = RREG32(GRBM_STATUS); 3919 if (tmp & (PA_BUSY | SC_BUSY | 3920 SH_BUSY | SX_BUSY | 3921 TA_BUSY | VGT_BUSY | 3922 DB_BUSY | CB_BUSY | 3923 SPI_BUSY | VGT_BUSY_NO_DMA)) 3924 reset_mask |= RADEON_RESET_GFX; 3925 3926 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3927 CP_BUSY | CP_COHERENCY_BUSY)) 3928 reset_mask |= RADEON_RESET_CP; 3929 3930 if (tmp & GRBM_EE_BUSY) 3931 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3932 3933 /* DMA_STATUS_REG */ 3934 tmp = RREG32(DMA_STATUS_REG); 3935 if (!(tmp & DMA_IDLE)) 3936 reset_mask |= RADEON_RESET_DMA; 3937 3938 /* SRBM_STATUS2 */ 3939 tmp = RREG32(SRBM_STATUS2); 3940 if (tmp & DMA_BUSY) 3941 reset_mask |= RADEON_RESET_DMA; 3942 3943 /* SRBM_STATUS */ 3944 tmp = RREG32(SRBM_STATUS); 3945 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3946 reset_mask |= RADEON_RESET_RLC; 3947 3948 if (tmp & IH_BUSY) 3949 reset_mask |= RADEON_RESET_IH; 3950 3951 if (tmp & SEM_BUSY) 3952 reset_mask |= RADEON_RESET_SEM; 3953 3954 if (tmp & GRBM_RQ_PENDING) 3955 reset_mask |= RADEON_RESET_GRBM; 3956 3957 if (tmp & VMC_BUSY) 3958 reset_mask |= RADEON_RESET_VMC; 3959 3960 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3961 MCC_BUSY | MCD_BUSY)) 3962 reset_mask |= RADEON_RESET_MC; 3963 3964 if (evergreen_is_display_hung(rdev)) 3965 reset_mask |= RADEON_RESET_DISPLAY; 3966 3967 /* VM_L2_STATUS */ 3968 tmp = RREG32(VM_L2_STATUS); 3969 if (tmp & L2_BUSY) 3970 reset_mask |= RADEON_RESET_VMC; 3971 3972 /* Skip MC reset as it's mostly likely not hung, just busy */ 3973 if (reset_mask & RADEON_RESET_MC) { 3974 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3975 reset_mask &= ~RADEON_RESET_MC; 3976 } 3977 3978 return reset_mask; 3979 } 3980 3981 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3982 { 3983 struct evergreen_mc_save save; 3984 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3985 u32 tmp; 3986 3987 if (reset_mask == 0) 3988 return; 3989 3990 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3991 3992 evergreen_print_gpu_status_regs(rdev); 3993 3994 /* Disable CP parsing/prefetching */ 3995 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3996 3997 if (reset_mask & RADEON_RESET_DMA) { 3998 /* Disable DMA */ 3999 tmp = RREG32(DMA_RB_CNTL); 4000 tmp &= ~DMA_RB_ENABLE; 4001 WREG32(DMA_RB_CNTL, tmp); 4002 } 4003 4004 udelay(50); 4005 4006 evergreen_mc_stop(rdev, &save); 4007 if (evergreen_mc_wait_for_idle(rdev)) { 4008 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 4009 } 4010 4011 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 4012 grbm_soft_reset |= SOFT_RESET_DB | 4013 SOFT_RESET_CB | 4014 SOFT_RESET_PA | 4015 SOFT_RESET_SC | 4016 SOFT_RESET_SPI | 4017 SOFT_RESET_SX | 4018 SOFT_RESET_SH | 4019 SOFT_RESET_TC | 4020 SOFT_RESET_TA | 4021 SOFT_RESET_VC | 4022 SOFT_RESET_VGT; 4023 } 4024 4025 if (reset_mask & RADEON_RESET_CP) { 4026 grbm_soft_reset |= SOFT_RESET_CP | 4027 SOFT_RESET_VGT; 4028 4029 srbm_soft_reset |= SOFT_RESET_GRBM; 4030 } 4031 4032 if (reset_mask & RADEON_RESET_DMA) 4033 srbm_soft_reset |= SOFT_RESET_DMA; 4034 4035 if (reset_mask & RADEON_RESET_DISPLAY) 4036 srbm_soft_reset |= SOFT_RESET_DC; 4037 4038 if (reset_mask & RADEON_RESET_RLC) 4039 srbm_soft_reset |= SOFT_RESET_RLC; 4040 4041 if (reset_mask & RADEON_RESET_SEM) 4042 srbm_soft_reset |= SOFT_RESET_SEM; 4043 4044 if (reset_mask & RADEON_RESET_IH) 4045 srbm_soft_reset |= SOFT_RESET_IH; 4046 4047 if (reset_mask & RADEON_RESET_GRBM) 4048 srbm_soft_reset |= SOFT_RESET_GRBM; 4049 4050 if (reset_mask & RADEON_RESET_VMC) 4051 srbm_soft_reset |= SOFT_RESET_VMC; 4052 4053 if (!(rdev->flags & RADEON_IS_IGP)) { 4054 if (reset_mask & RADEON_RESET_MC) 4055 srbm_soft_reset |= SOFT_RESET_MC; 4056 } 4057 4058 if (grbm_soft_reset) { 4059 tmp = RREG32(GRBM_SOFT_RESET); 4060 tmp |= grbm_soft_reset; 4061 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4062 WREG32(GRBM_SOFT_RESET, tmp); 4063 tmp = RREG32(GRBM_SOFT_RESET); 4064 4065 udelay(50); 4066 4067 tmp &= ~grbm_soft_reset; 4068 WREG32(GRBM_SOFT_RESET, tmp); 4069 tmp = RREG32(GRBM_SOFT_RESET); 4070 } 4071 4072 if (srbm_soft_reset) { 4073 tmp = RREG32(SRBM_SOFT_RESET); 4074 tmp |= srbm_soft_reset; 4075 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 4076 WREG32(SRBM_SOFT_RESET, tmp); 4077 tmp = RREG32(SRBM_SOFT_RESET); 4078 4079 udelay(50); 4080 4081 tmp &= ~srbm_soft_reset; 4082 WREG32(SRBM_SOFT_RESET, tmp); 4083 tmp = RREG32(SRBM_SOFT_RESET); 4084 } 4085 4086 /* Wait a little for things to settle down */ 4087 udelay(50); 4088 4089 evergreen_mc_resume(rdev, &save); 4090 udelay(50); 4091 4092 evergreen_print_gpu_status_regs(rdev); 4093 } 4094 4095 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) 4096 { 4097 struct evergreen_mc_save save; 4098 u32 tmp, i; 4099 4100 dev_info(rdev->dev, "GPU pci config reset\n"); 4101 4102 /* disable dpm? */ 4103 4104 /* Disable CP parsing/prefetching */ 4105 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 4106 udelay(50); 4107 /* Disable DMA */ 4108 tmp = RREG32(DMA_RB_CNTL); 4109 tmp &= ~DMA_RB_ENABLE; 4110 WREG32(DMA_RB_CNTL, tmp); 4111 /* XXX other engines? */ 4112 4113 /* halt the rlc */ 4114 r600_rlc_stop(rdev); 4115 4116 udelay(50); 4117 4118 /* set mclk/sclk to bypass */ 4119 rv770_set_clk_bypass_mode(rdev); 4120 /* disable BM */ 4121 pci_clear_master(rdev->pdev); 4122 /* disable mem access */ 4123 evergreen_mc_stop(rdev, &save); 4124 if (evergreen_mc_wait_for_idle(rdev)) { 4125 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 4126 } 4127 /* reset */ 4128 radeon_pci_config_reset(rdev); 4129 /* wait for asic to come out of reset */ 4130 for (i = 0; i < rdev->usec_timeout; i++) { 4131 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 4132 break; 4133 udelay(1); 4134 } 4135 } 4136 4137 int evergreen_asic_reset(struct radeon_device *rdev, bool hard) 4138 { 4139 u32 reset_mask; 4140 4141 if (hard) { 4142 evergreen_gpu_pci_config_reset(rdev); 4143 return 0; 4144 } 4145 4146 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4147 4148 if (reset_mask) 4149 r600_set_bios_scratch_engine_hung(rdev, true); 4150 4151 /* try soft reset */ 4152 evergreen_gpu_soft_reset(rdev, reset_mask); 4153 4154 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4155 4156 /* try pci config reset */ 4157 if (reset_mask && radeon_hard_reset) 4158 evergreen_gpu_pci_config_reset(rdev); 4159 4160 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4161 4162 if (!reset_mask) 4163 r600_set_bios_scratch_engine_hung(rdev, false); 4164 4165 return 0; 4166 } 4167 4168 /** 4169 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up 4170 * 4171 * @rdev: radeon_device pointer 4172 * @ring: radeon_ring structure holding ring information 4173 * 4174 * Check if the GFX engine is locked up. 4175 * Returns true if the engine appears to be locked up, false if not. 4176 */ 4177 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 4178 { 4179 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4180 4181 if (!(reset_mask & (RADEON_RESET_GFX | 4182 RADEON_RESET_COMPUTE | 4183 RADEON_RESET_CP))) { 4184 radeon_ring_lockup_update(rdev, ring); 4185 return false; 4186 } 4187 return radeon_ring_test_lockup(rdev, ring); 4188 } 4189 4190 /* 4191 * RLC 4192 */ 4193 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 4194 #define RLC_CLEAR_STATE_END_MARKER 0x00000001 4195 4196 void sumo_rlc_fini(struct radeon_device *rdev) 4197 { 4198 int r; 4199 4200 /* save restore block */ 4201 if (rdev->rlc.save_restore_obj) { 4202 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4203 if (unlikely(r != 0)) 4204 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 4205 radeon_bo_unpin(rdev->rlc.save_restore_obj); 4206 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4207 4208 radeon_bo_unref(&rdev->rlc.save_restore_obj); 4209 rdev->rlc.save_restore_obj = NULL; 4210 } 4211 4212 /* clear state block */ 4213 if (rdev->rlc.clear_state_obj) { 4214 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4215 if (unlikely(r != 0)) 4216 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 4217 radeon_bo_unpin(rdev->rlc.clear_state_obj); 4218 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4219 4220 radeon_bo_unref(&rdev->rlc.clear_state_obj); 4221 rdev->rlc.clear_state_obj = NULL; 4222 } 4223 4224 /* clear state block */ 4225 if (rdev->rlc.cp_table_obj) { 4226 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4227 if (unlikely(r != 0)) 4228 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4229 radeon_bo_unpin(rdev->rlc.cp_table_obj); 4230 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4231 4232 radeon_bo_unref(&rdev->rlc.cp_table_obj); 4233 rdev->rlc.cp_table_obj = NULL; 4234 } 4235 } 4236 4237 #define CP_ME_TABLE_SIZE 96 4238 4239 int sumo_rlc_init(struct radeon_device *rdev) 4240 { 4241 const u32 *src_ptr; 4242 volatile u32 *dst_ptr; 4243 u32 dws, data, i, j, k, reg_num; 4244 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; 4245 u64 reg_list_mc_addr; 4246 const struct cs_section_def *cs_data; 4247 int r; 4248 4249 src_ptr = rdev->rlc.reg_list; 4250 dws = rdev->rlc.reg_list_size; 4251 if (rdev->family >= CHIP_BONAIRE) { 4252 dws += (5 * 16) + 48 + 48 + 64; 4253 } 4254 cs_data = rdev->rlc.cs_data; 4255 4256 if (src_ptr) { 4257 /* save restore block */ 4258 if (rdev->rlc.save_restore_obj == NULL) { 4259 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4260 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4261 NULL, &rdev->rlc.save_restore_obj); 4262 if (r) { 4263 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 4264 return r; 4265 } 4266 } 4267 4268 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4269 if (unlikely(r != 0)) { 4270 sumo_rlc_fini(rdev); 4271 return r; 4272 } 4273 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 4274 &rdev->rlc.save_restore_gpu_addr); 4275 if (r) { 4276 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4277 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 4278 sumo_rlc_fini(rdev); 4279 return r; 4280 } 4281 4282 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr); 4283 if (r) { 4284 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); 4285 sumo_rlc_fini(rdev); 4286 return r; 4287 } 4288 /* write the sr buffer */ 4289 dst_ptr = rdev->rlc.sr_ptr; 4290 if (rdev->family >= CHIP_TAHITI) { 4291 /* SI */ 4292 for (i = 0; i < rdev->rlc.reg_list_size; i++) 4293 dst_ptr[i] = cpu_to_le32(src_ptr[i]); 4294 } else { 4295 /* ON/LN/TN */ 4296 /* format: 4297 * dw0: (reg2 << 16) | reg1 4298 * dw1: reg1 save space 4299 * dw2: reg2 save space 4300 */ 4301 for (i = 0; i < dws; i++) { 4302 data = src_ptr[i] >> 2; 4303 i++; 4304 if (i < dws) 4305 data |= (src_ptr[i] >> 2) << 16; 4306 j = (((i - 1) * 3) / 2); 4307 dst_ptr[j] = cpu_to_le32(data); 4308 } 4309 j = ((i * 3) / 2); 4310 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); 4311 } 4312 radeon_bo_kunmap(rdev->rlc.save_restore_obj); 4313 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4314 } 4315 4316 if (cs_data) { 4317 /* clear state block */ 4318 if (rdev->family >= CHIP_BONAIRE) { 4319 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); 4320 } else if (rdev->family >= CHIP_TAHITI) { 4321 rdev->rlc.clear_state_size = si_get_csb_size(rdev); 4322 dws = rdev->rlc.clear_state_size + (256 / 4); 4323 } else { 4324 reg_list_num = 0; 4325 dws = 0; 4326 for (i = 0; cs_data[i].section != NULL; i++) { 4327 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4328 reg_list_num++; 4329 dws += cs_data[i].section[j].reg_count; 4330 } 4331 } 4332 reg_list_blk_index = (3 * reg_list_num + 2); 4333 dws += reg_list_blk_index; 4334 rdev->rlc.clear_state_size = dws; 4335 } 4336 4337 if (rdev->rlc.clear_state_obj == NULL) { 4338 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4339 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4340 NULL, &rdev->rlc.clear_state_obj); 4341 if (r) { 4342 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 4343 sumo_rlc_fini(rdev); 4344 return r; 4345 } 4346 } 4347 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4348 if (unlikely(r != 0)) { 4349 sumo_rlc_fini(rdev); 4350 return r; 4351 } 4352 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 4353 &rdev->rlc.clear_state_gpu_addr); 4354 if (r) { 4355 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4356 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 4357 sumo_rlc_fini(rdev); 4358 return r; 4359 } 4360 4361 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr); 4362 if (r) { 4363 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); 4364 sumo_rlc_fini(rdev); 4365 return r; 4366 } 4367 /* set up the cs buffer */ 4368 dst_ptr = rdev->rlc.cs_ptr; 4369 if (rdev->family >= CHIP_BONAIRE) { 4370 cik_get_csb_buffer(rdev, dst_ptr); 4371 } else if (rdev->family >= CHIP_TAHITI) { 4372 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; 4373 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); 4374 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); 4375 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); 4376 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); 4377 } else { 4378 reg_list_hdr_blk_index = 0; 4379 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); 4380 data = upper_32_bits(reg_list_mc_addr); 4381 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4382 reg_list_hdr_blk_index++; 4383 for (i = 0; cs_data[i].section != NULL; i++) { 4384 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4385 reg_num = cs_data[i].section[j].reg_count; 4386 data = reg_list_mc_addr & 0xffffffff; 4387 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4388 reg_list_hdr_blk_index++; 4389 4390 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; 4391 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4392 reg_list_hdr_blk_index++; 4393 4394 data = 0x08000000 | (reg_num * 4); 4395 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4396 reg_list_hdr_blk_index++; 4397 4398 for (k = 0; k < reg_num; k++) { 4399 data = cs_data[i].section[j].extent[k]; 4400 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); 4401 } 4402 reg_list_mc_addr += reg_num * 4; 4403 reg_list_blk_index += reg_num; 4404 } 4405 } 4406 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); 4407 } 4408 radeon_bo_kunmap(rdev->rlc.clear_state_obj); 4409 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4410 } 4411 4412 if (rdev->rlc.cp_table_size) { 4413 if (rdev->rlc.cp_table_obj == NULL) { 4414 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, 4415 PAGE_SIZE, true, 4416 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4417 NULL, &rdev->rlc.cp_table_obj); 4418 if (r) { 4419 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); 4420 sumo_rlc_fini(rdev); 4421 return r; 4422 } 4423 } 4424 4425 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4426 if (unlikely(r != 0)) { 4427 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4428 sumo_rlc_fini(rdev); 4429 return r; 4430 } 4431 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, 4432 &rdev->rlc.cp_table_gpu_addr); 4433 if (r) { 4434 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4435 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); 4436 sumo_rlc_fini(rdev); 4437 return r; 4438 } 4439 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr); 4440 if (r) { 4441 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); 4442 sumo_rlc_fini(rdev); 4443 return r; 4444 } 4445 4446 cik_init_cp_pg_table(rdev); 4447 4448 radeon_bo_kunmap(rdev->rlc.cp_table_obj); 4449 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4450 4451 } 4452 4453 return 0; 4454 } 4455 4456 static void evergreen_rlc_start(struct radeon_device *rdev) 4457 { 4458 u32 mask = RLC_ENABLE; 4459 4460 if (rdev->flags & RADEON_IS_IGP) { 4461 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; 4462 } 4463 4464 WREG32(RLC_CNTL, mask); 4465 } 4466 4467 int evergreen_rlc_resume(struct radeon_device *rdev) 4468 { 4469 u32 i; 4470 const __be32 *fw_data; 4471 4472 if (!rdev->rlc_fw) 4473 return -EINVAL; 4474 4475 r600_rlc_stop(rdev); 4476 4477 WREG32(RLC_HB_CNTL, 0); 4478 4479 if (rdev->flags & RADEON_IS_IGP) { 4480 if (rdev->family == CHIP_ARUBA) { 4481 u32 always_on_bitmap = 4482 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); 4483 /* find out the number of active simds */ 4484 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 4485 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; 4486 tmp = hweight32(~tmp); 4487 if (tmp == rdev->config.cayman.max_simds_per_se) { 4488 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); 4489 WREG32(TN_RLC_LB_PARAMS, 0x00601004); 4490 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); 4491 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); 4492 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); 4493 } 4494 } else { 4495 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4496 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4497 } 4498 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4499 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4500 } else { 4501 WREG32(RLC_HB_BASE, 0); 4502 WREG32(RLC_HB_RPTR, 0); 4503 WREG32(RLC_HB_WPTR, 0); 4504 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4505 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4506 } 4507 WREG32(RLC_MC_CNTL, 0); 4508 WREG32(RLC_UCODE_CNTL, 0); 4509 4510 fw_data = (const __be32 *)rdev->rlc_fw->data; 4511 if (rdev->family >= CHIP_ARUBA) { 4512 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { 4513 WREG32(RLC_UCODE_ADDR, i); 4514 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4515 } 4516 } else if (rdev->family >= CHIP_CAYMAN) { 4517 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { 4518 WREG32(RLC_UCODE_ADDR, i); 4519 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4520 } 4521 } else { 4522 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { 4523 WREG32(RLC_UCODE_ADDR, i); 4524 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4525 } 4526 } 4527 WREG32(RLC_UCODE_ADDR, 0); 4528 4529 evergreen_rlc_start(rdev); 4530 4531 return 0; 4532 } 4533 4534 /* Interrupts */ 4535 4536 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 4537 { 4538 if (crtc >= rdev->num_crtc) 4539 return 0; 4540 else 4541 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); 4542 } 4543 4544 void evergreen_disable_interrupt_state(struct radeon_device *rdev) 4545 { 4546 u32 tmp; 4547 4548 if (rdev->family >= CHIP_CAYMAN) { 4549 cayman_cp_int_cntl_setup(rdev, 0, 4550 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4551 cayman_cp_int_cntl_setup(rdev, 1, 0); 4552 cayman_cp_int_cntl_setup(rdev, 2, 0); 4553 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4554 WREG32(CAYMAN_DMA1_CNTL, tmp); 4555 } else 4556 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4557 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4558 WREG32(DMA_CNTL, tmp); 4559 WREG32(GRBM_INT_CNTL, 0); 4560 WREG32(SRBM_INT_CNTL, 0); 4561 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4562 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4563 if (rdev->num_crtc >= 4) { 4564 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4565 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4566 } 4567 if (rdev->num_crtc >= 6) { 4568 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4569 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4570 } 4571 4572 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4573 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4574 if (rdev->num_crtc >= 4) { 4575 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4576 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4577 } 4578 if (rdev->num_crtc >= 6) { 4579 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4580 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4581 } 4582 4583 /* only one DAC on DCE5 */ 4584 if (!ASIC_IS_DCE5(rdev)) 4585 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 4586 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 4587 4588 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4589 WREG32(DC_HPD1_INT_CONTROL, tmp); 4590 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4591 WREG32(DC_HPD2_INT_CONTROL, tmp); 4592 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4593 WREG32(DC_HPD3_INT_CONTROL, tmp); 4594 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4595 WREG32(DC_HPD4_INT_CONTROL, tmp); 4596 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4597 WREG32(DC_HPD5_INT_CONTROL, tmp); 4598 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4599 WREG32(DC_HPD6_INT_CONTROL, tmp); 4600 4601 } 4602 4603 int evergreen_irq_set(struct radeon_device *rdev) 4604 { 4605 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 4606 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 4607 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 4608 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 4609 u32 grbm_int_cntl = 0; 4610 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; 4611 u32 dma_cntl, dma_cntl1 = 0; 4612 u32 thermal_int = 0; 4613 4614 if (!rdev->irq.installed) { 4615 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 4616 return -EINVAL; 4617 } 4618 /* don't enable anything if the ih is disabled */ 4619 if (!rdev->ih.enabled) { 4620 r600_disable_interrupts(rdev); 4621 /* force the active interrupt state to all disabled */ 4622 evergreen_disable_interrupt_state(rdev); 4623 return 0; 4624 } 4625 4626 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4627 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4628 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4629 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4630 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4631 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN); 4632 if (rdev->family == CHIP_ARUBA) 4633 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & 4634 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4635 else 4636 thermal_int = RREG32(CG_THERMAL_INT) & 4637 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4638 4639 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4640 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4641 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4642 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4643 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4644 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4645 4646 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4647 4648 if (rdev->family >= CHIP_CAYMAN) { 4649 /* enable CP interrupts on all rings */ 4650 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4651 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4652 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4653 } 4654 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 4655 DRM_DEBUG("evergreen_irq_set: sw int cp1\n"); 4656 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 4657 } 4658 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 4659 DRM_DEBUG("evergreen_irq_set: sw int cp2\n"); 4660 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 4661 } 4662 } else { 4663 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4664 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4665 cp_int_cntl |= RB_INT_ENABLE; 4666 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4667 } 4668 } 4669 4670 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 4671 DRM_DEBUG("r600_irq_set: sw int dma\n"); 4672 dma_cntl |= TRAP_ENABLE; 4673 } 4674 4675 if (rdev->family >= CHIP_CAYMAN) { 4676 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4677 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 4678 DRM_DEBUG("r600_irq_set: sw int dma1\n"); 4679 dma_cntl1 |= TRAP_ENABLE; 4680 } 4681 } 4682 4683 if (rdev->irq.dpm_thermal) { 4684 DRM_DEBUG("dpm thermal\n"); 4685 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 4686 } 4687 4688 if (rdev->irq.crtc_vblank_int[0] || 4689 atomic_read(&rdev->irq.pflip[0])) { 4690 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 4691 crtc1 |= VBLANK_INT_MASK; 4692 } 4693 if (rdev->irq.crtc_vblank_int[1] || 4694 atomic_read(&rdev->irq.pflip[1])) { 4695 DRM_DEBUG("evergreen_irq_set: vblank 1\n"); 4696 crtc2 |= VBLANK_INT_MASK; 4697 } 4698 if (rdev->irq.crtc_vblank_int[2] || 4699 atomic_read(&rdev->irq.pflip[2])) { 4700 DRM_DEBUG("evergreen_irq_set: vblank 2\n"); 4701 crtc3 |= VBLANK_INT_MASK; 4702 } 4703 if (rdev->irq.crtc_vblank_int[3] || 4704 atomic_read(&rdev->irq.pflip[3])) { 4705 DRM_DEBUG("evergreen_irq_set: vblank 3\n"); 4706 crtc4 |= VBLANK_INT_MASK; 4707 } 4708 if (rdev->irq.crtc_vblank_int[4] || 4709 atomic_read(&rdev->irq.pflip[4])) { 4710 DRM_DEBUG("evergreen_irq_set: vblank 4\n"); 4711 crtc5 |= VBLANK_INT_MASK; 4712 } 4713 if (rdev->irq.crtc_vblank_int[5] || 4714 atomic_read(&rdev->irq.pflip[5])) { 4715 DRM_DEBUG("evergreen_irq_set: vblank 5\n"); 4716 crtc6 |= VBLANK_INT_MASK; 4717 } 4718 if (rdev->irq.hpd[0]) { 4719 DRM_DEBUG("evergreen_irq_set: hpd 1\n"); 4720 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4721 } 4722 if (rdev->irq.hpd[1]) { 4723 DRM_DEBUG("evergreen_irq_set: hpd 2\n"); 4724 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4725 } 4726 if (rdev->irq.hpd[2]) { 4727 DRM_DEBUG("evergreen_irq_set: hpd 3\n"); 4728 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4729 } 4730 if (rdev->irq.hpd[3]) { 4731 DRM_DEBUG("evergreen_irq_set: hpd 4\n"); 4732 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4733 } 4734 if (rdev->irq.hpd[4]) { 4735 DRM_DEBUG("evergreen_irq_set: hpd 5\n"); 4736 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4737 } 4738 if (rdev->irq.hpd[5]) { 4739 DRM_DEBUG("evergreen_irq_set: hpd 6\n"); 4740 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN; 4741 } 4742 if (rdev->irq.afmt[0]) { 4743 DRM_DEBUG("evergreen_irq_set: hdmi 0\n"); 4744 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4745 } 4746 if (rdev->irq.afmt[1]) { 4747 DRM_DEBUG("evergreen_irq_set: hdmi 1\n"); 4748 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4749 } 4750 if (rdev->irq.afmt[2]) { 4751 DRM_DEBUG("evergreen_irq_set: hdmi 2\n"); 4752 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4753 } 4754 if (rdev->irq.afmt[3]) { 4755 DRM_DEBUG("evergreen_irq_set: hdmi 3\n"); 4756 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4757 } 4758 if (rdev->irq.afmt[4]) { 4759 DRM_DEBUG("evergreen_irq_set: hdmi 4\n"); 4760 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4761 } 4762 if (rdev->irq.afmt[5]) { 4763 DRM_DEBUG("evergreen_irq_set: hdmi 5\n"); 4764 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4765 } 4766 4767 if (rdev->family >= CHIP_CAYMAN) { 4768 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl); 4769 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1); 4770 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2); 4771 } else 4772 WREG32(CP_INT_CNTL, cp_int_cntl); 4773 4774 WREG32(DMA_CNTL, dma_cntl); 4775 4776 if (rdev->family >= CHIP_CAYMAN) 4777 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1); 4778 4779 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 4780 4781 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 4782 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 4783 if (rdev->num_crtc >= 4) { 4784 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 4785 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 4786 } 4787 if (rdev->num_crtc >= 6) { 4788 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 4789 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 4790 } 4791 4792 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 4793 GRPH_PFLIP_INT_MASK); 4794 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 4795 GRPH_PFLIP_INT_MASK); 4796 if (rdev->num_crtc >= 4) { 4797 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 4798 GRPH_PFLIP_INT_MASK); 4799 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 4800 GRPH_PFLIP_INT_MASK); 4801 } 4802 if (rdev->num_crtc >= 6) { 4803 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 4804 GRPH_PFLIP_INT_MASK); 4805 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 4806 GRPH_PFLIP_INT_MASK); 4807 } 4808 4809 WREG32(DC_HPD1_INT_CONTROL, hpd1); 4810 WREG32(DC_HPD2_INT_CONTROL, hpd2); 4811 WREG32(DC_HPD3_INT_CONTROL, hpd3); 4812 WREG32(DC_HPD4_INT_CONTROL, hpd4); 4813 WREG32(DC_HPD5_INT_CONTROL, hpd5); 4814 WREG32(DC_HPD6_INT_CONTROL, hpd6); 4815 if (rdev->family == CHIP_ARUBA) 4816 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); 4817 else 4818 WREG32(CG_THERMAL_INT, thermal_int); 4819 4820 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1); 4821 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2); 4822 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3); 4823 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4); 4824 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5); 4825 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6); 4826 4827 /* posting read */ 4828 RREG32(SRBM_STATUS); 4829 4830 return 0; 4831 } 4832 4833 static void evergreen_irq_ack(struct radeon_device *rdev) 4834 { 4835 u32 tmp; 4836 4837 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 4838 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 4839 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 4840 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 4841 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 4842 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 4843 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4844 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4845 if (rdev->num_crtc >= 4) { 4846 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4847 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4848 } 4849 if (rdev->num_crtc >= 6) { 4850 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4851 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4852 } 4853 4854 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4855 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4856 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4857 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4858 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4859 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4860 4861 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 4862 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4863 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 4864 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4865 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 4866 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 4867 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 4868 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 4869 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 4870 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 4871 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 4872 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 4873 4874 if (rdev->num_crtc >= 4) { 4875 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 4876 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4877 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 4878 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4879 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 4880 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 4881 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 4882 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 4883 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 4884 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 4885 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 4886 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 4887 } 4888 4889 if (rdev->num_crtc >= 6) { 4890 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 4891 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4892 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 4893 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4894 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 4895 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 4896 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 4897 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 4898 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 4899 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 4900 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 4901 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 4902 } 4903 4904 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 4905 tmp = RREG32(DC_HPD1_INT_CONTROL); 4906 tmp |= DC_HPDx_INT_ACK; 4907 WREG32(DC_HPD1_INT_CONTROL, tmp); 4908 } 4909 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 4910 tmp = RREG32(DC_HPD2_INT_CONTROL); 4911 tmp |= DC_HPDx_INT_ACK; 4912 WREG32(DC_HPD2_INT_CONTROL, tmp); 4913 } 4914 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 4915 tmp = RREG32(DC_HPD3_INT_CONTROL); 4916 tmp |= DC_HPDx_INT_ACK; 4917 WREG32(DC_HPD3_INT_CONTROL, tmp); 4918 } 4919 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 4920 tmp = RREG32(DC_HPD4_INT_CONTROL); 4921 tmp |= DC_HPDx_INT_ACK; 4922 WREG32(DC_HPD4_INT_CONTROL, tmp); 4923 } 4924 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 4925 tmp = RREG32(DC_HPD5_INT_CONTROL); 4926 tmp |= DC_HPDx_INT_ACK; 4927 WREG32(DC_HPD5_INT_CONTROL, tmp); 4928 } 4929 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 4930 tmp = RREG32(DC_HPD5_INT_CONTROL); 4931 tmp |= DC_HPDx_INT_ACK; 4932 WREG32(DC_HPD6_INT_CONTROL, tmp); 4933 } 4934 4935 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) { 4936 tmp = RREG32(DC_HPD1_INT_CONTROL); 4937 tmp |= DC_HPDx_RX_INT_ACK; 4938 WREG32(DC_HPD1_INT_CONTROL, tmp); 4939 } 4940 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) { 4941 tmp = RREG32(DC_HPD2_INT_CONTROL); 4942 tmp |= DC_HPDx_RX_INT_ACK; 4943 WREG32(DC_HPD2_INT_CONTROL, tmp); 4944 } 4945 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { 4946 tmp = RREG32(DC_HPD3_INT_CONTROL); 4947 tmp |= DC_HPDx_RX_INT_ACK; 4948 WREG32(DC_HPD3_INT_CONTROL, tmp); 4949 } 4950 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { 4951 tmp = RREG32(DC_HPD4_INT_CONTROL); 4952 tmp |= DC_HPDx_RX_INT_ACK; 4953 WREG32(DC_HPD4_INT_CONTROL, tmp); 4954 } 4955 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { 4956 tmp = RREG32(DC_HPD5_INT_CONTROL); 4957 tmp |= DC_HPDx_RX_INT_ACK; 4958 WREG32(DC_HPD5_INT_CONTROL, tmp); 4959 } 4960 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { 4961 tmp = RREG32(DC_HPD5_INT_CONTROL); 4962 tmp |= DC_HPDx_RX_INT_ACK; 4963 WREG32(DC_HPD6_INT_CONTROL, tmp); 4964 } 4965 4966 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 4967 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); 4968 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4969 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp); 4970 } 4971 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 4972 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); 4973 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4974 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp); 4975 } 4976 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 4977 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET); 4978 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4979 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp); 4980 } 4981 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 4982 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); 4983 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4984 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp); 4985 } 4986 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 4987 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET); 4988 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4989 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp); 4990 } 4991 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 4992 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 4993 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4994 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp); 4995 } 4996 } 4997 4998 static void evergreen_irq_disable(struct radeon_device *rdev) 4999 { 5000 r600_disable_interrupts(rdev); 5001 /* Wait and acknowledge irq */ 5002 mdelay(1); 5003 evergreen_irq_ack(rdev); 5004 evergreen_disable_interrupt_state(rdev); 5005 } 5006 5007 void evergreen_irq_suspend(struct radeon_device *rdev) 5008 { 5009 evergreen_irq_disable(rdev); 5010 r600_rlc_stop(rdev); 5011 } 5012 5013 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 5014 { 5015 u32 wptr, tmp; 5016 5017 if (rdev->wb.enabled) 5018 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 5019 else 5020 wptr = RREG32(IH_RB_WPTR); 5021 5022 if (wptr & RB_OVERFLOW) { 5023 wptr &= ~RB_OVERFLOW; 5024 /* When a ring buffer overflow happen start parsing interrupt 5025 * from the last not overwritten vector (wptr + 16). Hopefully 5026 * this should allow us to catchup. 5027 */ 5028 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 5029 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 5030 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 5031 tmp = RREG32(IH_RB_CNTL); 5032 tmp |= IH_WPTR_OVERFLOW_CLEAR; 5033 WREG32(IH_RB_CNTL, tmp); 5034 } 5035 return (wptr & rdev->ih.ptr_mask); 5036 } 5037 5038 int evergreen_irq_process(struct radeon_device *rdev) 5039 { 5040 u32 wptr; 5041 u32 rptr; 5042 u32 src_id, src_data; 5043 u32 ring_index; 5044 bool queue_hotplug = false; 5045 bool queue_hdmi = false; 5046 bool queue_dp = false; 5047 bool queue_thermal = false; 5048 u32 status, addr; 5049 5050 if (!rdev->ih.enabled || rdev->shutdown) 5051 return IRQ_NONE; 5052 5053 wptr = evergreen_get_ih_wptr(rdev); 5054 5055 restart_ih: 5056 /* is somebody else already processing irqs? */ 5057 if (atomic_xchg(&rdev->ih.lock, 1)) 5058 return IRQ_NONE; 5059 5060 rptr = rdev->ih.rptr; 5061 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 5062 5063 /* Order reading of wptr vs. reading of IH ring data */ 5064 rmb(); 5065 5066 /* display interrupts */ 5067 evergreen_irq_ack(rdev); 5068 5069 while (rptr != wptr) { 5070 /* wptr/rptr are in bytes! */ 5071 ring_index = rptr / 4; 5072 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 5073 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 5074 5075 switch (src_id) { 5076 case 1: /* D1 vblank/vline */ 5077 switch (src_data) { 5078 case 0: /* D1 vblank */ 5079 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)) 5080 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n"); 5081 5082 if (rdev->irq.crtc_vblank_int[0]) { 5083 drm_handle_vblank(rdev->ddev, 0); 5084 rdev->pm.vblank_sync = true; 5085 wake_up(&rdev->irq.vblank_queue); 5086 } 5087 if (atomic_read(&rdev->irq.pflip[0])) 5088 radeon_crtc_handle_vblank(rdev, 0); 5089 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 5090 DRM_DEBUG("IH: D1 vblank\n"); 5091 5092 break; 5093 case 1: /* D1 vline */ 5094 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)) 5095 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n"); 5096 5097 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 5098 DRM_DEBUG("IH: D1 vline\n"); 5099 5100 break; 5101 default: 5102 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5103 break; 5104 } 5105 break; 5106 case 2: /* D2 vblank/vline */ 5107 switch (src_data) { 5108 case 0: /* D2 vblank */ 5109 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) 5110 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n"); 5111 5112 if (rdev->irq.crtc_vblank_int[1]) { 5113 drm_handle_vblank(rdev->ddev, 1); 5114 rdev->pm.vblank_sync = true; 5115 wake_up(&rdev->irq.vblank_queue); 5116 } 5117 if (atomic_read(&rdev->irq.pflip[1])) 5118 radeon_crtc_handle_vblank(rdev, 1); 5119 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 5120 DRM_DEBUG("IH: D2 vblank\n"); 5121 5122 break; 5123 case 1: /* D2 vline */ 5124 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)) 5125 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n"); 5126 5127 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 5128 DRM_DEBUG("IH: D2 vline\n"); 5129 5130 break; 5131 default: 5132 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5133 break; 5134 } 5135 break; 5136 case 3: /* D3 vblank/vline */ 5137 switch (src_data) { 5138 case 0: /* D3 vblank */ 5139 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) 5140 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n"); 5141 5142 if (rdev->irq.crtc_vblank_int[2]) { 5143 drm_handle_vblank(rdev->ddev, 2); 5144 rdev->pm.vblank_sync = true; 5145 wake_up(&rdev->irq.vblank_queue); 5146 } 5147 if (atomic_read(&rdev->irq.pflip[2])) 5148 radeon_crtc_handle_vblank(rdev, 2); 5149 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 5150 DRM_DEBUG("IH: D3 vblank\n"); 5151 5152 break; 5153 case 1: /* D3 vline */ 5154 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) 5155 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n"); 5156 5157 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 5158 DRM_DEBUG("IH: D3 vline\n"); 5159 5160 break; 5161 default: 5162 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5163 break; 5164 } 5165 break; 5166 case 4: /* D4 vblank/vline */ 5167 switch (src_data) { 5168 case 0: /* D4 vblank */ 5169 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) 5170 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n"); 5171 5172 if (rdev->irq.crtc_vblank_int[3]) { 5173 drm_handle_vblank(rdev->ddev, 3); 5174 rdev->pm.vblank_sync = true; 5175 wake_up(&rdev->irq.vblank_queue); 5176 } 5177 if (atomic_read(&rdev->irq.pflip[3])) 5178 radeon_crtc_handle_vblank(rdev, 3); 5179 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 5180 DRM_DEBUG("IH: D4 vblank\n"); 5181 5182 break; 5183 case 1: /* D4 vline */ 5184 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) 5185 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n"); 5186 5187 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 5188 DRM_DEBUG("IH: D4 vline\n"); 5189 5190 break; 5191 default: 5192 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5193 break; 5194 } 5195 break; 5196 case 5: /* D5 vblank/vline */ 5197 switch (src_data) { 5198 case 0: /* D5 vblank */ 5199 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) 5200 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n"); 5201 5202 if (rdev->irq.crtc_vblank_int[4]) { 5203 drm_handle_vblank(rdev->ddev, 4); 5204 rdev->pm.vblank_sync = true; 5205 wake_up(&rdev->irq.vblank_queue); 5206 } 5207 if (atomic_read(&rdev->irq.pflip[4])) 5208 radeon_crtc_handle_vblank(rdev, 4); 5209 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 5210 DRM_DEBUG("IH: D5 vblank\n"); 5211 5212 break; 5213 case 1: /* D5 vline */ 5214 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) 5215 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n"); 5216 5217 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 5218 DRM_DEBUG("IH: D5 vline\n"); 5219 5220 break; 5221 default: 5222 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5223 break; 5224 } 5225 break; 5226 case 6: /* D6 vblank/vline */ 5227 switch (src_data) { 5228 case 0: /* D6 vblank */ 5229 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) 5230 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n"); 5231 5232 if (rdev->irq.crtc_vblank_int[5]) { 5233 drm_handle_vblank(rdev->ddev, 5); 5234 rdev->pm.vblank_sync = true; 5235 wake_up(&rdev->irq.vblank_queue); 5236 } 5237 if (atomic_read(&rdev->irq.pflip[5])) 5238 radeon_crtc_handle_vblank(rdev, 5); 5239 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 5240 DRM_DEBUG("IH: D6 vblank\n"); 5241 5242 break; 5243 case 1: /* D6 vline */ 5244 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) 5245 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n"); 5246 5247 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 5248 DRM_DEBUG("IH: D6 vline\n"); 5249 5250 break; 5251 default: 5252 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5253 break; 5254 } 5255 break; 5256 case 8: /* D1 page flip */ 5257 case 10: /* D2 page flip */ 5258 case 12: /* D3 page flip */ 5259 case 14: /* D4 page flip */ 5260 case 16: /* D5 page flip */ 5261 case 18: /* D6 page flip */ 5262 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 5263 if (radeon_use_pflipirq > 0) 5264 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 5265 break; 5266 case 42: /* HPD hotplug */ 5267 switch (src_data) { 5268 case 0: 5269 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT)) 5270 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5271 5272 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 5273 queue_hotplug = true; 5274 DRM_DEBUG("IH: HPD1\n"); 5275 break; 5276 case 1: 5277 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT)) 5278 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5279 5280 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 5281 queue_hotplug = true; 5282 DRM_DEBUG("IH: HPD2\n"); 5283 break; 5284 case 2: 5285 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT)) 5286 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5287 5288 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 5289 queue_hotplug = true; 5290 DRM_DEBUG("IH: HPD3\n"); 5291 break; 5292 case 3: 5293 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT)) 5294 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5295 5296 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 5297 queue_hotplug = true; 5298 DRM_DEBUG("IH: HPD4\n"); 5299 break; 5300 case 4: 5301 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT)) 5302 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5303 5304 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 5305 queue_hotplug = true; 5306 DRM_DEBUG("IH: HPD5\n"); 5307 break; 5308 case 5: 5309 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT)) 5310 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5311 5312 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 5313 queue_hotplug = true; 5314 DRM_DEBUG("IH: HPD6\n"); 5315 break; 5316 case 6: 5317 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT)) 5318 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5319 5320 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT; 5321 queue_dp = true; 5322 DRM_DEBUG("IH: HPD_RX 1\n"); 5323 break; 5324 case 7: 5325 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT)) 5326 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5327 5328 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; 5329 queue_dp = true; 5330 DRM_DEBUG("IH: HPD_RX 2\n"); 5331 break; 5332 case 8: 5333 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) 5334 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5335 5336 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; 5337 queue_dp = true; 5338 DRM_DEBUG("IH: HPD_RX 3\n"); 5339 break; 5340 case 9: 5341 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) 5342 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5343 5344 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; 5345 queue_dp = true; 5346 DRM_DEBUG("IH: HPD_RX 4\n"); 5347 break; 5348 case 10: 5349 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) 5350 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5351 5352 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; 5353 queue_dp = true; 5354 DRM_DEBUG("IH: HPD_RX 5\n"); 5355 break; 5356 case 11: 5357 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) 5358 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5359 5360 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; 5361 queue_dp = true; 5362 DRM_DEBUG("IH: HPD_RX 6\n"); 5363 break; 5364 default: 5365 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5366 break; 5367 } 5368 break; 5369 case 44: /* hdmi */ 5370 switch (src_data) { 5371 case 0: 5372 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG)) 5373 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5374 5375 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG; 5376 queue_hdmi = true; 5377 DRM_DEBUG("IH: HDMI0\n"); 5378 break; 5379 case 1: 5380 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG)) 5381 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5382 5383 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG; 5384 queue_hdmi = true; 5385 DRM_DEBUG("IH: HDMI1\n"); 5386 break; 5387 case 2: 5388 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG)) 5389 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5390 5391 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG; 5392 queue_hdmi = true; 5393 DRM_DEBUG("IH: HDMI2\n"); 5394 break; 5395 case 3: 5396 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG)) 5397 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5398 5399 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG; 5400 queue_hdmi = true; 5401 DRM_DEBUG("IH: HDMI3\n"); 5402 break; 5403 case 4: 5404 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG)) 5405 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5406 5407 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG; 5408 queue_hdmi = true; 5409 DRM_DEBUG("IH: HDMI4\n"); 5410 break; 5411 case 5: 5412 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG)) 5413 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 5414 5415 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG; 5416 queue_hdmi = true; 5417 DRM_DEBUG("IH: HDMI5\n"); 5418 break; 5419 default: 5420 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 5421 break; 5422 } 5423 case 96: 5424 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); 5425 WREG32(SRBM_INT_ACK, 0x1); 5426 break; 5427 case 124: /* UVD */ 5428 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 5429 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 5430 break; 5431 case 146: 5432 case 147: 5433 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 5434 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 5435 /* reset addr and status */ 5436 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 5437 if (addr == 0x0 && status == 0x0) 5438 break; 5439 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 5440 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5441 addr); 5442 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5443 status); 5444 cayman_vm_decode_fault(rdev, status, addr); 5445 break; 5446 case 176: /* CP_INT in ring buffer */ 5447 case 177: /* CP_INT in IB1 */ 5448 case 178: /* CP_INT in IB2 */ 5449 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 5450 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5451 break; 5452 case 181: /* CP EOP event */ 5453 DRM_DEBUG("IH: CP EOP\n"); 5454 if (rdev->family >= CHIP_CAYMAN) { 5455 switch (src_data) { 5456 case 0: 5457 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5458 break; 5459 case 1: 5460 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 5461 break; 5462 case 2: 5463 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 5464 break; 5465 } 5466 } else 5467 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5468 break; 5469 case 224: /* DMA trap event */ 5470 DRM_DEBUG("IH: DMA trap\n"); 5471 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 5472 break; 5473 case 230: /* thermal low to high */ 5474 DRM_DEBUG("IH: thermal low to high\n"); 5475 rdev->pm.dpm.thermal.high_to_low = false; 5476 queue_thermal = true; 5477 break; 5478 case 231: /* thermal high to low */ 5479 DRM_DEBUG("IH: thermal high to low\n"); 5480 rdev->pm.dpm.thermal.high_to_low = true; 5481 queue_thermal = true; 5482 break; 5483 case 233: /* GUI IDLE */ 5484 DRM_DEBUG("IH: GUI idle\n"); 5485 break; 5486 case 244: /* DMA trap event */ 5487 if (rdev->family >= CHIP_CAYMAN) { 5488 DRM_DEBUG("IH: DMA1 trap\n"); 5489 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 5490 } 5491 break; 5492 default: 5493 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5494 break; 5495 } 5496 5497 /* wptr/rptr are in bytes! */ 5498 rptr += 16; 5499 rptr &= rdev->ih.ptr_mask; 5500 WREG32(IH_RB_RPTR, rptr); 5501 } 5502 if (queue_dp) 5503 schedule_work(&rdev->dp_work); 5504 if (queue_hotplug) 5505 schedule_delayed_work(&rdev->hotplug_work, 0); 5506 if (queue_hdmi) 5507 schedule_work(&rdev->audio_work); 5508 if (queue_thermal && rdev->pm.dpm_enabled) 5509 schedule_work(&rdev->pm.dpm.thermal.work); 5510 rdev->ih.rptr = rptr; 5511 atomic_set(&rdev->ih.lock, 0); 5512 5513 /* make sure wptr hasn't changed while processing */ 5514 wptr = evergreen_get_ih_wptr(rdev); 5515 if (wptr != rptr) 5516 goto restart_ih; 5517 5518 return IRQ_HANDLED; 5519 } 5520 5521 static void evergreen_uvd_init(struct radeon_device *rdev) 5522 { 5523 int r; 5524 5525 if (!rdev->has_uvd) 5526 return; 5527 5528 r = radeon_uvd_init(rdev); 5529 if (r) { 5530 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); 5531 /* 5532 * At this point rdev->uvd.vcpu_bo is NULL which trickles down 5533 * to early fails uvd_v2_2_resume() and thus nothing happens 5534 * there. So it is pointless to try to go through that code 5535 * hence why we disable uvd here. 5536 */ 5537 rdev->has_uvd = 0; 5538 return; 5539 } 5540 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 5541 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); 5542 } 5543 5544 static void evergreen_uvd_start(struct radeon_device *rdev) 5545 { 5546 int r; 5547 5548 if (!rdev->has_uvd) 5549 return; 5550 5551 r = uvd_v2_2_resume(rdev); 5552 if (r) { 5553 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); 5554 goto error; 5555 } 5556 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); 5557 if (r) { 5558 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); 5559 goto error; 5560 } 5561 return; 5562 5563 error: 5564 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 5565 } 5566 5567 static void evergreen_uvd_resume(struct radeon_device *rdev) 5568 { 5569 struct radeon_ring *ring; 5570 int r; 5571 5572 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) 5573 return; 5574 5575 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 5576 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0)); 5577 if (r) { 5578 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); 5579 return; 5580 } 5581 r = uvd_v1_0_init(rdev); 5582 if (r) { 5583 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); 5584 return; 5585 } 5586 } 5587 5588 static int evergreen_startup(struct radeon_device *rdev) 5589 { 5590 struct radeon_ring *ring; 5591 int r; 5592 5593 /* enable pcie gen2 link */ 5594 evergreen_pcie_gen2_enable(rdev); 5595 /* enable aspm */ 5596 evergreen_program_aspm(rdev); 5597 5598 /* scratch needs to be initialized before MC */ 5599 r = r600_vram_scratch_init(rdev); 5600 if (r) 5601 return r; 5602 5603 evergreen_mc_program(rdev); 5604 5605 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { 5606 r = ni_mc_load_microcode(rdev); 5607 if (r) { 5608 DRM_ERROR("Failed to load MC firmware!\n"); 5609 return r; 5610 } 5611 } 5612 5613 if (rdev->flags & RADEON_IS_AGP) { 5614 evergreen_agp_enable(rdev); 5615 } else { 5616 r = evergreen_pcie_gart_enable(rdev); 5617 if (r) 5618 return r; 5619 } 5620 evergreen_gpu_init(rdev); 5621 5622 /* allocate rlc buffers */ 5623 if (rdev->flags & RADEON_IS_IGP) { 5624 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; 5625 rdev->rlc.reg_list_size = 5626 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); 5627 rdev->rlc.cs_data = evergreen_cs_data; 5628 r = sumo_rlc_init(rdev); 5629 if (r) { 5630 DRM_ERROR("Failed to init rlc BOs!\n"); 5631 return r; 5632 } 5633 } 5634 5635 /* allocate wb buffer */ 5636 r = radeon_wb_init(rdev); 5637 if (r) 5638 return r; 5639 5640 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 5641 if (r) { 5642 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 5643 return r; 5644 } 5645 5646 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 5647 if (r) { 5648 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 5649 return r; 5650 } 5651 5652 evergreen_uvd_start(rdev); 5653 5654 /* Enable IRQ */ 5655 if (!rdev->irq.installed) { 5656 r = radeon_irq_kms_init(rdev); 5657 if (r) 5658 return r; 5659 } 5660 5661 r = r600_irq_init(rdev); 5662 if (r) { 5663 DRM_ERROR("radeon: IH init failed (%d).\n", r); 5664 radeon_irq_kms_fini(rdev); 5665 return r; 5666 } 5667 evergreen_irq_set(rdev); 5668 5669 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 5670 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 5671 RADEON_CP_PACKET2); 5672 if (r) 5673 return r; 5674 5675 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 5676 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 5677 DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 5678 if (r) 5679 return r; 5680 5681 r = evergreen_cp_load_microcode(rdev); 5682 if (r) 5683 return r; 5684 r = evergreen_cp_resume(rdev); 5685 if (r) 5686 return r; 5687 r = r600_dma_resume(rdev); 5688 if (r) 5689 return r; 5690 5691 evergreen_uvd_resume(rdev); 5692 5693 r = radeon_ib_pool_init(rdev); 5694 if (r) { 5695 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 5696 return r; 5697 } 5698 5699 r = radeon_audio_init(rdev); 5700 if (r) { 5701 DRM_ERROR("radeon: audio init failed\n"); 5702 return r; 5703 } 5704 5705 return 0; 5706 } 5707 5708 int evergreen_resume(struct radeon_device *rdev) 5709 { 5710 int r; 5711 5712 /* reset the asic, the gfx blocks are often in a bad state 5713 * after the driver is unloaded or after a resume 5714 */ 5715 if (radeon_asic_reset(rdev)) 5716 dev_warn(rdev->dev, "GPU reset failed !\n"); 5717 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 5718 * posting will perform necessary task to bring back GPU into good 5719 * shape. 5720 */ 5721 /* post card */ 5722 atom_asic_init(rdev->mode_info.atom_context); 5723 5724 /* init golden registers */ 5725 evergreen_init_golden_registers(rdev); 5726 5727 if (rdev->pm.pm_method == PM_METHOD_DPM) 5728 radeon_pm_resume(rdev); 5729 5730 rdev->accel_working = true; 5731 r = evergreen_startup(rdev); 5732 if (r) { 5733 DRM_ERROR("evergreen startup failed on resume\n"); 5734 rdev->accel_working = false; 5735 return r; 5736 } 5737 5738 return r; 5739 5740 } 5741 5742 int evergreen_suspend(struct radeon_device *rdev) 5743 { 5744 radeon_pm_suspend(rdev); 5745 radeon_audio_fini(rdev); 5746 if (rdev->has_uvd) { 5747 uvd_v1_0_fini(rdev); 5748 radeon_uvd_suspend(rdev); 5749 } 5750 r700_cp_stop(rdev); 5751 r600_dma_stop(rdev); 5752 evergreen_irq_suspend(rdev); 5753 radeon_wb_disable(rdev); 5754 evergreen_pcie_gart_disable(rdev); 5755 5756 return 0; 5757 } 5758 5759 /* Plan is to move initialization in that function and use 5760 * helper function so that radeon_device_init pretty much 5761 * do nothing more than calling asic specific function. This 5762 * should also allow to remove a bunch of callback function 5763 * like vram_info. 5764 */ 5765 int evergreen_init(struct radeon_device *rdev) 5766 { 5767 int r; 5768 5769 /* Read BIOS */ 5770 if (!radeon_get_bios(rdev)) { 5771 if (ASIC_IS_AVIVO(rdev)) 5772 return -EINVAL; 5773 } 5774 /* Must be an ATOMBIOS */ 5775 if (!rdev->is_atom_bios) { 5776 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 5777 return -EINVAL; 5778 } 5779 r = radeon_atombios_init(rdev); 5780 if (r) 5781 return r; 5782 /* reset the asic, the gfx blocks are often in a bad state 5783 * after the driver is unloaded or after a resume 5784 */ 5785 if (radeon_asic_reset(rdev)) 5786 dev_warn(rdev->dev, "GPU reset failed !\n"); 5787 /* Post card if necessary */ 5788 if (!radeon_card_posted(rdev)) { 5789 if (!rdev->bios) { 5790 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 5791 return -EINVAL; 5792 } 5793 DRM_INFO("GPU not posted. posting now...\n"); 5794 atom_asic_init(rdev->mode_info.atom_context); 5795 } 5796 /* init golden registers */ 5797 evergreen_init_golden_registers(rdev); 5798 /* Initialize scratch registers */ 5799 r600_scratch_init(rdev); 5800 /* Initialize surface registers */ 5801 radeon_surface_init(rdev); 5802 /* Initialize clocks */ 5803 radeon_get_clock_info(rdev->ddev); 5804 /* Fence driver */ 5805 r = radeon_fence_driver_init(rdev); 5806 if (r) 5807 return r; 5808 /* initialize AGP */ 5809 if (rdev->flags & RADEON_IS_AGP) { 5810 r = radeon_agp_init(rdev); 5811 if (r) 5812 radeon_agp_disable(rdev); 5813 } 5814 /* initialize memory controller */ 5815 r = evergreen_mc_init(rdev); 5816 if (r) 5817 return r; 5818 /* Memory manager */ 5819 r = radeon_bo_init(rdev); 5820 if (r) 5821 return r; 5822 5823 if (ASIC_IS_DCE5(rdev)) { 5824 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 5825 r = ni_init_microcode(rdev); 5826 if (r) { 5827 DRM_ERROR("Failed to load firmware!\n"); 5828 return r; 5829 } 5830 } 5831 } else { 5832 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 5833 r = r600_init_microcode(rdev); 5834 if (r) { 5835 DRM_ERROR("Failed to load firmware!\n"); 5836 return r; 5837 } 5838 } 5839 } 5840 5841 /* Initialize power management */ 5842 radeon_pm_init(rdev); 5843 5844 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; 5845 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); 5846 5847 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; 5848 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); 5849 5850 evergreen_uvd_init(rdev); 5851 5852 rdev->ih.ring_obj = NULL; 5853 r600_ih_ring_init(rdev, 64 * 1024); 5854 5855 r = r600_pcie_gart_init(rdev); 5856 if (r) 5857 return r; 5858 5859 rdev->accel_working = true; 5860 r = evergreen_startup(rdev); 5861 if (r) { 5862 dev_err(rdev->dev, "disabling GPU acceleration\n"); 5863 r700_cp_fini(rdev); 5864 r600_dma_fini(rdev); 5865 r600_irq_fini(rdev); 5866 if (rdev->flags & RADEON_IS_IGP) 5867 sumo_rlc_fini(rdev); 5868 radeon_wb_fini(rdev); 5869 radeon_ib_pool_fini(rdev); 5870 radeon_irq_kms_fini(rdev); 5871 evergreen_pcie_gart_fini(rdev); 5872 rdev->accel_working = false; 5873 } 5874 5875 /* Don't start up if the MC ucode is missing on BTC parts. 5876 * The default clocks and voltages before the MC ucode 5877 * is loaded are not suffient for advanced operations. 5878 */ 5879 if (ASIC_IS_DCE5(rdev)) { 5880 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 5881 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 5882 return -EINVAL; 5883 } 5884 } 5885 5886 return 0; 5887 } 5888 5889 void evergreen_fini(struct radeon_device *rdev) 5890 { 5891 radeon_pm_fini(rdev); 5892 radeon_audio_fini(rdev); 5893 r700_cp_fini(rdev); 5894 r600_dma_fini(rdev); 5895 r600_irq_fini(rdev); 5896 if (rdev->flags & RADEON_IS_IGP) 5897 sumo_rlc_fini(rdev); 5898 radeon_wb_fini(rdev); 5899 radeon_ib_pool_fini(rdev); 5900 radeon_irq_kms_fini(rdev); 5901 uvd_v1_0_fini(rdev); 5902 radeon_uvd_fini(rdev); 5903 evergreen_pcie_gart_fini(rdev); 5904 r600_vram_scratch_fini(rdev); 5905 radeon_gem_fini(rdev); 5906 radeon_fence_driver_fini(rdev); 5907 radeon_agp_fini(rdev); 5908 radeon_bo_fini(rdev); 5909 radeon_atombios_fini(rdev); 5910 kfree(rdev->bios); 5911 rdev->bios = NULL; 5912 } 5913 5914 void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 5915 { 5916 u32 link_width_cntl, speed_cntl; 5917 5918 if (radeon_pcie_gen2 == 0) 5919 return; 5920 5921 if (rdev->flags & RADEON_IS_IGP) 5922 return; 5923 5924 if (!(rdev->flags & RADEON_IS_PCIE)) 5925 return; 5926 5927 /* x2 cards have a special sequence */ 5928 if (ASIC_IS_X2(rdev)) 5929 return; 5930 5931 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) && 5932 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT)) 5933 return; 5934 5935 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5936 if (speed_cntl & LC_CURRENT_DATA_RATE) { 5937 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 5938 return; 5939 } 5940 5941 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 5942 5943 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 5944 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 5945 5946 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5947 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5948 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5949 5950 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5951 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 5952 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5953 5954 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5955 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 5956 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5957 5958 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5959 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 5960 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5961 5962 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5963 speed_cntl |= LC_GEN2_EN_STRAP; 5964 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5965 5966 } else { 5967 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5968 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 5969 if (1) 5970 link_width_cntl |= LC_UPCONFIGURE_DIS; 5971 else 5972 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5973 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5974 } 5975 } 5976 5977 void evergreen_program_aspm(struct radeon_device *rdev) 5978 { 5979 u32 data, orig; 5980 u32 pcie_lc_cntl, pcie_lc_cntl_old; 5981 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; 5982 /* fusion_platform = true 5983 * if the system is a fusion system 5984 * (APU or DGPU in a fusion system). 5985 * todo: check if the system is a fusion platform. 5986 */ 5987 bool fusion_platform = false; 5988 5989 if (radeon_aspm == 0) 5990 return; 5991 5992 if (!(rdev->flags & RADEON_IS_PCIE)) 5993 return; 5994 5995 switch (rdev->family) { 5996 case CHIP_CYPRESS: 5997 case CHIP_HEMLOCK: 5998 case CHIP_JUNIPER: 5999 case CHIP_REDWOOD: 6000 case CHIP_CEDAR: 6001 case CHIP_SUMO: 6002 case CHIP_SUMO2: 6003 case CHIP_PALM: 6004 case CHIP_ARUBA: 6005 disable_l0s = true; 6006 break; 6007 default: 6008 disable_l0s = false; 6009 break; 6010 } 6011 6012 if (rdev->flags & RADEON_IS_IGP) 6013 fusion_platform = true; /* XXX also dGPUs in a fusion system */ 6014 6015 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); 6016 if (fusion_platform) 6017 data &= ~MULTI_PIF; 6018 else 6019 data |= MULTI_PIF; 6020 if (data != orig) 6021 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); 6022 6023 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); 6024 if (fusion_platform) 6025 data &= ~MULTI_PIF; 6026 else 6027 data |= MULTI_PIF; 6028 if (data != orig) 6029 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); 6030 6031 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); 6032 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 6033 if (!disable_l0s) { 6034 if (rdev->family >= CHIP_BARTS) 6035 pcie_lc_cntl |= LC_L0S_INACTIVITY(7); 6036 else 6037 pcie_lc_cntl |= LC_L0S_INACTIVITY(3); 6038 } 6039 6040 if (!disable_l1) { 6041 if (rdev->family >= CHIP_BARTS) 6042 pcie_lc_cntl |= LC_L1_INACTIVITY(7); 6043 else 6044 pcie_lc_cntl |= LC_L1_INACTIVITY(8); 6045 6046 if (!disable_plloff_in_l1) { 6047 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 6048 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 6049 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 6050 if (data != orig) 6051 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 6052 6053 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 6054 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 6055 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 6056 if (data != orig) 6057 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 6058 6059 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 6060 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 6061 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 6062 if (data != orig) 6063 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 6064 6065 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 6066 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 6067 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 6068 if (data != orig) 6069 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 6070 6071 if (rdev->family >= CHIP_BARTS) { 6072 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 6073 data &= ~PLL_RAMP_UP_TIME_0_MASK; 6074 data |= PLL_RAMP_UP_TIME_0(4); 6075 if (data != orig) 6076 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 6077 6078 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 6079 data &= ~PLL_RAMP_UP_TIME_1_MASK; 6080 data |= PLL_RAMP_UP_TIME_1(4); 6081 if (data != orig) 6082 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 6083 6084 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 6085 data &= ~PLL_RAMP_UP_TIME_0_MASK; 6086 data |= PLL_RAMP_UP_TIME_0(4); 6087 if (data != orig) 6088 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 6089 6090 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 6091 data &= ~PLL_RAMP_UP_TIME_1_MASK; 6092 data |= PLL_RAMP_UP_TIME_1(4); 6093 if (data != orig) 6094 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 6095 } 6096 6097 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 6098 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 6099 data |= LC_DYN_LANES_PWR_STATE(3); 6100 if (data != orig) 6101 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 6102 6103 if (rdev->family >= CHIP_BARTS) { 6104 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); 6105 data &= ~LS2_EXIT_TIME_MASK; 6106 data |= LS2_EXIT_TIME(1); 6107 if (data != orig) 6108 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 6109 6110 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); 6111 data &= ~LS2_EXIT_TIME_MASK; 6112 data |= LS2_EXIT_TIME(1); 6113 if (data != orig) 6114 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 6115 } 6116 } 6117 } 6118 6119 /* evergreen parts only */ 6120 if (rdev->family < CHIP_BARTS) 6121 pcie_lc_cntl |= LC_PMI_TO_L1_DIS; 6122 6123 if (pcie_lc_cntl != pcie_lc_cntl_old) 6124 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); 6125 } 6126