1 /* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <drm/drmP.h> 27 #include "radeon.h" 28 #include "radeon_asic.h" 29 #include "radeon_audio.h" 30 #include <drm/radeon_drm.h> 31 #include "evergreend.h" 32 #include "atom.h" 33 #include "avivod.h" 34 #include "evergreen_reg.h" 35 #include "evergreen_blit_shaders.h" 36 #include "radeon_ucode.h" 37 38 static const u32 crtc_offsets[6] = 39 { 40 EVERGREEN_CRTC0_REGISTER_OFFSET, 41 EVERGREEN_CRTC1_REGISTER_OFFSET, 42 EVERGREEN_CRTC2_REGISTER_OFFSET, 43 EVERGREEN_CRTC3_REGISTER_OFFSET, 44 EVERGREEN_CRTC4_REGISTER_OFFSET, 45 EVERGREEN_CRTC5_REGISTER_OFFSET 46 }; 47 48 #include "clearstate_evergreen.h" 49 50 static const u32 sumo_rlc_save_restore_register_list[] = 51 { 52 0x98fc, 53 0x9830, 54 0x9834, 55 0x9838, 56 0x9870, 57 0x9874, 58 0x8a14, 59 0x8b24, 60 0x8bcc, 61 0x8b10, 62 0x8d00, 63 0x8d04, 64 0x8c00, 65 0x8c04, 66 0x8c08, 67 0x8c0c, 68 0x8d8c, 69 0x8c20, 70 0x8c24, 71 0x8c28, 72 0x8c18, 73 0x8c1c, 74 0x8cf0, 75 0x8e2c, 76 0x8e38, 77 0x8c30, 78 0x9508, 79 0x9688, 80 0x9608, 81 0x960c, 82 0x9610, 83 0x9614, 84 0x88c4, 85 0x88d4, 86 0xa008, 87 0x900c, 88 0x9100, 89 0x913c, 90 0x98f8, 91 0x98f4, 92 0x9b7c, 93 0x3f8c, 94 0x8950, 95 0x8954, 96 0x8a18, 97 0x8b28, 98 0x9144, 99 0x9148, 100 0x914c, 101 0x3f90, 102 0x3f94, 103 0x915c, 104 0x9160, 105 0x9178, 106 0x917c, 107 0x9180, 108 0x918c, 109 0x9190, 110 0x9194, 111 0x9198, 112 0x919c, 113 0x91a8, 114 0x91ac, 115 0x91b0, 116 0x91b4, 117 0x91b8, 118 0x91c4, 119 0x91c8, 120 0x91cc, 121 0x91d0, 122 0x91d4, 123 0x91e0, 124 0x91e4, 125 0x91ec, 126 0x91f0, 127 0x91f4, 128 0x9200, 129 0x9204, 130 0x929c, 131 0x9150, 132 0x802c, 133 }; 134 135 static void evergreen_gpu_init(struct radeon_device *rdev); 136 void evergreen_fini(struct radeon_device *rdev); 137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 138 void evergreen_program_aspm(struct radeon_device *rdev); 139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev, 140 int ring, u32 cp_int_cntl); 141 extern void cayman_vm_decode_fault(struct radeon_device *rdev, 142 u32 status, u32 addr); 143 void cik_init_cp_pg_table(struct radeon_device *rdev); 144 145 extern u32 si_get_csb_size(struct radeon_device *rdev); 146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 147 extern u32 cik_get_csb_size(struct radeon_device *rdev); 148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 149 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev); 150 151 static const u32 evergreen_golden_registers[] = 152 { 153 0x3f90, 0xffff0000, 0xff000000, 154 0x9148, 0xffff0000, 0xff000000, 155 0x3f94, 0xffff0000, 0xff000000, 156 0x914c, 0xffff0000, 0xff000000, 157 0x9b7c, 0xffffffff, 0x00000000, 158 0x8a14, 0xffffffff, 0x00000007, 159 0x8b10, 0xffffffff, 0x00000000, 160 0x960c, 0xffffffff, 0x54763210, 161 0x88c4, 0xffffffff, 0x000000c2, 162 0x88d4, 0xffffffff, 0x00000010, 163 0x8974, 0xffffffff, 0x00000000, 164 0xc78, 0x00000080, 0x00000080, 165 0x5eb4, 0xffffffff, 0x00000002, 166 0x5e78, 0xffffffff, 0x001000f0, 167 0x6104, 0x01000300, 0x00000000, 168 0x5bc0, 0x00300000, 0x00000000, 169 0x7030, 0xffffffff, 0x00000011, 170 0x7c30, 0xffffffff, 0x00000011, 171 0x10830, 0xffffffff, 0x00000011, 172 0x11430, 0xffffffff, 0x00000011, 173 0x12030, 0xffffffff, 0x00000011, 174 0x12c30, 0xffffffff, 0x00000011, 175 0xd02c, 0xffffffff, 0x08421000, 176 0x240c, 0xffffffff, 0x00000380, 177 0x8b24, 0xffffffff, 0x00ff0fff, 178 0x28a4c, 0x06000000, 0x06000000, 179 0x10c, 0x00000001, 0x00000001, 180 0x8d00, 0xffffffff, 0x100e4848, 181 0x8d04, 0xffffffff, 0x00164745, 182 0x8c00, 0xffffffff, 0xe4000003, 183 0x8c04, 0xffffffff, 0x40600060, 184 0x8c08, 0xffffffff, 0x001c001c, 185 0x8cf0, 0xffffffff, 0x08e00620, 186 0x8c20, 0xffffffff, 0x00800080, 187 0x8c24, 0xffffffff, 0x00800080, 188 0x8c18, 0xffffffff, 0x20202078, 189 0x8c1c, 0xffffffff, 0x00001010, 190 0x28350, 0xffffffff, 0x00000000, 191 0xa008, 0xffffffff, 0x00010000, 192 0x5c4, 0xffffffff, 0x00000001, 193 0x9508, 0xffffffff, 0x00000002, 194 0x913c, 0x0000000f, 0x0000000a 195 }; 196 197 static const u32 evergreen_golden_registers2[] = 198 { 199 0x2f4c, 0xffffffff, 0x00000000, 200 0x54f4, 0xffffffff, 0x00000000, 201 0x54f0, 0xffffffff, 0x00000000, 202 0x5498, 0xffffffff, 0x00000000, 203 0x549c, 0xffffffff, 0x00000000, 204 0x5494, 0xffffffff, 0x00000000, 205 0x53cc, 0xffffffff, 0x00000000, 206 0x53c8, 0xffffffff, 0x00000000, 207 0x53c4, 0xffffffff, 0x00000000, 208 0x53c0, 0xffffffff, 0x00000000, 209 0x53bc, 0xffffffff, 0x00000000, 210 0x53b8, 0xffffffff, 0x00000000, 211 0x53b4, 0xffffffff, 0x00000000, 212 0x53b0, 0xffffffff, 0x00000000 213 }; 214 215 static const u32 cypress_mgcg_init[] = 216 { 217 0x802c, 0xffffffff, 0xc0000000, 218 0x5448, 0xffffffff, 0x00000100, 219 0x55e4, 0xffffffff, 0x00000100, 220 0x160c, 0xffffffff, 0x00000100, 221 0x5644, 0xffffffff, 0x00000100, 222 0xc164, 0xffffffff, 0x00000100, 223 0x8a18, 0xffffffff, 0x00000100, 224 0x897c, 0xffffffff, 0x06000100, 225 0x8b28, 0xffffffff, 0x00000100, 226 0x9144, 0xffffffff, 0x00000100, 227 0x9a60, 0xffffffff, 0x00000100, 228 0x9868, 0xffffffff, 0x00000100, 229 0x8d58, 0xffffffff, 0x00000100, 230 0x9510, 0xffffffff, 0x00000100, 231 0x949c, 0xffffffff, 0x00000100, 232 0x9654, 0xffffffff, 0x00000100, 233 0x9030, 0xffffffff, 0x00000100, 234 0x9034, 0xffffffff, 0x00000100, 235 0x9038, 0xffffffff, 0x00000100, 236 0x903c, 0xffffffff, 0x00000100, 237 0x9040, 0xffffffff, 0x00000100, 238 0xa200, 0xffffffff, 0x00000100, 239 0xa204, 0xffffffff, 0x00000100, 240 0xa208, 0xffffffff, 0x00000100, 241 0xa20c, 0xffffffff, 0x00000100, 242 0x971c, 0xffffffff, 0x00000100, 243 0x977c, 0xffffffff, 0x00000100, 244 0x3f80, 0xffffffff, 0x00000100, 245 0xa210, 0xffffffff, 0x00000100, 246 0xa214, 0xffffffff, 0x00000100, 247 0x4d8, 0xffffffff, 0x00000100, 248 0x9784, 0xffffffff, 0x00000100, 249 0x9698, 0xffffffff, 0x00000100, 250 0x4d4, 0xffffffff, 0x00000200, 251 0x30cc, 0xffffffff, 0x00000100, 252 0xd0c0, 0xffffffff, 0xff000100, 253 0x802c, 0xffffffff, 0x40000000, 254 0x915c, 0xffffffff, 0x00010000, 255 0x9160, 0xffffffff, 0x00030002, 256 0x9178, 0xffffffff, 0x00070000, 257 0x917c, 0xffffffff, 0x00030002, 258 0x9180, 0xffffffff, 0x00050004, 259 0x918c, 0xffffffff, 0x00010006, 260 0x9190, 0xffffffff, 0x00090008, 261 0x9194, 0xffffffff, 0x00070000, 262 0x9198, 0xffffffff, 0x00030002, 263 0x919c, 0xffffffff, 0x00050004, 264 0x91a8, 0xffffffff, 0x00010006, 265 0x91ac, 0xffffffff, 0x00090008, 266 0x91b0, 0xffffffff, 0x00070000, 267 0x91b4, 0xffffffff, 0x00030002, 268 0x91b8, 0xffffffff, 0x00050004, 269 0x91c4, 0xffffffff, 0x00010006, 270 0x91c8, 0xffffffff, 0x00090008, 271 0x91cc, 0xffffffff, 0x00070000, 272 0x91d0, 0xffffffff, 0x00030002, 273 0x91d4, 0xffffffff, 0x00050004, 274 0x91e0, 0xffffffff, 0x00010006, 275 0x91e4, 0xffffffff, 0x00090008, 276 0x91e8, 0xffffffff, 0x00000000, 277 0x91ec, 0xffffffff, 0x00070000, 278 0x91f0, 0xffffffff, 0x00030002, 279 0x91f4, 0xffffffff, 0x00050004, 280 0x9200, 0xffffffff, 0x00010006, 281 0x9204, 0xffffffff, 0x00090008, 282 0x9208, 0xffffffff, 0x00070000, 283 0x920c, 0xffffffff, 0x00030002, 284 0x9210, 0xffffffff, 0x00050004, 285 0x921c, 0xffffffff, 0x00010006, 286 0x9220, 0xffffffff, 0x00090008, 287 0x9224, 0xffffffff, 0x00070000, 288 0x9228, 0xffffffff, 0x00030002, 289 0x922c, 0xffffffff, 0x00050004, 290 0x9238, 0xffffffff, 0x00010006, 291 0x923c, 0xffffffff, 0x00090008, 292 0x9240, 0xffffffff, 0x00070000, 293 0x9244, 0xffffffff, 0x00030002, 294 0x9248, 0xffffffff, 0x00050004, 295 0x9254, 0xffffffff, 0x00010006, 296 0x9258, 0xffffffff, 0x00090008, 297 0x925c, 0xffffffff, 0x00070000, 298 0x9260, 0xffffffff, 0x00030002, 299 0x9264, 0xffffffff, 0x00050004, 300 0x9270, 0xffffffff, 0x00010006, 301 0x9274, 0xffffffff, 0x00090008, 302 0x9278, 0xffffffff, 0x00070000, 303 0x927c, 0xffffffff, 0x00030002, 304 0x9280, 0xffffffff, 0x00050004, 305 0x928c, 0xffffffff, 0x00010006, 306 0x9290, 0xffffffff, 0x00090008, 307 0x9294, 0xffffffff, 0x00000000, 308 0x929c, 0xffffffff, 0x00000001, 309 0x802c, 0xffffffff, 0x40010000, 310 0x915c, 0xffffffff, 0x00010000, 311 0x9160, 0xffffffff, 0x00030002, 312 0x9178, 0xffffffff, 0x00070000, 313 0x917c, 0xffffffff, 0x00030002, 314 0x9180, 0xffffffff, 0x00050004, 315 0x918c, 0xffffffff, 0x00010006, 316 0x9190, 0xffffffff, 0x00090008, 317 0x9194, 0xffffffff, 0x00070000, 318 0x9198, 0xffffffff, 0x00030002, 319 0x919c, 0xffffffff, 0x00050004, 320 0x91a8, 0xffffffff, 0x00010006, 321 0x91ac, 0xffffffff, 0x00090008, 322 0x91b0, 0xffffffff, 0x00070000, 323 0x91b4, 0xffffffff, 0x00030002, 324 0x91b8, 0xffffffff, 0x00050004, 325 0x91c4, 0xffffffff, 0x00010006, 326 0x91c8, 0xffffffff, 0x00090008, 327 0x91cc, 0xffffffff, 0x00070000, 328 0x91d0, 0xffffffff, 0x00030002, 329 0x91d4, 0xffffffff, 0x00050004, 330 0x91e0, 0xffffffff, 0x00010006, 331 0x91e4, 0xffffffff, 0x00090008, 332 0x91e8, 0xffffffff, 0x00000000, 333 0x91ec, 0xffffffff, 0x00070000, 334 0x91f0, 0xffffffff, 0x00030002, 335 0x91f4, 0xffffffff, 0x00050004, 336 0x9200, 0xffffffff, 0x00010006, 337 0x9204, 0xffffffff, 0x00090008, 338 0x9208, 0xffffffff, 0x00070000, 339 0x920c, 0xffffffff, 0x00030002, 340 0x9210, 0xffffffff, 0x00050004, 341 0x921c, 0xffffffff, 0x00010006, 342 0x9220, 0xffffffff, 0x00090008, 343 0x9224, 0xffffffff, 0x00070000, 344 0x9228, 0xffffffff, 0x00030002, 345 0x922c, 0xffffffff, 0x00050004, 346 0x9238, 0xffffffff, 0x00010006, 347 0x923c, 0xffffffff, 0x00090008, 348 0x9240, 0xffffffff, 0x00070000, 349 0x9244, 0xffffffff, 0x00030002, 350 0x9248, 0xffffffff, 0x00050004, 351 0x9254, 0xffffffff, 0x00010006, 352 0x9258, 0xffffffff, 0x00090008, 353 0x925c, 0xffffffff, 0x00070000, 354 0x9260, 0xffffffff, 0x00030002, 355 0x9264, 0xffffffff, 0x00050004, 356 0x9270, 0xffffffff, 0x00010006, 357 0x9274, 0xffffffff, 0x00090008, 358 0x9278, 0xffffffff, 0x00070000, 359 0x927c, 0xffffffff, 0x00030002, 360 0x9280, 0xffffffff, 0x00050004, 361 0x928c, 0xffffffff, 0x00010006, 362 0x9290, 0xffffffff, 0x00090008, 363 0x9294, 0xffffffff, 0x00000000, 364 0x929c, 0xffffffff, 0x00000001, 365 0x802c, 0xffffffff, 0xc0000000 366 }; 367 368 static const u32 redwood_mgcg_init[] = 369 { 370 0x802c, 0xffffffff, 0xc0000000, 371 0x5448, 0xffffffff, 0x00000100, 372 0x55e4, 0xffffffff, 0x00000100, 373 0x160c, 0xffffffff, 0x00000100, 374 0x5644, 0xffffffff, 0x00000100, 375 0xc164, 0xffffffff, 0x00000100, 376 0x8a18, 0xffffffff, 0x00000100, 377 0x897c, 0xffffffff, 0x06000100, 378 0x8b28, 0xffffffff, 0x00000100, 379 0x9144, 0xffffffff, 0x00000100, 380 0x9a60, 0xffffffff, 0x00000100, 381 0x9868, 0xffffffff, 0x00000100, 382 0x8d58, 0xffffffff, 0x00000100, 383 0x9510, 0xffffffff, 0x00000100, 384 0x949c, 0xffffffff, 0x00000100, 385 0x9654, 0xffffffff, 0x00000100, 386 0x9030, 0xffffffff, 0x00000100, 387 0x9034, 0xffffffff, 0x00000100, 388 0x9038, 0xffffffff, 0x00000100, 389 0x903c, 0xffffffff, 0x00000100, 390 0x9040, 0xffffffff, 0x00000100, 391 0xa200, 0xffffffff, 0x00000100, 392 0xa204, 0xffffffff, 0x00000100, 393 0xa208, 0xffffffff, 0x00000100, 394 0xa20c, 0xffffffff, 0x00000100, 395 0x971c, 0xffffffff, 0x00000100, 396 0x977c, 0xffffffff, 0x00000100, 397 0x3f80, 0xffffffff, 0x00000100, 398 0xa210, 0xffffffff, 0x00000100, 399 0xa214, 0xffffffff, 0x00000100, 400 0x4d8, 0xffffffff, 0x00000100, 401 0x9784, 0xffffffff, 0x00000100, 402 0x9698, 0xffffffff, 0x00000100, 403 0x4d4, 0xffffffff, 0x00000200, 404 0x30cc, 0xffffffff, 0x00000100, 405 0xd0c0, 0xffffffff, 0xff000100, 406 0x802c, 0xffffffff, 0x40000000, 407 0x915c, 0xffffffff, 0x00010000, 408 0x9160, 0xffffffff, 0x00030002, 409 0x9178, 0xffffffff, 0x00070000, 410 0x917c, 0xffffffff, 0x00030002, 411 0x9180, 0xffffffff, 0x00050004, 412 0x918c, 0xffffffff, 0x00010006, 413 0x9190, 0xffffffff, 0x00090008, 414 0x9194, 0xffffffff, 0x00070000, 415 0x9198, 0xffffffff, 0x00030002, 416 0x919c, 0xffffffff, 0x00050004, 417 0x91a8, 0xffffffff, 0x00010006, 418 0x91ac, 0xffffffff, 0x00090008, 419 0x91b0, 0xffffffff, 0x00070000, 420 0x91b4, 0xffffffff, 0x00030002, 421 0x91b8, 0xffffffff, 0x00050004, 422 0x91c4, 0xffffffff, 0x00010006, 423 0x91c8, 0xffffffff, 0x00090008, 424 0x91cc, 0xffffffff, 0x00070000, 425 0x91d0, 0xffffffff, 0x00030002, 426 0x91d4, 0xffffffff, 0x00050004, 427 0x91e0, 0xffffffff, 0x00010006, 428 0x91e4, 0xffffffff, 0x00090008, 429 0x91e8, 0xffffffff, 0x00000000, 430 0x91ec, 0xffffffff, 0x00070000, 431 0x91f0, 0xffffffff, 0x00030002, 432 0x91f4, 0xffffffff, 0x00050004, 433 0x9200, 0xffffffff, 0x00010006, 434 0x9204, 0xffffffff, 0x00090008, 435 0x9294, 0xffffffff, 0x00000000, 436 0x929c, 0xffffffff, 0x00000001, 437 0x802c, 0xffffffff, 0xc0000000 438 }; 439 440 static const u32 cedar_golden_registers[] = 441 { 442 0x3f90, 0xffff0000, 0xff000000, 443 0x9148, 0xffff0000, 0xff000000, 444 0x3f94, 0xffff0000, 0xff000000, 445 0x914c, 0xffff0000, 0xff000000, 446 0x9b7c, 0xffffffff, 0x00000000, 447 0x8a14, 0xffffffff, 0x00000007, 448 0x8b10, 0xffffffff, 0x00000000, 449 0x960c, 0xffffffff, 0x54763210, 450 0x88c4, 0xffffffff, 0x000000c2, 451 0x88d4, 0xffffffff, 0x00000000, 452 0x8974, 0xffffffff, 0x00000000, 453 0xc78, 0x00000080, 0x00000080, 454 0x5eb4, 0xffffffff, 0x00000002, 455 0x5e78, 0xffffffff, 0x001000f0, 456 0x6104, 0x01000300, 0x00000000, 457 0x5bc0, 0x00300000, 0x00000000, 458 0x7030, 0xffffffff, 0x00000011, 459 0x7c30, 0xffffffff, 0x00000011, 460 0x10830, 0xffffffff, 0x00000011, 461 0x11430, 0xffffffff, 0x00000011, 462 0xd02c, 0xffffffff, 0x08421000, 463 0x240c, 0xffffffff, 0x00000380, 464 0x8b24, 0xffffffff, 0x00ff0fff, 465 0x28a4c, 0x06000000, 0x06000000, 466 0x10c, 0x00000001, 0x00000001, 467 0x8d00, 0xffffffff, 0x100e4848, 468 0x8d04, 0xffffffff, 0x00164745, 469 0x8c00, 0xffffffff, 0xe4000003, 470 0x8c04, 0xffffffff, 0x40600060, 471 0x8c08, 0xffffffff, 0x001c001c, 472 0x8cf0, 0xffffffff, 0x08e00410, 473 0x8c20, 0xffffffff, 0x00800080, 474 0x8c24, 0xffffffff, 0x00800080, 475 0x8c18, 0xffffffff, 0x20202078, 476 0x8c1c, 0xffffffff, 0x00001010, 477 0x28350, 0xffffffff, 0x00000000, 478 0xa008, 0xffffffff, 0x00010000, 479 0x5c4, 0xffffffff, 0x00000001, 480 0x9508, 0xffffffff, 0x00000002 481 }; 482 483 static const u32 cedar_mgcg_init[] = 484 { 485 0x802c, 0xffffffff, 0xc0000000, 486 0x5448, 0xffffffff, 0x00000100, 487 0x55e4, 0xffffffff, 0x00000100, 488 0x160c, 0xffffffff, 0x00000100, 489 0x5644, 0xffffffff, 0x00000100, 490 0xc164, 0xffffffff, 0x00000100, 491 0x8a18, 0xffffffff, 0x00000100, 492 0x897c, 0xffffffff, 0x06000100, 493 0x8b28, 0xffffffff, 0x00000100, 494 0x9144, 0xffffffff, 0x00000100, 495 0x9a60, 0xffffffff, 0x00000100, 496 0x9868, 0xffffffff, 0x00000100, 497 0x8d58, 0xffffffff, 0x00000100, 498 0x9510, 0xffffffff, 0x00000100, 499 0x949c, 0xffffffff, 0x00000100, 500 0x9654, 0xffffffff, 0x00000100, 501 0x9030, 0xffffffff, 0x00000100, 502 0x9034, 0xffffffff, 0x00000100, 503 0x9038, 0xffffffff, 0x00000100, 504 0x903c, 0xffffffff, 0x00000100, 505 0x9040, 0xffffffff, 0x00000100, 506 0xa200, 0xffffffff, 0x00000100, 507 0xa204, 0xffffffff, 0x00000100, 508 0xa208, 0xffffffff, 0x00000100, 509 0xa20c, 0xffffffff, 0x00000100, 510 0x971c, 0xffffffff, 0x00000100, 511 0x977c, 0xffffffff, 0x00000100, 512 0x3f80, 0xffffffff, 0x00000100, 513 0xa210, 0xffffffff, 0x00000100, 514 0xa214, 0xffffffff, 0x00000100, 515 0x4d8, 0xffffffff, 0x00000100, 516 0x9784, 0xffffffff, 0x00000100, 517 0x9698, 0xffffffff, 0x00000100, 518 0x4d4, 0xffffffff, 0x00000200, 519 0x30cc, 0xffffffff, 0x00000100, 520 0xd0c0, 0xffffffff, 0xff000100, 521 0x802c, 0xffffffff, 0x40000000, 522 0x915c, 0xffffffff, 0x00010000, 523 0x9178, 0xffffffff, 0x00050000, 524 0x917c, 0xffffffff, 0x00030002, 525 0x918c, 0xffffffff, 0x00010004, 526 0x9190, 0xffffffff, 0x00070006, 527 0x9194, 0xffffffff, 0x00050000, 528 0x9198, 0xffffffff, 0x00030002, 529 0x91a8, 0xffffffff, 0x00010004, 530 0x91ac, 0xffffffff, 0x00070006, 531 0x91e8, 0xffffffff, 0x00000000, 532 0x9294, 0xffffffff, 0x00000000, 533 0x929c, 0xffffffff, 0x00000001, 534 0x802c, 0xffffffff, 0xc0000000 535 }; 536 537 static const u32 juniper_mgcg_init[] = 538 { 539 0x802c, 0xffffffff, 0xc0000000, 540 0x5448, 0xffffffff, 0x00000100, 541 0x55e4, 0xffffffff, 0x00000100, 542 0x160c, 0xffffffff, 0x00000100, 543 0x5644, 0xffffffff, 0x00000100, 544 0xc164, 0xffffffff, 0x00000100, 545 0x8a18, 0xffffffff, 0x00000100, 546 0x897c, 0xffffffff, 0x06000100, 547 0x8b28, 0xffffffff, 0x00000100, 548 0x9144, 0xffffffff, 0x00000100, 549 0x9a60, 0xffffffff, 0x00000100, 550 0x9868, 0xffffffff, 0x00000100, 551 0x8d58, 0xffffffff, 0x00000100, 552 0x9510, 0xffffffff, 0x00000100, 553 0x949c, 0xffffffff, 0x00000100, 554 0x9654, 0xffffffff, 0x00000100, 555 0x9030, 0xffffffff, 0x00000100, 556 0x9034, 0xffffffff, 0x00000100, 557 0x9038, 0xffffffff, 0x00000100, 558 0x903c, 0xffffffff, 0x00000100, 559 0x9040, 0xffffffff, 0x00000100, 560 0xa200, 0xffffffff, 0x00000100, 561 0xa204, 0xffffffff, 0x00000100, 562 0xa208, 0xffffffff, 0x00000100, 563 0xa20c, 0xffffffff, 0x00000100, 564 0x971c, 0xffffffff, 0x00000100, 565 0xd0c0, 0xffffffff, 0xff000100, 566 0x802c, 0xffffffff, 0x40000000, 567 0x915c, 0xffffffff, 0x00010000, 568 0x9160, 0xffffffff, 0x00030002, 569 0x9178, 0xffffffff, 0x00070000, 570 0x917c, 0xffffffff, 0x00030002, 571 0x9180, 0xffffffff, 0x00050004, 572 0x918c, 0xffffffff, 0x00010006, 573 0x9190, 0xffffffff, 0x00090008, 574 0x9194, 0xffffffff, 0x00070000, 575 0x9198, 0xffffffff, 0x00030002, 576 0x919c, 0xffffffff, 0x00050004, 577 0x91a8, 0xffffffff, 0x00010006, 578 0x91ac, 0xffffffff, 0x00090008, 579 0x91b0, 0xffffffff, 0x00070000, 580 0x91b4, 0xffffffff, 0x00030002, 581 0x91b8, 0xffffffff, 0x00050004, 582 0x91c4, 0xffffffff, 0x00010006, 583 0x91c8, 0xffffffff, 0x00090008, 584 0x91cc, 0xffffffff, 0x00070000, 585 0x91d0, 0xffffffff, 0x00030002, 586 0x91d4, 0xffffffff, 0x00050004, 587 0x91e0, 0xffffffff, 0x00010006, 588 0x91e4, 0xffffffff, 0x00090008, 589 0x91e8, 0xffffffff, 0x00000000, 590 0x91ec, 0xffffffff, 0x00070000, 591 0x91f0, 0xffffffff, 0x00030002, 592 0x91f4, 0xffffffff, 0x00050004, 593 0x9200, 0xffffffff, 0x00010006, 594 0x9204, 0xffffffff, 0x00090008, 595 0x9208, 0xffffffff, 0x00070000, 596 0x920c, 0xffffffff, 0x00030002, 597 0x9210, 0xffffffff, 0x00050004, 598 0x921c, 0xffffffff, 0x00010006, 599 0x9220, 0xffffffff, 0x00090008, 600 0x9224, 0xffffffff, 0x00070000, 601 0x9228, 0xffffffff, 0x00030002, 602 0x922c, 0xffffffff, 0x00050004, 603 0x9238, 0xffffffff, 0x00010006, 604 0x923c, 0xffffffff, 0x00090008, 605 0x9240, 0xffffffff, 0x00070000, 606 0x9244, 0xffffffff, 0x00030002, 607 0x9248, 0xffffffff, 0x00050004, 608 0x9254, 0xffffffff, 0x00010006, 609 0x9258, 0xffffffff, 0x00090008, 610 0x925c, 0xffffffff, 0x00070000, 611 0x9260, 0xffffffff, 0x00030002, 612 0x9264, 0xffffffff, 0x00050004, 613 0x9270, 0xffffffff, 0x00010006, 614 0x9274, 0xffffffff, 0x00090008, 615 0x9278, 0xffffffff, 0x00070000, 616 0x927c, 0xffffffff, 0x00030002, 617 0x9280, 0xffffffff, 0x00050004, 618 0x928c, 0xffffffff, 0x00010006, 619 0x9290, 0xffffffff, 0x00090008, 620 0x9294, 0xffffffff, 0x00000000, 621 0x929c, 0xffffffff, 0x00000001, 622 0x802c, 0xffffffff, 0xc0000000, 623 0x977c, 0xffffffff, 0x00000100, 624 0x3f80, 0xffffffff, 0x00000100, 625 0xa210, 0xffffffff, 0x00000100, 626 0xa214, 0xffffffff, 0x00000100, 627 0x4d8, 0xffffffff, 0x00000100, 628 0x9784, 0xffffffff, 0x00000100, 629 0x9698, 0xffffffff, 0x00000100, 630 0x4d4, 0xffffffff, 0x00000200, 631 0x30cc, 0xffffffff, 0x00000100, 632 0x802c, 0xffffffff, 0xc0000000 633 }; 634 635 static const u32 supersumo_golden_registers[] = 636 { 637 0x5eb4, 0xffffffff, 0x00000002, 638 0x5c4, 0xffffffff, 0x00000001, 639 0x7030, 0xffffffff, 0x00000011, 640 0x7c30, 0xffffffff, 0x00000011, 641 0x6104, 0x01000300, 0x00000000, 642 0x5bc0, 0x00300000, 0x00000000, 643 0x8c04, 0xffffffff, 0x40600060, 644 0x8c08, 0xffffffff, 0x001c001c, 645 0x8c20, 0xffffffff, 0x00800080, 646 0x8c24, 0xffffffff, 0x00800080, 647 0x8c18, 0xffffffff, 0x20202078, 648 0x8c1c, 0xffffffff, 0x00001010, 649 0x918c, 0xffffffff, 0x00010006, 650 0x91a8, 0xffffffff, 0x00010006, 651 0x91c4, 0xffffffff, 0x00010006, 652 0x91e0, 0xffffffff, 0x00010006, 653 0x9200, 0xffffffff, 0x00010006, 654 0x9150, 0xffffffff, 0x6e944040, 655 0x917c, 0xffffffff, 0x00030002, 656 0x9180, 0xffffffff, 0x00050004, 657 0x9198, 0xffffffff, 0x00030002, 658 0x919c, 0xffffffff, 0x00050004, 659 0x91b4, 0xffffffff, 0x00030002, 660 0x91b8, 0xffffffff, 0x00050004, 661 0x91d0, 0xffffffff, 0x00030002, 662 0x91d4, 0xffffffff, 0x00050004, 663 0x91f0, 0xffffffff, 0x00030002, 664 0x91f4, 0xffffffff, 0x00050004, 665 0x915c, 0xffffffff, 0x00010000, 666 0x9160, 0xffffffff, 0x00030002, 667 0x3f90, 0xffff0000, 0xff000000, 668 0x9178, 0xffffffff, 0x00070000, 669 0x9194, 0xffffffff, 0x00070000, 670 0x91b0, 0xffffffff, 0x00070000, 671 0x91cc, 0xffffffff, 0x00070000, 672 0x91ec, 0xffffffff, 0x00070000, 673 0x9148, 0xffff0000, 0xff000000, 674 0x9190, 0xffffffff, 0x00090008, 675 0x91ac, 0xffffffff, 0x00090008, 676 0x91c8, 0xffffffff, 0x00090008, 677 0x91e4, 0xffffffff, 0x00090008, 678 0x9204, 0xffffffff, 0x00090008, 679 0x3f94, 0xffff0000, 0xff000000, 680 0x914c, 0xffff0000, 0xff000000, 681 0x929c, 0xffffffff, 0x00000001, 682 0x8a18, 0xffffffff, 0x00000100, 683 0x8b28, 0xffffffff, 0x00000100, 684 0x9144, 0xffffffff, 0x00000100, 685 0x5644, 0xffffffff, 0x00000100, 686 0x9b7c, 0xffffffff, 0x00000000, 687 0x8030, 0xffffffff, 0x0000100a, 688 0x8a14, 0xffffffff, 0x00000007, 689 0x8b24, 0xffffffff, 0x00ff0fff, 690 0x8b10, 0xffffffff, 0x00000000, 691 0x28a4c, 0x06000000, 0x06000000, 692 0x4d8, 0xffffffff, 0x00000100, 693 0x913c, 0xffff000f, 0x0100000a, 694 0x960c, 0xffffffff, 0x54763210, 695 0x88c4, 0xffffffff, 0x000000c2, 696 0x88d4, 0xffffffff, 0x00000010, 697 0x8974, 0xffffffff, 0x00000000, 698 0xc78, 0x00000080, 0x00000080, 699 0x5e78, 0xffffffff, 0x001000f0, 700 0xd02c, 0xffffffff, 0x08421000, 701 0xa008, 0xffffffff, 0x00010000, 702 0x8d00, 0xffffffff, 0x100e4848, 703 0x8d04, 0xffffffff, 0x00164745, 704 0x8c00, 0xffffffff, 0xe4000003, 705 0x8cf0, 0x1fffffff, 0x08e00620, 706 0x28350, 0xffffffff, 0x00000000, 707 0x9508, 0xffffffff, 0x00000002 708 }; 709 710 static const u32 sumo_golden_registers[] = 711 { 712 0x900c, 0x00ffffff, 0x0017071f, 713 0x8c18, 0xffffffff, 0x10101060, 714 0x8c1c, 0xffffffff, 0x00001010, 715 0x8c30, 0x0000000f, 0x00000005, 716 0x9688, 0x0000000f, 0x00000007 717 }; 718 719 static const u32 wrestler_golden_registers[] = 720 { 721 0x5eb4, 0xffffffff, 0x00000002, 722 0x5c4, 0xffffffff, 0x00000001, 723 0x7030, 0xffffffff, 0x00000011, 724 0x7c30, 0xffffffff, 0x00000011, 725 0x6104, 0x01000300, 0x00000000, 726 0x5bc0, 0x00300000, 0x00000000, 727 0x918c, 0xffffffff, 0x00010006, 728 0x91a8, 0xffffffff, 0x00010006, 729 0x9150, 0xffffffff, 0x6e944040, 730 0x917c, 0xffffffff, 0x00030002, 731 0x9198, 0xffffffff, 0x00030002, 732 0x915c, 0xffffffff, 0x00010000, 733 0x3f90, 0xffff0000, 0xff000000, 734 0x9178, 0xffffffff, 0x00070000, 735 0x9194, 0xffffffff, 0x00070000, 736 0x9148, 0xffff0000, 0xff000000, 737 0x9190, 0xffffffff, 0x00090008, 738 0x91ac, 0xffffffff, 0x00090008, 739 0x3f94, 0xffff0000, 0xff000000, 740 0x914c, 0xffff0000, 0xff000000, 741 0x929c, 0xffffffff, 0x00000001, 742 0x8a18, 0xffffffff, 0x00000100, 743 0x8b28, 0xffffffff, 0x00000100, 744 0x9144, 0xffffffff, 0x00000100, 745 0x9b7c, 0xffffffff, 0x00000000, 746 0x8030, 0xffffffff, 0x0000100a, 747 0x8a14, 0xffffffff, 0x00000001, 748 0x8b24, 0xffffffff, 0x00ff0fff, 749 0x8b10, 0xffffffff, 0x00000000, 750 0x28a4c, 0x06000000, 0x06000000, 751 0x4d8, 0xffffffff, 0x00000100, 752 0x913c, 0xffff000f, 0x0100000a, 753 0x960c, 0xffffffff, 0x54763210, 754 0x88c4, 0xffffffff, 0x000000c2, 755 0x88d4, 0xffffffff, 0x00000010, 756 0x8974, 0xffffffff, 0x00000000, 757 0xc78, 0x00000080, 0x00000080, 758 0x5e78, 0xffffffff, 0x001000f0, 759 0xd02c, 0xffffffff, 0x08421000, 760 0xa008, 0xffffffff, 0x00010000, 761 0x8d00, 0xffffffff, 0x100e4848, 762 0x8d04, 0xffffffff, 0x00164745, 763 0x8c00, 0xffffffff, 0xe4000003, 764 0x8cf0, 0x1fffffff, 0x08e00410, 765 0x28350, 0xffffffff, 0x00000000, 766 0x9508, 0xffffffff, 0x00000002, 767 0x900c, 0xffffffff, 0x0017071f, 768 0x8c18, 0xffffffff, 0x10101060, 769 0x8c1c, 0xffffffff, 0x00001010 770 }; 771 772 static const u32 barts_golden_registers[] = 773 { 774 0x5eb4, 0xffffffff, 0x00000002, 775 0x5e78, 0x8f311ff1, 0x001000f0, 776 0x3f90, 0xffff0000, 0xff000000, 777 0x9148, 0xffff0000, 0xff000000, 778 0x3f94, 0xffff0000, 0xff000000, 779 0x914c, 0xffff0000, 0xff000000, 780 0xc78, 0x00000080, 0x00000080, 781 0xbd4, 0x70073777, 0x00010001, 782 0xd02c, 0xbfffff1f, 0x08421000, 783 0xd0b8, 0x03773777, 0x02011003, 784 0x5bc0, 0x00200000, 0x50100000, 785 0x98f8, 0x33773777, 0x02011003, 786 0x98fc, 0xffffffff, 0x76543210, 787 0x7030, 0x31000311, 0x00000011, 788 0x2f48, 0x00000007, 0x02011003, 789 0x6b28, 0x00000010, 0x00000012, 790 0x7728, 0x00000010, 0x00000012, 791 0x10328, 0x00000010, 0x00000012, 792 0x10f28, 0x00000010, 0x00000012, 793 0x11b28, 0x00000010, 0x00000012, 794 0x12728, 0x00000010, 0x00000012, 795 0x240c, 0x000007ff, 0x00000380, 796 0x8a14, 0xf000001f, 0x00000007, 797 0x8b24, 0x3fff3fff, 0x00ff0fff, 798 0x8b10, 0x0000ff0f, 0x00000000, 799 0x28a4c, 0x07ffffff, 0x06000000, 800 0x10c, 0x00000001, 0x00010003, 801 0xa02c, 0xffffffff, 0x0000009b, 802 0x913c, 0x0000000f, 0x0100000a, 803 0x8d00, 0xffff7f7f, 0x100e4848, 804 0x8d04, 0x00ffffff, 0x00164745, 805 0x8c00, 0xfffc0003, 0xe4000003, 806 0x8c04, 0xf8ff00ff, 0x40600060, 807 0x8c08, 0x00ff00ff, 0x001c001c, 808 0x8cf0, 0x1fff1fff, 0x08e00620, 809 0x8c20, 0x0fff0fff, 0x00800080, 810 0x8c24, 0x0fff0fff, 0x00800080, 811 0x8c18, 0xffffffff, 0x20202078, 812 0x8c1c, 0x0000ffff, 0x00001010, 813 0x28350, 0x00000f01, 0x00000000, 814 0x9508, 0x3700001f, 0x00000002, 815 0x960c, 0xffffffff, 0x54763210, 816 0x88c4, 0x001f3ae3, 0x000000c2, 817 0x88d4, 0x0000001f, 0x00000010, 818 0x8974, 0xffffffff, 0x00000000 819 }; 820 821 static const u32 turks_golden_registers[] = 822 { 823 0x5eb4, 0xffffffff, 0x00000002, 824 0x5e78, 0x8f311ff1, 0x001000f0, 825 0x8c8, 0x00003000, 0x00001070, 826 0x8cc, 0x000fffff, 0x00040035, 827 0x3f90, 0xffff0000, 0xfff00000, 828 0x9148, 0xffff0000, 0xfff00000, 829 0x3f94, 0xffff0000, 0xfff00000, 830 0x914c, 0xffff0000, 0xfff00000, 831 0xc78, 0x00000080, 0x00000080, 832 0xbd4, 0x00073007, 0x00010002, 833 0xd02c, 0xbfffff1f, 0x08421000, 834 0xd0b8, 0x03773777, 0x02010002, 835 0x5bc0, 0x00200000, 0x50100000, 836 0x98f8, 0x33773777, 0x00010002, 837 0x98fc, 0xffffffff, 0x33221100, 838 0x7030, 0x31000311, 0x00000011, 839 0x2f48, 0x33773777, 0x00010002, 840 0x6b28, 0x00000010, 0x00000012, 841 0x7728, 0x00000010, 0x00000012, 842 0x10328, 0x00000010, 0x00000012, 843 0x10f28, 0x00000010, 0x00000012, 844 0x11b28, 0x00000010, 0x00000012, 845 0x12728, 0x00000010, 0x00000012, 846 0x240c, 0x000007ff, 0x00000380, 847 0x8a14, 0xf000001f, 0x00000007, 848 0x8b24, 0x3fff3fff, 0x00ff0fff, 849 0x8b10, 0x0000ff0f, 0x00000000, 850 0x28a4c, 0x07ffffff, 0x06000000, 851 0x10c, 0x00000001, 0x00010003, 852 0xa02c, 0xffffffff, 0x0000009b, 853 0x913c, 0x0000000f, 0x0100000a, 854 0x8d00, 0xffff7f7f, 0x100e4848, 855 0x8d04, 0x00ffffff, 0x00164745, 856 0x8c00, 0xfffc0003, 0xe4000003, 857 0x8c04, 0xf8ff00ff, 0x40600060, 858 0x8c08, 0x00ff00ff, 0x001c001c, 859 0x8cf0, 0x1fff1fff, 0x08e00410, 860 0x8c20, 0x0fff0fff, 0x00800080, 861 0x8c24, 0x0fff0fff, 0x00800080, 862 0x8c18, 0xffffffff, 0x20202078, 863 0x8c1c, 0x0000ffff, 0x00001010, 864 0x28350, 0x00000f01, 0x00000000, 865 0x9508, 0x3700001f, 0x00000002, 866 0x960c, 0xffffffff, 0x54763210, 867 0x88c4, 0x001f3ae3, 0x000000c2, 868 0x88d4, 0x0000001f, 0x00000010, 869 0x8974, 0xffffffff, 0x00000000 870 }; 871 872 static const u32 caicos_golden_registers[] = 873 { 874 0x5eb4, 0xffffffff, 0x00000002, 875 0x5e78, 0x8f311ff1, 0x001000f0, 876 0x8c8, 0x00003420, 0x00001450, 877 0x8cc, 0x000fffff, 0x00040035, 878 0x3f90, 0xffff0000, 0xfffc0000, 879 0x9148, 0xffff0000, 0xfffc0000, 880 0x3f94, 0xffff0000, 0xfffc0000, 881 0x914c, 0xffff0000, 0xfffc0000, 882 0xc78, 0x00000080, 0x00000080, 883 0xbd4, 0x00073007, 0x00010001, 884 0xd02c, 0xbfffff1f, 0x08421000, 885 0xd0b8, 0x03773777, 0x02010001, 886 0x5bc0, 0x00200000, 0x50100000, 887 0x98f8, 0x33773777, 0x02010001, 888 0x98fc, 0xffffffff, 0x33221100, 889 0x7030, 0x31000311, 0x00000011, 890 0x2f48, 0x33773777, 0x02010001, 891 0x6b28, 0x00000010, 0x00000012, 892 0x7728, 0x00000010, 0x00000012, 893 0x10328, 0x00000010, 0x00000012, 894 0x10f28, 0x00000010, 0x00000012, 895 0x11b28, 0x00000010, 0x00000012, 896 0x12728, 0x00000010, 0x00000012, 897 0x240c, 0x000007ff, 0x00000380, 898 0x8a14, 0xf000001f, 0x00000001, 899 0x8b24, 0x3fff3fff, 0x00ff0fff, 900 0x8b10, 0x0000ff0f, 0x00000000, 901 0x28a4c, 0x07ffffff, 0x06000000, 902 0x10c, 0x00000001, 0x00010003, 903 0xa02c, 0xffffffff, 0x0000009b, 904 0x913c, 0x0000000f, 0x0100000a, 905 0x8d00, 0xffff7f7f, 0x100e4848, 906 0x8d04, 0x00ffffff, 0x00164745, 907 0x8c00, 0xfffc0003, 0xe4000003, 908 0x8c04, 0xf8ff00ff, 0x40600060, 909 0x8c08, 0x00ff00ff, 0x001c001c, 910 0x8cf0, 0x1fff1fff, 0x08e00410, 911 0x8c20, 0x0fff0fff, 0x00800080, 912 0x8c24, 0x0fff0fff, 0x00800080, 913 0x8c18, 0xffffffff, 0x20202078, 914 0x8c1c, 0x0000ffff, 0x00001010, 915 0x28350, 0x00000f01, 0x00000000, 916 0x9508, 0x3700001f, 0x00000002, 917 0x960c, 0xffffffff, 0x54763210, 918 0x88c4, 0x001f3ae3, 0x000000c2, 919 0x88d4, 0x0000001f, 0x00000010, 920 0x8974, 0xffffffff, 0x00000000 921 }; 922 923 static void evergreen_init_golden_registers(struct radeon_device *rdev) 924 { 925 switch (rdev->family) { 926 case CHIP_CYPRESS: 927 case CHIP_HEMLOCK: 928 radeon_program_register_sequence(rdev, 929 evergreen_golden_registers, 930 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 931 radeon_program_register_sequence(rdev, 932 evergreen_golden_registers2, 933 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 934 radeon_program_register_sequence(rdev, 935 cypress_mgcg_init, 936 (const u32)ARRAY_SIZE(cypress_mgcg_init)); 937 break; 938 case CHIP_JUNIPER: 939 radeon_program_register_sequence(rdev, 940 evergreen_golden_registers, 941 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 942 radeon_program_register_sequence(rdev, 943 evergreen_golden_registers2, 944 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 945 radeon_program_register_sequence(rdev, 946 juniper_mgcg_init, 947 (const u32)ARRAY_SIZE(juniper_mgcg_init)); 948 break; 949 case CHIP_REDWOOD: 950 radeon_program_register_sequence(rdev, 951 evergreen_golden_registers, 952 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 953 radeon_program_register_sequence(rdev, 954 evergreen_golden_registers2, 955 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 956 radeon_program_register_sequence(rdev, 957 redwood_mgcg_init, 958 (const u32)ARRAY_SIZE(redwood_mgcg_init)); 959 break; 960 case CHIP_CEDAR: 961 radeon_program_register_sequence(rdev, 962 cedar_golden_registers, 963 (const u32)ARRAY_SIZE(cedar_golden_registers)); 964 radeon_program_register_sequence(rdev, 965 evergreen_golden_registers2, 966 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 967 radeon_program_register_sequence(rdev, 968 cedar_mgcg_init, 969 (const u32)ARRAY_SIZE(cedar_mgcg_init)); 970 break; 971 case CHIP_PALM: 972 radeon_program_register_sequence(rdev, 973 wrestler_golden_registers, 974 (const u32)ARRAY_SIZE(wrestler_golden_registers)); 975 break; 976 case CHIP_SUMO: 977 radeon_program_register_sequence(rdev, 978 supersumo_golden_registers, 979 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 980 break; 981 case CHIP_SUMO2: 982 radeon_program_register_sequence(rdev, 983 supersumo_golden_registers, 984 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 985 radeon_program_register_sequence(rdev, 986 sumo_golden_registers, 987 (const u32)ARRAY_SIZE(sumo_golden_registers)); 988 break; 989 case CHIP_BARTS: 990 radeon_program_register_sequence(rdev, 991 barts_golden_registers, 992 (const u32)ARRAY_SIZE(barts_golden_registers)); 993 break; 994 case CHIP_TURKS: 995 radeon_program_register_sequence(rdev, 996 turks_golden_registers, 997 (const u32)ARRAY_SIZE(turks_golden_registers)); 998 break; 999 case CHIP_CAICOS: 1000 radeon_program_register_sequence(rdev, 1001 caicos_golden_registers, 1002 (const u32)ARRAY_SIZE(caicos_golden_registers)); 1003 break; 1004 default: 1005 break; 1006 } 1007 } 1008 1009 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, 1010 unsigned *bankh, unsigned *mtaspect, 1011 unsigned *tile_split) 1012 { 1013 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 1014 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 1015 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 1016 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 1017 switch (*bankw) { 1018 default: 1019 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break; 1020 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break; 1021 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break; 1022 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break; 1023 } 1024 switch (*bankh) { 1025 default: 1026 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break; 1027 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break; 1028 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break; 1029 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break; 1030 } 1031 switch (*mtaspect) { 1032 default: 1033 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break; 1034 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break; 1035 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break; 1036 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break; 1037 } 1038 } 1039 1040 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock, 1041 u32 cntl_reg, u32 status_reg) 1042 { 1043 int r, i; 1044 struct atom_clock_dividers dividers; 1045 1046 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, 1047 clock, false, ÷rs); 1048 if (r) 1049 return r; 1050 1051 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK)); 1052 1053 for (i = 0; i < 100; i++) { 1054 if (RREG32(status_reg) & DCLK_STATUS) 1055 break; 1056 mdelay(10); 1057 } 1058 if (i == 100) 1059 return -ETIMEDOUT; 1060 1061 return 0; 1062 } 1063 1064 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1065 { 1066 int r = 0; 1067 u32 cg_scratch = RREG32(CG_SCRATCH1); 1068 1069 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 1070 if (r) 1071 goto done; 1072 cg_scratch &= 0xffff0000; 1073 cg_scratch |= vclk / 100; /* Mhz */ 1074 1075 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 1076 if (r) 1077 goto done; 1078 cg_scratch &= 0x0000ffff; 1079 cg_scratch |= (dclk / 100) << 16; /* Mhz */ 1080 1081 done: 1082 WREG32(CG_SCRATCH1, cg_scratch); 1083 1084 return r; 1085 } 1086 1087 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1088 { 1089 /* start off with something large */ 1090 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 1091 int r; 1092 1093 /* bypass vclk and dclk with bclk */ 1094 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1095 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 1096 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1097 1098 /* put PLL in bypass mode */ 1099 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 1100 1101 if (!vclk || !dclk) { 1102 /* keep the Bypass mode, put PLL to sleep */ 1103 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1104 return 0; 1105 } 1106 1107 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 1108 16384, 0x03FFFFFF, 0, 128, 5, 1109 &fb_div, &vclk_div, &dclk_div); 1110 if (r) 1111 return r; 1112 1113 /* set VCO_MODE to 1 */ 1114 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 1115 1116 /* toggle UPLL_SLEEP to 1 then back to 0 */ 1117 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1118 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 1119 1120 /* deassert UPLL_RESET */ 1121 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1122 1123 mdelay(1); 1124 1125 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1126 if (r) 1127 return r; 1128 1129 /* assert UPLL_RESET again */ 1130 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 1131 1132 /* disable spread spectrum. */ 1133 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 1134 1135 /* set feedback divider */ 1136 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 1137 1138 /* set ref divider to 0 */ 1139 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 1140 1141 if (fb_div < 307200) 1142 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 1143 else 1144 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 1145 1146 /* set PDIV_A and PDIV_B */ 1147 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1148 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 1149 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 1150 1151 /* give the PLL some time to settle */ 1152 mdelay(15); 1153 1154 /* deassert PLL_RESET */ 1155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1156 1157 mdelay(15); 1158 1159 /* switch from bypass mode to normal mode */ 1160 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 1161 1162 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1163 if (r) 1164 return r; 1165 1166 /* switch VCLK and DCLK selection */ 1167 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1168 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 1169 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1170 1171 mdelay(100); 1172 1173 return 0; 1174 } 1175 1176 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 1177 { 1178 int readrq; 1179 u16 v; 1180 1181 readrq = pcie_get_readrq(rdev->pdev); 1182 v = ffs(readrq) - 8; 1183 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it 1184 * to avoid hangs or perfomance issues 1185 */ 1186 if ((v == 0) || (v == 6) || (v == 7)) 1187 pcie_set_readrq(rdev->pdev, 512); 1188 } 1189 1190 void dce4_program_fmt(struct drm_encoder *encoder) 1191 { 1192 struct drm_device *dev = encoder->dev; 1193 struct radeon_device *rdev = dev->dev_private; 1194 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 1195 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 1196 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 1197 int bpc = 0; 1198 u32 tmp = 0; 1199 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 1200 1201 if (connector) { 1202 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1203 bpc = radeon_get_monitor_bpc(connector); 1204 dither = radeon_connector->dither; 1205 } 1206 1207 /* LVDS/eDP FMT is set up by atom */ 1208 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 1209 return; 1210 1211 /* not needed for analog */ 1212 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 1213 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 1214 return; 1215 1216 if (bpc == 0) 1217 return; 1218 1219 switch (bpc) { 1220 case 6: 1221 if (dither == RADEON_FMT_DITHER_ENABLE) 1222 /* XXX sort out optimal dither settings */ 1223 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1224 FMT_SPATIAL_DITHER_EN); 1225 else 1226 tmp |= FMT_TRUNCATE_EN; 1227 break; 1228 case 8: 1229 if (dither == RADEON_FMT_DITHER_ENABLE) 1230 /* XXX sort out optimal dither settings */ 1231 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1232 FMT_RGB_RANDOM_ENABLE | 1233 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); 1234 else 1235 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); 1236 break; 1237 case 10: 1238 default: 1239 /* not needed */ 1240 break; 1241 } 1242 1243 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 1244 } 1245 1246 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) 1247 { 1248 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK) 1249 return true; 1250 else 1251 return false; 1252 } 1253 1254 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc) 1255 { 1256 u32 pos1, pos2; 1257 1258 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1259 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1260 1261 if (pos1 != pos2) 1262 return true; 1263 else 1264 return false; 1265 } 1266 1267 /** 1268 * dce4_wait_for_vblank - vblank wait asic callback. 1269 * 1270 * @rdev: radeon_device pointer 1271 * @crtc: crtc to wait for vblank on 1272 * 1273 * Wait for vblank on the requested crtc (evergreen+). 1274 */ 1275 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) 1276 { 1277 unsigned i = 0; 1278 1279 if (crtc >= rdev->num_crtc) 1280 return; 1281 1282 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN)) 1283 return; 1284 1285 /* depending on when we hit vblank, we may be close to active; if so, 1286 * wait for another frame. 1287 */ 1288 while (dce4_is_in_vblank(rdev, crtc)) { 1289 if (i++ % 100 == 0) { 1290 if (!dce4_is_counter_moving(rdev, crtc)) 1291 break; 1292 } 1293 } 1294 1295 while (!dce4_is_in_vblank(rdev, crtc)) { 1296 if (i++ % 100 == 0) { 1297 if (!dce4_is_counter_moving(rdev, crtc)) 1298 break; 1299 } 1300 } 1301 } 1302 1303 /** 1304 * evergreen_page_flip - pageflip callback. 1305 * 1306 * @rdev: radeon_device pointer 1307 * @crtc_id: crtc to cleanup pageflip on 1308 * @crtc_base: new address of the crtc (GPU MC address) 1309 * 1310 * Does the actual pageflip (evergreen+). 1311 * During vblank we take the crtc lock and wait for the update_pending 1312 * bit to go high, when it does, we release the lock, and allow the 1313 * double buffered update to take place. 1314 * Returns the current update pending status. 1315 */ 1316 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) 1317 { 1318 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1319 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset); 1320 int i; 1321 1322 /* Lock the graphics update lock */ 1323 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 1324 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 1325 1326 /* update the scanout addresses */ 1327 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1328 upper_32_bits(crtc_base)); 1329 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1330 (u32)crtc_base); 1331 1332 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1333 upper_32_bits(crtc_base)); 1334 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1335 (u32)crtc_base); 1336 1337 /* Wait for update_pending to go high. */ 1338 for (i = 0; i < rdev->usec_timeout; i++) { 1339 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) 1340 break; 1341 udelay(1); 1342 } 1343 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n"); 1344 1345 /* Unlock the lock, so double-buffering can take place inside vblank */ 1346 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 1347 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 1348 } 1349 1350 /** 1351 * evergreen_page_flip_pending - check if page flip is still pending 1352 * 1353 * @rdev: radeon_device pointer 1354 * @crtc_id: crtc to check 1355 * 1356 * Returns the current update pending status. 1357 */ 1358 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) 1359 { 1360 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1361 1362 /* Return current update_pending status: */ 1363 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & 1364 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); 1365 } 1366 1367 /* get temperature in millidegrees */ 1368 int evergreen_get_temp(struct radeon_device *rdev) 1369 { 1370 u32 temp, toffset; 1371 int actual_temp = 0; 1372 1373 if (rdev->family == CHIP_JUNIPER) { 1374 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 1375 TOFFSET_SHIFT; 1376 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 1377 TS0_ADC_DOUT_SHIFT; 1378 1379 if (toffset & 0x100) 1380 actual_temp = temp / 2 - (0x200 - toffset); 1381 else 1382 actual_temp = temp / 2 + toffset; 1383 1384 actual_temp = actual_temp * 1000; 1385 1386 } else { 1387 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 1388 ASIC_T_SHIFT; 1389 1390 if (temp & 0x400) 1391 actual_temp = -256; 1392 else if (temp & 0x200) 1393 actual_temp = 255; 1394 else if (temp & 0x100) { 1395 actual_temp = temp & 0x1ff; 1396 actual_temp |= ~0x1ff; 1397 } else 1398 actual_temp = temp & 0xff; 1399 1400 actual_temp = (actual_temp * 1000) / 2; 1401 } 1402 1403 return actual_temp; 1404 } 1405 1406 int sumo_get_temp(struct radeon_device *rdev) 1407 { 1408 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 1409 int actual_temp = temp - 49; 1410 1411 return actual_temp * 1000; 1412 } 1413 1414 /** 1415 * sumo_pm_init_profile - Initialize power profiles callback. 1416 * 1417 * @rdev: radeon_device pointer 1418 * 1419 * Initialize the power states used in profile mode 1420 * (sumo, trinity, SI). 1421 * Used for profile mode only. 1422 */ 1423 void sumo_pm_init_profile(struct radeon_device *rdev) 1424 { 1425 int idx; 1426 1427 /* default */ 1428 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1429 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1430 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1431 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0; 1432 1433 /* low,mid sh/mh */ 1434 if (rdev->flags & RADEON_IS_MOBILITY) 1435 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1436 else 1437 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1438 1439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1440 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1441 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1442 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1443 1444 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1445 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1446 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1447 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1448 1449 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1450 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1451 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1452 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0; 1453 1454 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1455 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1457 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0; 1458 1459 /* high sh/mh */ 1460 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1461 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1462 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1463 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1464 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 1465 rdev->pm.power_state[idx].num_clock_modes - 1; 1466 1467 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1468 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1469 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1470 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 1471 rdev->pm.power_state[idx].num_clock_modes - 1; 1472 } 1473 1474 /** 1475 * btc_pm_init_profile - Initialize power profiles callback. 1476 * 1477 * @rdev: radeon_device pointer 1478 * 1479 * Initialize the power states used in profile mode 1480 * (BTC, cayman). 1481 * Used for profile mode only. 1482 */ 1483 void btc_pm_init_profile(struct radeon_device *rdev) 1484 { 1485 int idx; 1486 1487 /* default */ 1488 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1491 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2; 1492 /* starting with BTC, there is one state that is used for both 1493 * MH and SH. Difference is that we always use the high clock index for 1494 * mclk. 1495 */ 1496 if (rdev->flags & RADEON_IS_MOBILITY) 1497 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1498 else 1499 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1500 /* low sh */ 1501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1504 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1505 /* mid sh */ 1506 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1507 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1508 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1509 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1; 1510 /* high sh */ 1511 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1512 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1513 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1514 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2; 1515 /* low mh */ 1516 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1519 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1520 /* mid mh */ 1521 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1522 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1523 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1524 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1; 1525 /* high mh */ 1526 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1527 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1529 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2; 1530 } 1531 1532 /** 1533 * evergreen_pm_misc - set additional pm hw parameters callback. 1534 * 1535 * @rdev: radeon_device pointer 1536 * 1537 * Set non-clock parameters associated with a power state 1538 * (voltage, etc.) (evergreen+). 1539 */ 1540 void evergreen_pm_misc(struct radeon_device *rdev) 1541 { 1542 int req_ps_idx = rdev->pm.requested_power_state_index; 1543 int req_cm_idx = rdev->pm.requested_clock_mode_index; 1544 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 1545 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 1546 1547 if (voltage->type == VOLTAGE_SW) { 1548 /* 0xff0x are flags rather then an actual voltage */ 1549 if ((voltage->voltage & 0xff00) == 0xff00) 1550 return; 1551 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 1552 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 1553 rdev->pm.current_vddc = voltage->voltage; 1554 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 1555 } 1556 1557 /* starting with BTC, there is one state that is used for both 1558 * MH and SH. Difference is that we always use the high clock index for 1559 * mclk and vddci. 1560 */ 1561 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) && 1562 (rdev->family >= CHIP_BARTS) && 1563 rdev->pm.active_crtc_count && 1564 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) || 1565 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX))) 1566 voltage = &rdev->pm.power_state[req_ps_idx]. 1567 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; 1568 1569 /* 0xff0x are flags rather then an actual voltage */ 1570 if ((voltage->vddci & 0xff00) == 0xff00) 1571 return; 1572 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 1573 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 1574 rdev->pm.current_vddci = voltage->vddci; 1575 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 1576 } 1577 } 1578 } 1579 1580 /** 1581 * evergreen_pm_prepare - pre-power state change callback. 1582 * 1583 * @rdev: radeon_device pointer 1584 * 1585 * Prepare for a power state change (evergreen+). 1586 */ 1587 void evergreen_pm_prepare(struct radeon_device *rdev) 1588 { 1589 struct drm_device *ddev = rdev->ddev; 1590 struct drm_crtc *crtc; 1591 struct radeon_crtc *radeon_crtc; 1592 u32 tmp; 1593 1594 /* disable any active CRTCs */ 1595 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1596 radeon_crtc = to_radeon_crtc(crtc); 1597 if (radeon_crtc->enabled) { 1598 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1599 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1600 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1601 } 1602 } 1603 } 1604 1605 /** 1606 * evergreen_pm_finish - post-power state change callback. 1607 * 1608 * @rdev: radeon_device pointer 1609 * 1610 * Clean up after a power state change (evergreen+). 1611 */ 1612 void evergreen_pm_finish(struct radeon_device *rdev) 1613 { 1614 struct drm_device *ddev = rdev->ddev; 1615 struct drm_crtc *crtc; 1616 struct radeon_crtc *radeon_crtc; 1617 u32 tmp; 1618 1619 /* enable any active CRTCs */ 1620 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1621 radeon_crtc = to_radeon_crtc(crtc); 1622 if (radeon_crtc->enabled) { 1623 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1624 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1625 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1626 } 1627 } 1628 } 1629 1630 /** 1631 * evergreen_hpd_sense - hpd sense callback. 1632 * 1633 * @rdev: radeon_device pointer 1634 * @hpd: hpd (hotplug detect) pin 1635 * 1636 * Checks if a digital monitor is connected (evergreen+). 1637 * Returns true if connected, false if not connected. 1638 */ 1639 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 1640 { 1641 bool connected = false; 1642 1643 switch (hpd) { 1644 case RADEON_HPD_1: 1645 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE) 1646 connected = true; 1647 break; 1648 case RADEON_HPD_2: 1649 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE) 1650 connected = true; 1651 break; 1652 case RADEON_HPD_3: 1653 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE) 1654 connected = true; 1655 break; 1656 case RADEON_HPD_4: 1657 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE) 1658 connected = true; 1659 break; 1660 case RADEON_HPD_5: 1661 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE) 1662 connected = true; 1663 break; 1664 case RADEON_HPD_6: 1665 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) 1666 connected = true; 1667 break; 1668 default: 1669 break; 1670 } 1671 1672 return connected; 1673 } 1674 1675 /** 1676 * evergreen_hpd_set_polarity - hpd set polarity callback. 1677 * 1678 * @rdev: radeon_device pointer 1679 * @hpd: hpd (hotplug detect) pin 1680 * 1681 * Set the polarity of the hpd pin (evergreen+). 1682 */ 1683 void evergreen_hpd_set_polarity(struct radeon_device *rdev, 1684 enum radeon_hpd_id hpd) 1685 { 1686 u32 tmp; 1687 bool connected = evergreen_hpd_sense(rdev, hpd); 1688 1689 switch (hpd) { 1690 case RADEON_HPD_1: 1691 tmp = RREG32(DC_HPD1_INT_CONTROL); 1692 if (connected) 1693 tmp &= ~DC_HPDx_INT_POLARITY; 1694 else 1695 tmp |= DC_HPDx_INT_POLARITY; 1696 WREG32(DC_HPD1_INT_CONTROL, tmp); 1697 break; 1698 case RADEON_HPD_2: 1699 tmp = RREG32(DC_HPD2_INT_CONTROL); 1700 if (connected) 1701 tmp &= ~DC_HPDx_INT_POLARITY; 1702 else 1703 tmp |= DC_HPDx_INT_POLARITY; 1704 WREG32(DC_HPD2_INT_CONTROL, tmp); 1705 break; 1706 case RADEON_HPD_3: 1707 tmp = RREG32(DC_HPD3_INT_CONTROL); 1708 if (connected) 1709 tmp &= ~DC_HPDx_INT_POLARITY; 1710 else 1711 tmp |= DC_HPDx_INT_POLARITY; 1712 WREG32(DC_HPD3_INT_CONTROL, tmp); 1713 break; 1714 case RADEON_HPD_4: 1715 tmp = RREG32(DC_HPD4_INT_CONTROL); 1716 if (connected) 1717 tmp &= ~DC_HPDx_INT_POLARITY; 1718 else 1719 tmp |= DC_HPDx_INT_POLARITY; 1720 WREG32(DC_HPD4_INT_CONTROL, tmp); 1721 break; 1722 case RADEON_HPD_5: 1723 tmp = RREG32(DC_HPD5_INT_CONTROL); 1724 if (connected) 1725 tmp &= ~DC_HPDx_INT_POLARITY; 1726 else 1727 tmp |= DC_HPDx_INT_POLARITY; 1728 WREG32(DC_HPD5_INT_CONTROL, tmp); 1729 break; 1730 case RADEON_HPD_6: 1731 tmp = RREG32(DC_HPD6_INT_CONTROL); 1732 if (connected) 1733 tmp &= ~DC_HPDx_INT_POLARITY; 1734 else 1735 tmp |= DC_HPDx_INT_POLARITY; 1736 WREG32(DC_HPD6_INT_CONTROL, tmp); 1737 break; 1738 default: 1739 break; 1740 } 1741 } 1742 1743 /** 1744 * evergreen_hpd_init - hpd setup callback. 1745 * 1746 * @rdev: radeon_device pointer 1747 * 1748 * Setup the hpd pins used by the card (evergreen+). 1749 * Enable the pin, set the polarity, and enable the hpd interrupts. 1750 */ 1751 void evergreen_hpd_init(struct radeon_device *rdev) 1752 { 1753 struct drm_device *dev = rdev->ddev; 1754 struct drm_connector *connector; 1755 unsigned enabled = 0; 1756 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 1757 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 1758 1759 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1760 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1761 1762 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || 1763 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { 1764 /* don't try to enable hpd on eDP or LVDS avoid breaking the 1765 * aux dp channel on imac and help (but not completely fix) 1766 * https://bugzilla.redhat.com/show_bug.cgi?id=726143 1767 * also avoid interrupt storms during dpms. 1768 */ 1769 continue; 1770 } 1771 switch (radeon_connector->hpd.hpd) { 1772 case RADEON_HPD_1: 1773 WREG32(DC_HPD1_CONTROL, tmp); 1774 break; 1775 case RADEON_HPD_2: 1776 WREG32(DC_HPD2_CONTROL, tmp); 1777 break; 1778 case RADEON_HPD_3: 1779 WREG32(DC_HPD3_CONTROL, tmp); 1780 break; 1781 case RADEON_HPD_4: 1782 WREG32(DC_HPD4_CONTROL, tmp); 1783 break; 1784 case RADEON_HPD_5: 1785 WREG32(DC_HPD5_CONTROL, tmp); 1786 break; 1787 case RADEON_HPD_6: 1788 WREG32(DC_HPD6_CONTROL, tmp); 1789 break; 1790 default: 1791 break; 1792 } 1793 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd); 1794 enabled |= 1 << radeon_connector->hpd.hpd; 1795 } 1796 radeon_irq_kms_enable_hpd(rdev, enabled); 1797 } 1798 1799 /** 1800 * evergreen_hpd_fini - hpd tear down callback. 1801 * 1802 * @rdev: radeon_device pointer 1803 * 1804 * Tear down the hpd pins used by the card (evergreen+). 1805 * Disable the hpd interrupts. 1806 */ 1807 void evergreen_hpd_fini(struct radeon_device *rdev) 1808 { 1809 struct drm_device *dev = rdev->ddev; 1810 struct drm_connector *connector; 1811 unsigned disabled = 0; 1812 1813 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1814 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1815 switch (radeon_connector->hpd.hpd) { 1816 case RADEON_HPD_1: 1817 WREG32(DC_HPD1_CONTROL, 0); 1818 break; 1819 case RADEON_HPD_2: 1820 WREG32(DC_HPD2_CONTROL, 0); 1821 break; 1822 case RADEON_HPD_3: 1823 WREG32(DC_HPD3_CONTROL, 0); 1824 break; 1825 case RADEON_HPD_4: 1826 WREG32(DC_HPD4_CONTROL, 0); 1827 break; 1828 case RADEON_HPD_5: 1829 WREG32(DC_HPD5_CONTROL, 0); 1830 break; 1831 case RADEON_HPD_6: 1832 WREG32(DC_HPD6_CONTROL, 0); 1833 break; 1834 default: 1835 break; 1836 } 1837 disabled |= 1 << radeon_connector->hpd.hpd; 1838 } 1839 radeon_irq_kms_disable_hpd(rdev, disabled); 1840 } 1841 1842 /* watermark setup */ 1843 1844 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 1845 struct radeon_crtc *radeon_crtc, 1846 struct drm_display_mode *mode, 1847 struct drm_display_mode *other_mode) 1848 { 1849 u32 tmp, buffer_alloc, i; 1850 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1851 /* 1852 * Line Buffer Setup 1853 * There are 3 line buffers, each one shared by 2 display controllers. 1854 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1855 * the display controllers. The paritioning is done via one of four 1856 * preset allocations specified in bits 2:0: 1857 * first display controller 1858 * 0 - first half of lb (3840 * 2) 1859 * 1 - first 3/4 of lb (5760 * 2) 1860 * 2 - whole lb (7680 * 2), other crtc must be disabled 1861 * 3 - first 1/4 of lb (1920 * 2) 1862 * second display controller 1863 * 4 - second half of lb (3840 * 2) 1864 * 5 - second 3/4 of lb (5760 * 2) 1865 * 6 - whole lb (7680 * 2), other crtc must be disabled 1866 * 7 - last 1/4 of lb (1920 * 2) 1867 */ 1868 /* this can get tricky if we have two large displays on a paired group 1869 * of crtcs. Ideally for multiple large displays we'd assign them to 1870 * non-linked crtcs for maximum line buffer allocation. 1871 */ 1872 if (radeon_crtc->base.enabled && mode) { 1873 if (other_mode) { 1874 tmp = 0; /* 1/2 */ 1875 buffer_alloc = 1; 1876 } else { 1877 tmp = 2; /* whole */ 1878 buffer_alloc = 2; 1879 } 1880 } else { 1881 tmp = 0; 1882 buffer_alloc = 0; 1883 } 1884 1885 /* second controller of the pair uses second half of the lb */ 1886 if (radeon_crtc->crtc_id % 2) 1887 tmp += 4; 1888 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 1889 1890 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { 1891 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1892 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1893 for (i = 0; i < rdev->usec_timeout; i++) { 1894 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1895 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1896 break; 1897 udelay(1); 1898 } 1899 } 1900 1901 if (radeon_crtc->base.enabled && mode) { 1902 switch (tmp) { 1903 case 0: 1904 case 4: 1905 default: 1906 if (ASIC_IS_DCE5(rdev)) 1907 return 4096 * 2; 1908 else 1909 return 3840 * 2; 1910 case 1: 1911 case 5: 1912 if (ASIC_IS_DCE5(rdev)) 1913 return 6144 * 2; 1914 else 1915 return 5760 * 2; 1916 case 2: 1917 case 6: 1918 if (ASIC_IS_DCE5(rdev)) 1919 return 8192 * 2; 1920 else 1921 return 7680 * 2; 1922 case 3: 1923 case 7: 1924 if (ASIC_IS_DCE5(rdev)) 1925 return 2048 * 2; 1926 else 1927 return 1920 * 2; 1928 } 1929 } 1930 1931 /* controller not enabled, so no lb used */ 1932 return 0; 1933 } 1934 1935 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 1936 { 1937 u32 tmp = RREG32(MC_SHARED_CHMAP); 1938 1939 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1940 case 0: 1941 default: 1942 return 1; 1943 case 1: 1944 return 2; 1945 case 2: 1946 return 4; 1947 case 3: 1948 return 8; 1949 } 1950 } 1951 1952 struct evergreen_wm_params { 1953 u32 dram_channels; /* number of dram channels */ 1954 u32 yclk; /* bandwidth per dram data pin in kHz */ 1955 u32 sclk; /* engine clock in kHz */ 1956 u32 disp_clk; /* display clock in kHz */ 1957 u32 src_width; /* viewport width */ 1958 u32 active_time; /* active display time in ns */ 1959 u32 blank_time; /* blank time in ns */ 1960 bool interlaced; /* mode is interlaced */ 1961 fixed20_12 vsc; /* vertical scale ratio */ 1962 u32 num_heads; /* number of active crtcs */ 1963 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1964 u32 lb_size; /* line buffer allocated to pipe */ 1965 u32 vtaps; /* vertical scaler taps */ 1966 }; 1967 1968 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 1969 { 1970 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1971 fixed20_12 dram_efficiency; /* 0.7 */ 1972 fixed20_12 yclk, dram_channels, bandwidth; 1973 fixed20_12 a; 1974 1975 a.full = dfixed_const(1000); 1976 yclk.full = dfixed_const(wm->yclk); 1977 yclk.full = dfixed_div(yclk, a); 1978 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1979 a.full = dfixed_const(10); 1980 dram_efficiency.full = dfixed_const(7); 1981 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1982 bandwidth.full = dfixed_mul(dram_channels, yclk); 1983 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1984 1985 return dfixed_trunc(bandwidth); 1986 } 1987 1988 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 1989 { 1990 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1991 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 1992 fixed20_12 yclk, dram_channels, bandwidth; 1993 fixed20_12 a; 1994 1995 a.full = dfixed_const(1000); 1996 yclk.full = dfixed_const(wm->yclk); 1997 yclk.full = dfixed_div(yclk, a); 1998 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1999 a.full = dfixed_const(10); 2000 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 2001 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 2002 bandwidth.full = dfixed_mul(dram_channels, yclk); 2003 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 2004 2005 return dfixed_trunc(bandwidth); 2006 } 2007 2008 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 2009 { 2010 /* Calculate the display Data return Bandwidth */ 2011 fixed20_12 return_efficiency; /* 0.8 */ 2012 fixed20_12 sclk, bandwidth; 2013 fixed20_12 a; 2014 2015 a.full = dfixed_const(1000); 2016 sclk.full = dfixed_const(wm->sclk); 2017 sclk.full = dfixed_div(sclk, a); 2018 a.full = dfixed_const(10); 2019 return_efficiency.full = dfixed_const(8); 2020 return_efficiency.full = dfixed_div(return_efficiency, a); 2021 a.full = dfixed_const(32); 2022 bandwidth.full = dfixed_mul(a, sclk); 2023 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 2024 2025 return dfixed_trunc(bandwidth); 2026 } 2027 2028 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 2029 { 2030 /* Calculate the DMIF Request Bandwidth */ 2031 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 2032 fixed20_12 disp_clk, bandwidth; 2033 fixed20_12 a; 2034 2035 a.full = dfixed_const(1000); 2036 disp_clk.full = dfixed_const(wm->disp_clk); 2037 disp_clk.full = dfixed_div(disp_clk, a); 2038 a.full = dfixed_const(10); 2039 disp_clk_request_efficiency.full = dfixed_const(8); 2040 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 2041 a.full = dfixed_const(32); 2042 bandwidth.full = dfixed_mul(a, disp_clk); 2043 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 2044 2045 return dfixed_trunc(bandwidth); 2046 } 2047 2048 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 2049 { 2050 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 2051 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 2052 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 2053 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 2054 2055 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 2056 } 2057 2058 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 2059 { 2060 /* Calculate the display mode Average Bandwidth 2061 * DisplayMode should contain the source and destination dimensions, 2062 * timing, etc. 2063 */ 2064 fixed20_12 bpp; 2065 fixed20_12 line_time; 2066 fixed20_12 src_width; 2067 fixed20_12 bandwidth; 2068 fixed20_12 a; 2069 2070 a.full = dfixed_const(1000); 2071 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 2072 line_time.full = dfixed_div(line_time, a); 2073 bpp.full = dfixed_const(wm->bytes_per_pixel); 2074 src_width.full = dfixed_const(wm->src_width); 2075 bandwidth.full = dfixed_mul(src_width, bpp); 2076 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 2077 bandwidth.full = dfixed_div(bandwidth, line_time); 2078 2079 return dfixed_trunc(bandwidth); 2080 } 2081 2082 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 2083 { 2084 /* First calcualte the latency in ns */ 2085 u32 mc_latency = 2000; /* 2000 ns. */ 2086 u32 available_bandwidth = evergreen_available_bandwidth(wm); 2087 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 2088 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 2089 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 2090 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 2091 (wm->num_heads * cursor_line_pair_return_time); 2092 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 2093 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 2094 fixed20_12 a, b, c; 2095 2096 if (wm->num_heads == 0) 2097 return 0; 2098 2099 a.full = dfixed_const(2); 2100 b.full = dfixed_const(1); 2101 if ((wm->vsc.full > a.full) || 2102 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 2103 (wm->vtaps >= 5) || 2104 ((wm->vsc.full >= a.full) && wm->interlaced)) 2105 max_src_lines_per_dst_line = 4; 2106 else 2107 max_src_lines_per_dst_line = 2; 2108 2109 a.full = dfixed_const(available_bandwidth); 2110 b.full = dfixed_const(wm->num_heads); 2111 a.full = dfixed_div(a, b); 2112 2113 b.full = dfixed_const(1000); 2114 c.full = dfixed_const(wm->disp_clk); 2115 b.full = dfixed_div(c, b); 2116 c.full = dfixed_const(wm->bytes_per_pixel); 2117 b.full = dfixed_mul(b, c); 2118 2119 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b)); 2120 2121 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 2122 b.full = dfixed_const(1000); 2123 c.full = dfixed_const(lb_fill_bw); 2124 b.full = dfixed_div(c, b); 2125 a.full = dfixed_div(a, b); 2126 line_fill_time = dfixed_trunc(a); 2127 2128 if (line_fill_time < wm->active_time) 2129 return latency; 2130 else 2131 return latency + (line_fill_time - wm->active_time); 2132 2133 } 2134 2135 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2136 { 2137 if (evergreen_average_bandwidth(wm) <= 2138 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 2139 return true; 2140 else 2141 return false; 2142 }; 2143 2144 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 2145 { 2146 if (evergreen_average_bandwidth(wm) <= 2147 (evergreen_available_bandwidth(wm) / wm->num_heads)) 2148 return true; 2149 else 2150 return false; 2151 }; 2152 2153 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 2154 { 2155 u32 lb_partitions = wm->lb_size / wm->src_width; 2156 u32 line_time = wm->active_time + wm->blank_time; 2157 u32 latency_tolerant_lines; 2158 u32 latency_hiding; 2159 fixed20_12 a; 2160 2161 a.full = dfixed_const(1); 2162 if (wm->vsc.full > a.full) 2163 latency_tolerant_lines = 1; 2164 else { 2165 if (lb_partitions <= (wm->vtaps + 1)) 2166 latency_tolerant_lines = 1; 2167 else 2168 latency_tolerant_lines = 2; 2169 } 2170 2171 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2172 2173 if (evergreen_latency_watermark(wm) <= latency_hiding) 2174 return true; 2175 else 2176 return false; 2177 } 2178 2179 static void evergreen_program_watermarks(struct radeon_device *rdev, 2180 struct radeon_crtc *radeon_crtc, 2181 u32 lb_size, u32 num_heads) 2182 { 2183 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2184 struct evergreen_wm_params wm_low, wm_high; 2185 u32 dram_channels; 2186 u32 pixel_period; 2187 u32 line_time = 0; 2188 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2189 u32 priority_a_mark = 0, priority_b_mark = 0; 2190 u32 priority_a_cnt = PRIORITY_OFF; 2191 u32 priority_b_cnt = PRIORITY_OFF; 2192 u32 pipe_offset = radeon_crtc->crtc_id * 16; 2193 u32 tmp, arb_control3; 2194 fixed20_12 a, b, c; 2195 2196 if (radeon_crtc->base.enabled && num_heads && mode) { 2197 pixel_period = 1000000 / (u32)mode->clock; 2198 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 2199 priority_a_cnt = 0; 2200 priority_b_cnt = 0; 2201 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2202 2203 /* watermark for high clocks */ 2204 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2205 wm_high.yclk = 2206 radeon_dpm_get_mclk(rdev, false) * 10; 2207 wm_high.sclk = 2208 radeon_dpm_get_sclk(rdev, false) * 10; 2209 } else { 2210 wm_high.yclk = rdev->pm.current_mclk * 10; 2211 wm_high.sclk = rdev->pm.current_sclk * 10; 2212 } 2213 2214 wm_high.disp_clk = mode->clock; 2215 wm_high.src_width = mode->crtc_hdisplay; 2216 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 2217 wm_high.blank_time = line_time - wm_high.active_time; 2218 wm_high.interlaced = false; 2219 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2220 wm_high.interlaced = true; 2221 wm_high.vsc = radeon_crtc->vsc; 2222 wm_high.vtaps = 1; 2223 if (radeon_crtc->rmx_type != RMX_OFF) 2224 wm_high.vtaps = 2; 2225 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2226 wm_high.lb_size = lb_size; 2227 wm_high.dram_channels = dram_channels; 2228 wm_high.num_heads = num_heads; 2229 2230 /* watermark for low clocks */ 2231 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2232 wm_low.yclk = 2233 radeon_dpm_get_mclk(rdev, true) * 10; 2234 wm_low.sclk = 2235 radeon_dpm_get_sclk(rdev, true) * 10; 2236 } else { 2237 wm_low.yclk = rdev->pm.current_mclk * 10; 2238 wm_low.sclk = rdev->pm.current_sclk * 10; 2239 } 2240 2241 wm_low.disp_clk = mode->clock; 2242 wm_low.src_width = mode->crtc_hdisplay; 2243 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 2244 wm_low.blank_time = line_time - wm_low.active_time; 2245 wm_low.interlaced = false; 2246 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2247 wm_low.interlaced = true; 2248 wm_low.vsc = radeon_crtc->vsc; 2249 wm_low.vtaps = 1; 2250 if (radeon_crtc->rmx_type != RMX_OFF) 2251 wm_low.vtaps = 2; 2252 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2253 wm_low.lb_size = lb_size; 2254 wm_low.dram_channels = dram_channels; 2255 wm_low.num_heads = num_heads; 2256 2257 /* set for high clocks */ 2258 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); 2259 /* set for low clocks */ 2260 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); 2261 2262 /* possibly force display priority to high */ 2263 /* should really do this at mode validation time... */ 2264 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2265 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || 2266 !evergreen_check_latency_hiding(&wm_high) || 2267 (rdev->disp_priority == 2)) { 2268 DRM_DEBUG_KMS("force priority a to high\n"); 2269 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2270 } 2271 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2272 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || 2273 !evergreen_check_latency_hiding(&wm_low) || 2274 (rdev->disp_priority == 2)) { 2275 DRM_DEBUG_KMS("force priority b to high\n"); 2276 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2277 } 2278 2279 a.full = dfixed_const(1000); 2280 b.full = dfixed_const(mode->clock); 2281 b.full = dfixed_div(b, a); 2282 c.full = dfixed_const(latency_watermark_a); 2283 c.full = dfixed_mul(c, b); 2284 c.full = dfixed_mul(c, radeon_crtc->hsc); 2285 c.full = dfixed_div(c, a); 2286 a.full = dfixed_const(16); 2287 c.full = dfixed_div(c, a); 2288 priority_a_mark = dfixed_trunc(c); 2289 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2290 2291 a.full = dfixed_const(1000); 2292 b.full = dfixed_const(mode->clock); 2293 b.full = dfixed_div(b, a); 2294 c.full = dfixed_const(latency_watermark_b); 2295 c.full = dfixed_mul(c, b); 2296 c.full = dfixed_mul(c, radeon_crtc->hsc); 2297 c.full = dfixed_div(c, a); 2298 a.full = dfixed_const(16); 2299 c.full = dfixed_div(c, a); 2300 priority_b_mark = dfixed_trunc(c); 2301 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2302 } 2303 2304 /* select wm A */ 2305 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2306 tmp = arb_control3; 2307 tmp &= ~LATENCY_WATERMARK_MASK(3); 2308 tmp |= LATENCY_WATERMARK_MASK(1); 2309 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2310 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2311 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2312 LATENCY_HIGH_WATERMARK(line_time))); 2313 /* select wm B */ 2314 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2315 tmp &= ~LATENCY_WATERMARK_MASK(3); 2316 tmp |= LATENCY_WATERMARK_MASK(2); 2317 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2318 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2319 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2320 LATENCY_HIGH_WATERMARK(line_time))); 2321 /* restore original selection */ 2322 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 2323 2324 /* write the priority marks */ 2325 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2326 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2327 2328 /* save values for DPM */ 2329 radeon_crtc->line_time = line_time; 2330 radeon_crtc->wm_high = latency_watermark_a; 2331 radeon_crtc->wm_low = latency_watermark_b; 2332 } 2333 2334 /** 2335 * evergreen_bandwidth_update - update display watermarks callback. 2336 * 2337 * @rdev: radeon_device pointer 2338 * 2339 * Update the display watermarks based on the requested mode(s) 2340 * (evergreen+). 2341 */ 2342 void evergreen_bandwidth_update(struct radeon_device *rdev) 2343 { 2344 struct drm_display_mode *mode0 = NULL; 2345 struct drm_display_mode *mode1 = NULL; 2346 u32 num_heads = 0, lb_size; 2347 int i; 2348 2349 if (!rdev->mode_info.mode_config_initialized) 2350 return; 2351 2352 radeon_update_display_priority(rdev); 2353 2354 for (i = 0; i < rdev->num_crtc; i++) { 2355 if (rdev->mode_info.crtcs[i]->base.enabled) 2356 num_heads++; 2357 } 2358 for (i = 0; i < rdev->num_crtc; i += 2) { 2359 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2360 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2361 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2362 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2363 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2364 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2365 } 2366 } 2367 2368 /** 2369 * evergreen_mc_wait_for_idle - wait for MC idle callback. 2370 * 2371 * @rdev: radeon_device pointer 2372 * 2373 * Wait for the MC (memory controller) to be idle. 2374 * (evergreen+). 2375 * Returns 0 if the MC is idle, -1 if not. 2376 */ 2377 int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 2378 { 2379 unsigned i; 2380 u32 tmp; 2381 2382 for (i = 0; i < rdev->usec_timeout; i++) { 2383 /* read MC_STATUS */ 2384 tmp = RREG32(SRBM_STATUS) & 0x1F00; 2385 if (!tmp) 2386 return 0; 2387 udelay(1); 2388 } 2389 return -1; 2390 } 2391 2392 /* 2393 * GART 2394 */ 2395 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 2396 { 2397 unsigned i; 2398 u32 tmp; 2399 2400 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 2401 2402 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 2403 for (i = 0; i < rdev->usec_timeout; i++) { 2404 /* read MC_STATUS */ 2405 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 2406 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 2407 if (tmp == 2) { 2408 printk(KERN_WARNING "[drm] r600 flush TLB failed\n"); 2409 return; 2410 } 2411 if (tmp) { 2412 return; 2413 } 2414 udelay(1); 2415 } 2416 } 2417 2418 static int evergreen_pcie_gart_enable(struct radeon_device *rdev) 2419 { 2420 u32 tmp; 2421 int r; 2422 2423 if (rdev->gart.robj == NULL) { 2424 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 2425 return -EINVAL; 2426 } 2427 r = radeon_gart_table_vram_pin(rdev); 2428 if (r) 2429 return r; 2430 /* Setup L2 cache */ 2431 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2432 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2433 EFFECTIVE_L2_QUEUE_SIZE(7)); 2434 WREG32(VM_L2_CNTL2, 0); 2435 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2436 /* Setup TLB control */ 2437 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2438 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2439 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2440 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2441 if (rdev->flags & RADEON_IS_IGP) { 2442 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 2443 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 2444 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 2445 } else { 2446 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2447 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2448 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2449 if ((rdev->family == CHIP_JUNIPER) || 2450 (rdev->family == CHIP_CYPRESS) || 2451 (rdev->family == CHIP_HEMLOCK) || 2452 (rdev->family == CHIP_BARTS)) 2453 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp); 2454 } 2455 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2456 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2457 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2458 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2459 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 2460 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 2461 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 2462 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 2463 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 2464 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 2465 (u32)(rdev->dummy_page.addr >> 12)); 2466 WREG32(VM_CONTEXT1_CNTL, 0); 2467 2468 evergreen_pcie_gart_tlb_flush(rdev); 2469 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 2470 (unsigned)(rdev->mc.gtt_size >> 20), 2471 (unsigned long long)rdev->gart.table_addr); 2472 rdev->gart.ready = true; 2473 return 0; 2474 } 2475 2476 static void evergreen_pcie_gart_disable(struct radeon_device *rdev) 2477 { 2478 u32 tmp; 2479 2480 /* Disable all tables */ 2481 WREG32(VM_CONTEXT0_CNTL, 0); 2482 WREG32(VM_CONTEXT1_CNTL, 0); 2483 2484 /* Setup L2 cache */ 2485 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 2486 EFFECTIVE_L2_QUEUE_SIZE(7)); 2487 WREG32(VM_L2_CNTL2, 0); 2488 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2489 /* Setup TLB control */ 2490 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2491 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2492 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2493 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2494 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2495 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2496 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2497 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2498 radeon_gart_table_vram_unpin(rdev); 2499 } 2500 2501 static void evergreen_pcie_gart_fini(struct radeon_device *rdev) 2502 { 2503 evergreen_pcie_gart_disable(rdev); 2504 radeon_gart_table_vram_free(rdev); 2505 radeon_gart_fini(rdev); 2506 } 2507 2508 2509 static void evergreen_agp_enable(struct radeon_device *rdev) 2510 { 2511 u32 tmp; 2512 2513 /* Setup L2 cache */ 2514 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2515 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2516 EFFECTIVE_L2_QUEUE_SIZE(7)); 2517 WREG32(VM_L2_CNTL2, 0); 2518 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2519 /* Setup TLB control */ 2520 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2521 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2522 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2523 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2524 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2525 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2526 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2527 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2528 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2529 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2530 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2531 WREG32(VM_CONTEXT0_CNTL, 0); 2532 WREG32(VM_CONTEXT1_CNTL, 0); 2533 } 2534 2535 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 2536 { 2537 u32 crtc_enabled, tmp, frame_count, blackout; 2538 int i, j; 2539 2540 if (!ASIC_IS_NODCE(rdev)) { 2541 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 2542 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 2543 2544 /* disable VGA render */ 2545 WREG32(VGA_RENDER_CONTROL, 0); 2546 } 2547 /* blank the display controllers */ 2548 for (i = 0; i < rdev->num_crtc; i++) { 2549 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; 2550 if (crtc_enabled) { 2551 save->crtc_enabled[i] = true; 2552 if (ASIC_IS_DCE6(rdev)) { 2553 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2554 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) { 2555 radeon_wait_for_vblank(rdev, i); 2556 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2557 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 2558 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2559 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2560 } 2561 } else { 2562 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2563 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) { 2564 radeon_wait_for_vblank(rdev, i); 2565 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2566 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2567 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2568 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2569 } 2570 } 2571 /* wait for the next frame */ 2572 frame_count = radeon_get_vblank_counter(rdev, i); 2573 for (j = 0; j < rdev->usec_timeout; j++) { 2574 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2575 break; 2576 udelay(1); 2577 } 2578 2579 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ 2580 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2581 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2582 tmp &= ~EVERGREEN_CRTC_MASTER_EN; 2583 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2584 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2585 save->crtc_enabled[i] = false; 2586 /* ***** */ 2587 } else { 2588 save->crtc_enabled[i] = false; 2589 } 2590 } 2591 2592 radeon_mc_wait_for_idle(rdev); 2593 2594 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 2595 if ((blackout & BLACKOUT_MODE_MASK) != 1) { 2596 /* Block CPU access */ 2597 WREG32(BIF_FB_EN, 0); 2598 /* blackout the MC */ 2599 blackout &= ~BLACKOUT_MODE_MASK; 2600 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 2601 } 2602 /* wait for the MC to settle */ 2603 udelay(100); 2604 2605 /* lock double buffered regs */ 2606 for (i = 0; i < rdev->num_crtc; i++) { 2607 if (save->crtc_enabled[i]) { 2608 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2609 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) { 2610 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 2611 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2612 } 2613 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2614 if (!(tmp & 1)) { 2615 tmp |= 1; 2616 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2617 } 2618 } 2619 } 2620 } 2621 2622 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 2623 { 2624 u32 tmp, frame_count; 2625 int i, j; 2626 2627 /* update crtc base addresses */ 2628 for (i = 0; i < rdev->num_crtc; i++) { 2629 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2630 upper_32_bits(rdev->mc.vram_start)); 2631 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2632 upper_32_bits(rdev->mc.vram_start)); 2633 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], 2634 (u32)rdev->mc.vram_start); 2635 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], 2636 (u32)rdev->mc.vram_start); 2637 } 2638 2639 if (!ASIC_IS_NODCE(rdev)) { 2640 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 2641 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 2642 } 2643 2644 /* unlock regs and wait for update */ 2645 for (i = 0; i < rdev->num_crtc; i++) { 2646 if (save->crtc_enabled[i]) { 2647 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); 2648 if ((tmp & 0x7) != 3) { 2649 tmp &= ~0x7; 2650 tmp |= 0x3; 2651 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); 2652 } 2653 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2654 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) { 2655 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 2656 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2657 } 2658 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2659 if (tmp & 1) { 2660 tmp &= ~1; 2661 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2662 } 2663 for (j = 0; j < rdev->usec_timeout; j++) { 2664 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2665 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0) 2666 break; 2667 udelay(1); 2668 } 2669 } 2670 } 2671 2672 /* unblackout the MC */ 2673 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL); 2674 tmp &= ~BLACKOUT_MODE_MASK; 2675 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp); 2676 /* allow CPU access */ 2677 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2678 2679 for (i = 0; i < rdev->num_crtc; i++) { 2680 if (save->crtc_enabled[i]) { 2681 if (ASIC_IS_DCE6(rdev)) { 2682 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2683 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN; 2684 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2685 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2686 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2687 } else { 2688 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2689 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2690 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2691 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2692 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2693 } 2694 /* wait for the next frame */ 2695 frame_count = radeon_get_vblank_counter(rdev, i); 2696 for (j = 0; j < rdev->usec_timeout; j++) { 2697 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2698 break; 2699 udelay(1); 2700 } 2701 } 2702 } 2703 if (!ASIC_IS_NODCE(rdev)) { 2704 /* Unlock vga access */ 2705 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 2706 mdelay(1); 2707 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 2708 } 2709 } 2710 2711 void evergreen_mc_program(struct radeon_device *rdev) 2712 { 2713 struct evergreen_mc_save save; 2714 u32 tmp; 2715 int i, j; 2716 2717 /* Initialize HDP */ 2718 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2719 WREG32((0x2c14 + j), 0x00000000); 2720 WREG32((0x2c18 + j), 0x00000000); 2721 WREG32((0x2c1c + j), 0x00000000); 2722 WREG32((0x2c20 + j), 0x00000000); 2723 WREG32((0x2c24 + j), 0x00000000); 2724 } 2725 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 2726 2727 evergreen_mc_stop(rdev, &save); 2728 if (evergreen_mc_wait_for_idle(rdev)) { 2729 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2730 } 2731 /* Lockout access through VGA aperture*/ 2732 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 2733 /* Update configuration */ 2734 if (rdev->flags & RADEON_IS_AGP) { 2735 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 2736 /* VRAM before AGP */ 2737 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2738 rdev->mc.vram_start >> 12); 2739 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2740 rdev->mc.gtt_end >> 12); 2741 } else { 2742 /* VRAM after AGP */ 2743 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2744 rdev->mc.gtt_start >> 12); 2745 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2746 rdev->mc.vram_end >> 12); 2747 } 2748 } else { 2749 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2750 rdev->mc.vram_start >> 12); 2751 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2752 rdev->mc.vram_end >> 12); 2753 } 2754 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12); 2755 /* llano/ontario only */ 2756 if ((rdev->family == CHIP_PALM) || 2757 (rdev->family == CHIP_SUMO) || 2758 (rdev->family == CHIP_SUMO2)) { 2759 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 2760 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 2761 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 2762 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 2763 } 2764 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 2765 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 2766 WREG32(MC_VM_FB_LOCATION, tmp); 2767 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 2768 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 2769 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 2770 if (rdev->flags & RADEON_IS_AGP) { 2771 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 2772 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 2773 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 2774 } else { 2775 WREG32(MC_VM_AGP_BASE, 0); 2776 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 2777 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 2778 } 2779 if (evergreen_mc_wait_for_idle(rdev)) { 2780 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2781 } 2782 evergreen_mc_resume(rdev, &save); 2783 /* we need to own VRAM, so turn off the VGA renderer here 2784 * to stop it overwriting our objects */ 2785 rv515_vga_render_disable(rdev); 2786 } 2787 2788 /* 2789 * CP. 2790 */ 2791 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 2792 { 2793 struct radeon_ring *ring = &rdev->ring[ib->ring]; 2794 u32 next_rptr; 2795 2796 /* set to DX10/11 mode */ 2797 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 2798 radeon_ring_write(ring, 1); 2799 2800 if (ring->rptr_save_reg) { 2801 next_rptr = ring->wptr + 3 + 4; 2802 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 2803 radeon_ring_write(ring, ((ring->rptr_save_reg - 2804 PACKET3_SET_CONFIG_REG_START) >> 2)); 2805 radeon_ring_write(ring, next_rptr); 2806 } else if (rdev->wb.enabled) { 2807 next_rptr = ring->wptr + 5 + 4; 2808 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3)); 2809 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 2810 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18)); 2811 radeon_ring_write(ring, next_rptr); 2812 radeon_ring_write(ring, 0); 2813 } 2814 2815 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 2816 radeon_ring_write(ring, 2817 #ifdef __BIG_ENDIAN 2818 (2 << 0) | 2819 #endif 2820 (ib->gpu_addr & 0xFFFFFFFC)); 2821 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 2822 radeon_ring_write(ring, ib->length_dw); 2823 } 2824 2825 2826 static int evergreen_cp_load_microcode(struct radeon_device *rdev) 2827 { 2828 const __be32 *fw_data; 2829 int i; 2830 2831 if (!rdev->me_fw || !rdev->pfp_fw) 2832 return -EINVAL; 2833 2834 r700_cp_stop(rdev); 2835 WREG32(CP_RB_CNTL, 2836 #ifdef __BIG_ENDIAN 2837 BUF_SWAP_32BIT | 2838 #endif 2839 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 2840 2841 fw_data = (const __be32 *)rdev->pfp_fw->data; 2842 WREG32(CP_PFP_UCODE_ADDR, 0); 2843 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 2844 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 2845 WREG32(CP_PFP_UCODE_ADDR, 0); 2846 2847 fw_data = (const __be32 *)rdev->me_fw->data; 2848 WREG32(CP_ME_RAM_WADDR, 0); 2849 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 2850 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 2851 2852 WREG32(CP_PFP_UCODE_ADDR, 0); 2853 WREG32(CP_ME_RAM_WADDR, 0); 2854 WREG32(CP_ME_RAM_RADDR, 0); 2855 return 0; 2856 } 2857 2858 static int evergreen_cp_start(struct radeon_device *rdev) 2859 { 2860 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 2861 int r, i; 2862 uint32_t cp_me; 2863 2864 r = radeon_ring_lock(rdev, ring, 7); 2865 if (r) { 2866 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 2867 return r; 2868 } 2869 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 2870 radeon_ring_write(ring, 0x1); 2871 radeon_ring_write(ring, 0x0); 2872 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1); 2873 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 2874 radeon_ring_write(ring, 0); 2875 radeon_ring_write(ring, 0); 2876 radeon_ring_unlock_commit(rdev, ring, false); 2877 2878 cp_me = 0xff; 2879 WREG32(CP_ME_CNTL, cp_me); 2880 2881 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19); 2882 if (r) { 2883 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 2884 return r; 2885 } 2886 2887 /* setup clear context state */ 2888 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2889 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 2890 2891 for (i = 0; i < evergreen_default_size; i++) 2892 radeon_ring_write(ring, evergreen_default_state[i]); 2893 2894 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 2895 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 2896 2897 /* set clear context state */ 2898 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 2899 radeon_ring_write(ring, 0); 2900 2901 /* SQ_VTX_BASE_VTX_LOC */ 2902 radeon_ring_write(ring, 0xc0026f00); 2903 radeon_ring_write(ring, 0x00000000); 2904 radeon_ring_write(ring, 0x00000000); 2905 radeon_ring_write(ring, 0x00000000); 2906 2907 /* Clear consts */ 2908 radeon_ring_write(ring, 0xc0036f00); 2909 radeon_ring_write(ring, 0x00000bc4); 2910 radeon_ring_write(ring, 0xffffffff); 2911 radeon_ring_write(ring, 0xffffffff); 2912 radeon_ring_write(ring, 0xffffffff); 2913 2914 radeon_ring_write(ring, 0xc0026900); 2915 radeon_ring_write(ring, 0x00000316); 2916 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 2917 radeon_ring_write(ring, 0x00000010); /* */ 2918 2919 radeon_ring_unlock_commit(rdev, ring, false); 2920 2921 return 0; 2922 } 2923 2924 static int evergreen_cp_resume(struct radeon_device *rdev) 2925 { 2926 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 2927 u32 tmp; 2928 u32 rb_bufsz; 2929 int r; 2930 2931 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 2932 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 2933 SOFT_RESET_PA | 2934 SOFT_RESET_SH | 2935 SOFT_RESET_VGT | 2936 SOFT_RESET_SPI | 2937 SOFT_RESET_SX)); 2938 RREG32(GRBM_SOFT_RESET); 2939 mdelay(15); 2940 WREG32(GRBM_SOFT_RESET, 0); 2941 RREG32(GRBM_SOFT_RESET); 2942 2943 /* Set ring buffer size */ 2944 rb_bufsz = order_base_2(ring->ring_size / 8); 2945 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 2946 #ifdef __BIG_ENDIAN 2947 tmp |= BUF_SWAP_32BIT; 2948 #endif 2949 WREG32(CP_RB_CNTL, tmp); 2950 WREG32(CP_SEM_WAIT_TIMER, 0x0); 2951 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 2952 2953 /* Set the write pointer delay */ 2954 WREG32(CP_RB_WPTR_DELAY, 0); 2955 2956 /* Initialize the ring buffer's read and write pointers */ 2957 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 2958 WREG32(CP_RB_RPTR_WR, 0); 2959 ring->wptr = 0; 2960 WREG32(CP_RB_WPTR, ring->wptr); 2961 2962 /* set the wb address whether it's enabled or not */ 2963 WREG32(CP_RB_RPTR_ADDR, 2964 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 2965 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 2966 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 2967 2968 if (rdev->wb.enabled) 2969 WREG32(SCRATCH_UMSK, 0xff); 2970 else { 2971 tmp |= RB_NO_UPDATE; 2972 WREG32(SCRATCH_UMSK, 0); 2973 } 2974 2975 mdelay(1); 2976 WREG32(CP_RB_CNTL, tmp); 2977 2978 WREG32(CP_RB_BASE, ring->gpu_addr >> 8); 2979 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 2980 2981 evergreen_cp_start(rdev); 2982 ring->ready = true; 2983 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); 2984 if (r) { 2985 ring->ready = false; 2986 return r; 2987 } 2988 return 0; 2989 } 2990 2991 /* 2992 * Core functions 2993 */ 2994 static void evergreen_gpu_init(struct radeon_device *rdev) 2995 { 2996 u32 gb_addr_config; 2997 u32 mc_shared_chmap, mc_arb_ramcfg; 2998 u32 sx_debug_1; 2999 u32 smx_dc_ctl0; 3000 u32 sq_config; 3001 u32 sq_lds_resource_mgmt; 3002 u32 sq_gpr_resource_mgmt_1; 3003 u32 sq_gpr_resource_mgmt_2; 3004 u32 sq_gpr_resource_mgmt_3; 3005 u32 sq_thread_resource_mgmt; 3006 u32 sq_thread_resource_mgmt_2; 3007 u32 sq_stack_resource_mgmt_1; 3008 u32 sq_stack_resource_mgmt_2; 3009 u32 sq_stack_resource_mgmt_3; 3010 u32 vgt_cache_invalidation; 3011 u32 hdp_host_path_cntl, tmp; 3012 u32 disabled_rb_mask; 3013 int i, j, ps_thread_count; 3014 3015 switch (rdev->family) { 3016 case CHIP_CYPRESS: 3017 case CHIP_HEMLOCK: 3018 rdev->config.evergreen.num_ses = 2; 3019 rdev->config.evergreen.max_pipes = 4; 3020 rdev->config.evergreen.max_tile_pipes = 8; 3021 rdev->config.evergreen.max_simds = 10; 3022 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3023 rdev->config.evergreen.max_gprs = 256; 3024 rdev->config.evergreen.max_threads = 248; 3025 rdev->config.evergreen.max_gs_threads = 32; 3026 rdev->config.evergreen.max_stack_entries = 512; 3027 rdev->config.evergreen.sx_num_of_sets = 4; 3028 rdev->config.evergreen.sx_max_export_size = 256; 3029 rdev->config.evergreen.sx_max_export_pos_size = 64; 3030 rdev->config.evergreen.sx_max_export_smx_size = 192; 3031 rdev->config.evergreen.max_hw_contexts = 8; 3032 rdev->config.evergreen.sq_num_cf_insts = 2; 3033 3034 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3035 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3036 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3037 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN; 3038 break; 3039 case CHIP_JUNIPER: 3040 rdev->config.evergreen.num_ses = 1; 3041 rdev->config.evergreen.max_pipes = 4; 3042 rdev->config.evergreen.max_tile_pipes = 4; 3043 rdev->config.evergreen.max_simds = 10; 3044 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3045 rdev->config.evergreen.max_gprs = 256; 3046 rdev->config.evergreen.max_threads = 248; 3047 rdev->config.evergreen.max_gs_threads = 32; 3048 rdev->config.evergreen.max_stack_entries = 512; 3049 rdev->config.evergreen.sx_num_of_sets = 4; 3050 rdev->config.evergreen.sx_max_export_size = 256; 3051 rdev->config.evergreen.sx_max_export_pos_size = 64; 3052 rdev->config.evergreen.sx_max_export_smx_size = 192; 3053 rdev->config.evergreen.max_hw_contexts = 8; 3054 rdev->config.evergreen.sq_num_cf_insts = 2; 3055 3056 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3057 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3058 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3059 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN; 3060 break; 3061 case CHIP_REDWOOD: 3062 rdev->config.evergreen.num_ses = 1; 3063 rdev->config.evergreen.max_pipes = 4; 3064 rdev->config.evergreen.max_tile_pipes = 4; 3065 rdev->config.evergreen.max_simds = 5; 3066 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3067 rdev->config.evergreen.max_gprs = 256; 3068 rdev->config.evergreen.max_threads = 248; 3069 rdev->config.evergreen.max_gs_threads = 32; 3070 rdev->config.evergreen.max_stack_entries = 256; 3071 rdev->config.evergreen.sx_num_of_sets = 4; 3072 rdev->config.evergreen.sx_max_export_size = 256; 3073 rdev->config.evergreen.sx_max_export_pos_size = 64; 3074 rdev->config.evergreen.sx_max_export_smx_size = 192; 3075 rdev->config.evergreen.max_hw_contexts = 8; 3076 rdev->config.evergreen.sq_num_cf_insts = 2; 3077 3078 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3079 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3080 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3081 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN; 3082 break; 3083 case CHIP_CEDAR: 3084 default: 3085 rdev->config.evergreen.num_ses = 1; 3086 rdev->config.evergreen.max_pipes = 2; 3087 rdev->config.evergreen.max_tile_pipes = 2; 3088 rdev->config.evergreen.max_simds = 2; 3089 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3090 rdev->config.evergreen.max_gprs = 256; 3091 rdev->config.evergreen.max_threads = 192; 3092 rdev->config.evergreen.max_gs_threads = 16; 3093 rdev->config.evergreen.max_stack_entries = 256; 3094 rdev->config.evergreen.sx_num_of_sets = 4; 3095 rdev->config.evergreen.sx_max_export_size = 128; 3096 rdev->config.evergreen.sx_max_export_pos_size = 32; 3097 rdev->config.evergreen.sx_max_export_smx_size = 96; 3098 rdev->config.evergreen.max_hw_contexts = 4; 3099 rdev->config.evergreen.sq_num_cf_insts = 1; 3100 3101 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3102 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3103 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3104 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3105 break; 3106 case CHIP_PALM: 3107 rdev->config.evergreen.num_ses = 1; 3108 rdev->config.evergreen.max_pipes = 2; 3109 rdev->config.evergreen.max_tile_pipes = 2; 3110 rdev->config.evergreen.max_simds = 2; 3111 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3112 rdev->config.evergreen.max_gprs = 256; 3113 rdev->config.evergreen.max_threads = 192; 3114 rdev->config.evergreen.max_gs_threads = 16; 3115 rdev->config.evergreen.max_stack_entries = 256; 3116 rdev->config.evergreen.sx_num_of_sets = 4; 3117 rdev->config.evergreen.sx_max_export_size = 128; 3118 rdev->config.evergreen.sx_max_export_pos_size = 32; 3119 rdev->config.evergreen.sx_max_export_smx_size = 96; 3120 rdev->config.evergreen.max_hw_contexts = 4; 3121 rdev->config.evergreen.sq_num_cf_insts = 1; 3122 3123 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3124 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3125 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3126 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3127 break; 3128 case CHIP_SUMO: 3129 rdev->config.evergreen.num_ses = 1; 3130 rdev->config.evergreen.max_pipes = 4; 3131 rdev->config.evergreen.max_tile_pipes = 4; 3132 if (rdev->pdev->device == 0x9648) 3133 rdev->config.evergreen.max_simds = 3; 3134 else if ((rdev->pdev->device == 0x9647) || 3135 (rdev->pdev->device == 0x964a)) 3136 rdev->config.evergreen.max_simds = 4; 3137 else 3138 rdev->config.evergreen.max_simds = 5; 3139 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3140 rdev->config.evergreen.max_gprs = 256; 3141 rdev->config.evergreen.max_threads = 248; 3142 rdev->config.evergreen.max_gs_threads = 32; 3143 rdev->config.evergreen.max_stack_entries = 256; 3144 rdev->config.evergreen.sx_num_of_sets = 4; 3145 rdev->config.evergreen.sx_max_export_size = 256; 3146 rdev->config.evergreen.sx_max_export_pos_size = 64; 3147 rdev->config.evergreen.sx_max_export_smx_size = 192; 3148 rdev->config.evergreen.max_hw_contexts = 8; 3149 rdev->config.evergreen.sq_num_cf_insts = 2; 3150 3151 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3152 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3153 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3154 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN; 3155 break; 3156 case CHIP_SUMO2: 3157 rdev->config.evergreen.num_ses = 1; 3158 rdev->config.evergreen.max_pipes = 4; 3159 rdev->config.evergreen.max_tile_pipes = 4; 3160 rdev->config.evergreen.max_simds = 2; 3161 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3162 rdev->config.evergreen.max_gprs = 256; 3163 rdev->config.evergreen.max_threads = 248; 3164 rdev->config.evergreen.max_gs_threads = 32; 3165 rdev->config.evergreen.max_stack_entries = 512; 3166 rdev->config.evergreen.sx_num_of_sets = 4; 3167 rdev->config.evergreen.sx_max_export_size = 256; 3168 rdev->config.evergreen.sx_max_export_pos_size = 64; 3169 rdev->config.evergreen.sx_max_export_smx_size = 192; 3170 rdev->config.evergreen.max_hw_contexts = 4; 3171 rdev->config.evergreen.sq_num_cf_insts = 2; 3172 3173 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3174 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3175 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3176 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN; 3177 break; 3178 case CHIP_BARTS: 3179 rdev->config.evergreen.num_ses = 2; 3180 rdev->config.evergreen.max_pipes = 4; 3181 rdev->config.evergreen.max_tile_pipes = 8; 3182 rdev->config.evergreen.max_simds = 7; 3183 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3184 rdev->config.evergreen.max_gprs = 256; 3185 rdev->config.evergreen.max_threads = 248; 3186 rdev->config.evergreen.max_gs_threads = 32; 3187 rdev->config.evergreen.max_stack_entries = 512; 3188 rdev->config.evergreen.sx_num_of_sets = 4; 3189 rdev->config.evergreen.sx_max_export_size = 256; 3190 rdev->config.evergreen.sx_max_export_pos_size = 64; 3191 rdev->config.evergreen.sx_max_export_smx_size = 192; 3192 rdev->config.evergreen.max_hw_contexts = 8; 3193 rdev->config.evergreen.sq_num_cf_insts = 2; 3194 3195 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3196 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3197 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3198 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN; 3199 break; 3200 case CHIP_TURKS: 3201 rdev->config.evergreen.num_ses = 1; 3202 rdev->config.evergreen.max_pipes = 4; 3203 rdev->config.evergreen.max_tile_pipes = 4; 3204 rdev->config.evergreen.max_simds = 6; 3205 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3206 rdev->config.evergreen.max_gprs = 256; 3207 rdev->config.evergreen.max_threads = 248; 3208 rdev->config.evergreen.max_gs_threads = 32; 3209 rdev->config.evergreen.max_stack_entries = 256; 3210 rdev->config.evergreen.sx_num_of_sets = 4; 3211 rdev->config.evergreen.sx_max_export_size = 256; 3212 rdev->config.evergreen.sx_max_export_pos_size = 64; 3213 rdev->config.evergreen.sx_max_export_smx_size = 192; 3214 rdev->config.evergreen.max_hw_contexts = 8; 3215 rdev->config.evergreen.sq_num_cf_insts = 2; 3216 3217 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3218 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3219 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3220 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN; 3221 break; 3222 case CHIP_CAICOS: 3223 rdev->config.evergreen.num_ses = 1; 3224 rdev->config.evergreen.max_pipes = 2; 3225 rdev->config.evergreen.max_tile_pipes = 2; 3226 rdev->config.evergreen.max_simds = 2; 3227 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3228 rdev->config.evergreen.max_gprs = 256; 3229 rdev->config.evergreen.max_threads = 192; 3230 rdev->config.evergreen.max_gs_threads = 16; 3231 rdev->config.evergreen.max_stack_entries = 256; 3232 rdev->config.evergreen.sx_num_of_sets = 4; 3233 rdev->config.evergreen.sx_max_export_size = 128; 3234 rdev->config.evergreen.sx_max_export_pos_size = 32; 3235 rdev->config.evergreen.sx_max_export_smx_size = 96; 3236 rdev->config.evergreen.max_hw_contexts = 4; 3237 rdev->config.evergreen.sq_num_cf_insts = 1; 3238 3239 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3240 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3241 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3242 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN; 3243 break; 3244 } 3245 3246 /* Initialize HDP */ 3247 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3248 WREG32((0x2c14 + j), 0x00000000); 3249 WREG32((0x2c18 + j), 0x00000000); 3250 WREG32((0x2c1c + j), 0x00000000); 3251 WREG32((0x2c20 + j), 0x00000000); 3252 WREG32((0x2c24 + j), 0x00000000); 3253 } 3254 3255 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3256 3257 evergreen_fix_pci_max_read_req_size(rdev); 3258 3259 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3260 if ((rdev->family == CHIP_PALM) || 3261 (rdev->family == CHIP_SUMO) || 3262 (rdev->family == CHIP_SUMO2)) 3263 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 3264 else 3265 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3266 3267 /* setup tiling info dword. gb_addr_config is not adequate since it does 3268 * not have bank info, so create a custom tiling dword. 3269 * bits 3:0 num_pipes 3270 * bits 7:4 num_banks 3271 * bits 11:8 group_size 3272 * bits 15:12 row_size 3273 */ 3274 rdev->config.evergreen.tile_config = 0; 3275 switch (rdev->config.evergreen.max_tile_pipes) { 3276 case 1: 3277 default: 3278 rdev->config.evergreen.tile_config |= (0 << 0); 3279 break; 3280 case 2: 3281 rdev->config.evergreen.tile_config |= (1 << 0); 3282 break; 3283 case 4: 3284 rdev->config.evergreen.tile_config |= (2 << 0); 3285 break; 3286 case 8: 3287 rdev->config.evergreen.tile_config |= (3 << 0); 3288 break; 3289 } 3290 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 3291 if (rdev->flags & RADEON_IS_IGP) 3292 rdev->config.evergreen.tile_config |= 1 << 4; 3293 else { 3294 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3295 case 0: /* four banks */ 3296 rdev->config.evergreen.tile_config |= 0 << 4; 3297 break; 3298 case 1: /* eight banks */ 3299 rdev->config.evergreen.tile_config |= 1 << 4; 3300 break; 3301 case 2: /* sixteen banks */ 3302 default: 3303 rdev->config.evergreen.tile_config |= 2 << 4; 3304 break; 3305 } 3306 } 3307 rdev->config.evergreen.tile_config |= 0 << 8; 3308 rdev->config.evergreen.tile_config |= 3309 ((gb_addr_config & 0x30000000) >> 28) << 12; 3310 3311 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { 3312 u32 efuse_straps_4; 3313 u32 efuse_straps_3; 3314 3315 efuse_straps_4 = RREG32_RCU(0x204); 3316 efuse_straps_3 = RREG32_RCU(0x203); 3317 tmp = (((efuse_straps_4 & 0xf) << 4) | 3318 ((efuse_straps_3 & 0xf0000000) >> 28)); 3319 } else { 3320 tmp = 0; 3321 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) { 3322 u32 rb_disable_bitmap; 3323 3324 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3325 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3326 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 3327 tmp <<= 4; 3328 tmp |= rb_disable_bitmap; 3329 } 3330 } 3331 /* enabled rb are just the one not disabled :) */ 3332 disabled_rb_mask = tmp; 3333 tmp = 0; 3334 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3335 tmp |= (1 << i); 3336 /* if all the backends are disabled, fix it up here */ 3337 if ((disabled_rb_mask & tmp) == tmp) { 3338 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3339 disabled_rb_mask &= ~(1 << i); 3340 } 3341 3342 for (i = 0; i < rdev->config.evergreen.num_ses; i++) { 3343 u32 simd_disable_bitmap; 3344 3345 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3346 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3347 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 3348 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; 3349 tmp <<= 16; 3350 tmp |= simd_disable_bitmap; 3351 } 3352 rdev->config.evergreen.active_simds = hweight32(~tmp); 3353 3354 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3355 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3356 3357 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3358 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3359 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3360 WREG32(DMA_TILING_CONFIG, gb_addr_config); 3361 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3362 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3363 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3364 3365 if ((rdev->config.evergreen.max_backends == 1) && 3366 (rdev->flags & RADEON_IS_IGP)) { 3367 if ((disabled_rb_mask & 3) == 1) { 3368 /* RB0 disabled, RB1 enabled */ 3369 tmp = 0x11111111; 3370 } else { 3371 /* RB1 disabled, RB0 enabled */ 3372 tmp = 0x00000000; 3373 } 3374 } else { 3375 tmp = gb_addr_config & NUM_PIPES_MASK; 3376 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends, 3377 EVERGREEN_MAX_BACKENDS, disabled_rb_mask); 3378 } 3379 WREG32(GB_BACKEND_MAP, tmp); 3380 3381 WREG32(CGTS_SYS_TCC_DISABLE, 0); 3382 WREG32(CGTS_TCC_DISABLE, 0); 3383 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 3384 WREG32(CGTS_USER_TCC_DISABLE, 0); 3385 3386 /* set HW defaults for 3D engine */ 3387 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3388 ROQ_IB2_START(0x2b))); 3389 3390 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 3391 3392 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 3393 SYNC_GRADIENT | 3394 SYNC_WALKER | 3395 SYNC_ALIGNER)); 3396 3397 sx_debug_1 = RREG32(SX_DEBUG_1); 3398 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 3399 WREG32(SX_DEBUG_1, sx_debug_1); 3400 3401 3402 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 3403 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 3404 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 3405 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 3406 3407 if (rdev->family <= CHIP_SUMO2) 3408 WREG32(SMX_SAR_CTL0, 0x00010000); 3409 3410 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 3411 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 3412 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 3413 3414 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 3415 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 3416 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 3417 3418 WREG32(VGT_NUM_INSTANCES, 1); 3419 WREG32(SPI_CONFIG_CNTL, 0); 3420 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3421 WREG32(CP_PERFMON_CNTL, 0); 3422 3423 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 3424 FETCH_FIFO_HIWATER(0x4) | 3425 DONE_FIFO_HIWATER(0xe0) | 3426 ALU_UPDATE_FIFO_HIWATER(0x8))); 3427 3428 sq_config = RREG32(SQ_CONFIG); 3429 sq_config &= ~(PS_PRIO(3) | 3430 VS_PRIO(3) | 3431 GS_PRIO(3) | 3432 ES_PRIO(3)); 3433 sq_config |= (VC_ENABLE | 3434 EXPORT_SRC_C | 3435 PS_PRIO(0) | 3436 VS_PRIO(1) | 3437 GS_PRIO(2) | 3438 ES_PRIO(3)); 3439 3440 switch (rdev->family) { 3441 case CHIP_CEDAR: 3442 case CHIP_PALM: 3443 case CHIP_SUMO: 3444 case CHIP_SUMO2: 3445 case CHIP_CAICOS: 3446 /* no vertex cache */ 3447 sq_config &= ~VC_ENABLE; 3448 break; 3449 default: 3450 break; 3451 } 3452 3453 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 3454 3455 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 3456 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 3457 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 3458 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3459 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3460 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3461 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3462 3463 switch (rdev->family) { 3464 case CHIP_CEDAR: 3465 case CHIP_PALM: 3466 case CHIP_SUMO: 3467 case CHIP_SUMO2: 3468 ps_thread_count = 96; 3469 break; 3470 default: 3471 ps_thread_count = 128; 3472 break; 3473 } 3474 3475 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 3476 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3477 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3478 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3479 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3480 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3481 3482 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3483 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3484 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3485 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3486 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3487 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3488 3489 WREG32(SQ_CONFIG, sq_config); 3490 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 3491 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 3492 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 3493 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 3494 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 3495 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 3496 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 3497 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 3498 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 3499 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 3500 3501 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3502 FORCE_EOV_MAX_REZ_CNT(255))); 3503 3504 switch (rdev->family) { 3505 case CHIP_CEDAR: 3506 case CHIP_PALM: 3507 case CHIP_SUMO: 3508 case CHIP_SUMO2: 3509 case CHIP_CAICOS: 3510 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 3511 break; 3512 default: 3513 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 3514 break; 3515 } 3516 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 3517 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 3518 3519 WREG32(VGT_GS_VERTEX_REUSE, 16); 3520 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 3521 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3522 3523 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 3524 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 3525 3526 WREG32(CB_PERF_CTR0_SEL_0, 0); 3527 WREG32(CB_PERF_CTR0_SEL_1, 0); 3528 WREG32(CB_PERF_CTR1_SEL_0, 0); 3529 WREG32(CB_PERF_CTR1_SEL_1, 0); 3530 WREG32(CB_PERF_CTR2_SEL_0, 0); 3531 WREG32(CB_PERF_CTR2_SEL_1, 0); 3532 WREG32(CB_PERF_CTR3_SEL_0, 0); 3533 WREG32(CB_PERF_CTR3_SEL_1, 0); 3534 3535 /* clear render buffer base addresses */ 3536 WREG32(CB_COLOR0_BASE, 0); 3537 WREG32(CB_COLOR1_BASE, 0); 3538 WREG32(CB_COLOR2_BASE, 0); 3539 WREG32(CB_COLOR3_BASE, 0); 3540 WREG32(CB_COLOR4_BASE, 0); 3541 WREG32(CB_COLOR5_BASE, 0); 3542 WREG32(CB_COLOR6_BASE, 0); 3543 WREG32(CB_COLOR7_BASE, 0); 3544 WREG32(CB_COLOR8_BASE, 0); 3545 WREG32(CB_COLOR9_BASE, 0); 3546 WREG32(CB_COLOR10_BASE, 0); 3547 WREG32(CB_COLOR11_BASE, 0); 3548 3549 /* set the shader const cache sizes to 0 */ 3550 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 3551 WREG32(i, 0); 3552 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 3553 WREG32(i, 0); 3554 3555 tmp = RREG32(HDP_MISC_CNTL); 3556 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3557 WREG32(HDP_MISC_CNTL, tmp); 3558 3559 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3560 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3561 3562 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3563 3564 udelay(50); 3565 3566 } 3567 3568 int evergreen_mc_init(struct radeon_device *rdev) 3569 { 3570 u32 tmp; 3571 int chansize, numchan; 3572 3573 /* Get VRAM informations */ 3574 rdev->mc.vram_is_ddr = true; 3575 if ((rdev->family == CHIP_PALM) || 3576 (rdev->family == CHIP_SUMO) || 3577 (rdev->family == CHIP_SUMO2)) 3578 tmp = RREG32(FUS_MC_ARB_RAMCFG); 3579 else 3580 tmp = RREG32(MC_ARB_RAMCFG); 3581 if (tmp & CHANSIZE_OVERRIDE) { 3582 chansize = 16; 3583 } else if (tmp & CHANSIZE_MASK) { 3584 chansize = 64; 3585 } else { 3586 chansize = 32; 3587 } 3588 tmp = RREG32(MC_SHARED_CHMAP); 3589 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3590 case 0: 3591 default: 3592 numchan = 1; 3593 break; 3594 case 1: 3595 numchan = 2; 3596 break; 3597 case 2: 3598 numchan = 4; 3599 break; 3600 case 3: 3601 numchan = 8; 3602 break; 3603 } 3604 rdev->mc.vram_width = numchan * chansize; 3605 /* Could aper size report 0 ? */ 3606 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 3607 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 3608 /* Setup GPU memory space */ 3609 if ((rdev->family == CHIP_PALM) || 3610 (rdev->family == CHIP_SUMO) || 3611 (rdev->family == CHIP_SUMO2)) { 3612 /* size in bytes on fusion */ 3613 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 3614 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 3615 } else { 3616 /* size in MB on evergreen/cayman/tn */ 3617 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3618 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3619 } 3620 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3621 r700_vram_gtt_location(rdev, &rdev->mc); 3622 radeon_update_bandwidth_info(rdev); 3623 3624 return 0; 3625 } 3626 3627 void evergreen_print_gpu_status_regs(struct radeon_device *rdev) 3628 { 3629 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", 3630 RREG32(GRBM_STATUS)); 3631 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", 3632 RREG32(GRBM_STATUS_SE0)); 3633 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", 3634 RREG32(GRBM_STATUS_SE1)); 3635 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", 3636 RREG32(SRBM_STATUS)); 3637 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", 3638 RREG32(SRBM_STATUS2)); 3639 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", 3640 RREG32(CP_STALLED_STAT1)); 3641 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", 3642 RREG32(CP_STALLED_STAT2)); 3643 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", 3644 RREG32(CP_BUSY_STAT)); 3645 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", 3646 RREG32(CP_STAT)); 3647 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", 3648 RREG32(DMA_STATUS_REG)); 3649 if (rdev->family >= CHIP_CAYMAN) { 3650 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", 3651 RREG32(DMA_STATUS_REG + 0x800)); 3652 } 3653 } 3654 3655 bool evergreen_is_display_hung(struct radeon_device *rdev) 3656 { 3657 u32 crtc_hung = 0; 3658 u32 crtc_status[6]; 3659 u32 i, j, tmp; 3660 3661 for (i = 0; i < rdev->num_crtc; i++) { 3662 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { 3663 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3664 crtc_hung |= (1 << i); 3665 } 3666 } 3667 3668 for (j = 0; j < 10; j++) { 3669 for (i = 0; i < rdev->num_crtc; i++) { 3670 if (crtc_hung & (1 << i)) { 3671 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3672 if (tmp != crtc_status[i]) 3673 crtc_hung &= ~(1 << i); 3674 } 3675 } 3676 if (crtc_hung == 0) 3677 return false; 3678 udelay(100); 3679 } 3680 3681 return true; 3682 } 3683 3684 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) 3685 { 3686 u32 reset_mask = 0; 3687 u32 tmp; 3688 3689 /* GRBM_STATUS */ 3690 tmp = RREG32(GRBM_STATUS); 3691 if (tmp & (PA_BUSY | SC_BUSY | 3692 SH_BUSY | SX_BUSY | 3693 TA_BUSY | VGT_BUSY | 3694 DB_BUSY | CB_BUSY | 3695 SPI_BUSY | VGT_BUSY_NO_DMA)) 3696 reset_mask |= RADEON_RESET_GFX; 3697 3698 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3699 CP_BUSY | CP_COHERENCY_BUSY)) 3700 reset_mask |= RADEON_RESET_CP; 3701 3702 if (tmp & GRBM_EE_BUSY) 3703 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3704 3705 /* DMA_STATUS_REG */ 3706 tmp = RREG32(DMA_STATUS_REG); 3707 if (!(tmp & DMA_IDLE)) 3708 reset_mask |= RADEON_RESET_DMA; 3709 3710 /* SRBM_STATUS2 */ 3711 tmp = RREG32(SRBM_STATUS2); 3712 if (tmp & DMA_BUSY) 3713 reset_mask |= RADEON_RESET_DMA; 3714 3715 /* SRBM_STATUS */ 3716 tmp = RREG32(SRBM_STATUS); 3717 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3718 reset_mask |= RADEON_RESET_RLC; 3719 3720 if (tmp & IH_BUSY) 3721 reset_mask |= RADEON_RESET_IH; 3722 3723 if (tmp & SEM_BUSY) 3724 reset_mask |= RADEON_RESET_SEM; 3725 3726 if (tmp & GRBM_RQ_PENDING) 3727 reset_mask |= RADEON_RESET_GRBM; 3728 3729 if (tmp & VMC_BUSY) 3730 reset_mask |= RADEON_RESET_VMC; 3731 3732 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3733 MCC_BUSY | MCD_BUSY)) 3734 reset_mask |= RADEON_RESET_MC; 3735 3736 if (evergreen_is_display_hung(rdev)) 3737 reset_mask |= RADEON_RESET_DISPLAY; 3738 3739 /* VM_L2_STATUS */ 3740 tmp = RREG32(VM_L2_STATUS); 3741 if (tmp & L2_BUSY) 3742 reset_mask |= RADEON_RESET_VMC; 3743 3744 /* Skip MC reset as it's mostly likely not hung, just busy */ 3745 if (reset_mask & RADEON_RESET_MC) { 3746 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3747 reset_mask &= ~RADEON_RESET_MC; 3748 } 3749 3750 return reset_mask; 3751 } 3752 3753 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3754 { 3755 struct evergreen_mc_save save; 3756 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3757 u32 tmp; 3758 3759 if (reset_mask == 0) 3760 return; 3761 3762 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3763 3764 evergreen_print_gpu_status_regs(rdev); 3765 3766 /* Disable CP parsing/prefetching */ 3767 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3768 3769 if (reset_mask & RADEON_RESET_DMA) { 3770 /* Disable DMA */ 3771 tmp = RREG32(DMA_RB_CNTL); 3772 tmp &= ~DMA_RB_ENABLE; 3773 WREG32(DMA_RB_CNTL, tmp); 3774 } 3775 3776 udelay(50); 3777 3778 evergreen_mc_stop(rdev, &save); 3779 if (evergreen_mc_wait_for_idle(rdev)) { 3780 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3781 } 3782 3783 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 3784 grbm_soft_reset |= SOFT_RESET_DB | 3785 SOFT_RESET_CB | 3786 SOFT_RESET_PA | 3787 SOFT_RESET_SC | 3788 SOFT_RESET_SPI | 3789 SOFT_RESET_SX | 3790 SOFT_RESET_SH | 3791 SOFT_RESET_TC | 3792 SOFT_RESET_TA | 3793 SOFT_RESET_VC | 3794 SOFT_RESET_VGT; 3795 } 3796 3797 if (reset_mask & RADEON_RESET_CP) { 3798 grbm_soft_reset |= SOFT_RESET_CP | 3799 SOFT_RESET_VGT; 3800 3801 srbm_soft_reset |= SOFT_RESET_GRBM; 3802 } 3803 3804 if (reset_mask & RADEON_RESET_DMA) 3805 srbm_soft_reset |= SOFT_RESET_DMA; 3806 3807 if (reset_mask & RADEON_RESET_DISPLAY) 3808 srbm_soft_reset |= SOFT_RESET_DC; 3809 3810 if (reset_mask & RADEON_RESET_RLC) 3811 srbm_soft_reset |= SOFT_RESET_RLC; 3812 3813 if (reset_mask & RADEON_RESET_SEM) 3814 srbm_soft_reset |= SOFT_RESET_SEM; 3815 3816 if (reset_mask & RADEON_RESET_IH) 3817 srbm_soft_reset |= SOFT_RESET_IH; 3818 3819 if (reset_mask & RADEON_RESET_GRBM) 3820 srbm_soft_reset |= SOFT_RESET_GRBM; 3821 3822 if (reset_mask & RADEON_RESET_VMC) 3823 srbm_soft_reset |= SOFT_RESET_VMC; 3824 3825 if (!(rdev->flags & RADEON_IS_IGP)) { 3826 if (reset_mask & RADEON_RESET_MC) 3827 srbm_soft_reset |= SOFT_RESET_MC; 3828 } 3829 3830 if (grbm_soft_reset) { 3831 tmp = RREG32(GRBM_SOFT_RESET); 3832 tmp |= grbm_soft_reset; 3833 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3834 WREG32(GRBM_SOFT_RESET, tmp); 3835 tmp = RREG32(GRBM_SOFT_RESET); 3836 3837 udelay(50); 3838 3839 tmp &= ~grbm_soft_reset; 3840 WREG32(GRBM_SOFT_RESET, tmp); 3841 tmp = RREG32(GRBM_SOFT_RESET); 3842 } 3843 3844 if (srbm_soft_reset) { 3845 tmp = RREG32(SRBM_SOFT_RESET); 3846 tmp |= srbm_soft_reset; 3847 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3848 WREG32(SRBM_SOFT_RESET, tmp); 3849 tmp = RREG32(SRBM_SOFT_RESET); 3850 3851 udelay(50); 3852 3853 tmp &= ~srbm_soft_reset; 3854 WREG32(SRBM_SOFT_RESET, tmp); 3855 tmp = RREG32(SRBM_SOFT_RESET); 3856 } 3857 3858 /* Wait a little for things to settle down */ 3859 udelay(50); 3860 3861 evergreen_mc_resume(rdev, &save); 3862 udelay(50); 3863 3864 evergreen_print_gpu_status_regs(rdev); 3865 } 3866 3867 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) 3868 { 3869 struct evergreen_mc_save save; 3870 u32 tmp, i; 3871 3872 dev_info(rdev->dev, "GPU pci config reset\n"); 3873 3874 /* disable dpm? */ 3875 3876 /* Disable CP parsing/prefetching */ 3877 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3878 udelay(50); 3879 /* Disable DMA */ 3880 tmp = RREG32(DMA_RB_CNTL); 3881 tmp &= ~DMA_RB_ENABLE; 3882 WREG32(DMA_RB_CNTL, tmp); 3883 /* XXX other engines? */ 3884 3885 /* halt the rlc */ 3886 r600_rlc_stop(rdev); 3887 3888 udelay(50); 3889 3890 /* set mclk/sclk to bypass */ 3891 rv770_set_clk_bypass_mode(rdev); 3892 /* disable BM */ 3893 pci_clear_master(rdev->pdev); 3894 /* disable mem access */ 3895 evergreen_mc_stop(rdev, &save); 3896 if (evergreen_mc_wait_for_idle(rdev)) { 3897 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 3898 } 3899 /* reset */ 3900 radeon_pci_config_reset(rdev); 3901 /* wait for asic to come out of reset */ 3902 for (i = 0; i < rdev->usec_timeout; i++) { 3903 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 3904 break; 3905 udelay(1); 3906 } 3907 } 3908 3909 int evergreen_asic_reset(struct radeon_device *rdev) 3910 { 3911 u32 reset_mask; 3912 3913 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3914 3915 if (reset_mask) 3916 r600_set_bios_scratch_engine_hung(rdev, true); 3917 3918 /* try soft reset */ 3919 evergreen_gpu_soft_reset(rdev, reset_mask); 3920 3921 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3922 3923 /* try pci config reset */ 3924 if (reset_mask && radeon_hard_reset) 3925 evergreen_gpu_pci_config_reset(rdev); 3926 3927 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3928 3929 if (!reset_mask) 3930 r600_set_bios_scratch_engine_hung(rdev, false); 3931 3932 return 0; 3933 } 3934 3935 /** 3936 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up 3937 * 3938 * @rdev: radeon_device pointer 3939 * @ring: radeon_ring structure holding ring information 3940 * 3941 * Check if the GFX engine is locked up. 3942 * Returns true if the engine appears to be locked up, false if not. 3943 */ 3944 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 3945 { 3946 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 3947 3948 if (!(reset_mask & (RADEON_RESET_GFX | 3949 RADEON_RESET_COMPUTE | 3950 RADEON_RESET_CP))) { 3951 radeon_ring_lockup_update(rdev, ring); 3952 return false; 3953 } 3954 return radeon_ring_test_lockup(rdev, ring); 3955 } 3956 3957 /* 3958 * RLC 3959 */ 3960 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 3961 #define RLC_CLEAR_STATE_END_MARKER 0x00000001 3962 3963 void sumo_rlc_fini(struct radeon_device *rdev) 3964 { 3965 int r; 3966 3967 /* save restore block */ 3968 if (rdev->rlc.save_restore_obj) { 3969 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 3970 if (unlikely(r != 0)) 3971 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 3972 radeon_bo_unpin(rdev->rlc.save_restore_obj); 3973 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 3974 3975 radeon_bo_unref(&rdev->rlc.save_restore_obj); 3976 rdev->rlc.save_restore_obj = NULL; 3977 } 3978 3979 /* clear state block */ 3980 if (rdev->rlc.clear_state_obj) { 3981 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 3982 if (unlikely(r != 0)) 3983 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 3984 radeon_bo_unpin(rdev->rlc.clear_state_obj); 3985 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 3986 3987 radeon_bo_unref(&rdev->rlc.clear_state_obj); 3988 rdev->rlc.clear_state_obj = NULL; 3989 } 3990 3991 /* clear state block */ 3992 if (rdev->rlc.cp_table_obj) { 3993 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 3994 if (unlikely(r != 0)) 3995 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 3996 radeon_bo_unpin(rdev->rlc.cp_table_obj); 3997 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 3998 3999 radeon_bo_unref(&rdev->rlc.cp_table_obj); 4000 rdev->rlc.cp_table_obj = NULL; 4001 } 4002 } 4003 4004 #define CP_ME_TABLE_SIZE 96 4005 4006 int sumo_rlc_init(struct radeon_device *rdev) 4007 { 4008 const u32 *src_ptr; 4009 volatile u32 *dst_ptr; 4010 u32 dws, data, i, j, k, reg_num; 4011 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; 4012 u64 reg_list_mc_addr; 4013 const struct cs_section_def *cs_data; 4014 int r; 4015 4016 src_ptr = rdev->rlc.reg_list; 4017 dws = rdev->rlc.reg_list_size; 4018 if (rdev->family >= CHIP_BONAIRE) { 4019 dws += (5 * 16) + 48 + 48 + 64; 4020 } 4021 cs_data = rdev->rlc.cs_data; 4022 4023 if (src_ptr) { 4024 /* save restore block */ 4025 if (rdev->rlc.save_restore_obj == NULL) { 4026 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4027 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4028 NULL, &rdev->rlc.save_restore_obj); 4029 if (r) { 4030 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 4031 return r; 4032 } 4033 } 4034 4035 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4036 if (unlikely(r != 0)) { 4037 sumo_rlc_fini(rdev); 4038 return r; 4039 } 4040 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 4041 &rdev->rlc.save_restore_gpu_addr); 4042 if (r) { 4043 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4044 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 4045 sumo_rlc_fini(rdev); 4046 return r; 4047 } 4048 4049 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr); 4050 if (r) { 4051 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); 4052 sumo_rlc_fini(rdev); 4053 return r; 4054 } 4055 /* write the sr buffer */ 4056 dst_ptr = rdev->rlc.sr_ptr; 4057 if (rdev->family >= CHIP_TAHITI) { 4058 /* SI */ 4059 for (i = 0; i < rdev->rlc.reg_list_size; i++) 4060 dst_ptr[i] = cpu_to_le32(src_ptr[i]); 4061 } else { 4062 /* ON/LN/TN */ 4063 /* format: 4064 * dw0: (reg2 << 16) | reg1 4065 * dw1: reg1 save space 4066 * dw2: reg2 save space 4067 */ 4068 for (i = 0; i < dws; i++) { 4069 data = src_ptr[i] >> 2; 4070 i++; 4071 if (i < dws) 4072 data |= (src_ptr[i] >> 2) << 16; 4073 j = (((i - 1) * 3) / 2); 4074 dst_ptr[j] = cpu_to_le32(data); 4075 } 4076 j = ((i * 3) / 2); 4077 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); 4078 } 4079 radeon_bo_kunmap(rdev->rlc.save_restore_obj); 4080 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4081 } 4082 4083 if (cs_data) { 4084 /* clear state block */ 4085 if (rdev->family >= CHIP_BONAIRE) { 4086 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); 4087 } else if (rdev->family >= CHIP_TAHITI) { 4088 rdev->rlc.clear_state_size = si_get_csb_size(rdev); 4089 dws = rdev->rlc.clear_state_size + (256 / 4); 4090 } else { 4091 reg_list_num = 0; 4092 dws = 0; 4093 for (i = 0; cs_data[i].section != NULL; i++) { 4094 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4095 reg_list_num++; 4096 dws += cs_data[i].section[j].reg_count; 4097 } 4098 } 4099 reg_list_blk_index = (3 * reg_list_num + 2); 4100 dws += reg_list_blk_index; 4101 rdev->rlc.clear_state_size = dws; 4102 } 4103 4104 if (rdev->rlc.clear_state_obj == NULL) { 4105 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4106 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4107 NULL, &rdev->rlc.clear_state_obj); 4108 if (r) { 4109 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 4110 sumo_rlc_fini(rdev); 4111 return r; 4112 } 4113 } 4114 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4115 if (unlikely(r != 0)) { 4116 sumo_rlc_fini(rdev); 4117 return r; 4118 } 4119 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 4120 &rdev->rlc.clear_state_gpu_addr); 4121 if (r) { 4122 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4123 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 4124 sumo_rlc_fini(rdev); 4125 return r; 4126 } 4127 4128 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr); 4129 if (r) { 4130 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); 4131 sumo_rlc_fini(rdev); 4132 return r; 4133 } 4134 /* set up the cs buffer */ 4135 dst_ptr = rdev->rlc.cs_ptr; 4136 if (rdev->family >= CHIP_BONAIRE) { 4137 cik_get_csb_buffer(rdev, dst_ptr); 4138 } else if (rdev->family >= CHIP_TAHITI) { 4139 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; 4140 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); 4141 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); 4142 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); 4143 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); 4144 } else { 4145 reg_list_hdr_blk_index = 0; 4146 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); 4147 data = upper_32_bits(reg_list_mc_addr); 4148 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4149 reg_list_hdr_blk_index++; 4150 for (i = 0; cs_data[i].section != NULL; i++) { 4151 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4152 reg_num = cs_data[i].section[j].reg_count; 4153 data = reg_list_mc_addr & 0xffffffff; 4154 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4155 reg_list_hdr_blk_index++; 4156 4157 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; 4158 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4159 reg_list_hdr_blk_index++; 4160 4161 data = 0x08000000 | (reg_num * 4); 4162 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4163 reg_list_hdr_blk_index++; 4164 4165 for (k = 0; k < reg_num; k++) { 4166 data = cs_data[i].section[j].extent[k]; 4167 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); 4168 } 4169 reg_list_mc_addr += reg_num * 4; 4170 reg_list_blk_index += reg_num; 4171 } 4172 } 4173 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); 4174 } 4175 radeon_bo_kunmap(rdev->rlc.clear_state_obj); 4176 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4177 } 4178 4179 if (rdev->rlc.cp_table_size) { 4180 if (rdev->rlc.cp_table_obj == NULL) { 4181 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, 4182 PAGE_SIZE, true, 4183 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4184 NULL, &rdev->rlc.cp_table_obj); 4185 if (r) { 4186 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); 4187 sumo_rlc_fini(rdev); 4188 return r; 4189 } 4190 } 4191 4192 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4193 if (unlikely(r != 0)) { 4194 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4195 sumo_rlc_fini(rdev); 4196 return r; 4197 } 4198 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, 4199 &rdev->rlc.cp_table_gpu_addr); 4200 if (r) { 4201 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4202 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); 4203 sumo_rlc_fini(rdev); 4204 return r; 4205 } 4206 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr); 4207 if (r) { 4208 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); 4209 sumo_rlc_fini(rdev); 4210 return r; 4211 } 4212 4213 cik_init_cp_pg_table(rdev); 4214 4215 radeon_bo_kunmap(rdev->rlc.cp_table_obj); 4216 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4217 4218 } 4219 4220 return 0; 4221 } 4222 4223 static void evergreen_rlc_start(struct radeon_device *rdev) 4224 { 4225 u32 mask = RLC_ENABLE; 4226 4227 if (rdev->flags & RADEON_IS_IGP) { 4228 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; 4229 } 4230 4231 WREG32(RLC_CNTL, mask); 4232 } 4233 4234 int evergreen_rlc_resume(struct radeon_device *rdev) 4235 { 4236 u32 i; 4237 const __be32 *fw_data; 4238 4239 if (!rdev->rlc_fw) 4240 return -EINVAL; 4241 4242 r600_rlc_stop(rdev); 4243 4244 WREG32(RLC_HB_CNTL, 0); 4245 4246 if (rdev->flags & RADEON_IS_IGP) { 4247 if (rdev->family == CHIP_ARUBA) { 4248 u32 always_on_bitmap = 4249 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); 4250 /* find out the number of active simds */ 4251 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 4252 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; 4253 tmp = hweight32(~tmp); 4254 if (tmp == rdev->config.cayman.max_simds_per_se) { 4255 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); 4256 WREG32(TN_RLC_LB_PARAMS, 0x00601004); 4257 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); 4258 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); 4259 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); 4260 } 4261 } else { 4262 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4263 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4264 } 4265 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4266 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4267 } else { 4268 WREG32(RLC_HB_BASE, 0); 4269 WREG32(RLC_HB_RPTR, 0); 4270 WREG32(RLC_HB_WPTR, 0); 4271 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4272 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4273 } 4274 WREG32(RLC_MC_CNTL, 0); 4275 WREG32(RLC_UCODE_CNTL, 0); 4276 4277 fw_data = (const __be32 *)rdev->rlc_fw->data; 4278 if (rdev->family >= CHIP_ARUBA) { 4279 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { 4280 WREG32(RLC_UCODE_ADDR, i); 4281 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4282 } 4283 } else if (rdev->family >= CHIP_CAYMAN) { 4284 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { 4285 WREG32(RLC_UCODE_ADDR, i); 4286 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4287 } 4288 } else { 4289 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { 4290 WREG32(RLC_UCODE_ADDR, i); 4291 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4292 } 4293 } 4294 WREG32(RLC_UCODE_ADDR, 0); 4295 4296 evergreen_rlc_start(rdev); 4297 4298 return 0; 4299 } 4300 4301 /* Interrupts */ 4302 4303 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 4304 { 4305 if (crtc >= rdev->num_crtc) 4306 return 0; 4307 else 4308 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); 4309 } 4310 4311 void evergreen_disable_interrupt_state(struct radeon_device *rdev) 4312 { 4313 u32 tmp; 4314 4315 if (rdev->family >= CHIP_CAYMAN) { 4316 cayman_cp_int_cntl_setup(rdev, 0, 4317 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4318 cayman_cp_int_cntl_setup(rdev, 1, 0); 4319 cayman_cp_int_cntl_setup(rdev, 2, 0); 4320 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4321 WREG32(CAYMAN_DMA1_CNTL, tmp); 4322 } else 4323 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4324 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4325 WREG32(DMA_CNTL, tmp); 4326 WREG32(GRBM_INT_CNTL, 0); 4327 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4328 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4329 if (rdev->num_crtc >= 4) { 4330 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4331 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4332 } 4333 if (rdev->num_crtc >= 6) { 4334 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4335 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4336 } 4337 4338 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 4339 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 4340 if (rdev->num_crtc >= 4) { 4341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 4342 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 4343 } 4344 if (rdev->num_crtc >= 6) { 4345 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 4346 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 4347 } 4348 4349 /* only one DAC on DCE5 */ 4350 if (!ASIC_IS_DCE5(rdev)) 4351 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 4352 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 4353 4354 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4355 WREG32(DC_HPD1_INT_CONTROL, tmp); 4356 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4357 WREG32(DC_HPD2_INT_CONTROL, tmp); 4358 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4359 WREG32(DC_HPD3_INT_CONTROL, tmp); 4360 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4361 WREG32(DC_HPD4_INT_CONTROL, tmp); 4362 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4363 WREG32(DC_HPD5_INT_CONTROL, tmp); 4364 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 4365 WREG32(DC_HPD6_INT_CONTROL, tmp); 4366 4367 } 4368 4369 int evergreen_irq_set(struct radeon_device *rdev) 4370 { 4371 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 4372 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 4373 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 4374 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 4375 u32 grbm_int_cntl = 0; 4376 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0; 4377 u32 dma_cntl, dma_cntl1 = 0; 4378 u32 thermal_int = 0; 4379 4380 if (!rdev->irq.installed) { 4381 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 4382 return -EINVAL; 4383 } 4384 /* don't enable anything if the ih is disabled */ 4385 if (!rdev->ih.enabled) { 4386 r600_disable_interrupts(rdev); 4387 /* force the active interrupt state to all disabled */ 4388 evergreen_disable_interrupt_state(rdev); 4389 return 0; 4390 } 4391 4392 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 4393 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 4394 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 4395 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 4396 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 4397 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 4398 if (rdev->family == CHIP_ARUBA) 4399 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & 4400 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4401 else 4402 thermal_int = RREG32(CG_THERMAL_INT) & 4403 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4404 4405 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4406 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4407 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4408 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4409 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4410 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK; 4411 4412 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4413 4414 if (rdev->family >= CHIP_CAYMAN) { 4415 /* enable CP interrupts on all rings */ 4416 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4417 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4418 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4419 } 4420 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 4421 DRM_DEBUG("evergreen_irq_set: sw int cp1\n"); 4422 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 4423 } 4424 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 4425 DRM_DEBUG("evergreen_irq_set: sw int cp2\n"); 4426 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 4427 } 4428 } else { 4429 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4430 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4431 cp_int_cntl |= RB_INT_ENABLE; 4432 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4433 } 4434 } 4435 4436 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 4437 DRM_DEBUG("r600_irq_set: sw int dma\n"); 4438 dma_cntl |= TRAP_ENABLE; 4439 } 4440 4441 if (rdev->family >= CHIP_CAYMAN) { 4442 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4443 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 4444 DRM_DEBUG("r600_irq_set: sw int dma1\n"); 4445 dma_cntl1 |= TRAP_ENABLE; 4446 } 4447 } 4448 4449 if (rdev->irq.dpm_thermal) { 4450 DRM_DEBUG("dpm thermal\n"); 4451 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 4452 } 4453 4454 if (rdev->irq.crtc_vblank_int[0] || 4455 atomic_read(&rdev->irq.pflip[0])) { 4456 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 4457 crtc1 |= VBLANK_INT_MASK; 4458 } 4459 if (rdev->irq.crtc_vblank_int[1] || 4460 atomic_read(&rdev->irq.pflip[1])) { 4461 DRM_DEBUG("evergreen_irq_set: vblank 1\n"); 4462 crtc2 |= VBLANK_INT_MASK; 4463 } 4464 if (rdev->irq.crtc_vblank_int[2] || 4465 atomic_read(&rdev->irq.pflip[2])) { 4466 DRM_DEBUG("evergreen_irq_set: vblank 2\n"); 4467 crtc3 |= VBLANK_INT_MASK; 4468 } 4469 if (rdev->irq.crtc_vblank_int[3] || 4470 atomic_read(&rdev->irq.pflip[3])) { 4471 DRM_DEBUG("evergreen_irq_set: vblank 3\n"); 4472 crtc4 |= VBLANK_INT_MASK; 4473 } 4474 if (rdev->irq.crtc_vblank_int[4] || 4475 atomic_read(&rdev->irq.pflip[4])) { 4476 DRM_DEBUG("evergreen_irq_set: vblank 4\n"); 4477 crtc5 |= VBLANK_INT_MASK; 4478 } 4479 if (rdev->irq.crtc_vblank_int[5] || 4480 atomic_read(&rdev->irq.pflip[5])) { 4481 DRM_DEBUG("evergreen_irq_set: vblank 5\n"); 4482 crtc6 |= VBLANK_INT_MASK; 4483 } 4484 if (rdev->irq.hpd[0]) { 4485 DRM_DEBUG("evergreen_irq_set: hpd 1\n"); 4486 hpd1 |= DC_HPDx_INT_EN; 4487 } 4488 if (rdev->irq.hpd[1]) { 4489 DRM_DEBUG("evergreen_irq_set: hpd 2\n"); 4490 hpd2 |= DC_HPDx_INT_EN; 4491 } 4492 if (rdev->irq.hpd[2]) { 4493 DRM_DEBUG("evergreen_irq_set: hpd 3\n"); 4494 hpd3 |= DC_HPDx_INT_EN; 4495 } 4496 if (rdev->irq.hpd[3]) { 4497 DRM_DEBUG("evergreen_irq_set: hpd 4\n"); 4498 hpd4 |= DC_HPDx_INT_EN; 4499 } 4500 if (rdev->irq.hpd[4]) { 4501 DRM_DEBUG("evergreen_irq_set: hpd 5\n"); 4502 hpd5 |= DC_HPDx_INT_EN; 4503 } 4504 if (rdev->irq.hpd[5]) { 4505 DRM_DEBUG("evergreen_irq_set: hpd 6\n"); 4506 hpd6 |= DC_HPDx_INT_EN; 4507 } 4508 if (rdev->irq.afmt[0]) { 4509 DRM_DEBUG("evergreen_irq_set: hdmi 0\n"); 4510 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4511 } 4512 if (rdev->irq.afmt[1]) { 4513 DRM_DEBUG("evergreen_irq_set: hdmi 1\n"); 4514 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4515 } 4516 if (rdev->irq.afmt[2]) { 4517 DRM_DEBUG("evergreen_irq_set: hdmi 2\n"); 4518 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4519 } 4520 if (rdev->irq.afmt[3]) { 4521 DRM_DEBUG("evergreen_irq_set: hdmi 3\n"); 4522 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4523 } 4524 if (rdev->irq.afmt[4]) { 4525 DRM_DEBUG("evergreen_irq_set: hdmi 4\n"); 4526 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4527 } 4528 if (rdev->irq.afmt[5]) { 4529 DRM_DEBUG("evergreen_irq_set: hdmi 5\n"); 4530 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK; 4531 } 4532 4533 if (rdev->family >= CHIP_CAYMAN) { 4534 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl); 4535 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1); 4536 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2); 4537 } else 4538 WREG32(CP_INT_CNTL, cp_int_cntl); 4539 4540 WREG32(DMA_CNTL, dma_cntl); 4541 4542 if (rdev->family >= CHIP_CAYMAN) 4543 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1); 4544 4545 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 4546 4547 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 4548 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 4549 if (rdev->num_crtc >= 4) { 4550 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 4551 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 4552 } 4553 if (rdev->num_crtc >= 6) { 4554 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 4555 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 4556 } 4557 4558 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 4559 GRPH_PFLIP_INT_MASK); 4560 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 4561 GRPH_PFLIP_INT_MASK); 4562 if (rdev->num_crtc >= 4) { 4563 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 4564 GRPH_PFLIP_INT_MASK); 4565 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 4566 GRPH_PFLIP_INT_MASK); 4567 } 4568 if (rdev->num_crtc >= 6) { 4569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 4570 GRPH_PFLIP_INT_MASK); 4571 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 4572 GRPH_PFLIP_INT_MASK); 4573 } 4574 4575 WREG32(DC_HPD1_INT_CONTROL, hpd1); 4576 WREG32(DC_HPD2_INT_CONTROL, hpd2); 4577 WREG32(DC_HPD3_INT_CONTROL, hpd3); 4578 WREG32(DC_HPD4_INT_CONTROL, hpd4); 4579 WREG32(DC_HPD5_INT_CONTROL, hpd5); 4580 WREG32(DC_HPD6_INT_CONTROL, hpd6); 4581 if (rdev->family == CHIP_ARUBA) 4582 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); 4583 else 4584 WREG32(CG_THERMAL_INT, thermal_int); 4585 4586 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1); 4587 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2); 4588 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3); 4589 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4); 4590 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5); 4591 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6); 4592 4593 return 0; 4594 } 4595 4596 static void evergreen_irq_ack(struct radeon_device *rdev) 4597 { 4598 u32 tmp; 4599 4600 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 4601 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 4602 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 4603 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 4604 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 4605 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 4606 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4607 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4608 if (rdev->num_crtc >= 4) { 4609 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4610 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4611 } 4612 if (rdev->num_crtc >= 6) { 4613 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4614 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4615 } 4616 4617 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 4618 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 4619 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 4620 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 4621 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 4622 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 4623 4624 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 4625 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4626 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 4627 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4628 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 4629 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 4630 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 4631 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 4632 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 4633 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 4634 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 4635 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 4636 4637 if (rdev->num_crtc >= 4) { 4638 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 4639 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4640 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 4641 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4642 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 4643 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 4644 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 4645 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 4646 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 4647 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 4648 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 4649 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 4650 } 4651 4652 if (rdev->num_crtc >= 6) { 4653 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 4654 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4655 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 4656 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 4657 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 4658 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 4659 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 4660 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 4661 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 4662 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 4663 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 4664 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 4665 } 4666 4667 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 4668 tmp = RREG32(DC_HPD1_INT_CONTROL); 4669 tmp |= DC_HPDx_INT_ACK; 4670 WREG32(DC_HPD1_INT_CONTROL, tmp); 4671 } 4672 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 4673 tmp = RREG32(DC_HPD2_INT_CONTROL); 4674 tmp |= DC_HPDx_INT_ACK; 4675 WREG32(DC_HPD2_INT_CONTROL, tmp); 4676 } 4677 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 4678 tmp = RREG32(DC_HPD3_INT_CONTROL); 4679 tmp |= DC_HPDx_INT_ACK; 4680 WREG32(DC_HPD3_INT_CONTROL, tmp); 4681 } 4682 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 4683 tmp = RREG32(DC_HPD4_INT_CONTROL); 4684 tmp |= DC_HPDx_INT_ACK; 4685 WREG32(DC_HPD4_INT_CONTROL, tmp); 4686 } 4687 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 4688 tmp = RREG32(DC_HPD5_INT_CONTROL); 4689 tmp |= DC_HPDx_INT_ACK; 4690 WREG32(DC_HPD5_INT_CONTROL, tmp); 4691 } 4692 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 4693 tmp = RREG32(DC_HPD5_INT_CONTROL); 4694 tmp |= DC_HPDx_INT_ACK; 4695 WREG32(DC_HPD6_INT_CONTROL, tmp); 4696 } 4697 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 4698 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); 4699 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4700 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp); 4701 } 4702 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 4703 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); 4704 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4705 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp); 4706 } 4707 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 4708 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET); 4709 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4710 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp); 4711 } 4712 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 4713 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); 4714 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4715 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp); 4716 } 4717 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 4718 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET); 4719 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4720 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp); 4721 } 4722 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 4723 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 4724 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK; 4725 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp); 4726 } 4727 } 4728 4729 static void evergreen_irq_disable(struct radeon_device *rdev) 4730 { 4731 r600_disable_interrupts(rdev); 4732 /* Wait and acknowledge irq */ 4733 mdelay(1); 4734 evergreen_irq_ack(rdev); 4735 evergreen_disable_interrupt_state(rdev); 4736 } 4737 4738 void evergreen_irq_suspend(struct radeon_device *rdev) 4739 { 4740 evergreen_irq_disable(rdev); 4741 r600_rlc_stop(rdev); 4742 } 4743 4744 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 4745 { 4746 u32 wptr, tmp; 4747 4748 if (rdev->wb.enabled) 4749 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 4750 else 4751 wptr = RREG32(IH_RB_WPTR); 4752 4753 if (wptr & RB_OVERFLOW) { 4754 wptr &= ~RB_OVERFLOW; 4755 /* When a ring buffer overflow happen start parsing interrupt 4756 * from the last not overwritten vector (wptr + 16). Hopefully 4757 * this should allow us to catchup. 4758 */ 4759 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 4760 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 4761 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 4762 tmp = RREG32(IH_RB_CNTL); 4763 tmp |= IH_WPTR_OVERFLOW_CLEAR; 4764 WREG32(IH_RB_CNTL, tmp); 4765 } 4766 return (wptr & rdev->ih.ptr_mask); 4767 } 4768 4769 int evergreen_irq_process(struct radeon_device *rdev) 4770 { 4771 u32 wptr; 4772 u32 rptr; 4773 u32 src_id, src_data; 4774 u32 ring_index; 4775 bool queue_hotplug = false; 4776 bool queue_hdmi = false; 4777 bool queue_thermal = false; 4778 u32 status, addr; 4779 4780 if (!rdev->ih.enabled || rdev->shutdown) 4781 return IRQ_NONE; 4782 4783 wptr = evergreen_get_ih_wptr(rdev); 4784 4785 restart_ih: 4786 /* is somebody else already processing irqs? */ 4787 if (atomic_xchg(&rdev->ih.lock, 1)) 4788 return IRQ_NONE; 4789 4790 rptr = rdev->ih.rptr; 4791 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 4792 4793 /* Order reading of wptr vs. reading of IH ring data */ 4794 rmb(); 4795 4796 /* display interrupts */ 4797 evergreen_irq_ack(rdev); 4798 4799 while (rptr != wptr) { 4800 /* wptr/rptr are in bytes! */ 4801 ring_index = rptr / 4; 4802 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 4803 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 4804 4805 switch (src_id) { 4806 case 1: /* D1 vblank/vline */ 4807 switch (src_data) { 4808 case 0: /* D1 vblank */ 4809 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 4810 if (rdev->irq.crtc_vblank_int[0]) { 4811 drm_handle_vblank(rdev->ddev, 0); 4812 rdev->pm.vblank_sync = true; 4813 wake_up(&rdev->irq.vblank_queue); 4814 } 4815 if (atomic_read(&rdev->irq.pflip[0])) 4816 radeon_crtc_handle_vblank(rdev, 0); 4817 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 4818 DRM_DEBUG("IH: D1 vblank\n"); 4819 } 4820 break; 4821 case 1: /* D1 vline */ 4822 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 4823 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 4824 DRM_DEBUG("IH: D1 vline\n"); 4825 } 4826 break; 4827 default: 4828 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4829 break; 4830 } 4831 break; 4832 case 2: /* D2 vblank/vline */ 4833 switch (src_data) { 4834 case 0: /* D2 vblank */ 4835 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 4836 if (rdev->irq.crtc_vblank_int[1]) { 4837 drm_handle_vblank(rdev->ddev, 1); 4838 rdev->pm.vblank_sync = true; 4839 wake_up(&rdev->irq.vblank_queue); 4840 } 4841 if (atomic_read(&rdev->irq.pflip[1])) 4842 radeon_crtc_handle_vblank(rdev, 1); 4843 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 4844 DRM_DEBUG("IH: D2 vblank\n"); 4845 } 4846 break; 4847 case 1: /* D2 vline */ 4848 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 4849 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 4850 DRM_DEBUG("IH: D2 vline\n"); 4851 } 4852 break; 4853 default: 4854 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4855 break; 4856 } 4857 break; 4858 case 3: /* D3 vblank/vline */ 4859 switch (src_data) { 4860 case 0: /* D3 vblank */ 4861 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 4862 if (rdev->irq.crtc_vblank_int[2]) { 4863 drm_handle_vblank(rdev->ddev, 2); 4864 rdev->pm.vblank_sync = true; 4865 wake_up(&rdev->irq.vblank_queue); 4866 } 4867 if (atomic_read(&rdev->irq.pflip[2])) 4868 radeon_crtc_handle_vblank(rdev, 2); 4869 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 4870 DRM_DEBUG("IH: D3 vblank\n"); 4871 } 4872 break; 4873 case 1: /* D3 vline */ 4874 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 4875 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 4876 DRM_DEBUG("IH: D3 vline\n"); 4877 } 4878 break; 4879 default: 4880 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4881 break; 4882 } 4883 break; 4884 case 4: /* D4 vblank/vline */ 4885 switch (src_data) { 4886 case 0: /* D4 vblank */ 4887 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 4888 if (rdev->irq.crtc_vblank_int[3]) { 4889 drm_handle_vblank(rdev->ddev, 3); 4890 rdev->pm.vblank_sync = true; 4891 wake_up(&rdev->irq.vblank_queue); 4892 } 4893 if (atomic_read(&rdev->irq.pflip[3])) 4894 radeon_crtc_handle_vblank(rdev, 3); 4895 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 4896 DRM_DEBUG("IH: D4 vblank\n"); 4897 } 4898 break; 4899 case 1: /* D4 vline */ 4900 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 4901 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 4902 DRM_DEBUG("IH: D4 vline\n"); 4903 } 4904 break; 4905 default: 4906 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4907 break; 4908 } 4909 break; 4910 case 5: /* D5 vblank/vline */ 4911 switch (src_data) { 4912 case 0: /* D5 vblank */ 4913 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 4914 if (rdev->irq.crtc_vblank_int[4]) { 4915 drm_handle_vblank(rdev->ddev, 4); 4916 rdev->pm.vblank_sync = true; 4917 wake_up(&rdev->irq.vblank_queue); 4918 } 4919 if (atomic_read(&rdev->irq.pflip[4])) 4920 radeon_crtc_handle_vblank(rdev, 4); 4921 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 4922 DRM_DEBUG("IH: D5 vblank\n"); 4923 } 4924 break; 4925 case 1: /* D5 vline */ 4926 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 4927 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 4928 DRM_DEBUG("IH: D5 vline\n"); 4929 } 4930 break; 4931 default: 4932 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4933 break; 4934 } 4935 break; 4936 case 6: /* D6 vblank/vline */ 4937 switch (src_data) { 4938 case 0: /* D6 vblank */ 4939 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 4940 if (rdev->irq.crtc_vblank_int[5]) { 4941 drm_handle_vblank(rdev->ddev, 5); 4942 rdev->pm.vblank_sync = true; 4943 wake_up(&rdev->irq.vblank_queue); 4944 } 4945 if (atomic_read(&rdev->irq.pflip[5])) 4946 radeon_crtc_handle_vblank(rdev, 5); 4947 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 4948 DRM_DEBUG("IH: D6 vblank\n"); 4949 } 4950 break; 4951 case 1: /* D6 vline */ 4952 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 4953 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 4954 DRM_DEBUG("IH: D6 vline\n"); 4955 } 4956 break; 4957 default: 4958 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4959 break; 4960 } 4961 break; 4962 case 8: /* D1 page flip */ 4963 case 10: /* D2 page flip */ 4964 case 12: /* D3 page flip */ 4965 case 14: /* D4 page flip */ 4966 case 16: /* D5 page flip */ 4967 case 18: /* D6 page flip */ 4968 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 4969 if (radeon_use_pflipirq > 0) 4970 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 4971 break; 4972 case 42: /* HPD hotplug */ 4973 switch (src_data) { 4974 case 0: 4975 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 4976 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 4977 queue_hotplug = true; 4978 DRM_DEBUG("IH: HPD1\n"); 4979 } 4980 break; 4981 case 1: 4982 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 4983 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 4984 queue_hotplug = true; 4985 DRM_DEBUG("IH: HPD2\n"); 4986 } 4987 break; 4988 case 2: 4989 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 4990 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 4991 queue_hotplug = true; 4992 DRM_DEBUG("IH: HPD3\n"); 4993 } 4994 break; 4995 case 3: 4996 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 4997 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 4998 queue_hotplug = true; 4999 DRM_DEBUG("IH: HPD4\n"); 5000 } 5001 break; 5002 case 4: 5003 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 5004 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 5005 queue_hotplug = true; 5006 DRM_DEBUG("IH: HPD5\n"); 5007 } 5008 break; 5009 case 5: 5010 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 5011 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 5012 queue_hotplug = true; 5013 DRM_DEBUG("IH: HPD6\n"); 5014 } 5015 break; 5016 default: 5017 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5018 break; 5019 } 5020 break; 5021 case 44: /* hdmi */ 5022 switch (src_data) { 5023 case 0: 5024 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) { 5025 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG; 5026 queue_hdmi = true; 5027 DRM_DEBUG("IH: HDMI0\n"); 5028 } 5029 break; 5030 case 1: 5031 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) { 5032 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG; 5033 queue_hdmi = true; 5034 DRM_DEBUG("IH: HDMI1\n"); 5035 } 5036 break; 5037 case 2: 5038 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) { 5039 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG; 5040 queue_hdmi = true; 5041 DRM_DEBUG("IH: HDMI2\n"); 5042 } 5043 break; 5044 case 3: 5045 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) { 5046 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG; 5047 queue_hdmi = true; 5048 DRM_DEBUG("IH: HDMI3\n"); 5049 } 5050 break; 5051 case 4: 5052 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) { 5053 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG; 5054 queue_hdmi = true; 5055 DRM_DEBUG("IH: HDMI4\n"); 5056 } 5057 break; 5058 case 5: 5059 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) { 5060 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG; 5061 queue_hdmi = true; 5062 DRM_DEBUG("IH: HDMI5\n"); 5063 } 5064 break; 5065 default: 5066 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data); 5067 break; 5068 } 5069 case 124: /* UVD */ 5070 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 5071 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 5072 break; 5073 case 146: 5074 case 147: 5075 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 5076 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 5077 /* reset addr and status */ 5078 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 5079 if (addr == 0x0 && status == 0x0) 5080 break; 5081 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 5082 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 5083 addr); 5084 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 5085 status); 5086 cayman_vm_decode_fault(rdev, status, addr); 5087 break; 5088 case 176: /* CP_INT in ring buffer */ 5089 case 177: /* CP_INT in IB1 */ 5090 case 178: /* CP_INT in IB2 */ 5091 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 5092 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5093 break; 5094 case 181: /* CP EOP event */ 5095 DRM_DEBUG("IH: CP EOP\n"); 5096 if (rdev->family >= CHIP_CAYMAN) { 5097 switch (src_data) { 5098 case 0: 5099 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5100 break; 5101 case 1: 5102 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 5103 break; 5104 case 2: 5105 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 5106 break; 5107 } 5108 } else 5109 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 5110 break; 5111 case 224: /* DMA trap event */ 5112 DRM_DEBUG("IH: DMA trap\n"); 5113 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 5114 break; 5115 case 230: /* thermal low to high */ 5116 DRM_DEBUG("IH: thermal low to high\n"); 5117 rdev->pm.dpm.thermal.high_to_low = false; 5118 queue_thermal = true; 5119 break; 5120 case 231: /* thermal high to low */ 5121 DRM_DEBUG("IH: thermal high to low\n"); 5122 rdev->pm.dpm.thermal.high_to_low = true; 5123 queue_thermal = true; 5124 break; 5125 case 233: /* GUI IDLE */ 5126 DRM_DEBUG("IH: GUI idle\n"); 5127 break; 5128 case 244: /* DMA trap event */ 5129 if (rdev->family >= CHIP_CAYMAN) { 5130 DRM_DEBUG("IH: DMA1 trap\n"); 5131 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 5132 } 5133 break; 5134 default: 5135 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 5136 break; 5137 } 5138 5139 /* wptr/rptr are in bytes! */ 5140 rptr += 16; 5141 rptr &= rdev->ih.ptr_mask; 5142 WREG32(IH_RB_RPTR, rptr); 5143 } 5144 if (queue_hotplug) 5145 schedule_work(&rdev->hotplug_work); 5146 if (queue_hdmi) 5147 schedule_work(&rdev->audio_work); 5148 if (queue_thermal && rdev->pm.dpm_enabled) 5149 schedule_work(&rdev->pm.dpm.thermal.work); 5150 rdev->ih.rptr = rptr; 5151 atomic_set(&rdev->ih.lock, 0); 5152 5153 /* make sure wptr hasn't changed while processing */ 5154 wptr = evergreen_get_ih_wptr(rdev); 5155 if (wptr != rptr) 5156 goto restart_ih; 5157 5158 return IRQ_HANDLED; 5159 } 5160 5161 static int evergreen_startup(struct radeon_device *rdev) 5162 { 5163 struct radeon_ring *ring; 5164 int r; 5165 5166 /* enable pcie gen2 link */ 5167 evergreen_pcie_gen2_enable(rdev); 5168 /* enable aspm */ 5169 evergreen_program_aspm(rdev); 5170 5171 /* scratch needs to be initialized before MC */ 5172 r = r600_vram_scratch_init(rdev); 5173 if (r) 5174 return r; 5175 5176 evergreen_mc_program(rdev); 5177 5178 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { 5179 r = ni_mc_load_microcode(rdev); 5180 if (r) { 5181 DRM_ERROR("Failed to load MC firmware!\n"); 5182 return r; 5183 } 5184 } 5185 5186 if (rdev->flags & RADEON_IS_AGP) { 5187 evergreen_agp_enable(rdev); 5188 } else { 5189 r = evergreen_pcie_gart_enable(rdev); 5190 if (r) 5191 return r; 5192 } 5193 evergreen_gpu_init(rdev); 5194 5195 /* allocate rlc buffers */ 5196 if (rdev->flags & RADEON_IS_IGP) { 5197 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; 5198 rdev->rlc.reg_list_size = 5199 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); 5200 rdev->rlc.cs_data = evergreen_cs_data; 5201 r = sumo_rlc_init(rdev); 5202 if (r) { 5203 DRM_ERROR("Failed to init rlc BOs!\n"); 5204 return r; 5205 } 5206 } 5207 5208 /* allocate wb buffer */ 5209 r = radeon_wb_init(rdev); 5210 if (r) 5211 return r; 5212 5213 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 5214 if (r) { 5215 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 5216 return r; 5217 } 5218 5219 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 5220 if (r) { 5221 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 5222 return r; 5223 } 5224 5225 r = uvd_v2_2_resume(rdev); 5226 if (!r) { 5227 r = radeon_fence_driver_start_ring(rdev, 5228 R600_RING_TYPE_UVD_INDEX); 5229 if (r) 5230 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 5231 } 5232 5233 if (r) 5234 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 5235 5236 /* Enable IRQ */ 5237 if (!rdev->irq.installed) { 5238 r = radeon_irq_kms_init(rdev); 5239 if (r) 5240 return r; 5241 } 5242 5243 r = r600_irq_init(rdev); 5244 if (r) { 5245 DRM_ERROR("radeon: IH init failed (%d).\n", r); 5246 radeon_irq_kms_fini(rdev); 5247 return r; 5248 } 5249 evergreen_irq_set(rdev); 5250 5251 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 5252 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 5253 RADEON_CP_PACKET2); 5254 if (r) 5255 return r; 5256 5257 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 5258 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 5259 DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 5260 if (r) 5261 return r; 5262 5263 r = evergreen_cp_load_microcode(rdev); 5264 if (r) 5265 return r; 5266 r = evergreen_cp_resume(rdev); 5267 if (r) 5268 return r; 5269 r = r600_dma_resume(rdev); 5270 if (r) 5271 return r; 5272 5273 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 5274 if (ring->ring_size) { 5275 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 5276 RADEON_CP_PACKET2); 5277 if (!r) 5278 r = uvd_v1_0_init(rdev); 5279 5280 if (r) 5281 DRM_ERROR("radeon: error initializing UVD (%d).\n", r); 5282 } 5283 5284 r = radeon_ib_pool_init(rdev); 5285 if (r) { 5286 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 5287 return r; 5288 } 5289 5290 r = radeon_audio_init(rdev); 5291 if (r) { 5292 DRM_ERROR("radeon: audio init failed\n"); 5293 return r; 5294 } 5295 5296 return 0; 5297 } 5298 5299 int evergreen_resume(struct radeon_device *rdev) 5300 { 5301 int r; 5302 5303 /* reset the asic, the gfx blocks are often in a bad state 5304 * after the driver is unloaded or after a resume 5305 */ 5306 if (radeon_asic_reset(rdev)) 5307 dev_warn(rdev->dev, "GPU reset failed !\n"); 5308 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 5309 * posting will perform necessary task to bring back GPU into good 5310 * shape. 5311 */ 5312 /* post card */ 5313 atom_asic_init(rdev->mode_info.atom_context); 5314 5315 /* init golden registers */ 5316 evergreen_init_golden_registers(rdev); 5317 5318 if (rdev->pm.pm_method == PM_METHOD_DPM) 5319 radeon_pm_resume(rdev); 5320 5321 rdev->accel_working = true; 5322 r = evergreen_startup(rdev); 5323 if (r) { 5324 DRM_ERROR("evergreen startup failed on resume\n"); 5325 rdev->accel_working = false; 5326 return r; 5327 } 5328 5329 return r; 5330 5331 } 5332 5333 int evergreen_suspend(struct radeon_device *rdev) 5334 { 5335 radeon_pm_suspend(rdev); 5336 radeon_audio_fini(rdev); 5337 uvd_v1_0_fini(rdev); 5338 radeon_uvd_suspend(rdev); 5339 r700_cp_stop(rdev); 5340 r600_dma_stop(rdev); 5341 evergreen_irq_suspend(rdev); 5342 radeon_wb_disable(rdev); 5343 evergreen_pcie_gart_disable(rdev); 5344 5345 return 0; 5346 } 5347 5348 /* Plan is to move initialization in that function and use 5349 * helper function so that radeon_device_init pretty much 5350 * do nothing more than calling asic specific function. This 5351 * should also allow to remove a bunch of callback function 5352 * like vram_info. 5353 */ 5354 int evergreen_init(struct radeon_device *rdev) 5355 { 5356 int r; 5357 5358 /* Read BIOS */ 5359 if (!radeon_get_bios(rdev)) { 5360 if (ASIC_IS_AVIVO(rdev)) 5361 return -EINVAL; 5362 } 5363 /* Must be an ATOMBIOS */ 5364 if (!rdev->is_atom_bios) { 5365 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 5366 return -EINVAL; 5367 } 5368 r = radeon_atombios_init(rdev); 5369 if (r) 5370 return r; 5371 /* reset the asic, the gfx blocks are often in a bad state 5372 * after the driver is unloaded or after a resume 5373 */ 5374 if (radeon_asic_reset(rdev)) 5375 dev_warn(rdev->dev, "GPU reset failed !\n"); 5376 /* Post card if necessary */ 5377 if (!radeon_card_posted(rdev)) { 5378 if (!rdev->bios) { 5379 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 5380 return -EINVAL; 5381 } 5382 DRM_INFO("GPU not posted. posting now...\n"); 5383 atom_asic_init(rdev->mode_info.atom_context); 5384 } 5385 /* init golden registers */ 5386 evergreen_init_golden_registers(rdev); 5387 /* Initialize scratch registers */ 5388 r600_scratch_init(rdev); 5389 /* Initialize surface registers */ 5390 radeon_surface_init(rdev); 5391 /* Initialize clocks */ 5392 radeon_get_clock_info(rdev->ddev); 5393 /* Fence driver */ 5394 r = radeon_fence_driver_init(rdev); 5395 if (r) 5396 return r; 5397 /* initialize AGP */ 5398 if (rdev->flags & RADEON_IS_AGP) { 5399 r = radeon_agp_init(rdev); 5400 if (r) 5401 radeon_agp_disable(rdev); 5402 } 5403 /* initialize memory controller */ 5404 r = evergreen_mc_init(rdev); 5405 if (r) 5406 return r; 5407 /* Memory manager */ 5408 r = radeon_bo_init(rdev); 5409 if (r) 5410 return r; 5411 5412 if (ASIC_IS_DCE5(rdev)) { 5413 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 5414 r = ni_init_microcode(rdev); 5415 if (r) { 5416 DRM_ERROR("Failed to load firmware!\n"); 5417 return r; 5418 } 5419 } 5420 } else { 5421 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 5422 r = r600_init_microcode(rdev); 5423 if (r) { 5424 DRM_ERROR("Failed to load firmware!\n"); 5425 return r; 5426 } 5427 } 5428 } 5429 5430 /* Initialize power management */ 5431 radeon_pm_init(rdev); 5432 5433 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; 5434 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); 5435 5436 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; 5437 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); 5438 5439 r = radeon_uvd_init(rdev); 5440 if (!r) { 5441 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 5442 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 5443 4096); 5444 } 5445 5446 rdev->ih.ring_obj = NULL; 5447 r600_ih_ring_init(rdev, 64 * 1024); 5448 5449 r = r600_pcie_gart_init(rdev); 5450 if (r) 5451 return r; 5452 5453 rdev->accel_working = true; 5454 r = evergreen_startup(rdev); 5455 if (r) { 5456 dev_err(rdev->dev, "disabling GPU acceleration\n"); 5457 r700_cp_fini(rdev); 5458 r600_dma_fini(rdev); 5459 r600_irq_fini(rdev); 5460 if (rdev->flags & RADEON_IS_IGP) 5461 sumo_rlc_fini(rdev); 5462 radeon_wb_fini(rdev); 5463 radeon_ib_pool_fini(rdev); 5464 radeon_irq_kms_fini(rdev); 5465 evergreen_pcie_gart_fini(rdev); 5466 rdev->accel_working = false; 5467 } 5468 5469 /* Don't start up if the MC ucode is missing on BTC parts. 5470 * The default clocks and voltages before the MC ucode 5471 * is loaded are not suffient for advanced operations. 5472 */ 5473 if (ASIC_IS_DCE5(rdev)) { 5474 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 5475 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 5476 return -EINVAL; 5477 } 5478 } 5479 5480 return 0; 5481 } 5482 5483 void evergreen_fini(struct radeon_device *rdev) 5484 { 5485 radeon_pm_fini(rdev); 5486 radeon_audio_fini(rdev); 5487 r700_cp_fini(rdev); 5488 r600_dma_fini(rdev); 5489 r600_irq_fini(rdev); 5490 if (rdev->flags & RADEON_IS_IGP) 5491 sumo_rlc_fini(rdev); 5492 radeon_wb_fini(rdev); 5493 radeon_ib_pool_fini(rdev); 5494 radeon_irq_kms_fini(rdev); 5495 uvd_v1_0_fini(rdev); 5496 radeon_uvd_fini(rdev); 5497 evergreen_pcie_gart_fini(rdev); 5498 r600_vram_scratch_fini(rdev); 5499 radeon_gem_fini(rdev); 5500 radeon_fence_driver_fini(rdev); 5501 radeon_agp_fini(rdev); 5502 radeon_bo_fini(rdev); 5503 radeon_atombios_fini(rdev); 5504 kfree(rdev->bios); 5505 rdev->bios = NULL; 5506 } 5507 5508 void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 5509 { 5510 u32 link_width_cntl, speed_cntl; 5511 5512 if (radeon_pcie_gen2 == 0) 5513 return; 5514 5515 if (rdev->flags & RADEON_IS_IGP) 5516 return; 5517 5518 if (!(rdev->flags & RADEON_IS_PCIE)) 5519 return; 5520 5521 /* x2 cards have a special sequence */ 5522 if (ASIC_IS_X2(rdev)) 5523 return; 5524 5525 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) && 5526 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT)) 5527 return; 5528 5529 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5530 if (speed_cntl & LC_CURRENT_DATA_RATE) { 5531 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 5532 return; 5533 } 5534 5535 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 5536 5537 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 5538 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 5539 5540 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5541 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5542 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5543 5544 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5545 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 5546 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5547 5548 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5549 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 5550 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5551 5552 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5553 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 5554 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5555 5556 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5557 speed_cntl |= LC_GEN2_EN_STRAP; 5558 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5559 5560 } else { 5561 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5562 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 5563 if (1) 5564 link_width_cntl |= LC_UPCONFIGURE_DIS; 5565 else 5566 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5567 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5568 } 5569 } 5570 5571 void evergreen_program_aspm(struct radeon_device *rdev) 5572 { 5573 u32 data, orig; 5574 u32 pcie_lc_cntl, pcie_lc_cntl_old; 5575 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; 5576 /* fusion_platform = true 5577 * if the system is a fusion system 5578 * (APU or DGPU in a fusion system). 5579 * todo: check if the system is a fusion platform. 5580 */ 5581 bool fusion_platform = false; 5582 5583 if (radeon_aspm == 0) 5584 return; 5585 5586 if (!(rdev->flags & RADEON_IS_PCIE)) 5587 return; 5588 5589 switch (rdev->family) { 5590 case CHIP_CYPRESS: 5591 case CHIP_HEMLOCK: 5592 case CHIP_JUNIPER: 5593 case CHIP_REDWOOD: 5594 case CHIP_CEDAR: 5595 case CHIP_SUMO: 5596 case CHIP_SUMO2: 5597 case CHIP_PALM: 5598 case CHIP_ARUBA: 5599 disable_l0s = true; 5600 break; 5601 default: 5602 disable_l0s = false; 5603 break; 5604 } 5605 5606 if (rdev->flags & RADEON_IS_IGP) 5607 fusion_platform = true; /* XXX also dGPUs in a fusion system */ 5608 5609 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); 5610 if (fusion_platform) 5611 data &= ~MULTI_PIF; 5612 else 5613 data |= MULTI_PIF; 5614 if (data != orig) 5615 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); 5616 5617 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); 5618 if (fusion_platform) 5619 data &= ~MULTI_PIF; 5620 else 5621 data |= MULTI_PIF; 5622 if (data != orig) 5623 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); 5624 5625 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); 5626 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 5627 if (!disable_l0s) { 5628 if (rdev->family >= CHIP_BARTS) 5629 pcie_lc_cntl |= LC_L0S_INACTIVITY(7); 5630 else 5631 pcie_lc_cntl |= LC_L0S_INACTIVITY(3); 5632 } 5633 5634 if (!disable_l1) { 5635 if (rdev->family >= CHIP_BARTS) 5636 pcie_lc_cntl |= LC_L1_INACTIVITY(7); 5637 else 5638 pcie_lc_cntl |= LC_L1_INACTIVITY(8); 5639 5640 if (!disable_plloff_in_l1) { 5641 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5642 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5643 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5644 if (data != orig) 5645 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5646 5647 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5648 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5649 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5650 if (data != orig) 5651 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5652 5653 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5654 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5655 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5656 if (data != orig) 5657 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5658 5659 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5660 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5661 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5662 if (data != orig) 5663 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5664 5665 if (rdev->family >= CHIP_BARTS) { 5666 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5667 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5668 data |= PLL_RAMP_UP_TIME_0(4); 5669 if (data != orig) 5670 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5671 5672 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5673 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5674 data |= PLL_RAMP_UP_TIME_1(4); 5675 if (data != orig) 5676 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5677 5678 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5679 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5680 data |= PLL_RAMP_UP_TIME_0(4); 5681 if (data != orig) 5682 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5683 5684 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5685 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5686 data |= PLL_RAMP_UP_TIME_1(4); 5687 if (data != orig) 5688 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5689 } 5690 5691 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5692 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 5693 data |= LC_DYN_LANES_PWR_STATE(3); 5694 if (data != orig) 5695 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 5696 5697 if (rdev->family >= CHIP_BARTS) { 5698 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); 5699 data &= ~LS2_EXIT_TIME_MASK; 5700 data |= LS2_EXIT_TIME(1); 5701 if (data != orig) 5702 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 5703 5704 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); 5705 data &= ~LS2_EXIT_TIME_MASK; 5706 data |= LS2_EXIT_TIME(1); 5707 if (data != orig) 5708 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 5709 } 5710 } 5711 } 5712 5713 /* evergreen parts only */ 5714 if (rdev->family < CHIP_BARTS) 5715 pcie_lc_cntl |= LC_PMI_TO_L1_DIS; 5716 5717 if (pcie_lc_cntl != pcie_lc_cntl_old) 5718 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); 5719 } 5720