1 /* 2 * Copyright 2011 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include <drm/drmP.h> 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include <drm/radeon_drm.h> 31 #include "sid.h" 32 #include "atom.h" 33 #include "si_blit_shaders.h" 34 #include "clearstate_si.h" 35 #include "radeon_ucode.h" 36 37 38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin"); 39 MODULE_FIRMWARE("radeon/TAHITI_me.bin"); 40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin"); 41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin"); 42 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin"); 43 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin"); 44 MODULE_FIRMWARE("radeon/TAHITI_smc.bin"); 45 46 MODULE_FIRMWARE("radeon/tahiti_pfp.bin"); 47 MODULE_FIRMWARE("radeon/tahiti_me.bin"); 48 MODULE_FIRMWARE("radeon/tahiti_ce.bin"); 49 MODULE_FIRMWARE("radeon/tahiti_mc.bin"); 50 MODULE_FIRMWARE("radeon/tahiti_rlc.bin"); 51 MODULE_FIRMWARE("radeon/tahiti_smc.bin"); 52 53 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin"); 54 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin"); 55 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin"); 56 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin"); 57 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin"); 58 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin"); 59 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin"); 60 61 MODULE_FIRMWARE("radeon/pitcairn_pfp.bin"); 62 MODULE_FIRMWARE("radeon/pitcairn_me.bin"); 63 MODULE_FIRMWARE("radeon/pitcairn_ce.bin"); 64 MODULE_FIRMWARE("radeon/pitcairn_mc.bin"); 65 MODULE_FIRMWARE("radeon/pitcairn_rlc.bin"); 66 MODULE_FIRMWARE("radeon/pitcairn_smc.bin"); 67 68 MODULE_FIRMWARE("radeon/VERDE_pfp.bin"); 69 MODULE_FIRMWARE("radeon/VERDE_me.bin"); 70 MODULE_FIRMWARE("radeon/VERDE_ce.bin"); 71 MODULE_FIRMWARE("radeon/VERDE_mc.bin"); 72 MODULE_FIRMWARE("radeon/VERDE_mc2.bin"); 73 MODULE_FIRMWARE("radeon/VERDE_rlc.bin"); 74 MODULE_FIRMWARE("radeon/VERDE_smc.bin"); 75 76 MODULE_FIRMWARE("radeon/verde_pfp.bin"); 77 MODULE_FIRMWARE("radeon/verde_me.bin"); 78 MODULE_FIRMWARE("radeon/verde_ce.bin"); 79 MODULE_FIRMWARE("radeon/verde_mc.bin"); 80 MODULE_FIRMWARE("radeon/verde_rlc.bin"); 81 MODULE_FIRMWARE("radeon/verde_smc.bin"); 82 83 MODULE_FIRMWARE("radeon/OLAND_pfp.bin"); 84 MODULE_FIRMWARE("radeon/OLAND_me.bin"); 85 MODULE_FIRMWARE("radeon/OLAND_ce.bin"); 86 MODULE_FIRMWARE("radeon/OLAND_mc.bin"); 87 MODULE_FIRMWARE("radeon/OLAND_mc2.bin"); 88 MODULE_FIRMWARE("radeon/OLAND_rlc.bin"); 89 MODULE_FIRMWARE("radeon/OLAND_smc.bin"); 90 91 MODULE_FIRMWARE("radeon/oland_pfp.bin"); 92 MODULE_FIRMWARE("radeon/oland_me.bin"); 93 MODULE_FIRMWARE("radeon/oland_ce.bin"); 94 MODULE_FIRMWARE("radeon/oland_mc.bin"); 95 MODULE_FIRMWARE("radeon/oland_rlc.bin"); 96 MODULE_FIRMWARE("radeon/oland_smc.bin"); 97 98 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin"); 99 MODULE_FIRMWARE("radeon/HAINAN_me.bin"); 100 MODULE_FIRMWARE("radeon/HAINAN_ce.bin"); 101 MODULE_FIRMWARE("radeon/HAINAN_mc.bin"); 102 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin"); 103 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin"); 104 MODULE_FIRMWARE("radeon/HAINAN_smc.bin"); 105 106 MODULE_FIRMWARE("radeon/hainan_pfp.bin"); 107 MODULE_FIRMWARE("radeon/hainan_me.bin"); 108 MODULE_FIRMWARE("radeon/hainan_ce.bin"); 109 MODULE_FIRMWARE("radeon/hainan_mc.bin"); 110 MODULE_FIRMWARE("radeon/hainan_rlc.bin"); 111 MODULE_FIRMWARE("radeon/hainan_smc.bin"); 112 113 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh); 114 static void si_pcie_gen3_enable(struct radeon_device *rdev); 115 static void si_program_aspm(struct radeon_device *rdev); 116 extern void sumo_rlc_fini(struct radeon_device *rdev); 117 extern int sumo_rlc_init(struct radeon_device *rdev); 118 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 119 extern void r600_ih_ring_fini(struct radeon_device *rdev); 120 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev); 121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 123 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev); 124 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev); 125 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 126 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, 127 bool enable); 128 static void si_init_pg(struct radeon_device *rdev); 129 static void si_init_cg(struct radeon_device *rdev); 130 static void si_fini_pg(struct radeon_device *rdev); 131 static void si_fini_cg(struct radeon_device *rdev); 132 static void si_rlc_stop(struct radeon_device *rdev); 133 134 static const u32 verde_rlc_save_restore_register_list[] = 135 { 136 (0x8000 << 16) | (0x98f4 >> 2), 137 0x00000000, 138 (0x8040 << 16) | (0x98f4 >> 2), 139 0x00000000, 140 (0x8000 << 16) | (0xe80 >> 2), 141 0x00000000, 142 (0x8040 << 16) | (0xe80 >> 2), 143 0x00000000, 144 (0x8000 << 16) | (0x89bc >> 2), 145 0x00000000, 146 (0x8040 << 16) | (0x89bc >> 2), 147 0x00000000, 148 (0x8000 << 16) | (0x8c1c >> 2), 149 0x00000000, 150 (0x8040 << 16) | (0x8c1c >> 2), 151 0x00000000, 152 (0x9c00 << 16) | (0x98f0 >> 2), 153 0x00000000, 154 (0x9c00 << 16) | (0xe7c >> 2), 155 0x00000000, 156 (0x8000 << 16) | (0x9148 >> 2), 157 0x00000000, 158 (0x8040 << 16) | (0x9148 >> 2), 159 0x00000000, 160 (0x9c00 << 16) | (0x9150 >> 2), 161 0x00000000, 162 (0x9c00 << 16) | (0x897c >> 2), 163 0x00000000, 164 (0x9c00 << 16) | (0x8d8c >> 2), 165 0x00000000, 166 (0x9c00 << 16) | (0xac54 >> 2), 167 0X00000000, 168 0x3, 169 (0x9c00 << 16) | (0x98f8 >> 2), 170 0x00000000, 171 (0x9c00 << 16) | (0x9910 >> 2), 172 0x00000000, 173 (0x9c00 << 16) | (0x9914 >> 2), 174 0x00000000, 175 (0x9c00 << 16) | (0x9918 >> 2), 176 0x00000000, 177 (0x9c00 << 16) | (0x991c >> 2), 178 0x00000000, 179 (0x9c00 << 16) | (0x9920 >> 2), 180 0x00000000, 181 (0x9c00 << 16) | (0x9924 >> 2), 182 0x00000000, 183 (0x9c00 << 16) | (0x9928 >> 2), 184 0x00000000, 185 (0x9c00 << 16) | (0x992c >> 2), 186 0x00000000, 187 (0x9c00 << 16) | (0x9930 >> 2), 188 0x00000000, 189 (0x9c00 << 16) | (0x9934 >> 2), 190 0x00000000, 191 (0x9c00 << 16) | (0x9938 >> 2), 192 0x00000000, 193 (0x9c00 << 16) | (0x993c >> 2), 194 0x00000000, 195 (0x9c00 << 16) | (0x9940 >> 2), 196 0x00000000, 197 (0x9c00 << 16) | (0x9944 >> 2), 198 0x00000000, 199 (0x9c00 << 16) | (0x9948 >> 2), 200 0x00000000, 201 (0x9c00 << 16) | (0x994c >> 2), 202 0x00000000, 203 (0x9c00 << 16) | (0x9950 >> 2), 204 0x00000000, 205 (0x9c00 << 16) | (0x9954 >> 2), 206 0x00000000, 207 (0x9c00 << 16) | (0x9958 >> 2), 208 0x00000000, 209 (0x9c00 << 16) | (0x995c >> 2), 210 0x00000000, 211 (0x9c00 << 16) | (0x9960 >> 2), 212 0x00000000, 213 (0x9c00 << 16) | (0x9964 >> 2), 214 0x00000000, 215 (0x9c00 << 16) | (0x9968 >> 2), 216 0x00000000, 217 (0x9c00 << 16) | (0x996c >> 2), 218 0x00000000, 219 (0x9c00 << 16) | (0x9970 >> 2), 220 0x00000000, 221 (0x9c00 << 16) | (0x9974 >> 2), 222 0x00000000, 223 (0x9c00 << 16) | (0x9978 >> 2), 224 0x00000000, 225 (0x9c00 << 16) | (0x997c >> 2), 226 0x00000000, 227 (0x9c00 << 16) | (0x9980 >> 2), 228 0x00000000, 229 (0x9c00 << 16) | (0x9984 >> 2), 230 0x00000000, 231 (0x9c00 << 16) | (0x9988 >> 2), 232 0x00000000, 233 (0x9c00 << 16) | (0x998c >> 2), 234 0x00000000, 235 (0x9c00 << 16) | (0x8c00 >> 2), 236 0x00000000, 237 (0x9c00 << 16) | (0x8c14 >> 2), 238 0x00000000, 239 (0x9c00 << 16) | (0x8c04 >> 2), 240 0x00000000, 241 (0x9c00 << 16) | (0x8c08 >> 2), 242 0x00000000, 243 (0x8000 << 16) | (0x9b7c >> 2), 244 0x00000000, 245 (0x8040 << 16) | (0x9b7c >> 2), 246 0x00000000, 247 (0x8000 << 16) | (0xe84 >> 2), 248 0x00000000, 249 (0x8040 << 16) | (0xe84 >> 2), 250 0x00000000, 251 (0x8000 << 16) | (0x89c0 >> 2), 252 0x00000000, 253 (0x8040 << 16) | (0x89c0 >> 2), 254 0x00000000, 255 (0x8000 << 16) | (0x914c >> 2), 256 0x00000000, 257 (0x8040 << 16) | (0x914c >> 2), 258 0x00000000, 259 (0x8000 << 16) | (0x8c20 >> 2), 260 0x00000000, 261 (0x8040 << 16) | (0x8c20 >> 2), 262 0x00000000, 263 (0x8000 << 16) | (0x9354 >> 2), 264 0x00000000, 265 (0x8040 << 16) | (0x9354 >> 2), 266 0x00000000, 267 (0x9c00 << 16) | (0x9060 >> 2), 268 0x00000000, 269 (0x9c00 << 16) | (0x9364 >> 2), 270 0x00000000, 271 (0x9c00 << 16) | (0x9100 >> 2), 272 0x00000000, 273 (0x9c00 << 16) | (0x913c >> 2), 274 0x00000000, 275 (0x8000 << 16) | (0x90e0 >> 2), 276 0x00000000, 277 (0x8000 << 16) | (0x90e4 >> 2), 278 0x00000000, 279 (0x8000 << 16) | (0x90e8 >> 2), 280 0x00000000, 281 (0x8040 << 16) | (0x90e0 >> 2), 282 0x00000000, 283 (0x8040 << 16) | (0x90e4 >> 2), 284 0x00000000, 285 (0x8040 << 16) | (0x90e8 >> 2), 286 0x00000000, 287 (0x9c00 << 16) | (0x8bcc >> 2), 288 0x00000000, 289 (0x9c00 << 16) | (0x8b24 >> 2), 290 0x00000000, 291 (0x9c00 << 16) | (0x88c4 >> 2), 292 0x00000000, 293 (0x9c00 << 16) | (0x8e50 >> 2), 294 0x00000000, 295 (0x9c00 << 16) | (0x8c0c >> 2), 296 0x00000000, 297 (0x9c00 << 16) | (0x8e58 >> 2), 298 0x00000000, 299 (0x9c00 << 16) | (0x8e5c >> 2), 300 0x00000000, 301 (0x9c00 << 16) | (0x9508 >> 2), 302 0x00000000, 303 (0x9c00 << 16) | (0x950c >> 2), 304 0x00000000, 305 (0x9c00 << 16) | (0x9494 >> 2), 306 0x00000000, 307 (0x9c00 << 16) | (0xac0c >> 2), 308 0x00000000, 309 (0x9c00 << 16) | (0xac10 >> 2), 310 0x00000000, 311 (0x9c00 << 16) | (0xac14 >> 2), 312 0x00000000, 313 (0x9c00 << 16) | (0xae00 >> 2), 314 0x00000000, 315 (0x9c00 << 16) | (0xac08 >> 2), 316 0x00000000, 317 (0x9c00 << 16) | (0x88d4 >> 2), 318 0x00000000, 319 (0x9c00 << 16) | (0x88c8 >> 2), 320 0x00000000, 321 (0x9c00 << 16) | (0x88cc >> 2), 322 0x00000000, 323 (0x9c00 << 16) | (0x89b0 >> 2), 324 0x00000000, 325 (0x9c00 << 16) | (0x8b10 >> 2), 326 0x00000000, 327 (0x9c00 << 16) | (0x8a14 >> 2), 328 0x00000000, 329 (0x9c00 << 16) | (0x9830 >> 2), 330 0x00000000, 331 (0x9c00 << 16) | (0x9834 >> 2), 332 0x00000000, 333 (0x9c00 << 16) | (0x9838 >> 2), 334 0x00000000, 335 (0x9c00 << 16) | (0x9a10 >> 2), 336 0x00000000, 337 (0x8000 << 16) | (0x9870 >> 2), 338 0x00000000, 339 (0x8000 << 16) | (0x9874 >> 2), 340 0x00000000, 341 (0x8001 << 16) | (0x9870 >> 2), 342 0x00000000, 343 (0x8001 << 16) | (0x9874 >> 2), 344 0x00000000, 345 (0x8040 << 16) | (0x9870 >> 2), 346 0x00000000, 347 (0x8040 << 16) | (0x9874 >> 2), 348 0x00000000, 349 (0x8041 << 16) | (0x9870 >> 2), 350 0x00000000, 351 (0x8041 << 16) | (0x9874 >> 2), 352 0x00000000, 353 0x00000000 354 }; 355 356 static const u32 tahiti_golden_rlc_registers[] = 357 { 358 0xc424, 0xffffffff, 0x00601005, 359 0xc47c, 0xffffffff, 0x10104040, 360 0xc488, 0xffffffff, 0x0100000a, 361 0xc314, 0xffffffff, 0x00000800, 362 0xc30c, 0xffffffff, 0x800000f4, 363 0xf4a8, 0xffffffff, 0x00000000 364 }; 365 366 static const u32 tahiti_golden_registers[] = 367 { 368 0x9a10, 0x00010000, 0x00018208, 369 0x9830, 0xffffffff, 0x00000000, 370 0x9834, 0xf00fffff, 0x00000400, 371 0x9838, 0x0002021c, 0x00020200, 372 0xc78, 0x00000080, 0x00000000, 373 0xd030, 0x000300c0, 0x00800040, 374 0xd830, 0x000300c0, 0x00800040, 375 0x5bb0, 0x000000f0, 0x00000070, 376 0x5bc0, 0x00200000, 0x50100000, 377 0x7030, 0x31000311, 0x00000011, 378 0x277c, 0x00000003, 0x000007ff, 379 0x240c, 0x000007ff, 0x00000000, 380 0x8a14, 0xf000001f, 0x00000007, 381 0x8b24, 0xffffffff, 0x00ffffff, 382 0x8b10, 0x0000ff0f, 0x00000000, 383 0x28a4c, 0x07ffffff, 0x4e000000, 384 0x28350, 0x3f3f3fff, 0x2a00126a, 385 0x30, 0x000000ff, 0x0040, 386 0x34, 0x00000040, 0x00004040, 387 0x9100, 0x07ffffff, 0x03000000, 388 0x8e88, 0x01ff1f3f, 0x00000000, 389 0x8e84, 0x01ff1f3f, 0x00000000, 390 0x9060, 0x0000007f, 0x00000020, 391 0x9508, 0x00010000, 0x00010000, 392 0xac14, 0x00000200, 0x000002fb, 393 0xac10, 0xffffffff, 0x0000543b, 394 0xac0c, 0xffffffff, 0xa9210876, 395 0x88d0, 0xffffffff, 0x000fff40, 396 0x88d4, 0x0000001f, 0x00000010, 397 0x1410, 0x20000000, 0x20fffed8, 398 0x15c0, 0x000c0fc0, 0x000c0400 399 }; 400 401 static const u32 tahiti_golden_registers2[] = 402 { 403 0xc64, 0x00000001, 0x00000001 404 }; 405 406 static const u32 pitcairn_golden_rlc_registers[] = 407 { 408 0xc424, 0xffffffff, 0x00601004, 409 0xc47c, 0xffffffff, 0x10102020, 410 0xc488, 0xffffffff, 0x01000020, 411 0xc314, 0xffffffff, 0x00000800, 412 0xc30c, 0xffffffff, 0x800000a4 413 }; 414 415 static const u32 pitcairn_golden_registers[] = 416 { 417 0x9a10, 0x00010000, 0x00018208, 418 0x9830, 0xffffffff, 0x00000000, 419 0x9834, 0xf00fffff, 0x00000400, 420 0x9838, 0x0002021c, 0x00020200, 421 0xc78, 0x00000080, 0x00000000, 422 0xd030, 0x000300c0, 0x00800040, 423 0xd830, 0x000300c0, 0x00800040, 424 0x5bb0, 0x000000f0, 0x00000070, 425 0x5bc0, 0x00200000, 0x50100000, 426 0x7030, 0x31000311, 0x00000011, 427 0x2ae4, 0x00073ffe, 0x000022a2, 428 0x240c, 0x000007ff, 0x00000000, 429 0x8a14, 0xf000001f, 0x00000007, 430 0x8b24, 0xffffffff, 0x00ffffff, 431 0x8b10, 0x0000ff0f, 0x00000000, 432 0x28a4c, 0x07ffffff, 0x4e000000, 433 0x28350, 0x3f3f3fff, 0x2a00126a, 434 0x30, 0x000000ff, 0x0040, 435 0x34, 0x00000040, 0x00004040, 436 0x9100, 0x07ffffff, 0x03000000, 437 0x9060, 0x0000007f, 0x00000020, 438 0x9508, 0x00010000, 0x00010000, 439 0xac14, 0x000003ff, 0x000000f7, 440 0xac10, 0xffffffff, 0x00000000, 441 0xac0c, 0xffffffff, 0x32761054, 442 0x88d4, 0x0000001f, 0x00000010, 443 0x15c0, 0x000c0fc0, 0x000c0400 444 }; 445 446 static const u32 verde_golden_rlc_registers[] = 447 { 448 0xc424, 0xffffffff, 0x033f1005, 449 0xc47c, 0xffffffff, 0x10808020, 450 0xc488, 0xffffffff, 0x00800008, 451 0xc314, 0xffffffff, 0x00001000, 452 0xc30c, 0xffffffff, 0x80010014 453 }; 454 455 static const u32 verde_golden_registers[] = 456 { 457 0x9a10, 0x00010000, 0x00018208, 458 0x9830, 0xffffffff, 0x00000000, 459 0x9834, 0xf00fffff, 0x00000400, 460 0x9838, 0x0002021c, 0x00020200, 461 0xc78, 0x00000080, 0x00000000, 462 0xd030, 0x000300c0, 0x00800040, 463 0xd030, 0x000300c0, 0x00800040, 464 0xd830, 0x000300c0, 0x00800040, 465 0xd830, 0x000300c0, 0x00800040, 466 0x5bb0, 0x000000f0, 0x00000070, 467 0x5bc0, 0x00200000, 0x50100000, 468 0x7030, 0x31000311, 0x00000011, 469 0x2ae4, 0x00073ffe, 0x000022a2, 470 0x2ae4, 0x00073ffe, 0x000022a2, 471 0x2ae4, 0x00073ffe, 0x000022a2, 472 0x240c, 0x000007ff, 0x00000000, 473 0x240c, 0x000007ff, 0x00000000, 474 0x240c, 0x000007ff, 0x00000000, 475 0x8a14, 0xf000001f, 0x00000007, 476 0x8a14, 0xf000001f, 0x00000007, 477 0x8a14, 0xf000001f, 0x00000007, 478 0x8b24, 0xffffffff, 0x00ffffff, 479 0x8b10, 0x0000ff0f, 0x00000000, 480 0x28a4c, 0x07ffffff, 0x4e000000, 481 0x28350, 0x3f3f3fff, 0x0000124a, 482 0x28350, 0x3f3f3fff, 0x0000124a, 483 0x28350, 0x3f3f3fff, 0x0000124a, 484 0x30, 0x000000ff, 0x0040, 485 0x34, 0x00000040, 0x00004040, 486 0x9100, 0x07ffffff, 0x03000000, 487 0x9100, 0x07ffffff, 0x03000000, 488 0x8e88, 0x01ff1f3f, 0x00000000, 489 0x8e88, 0x01ff1f3f, 0x00000000, 490 0x8e88, 0x01ff1f3f, 0x00000000, 491 0x8e84, 0x01ff1f3f, 0x00000000, 492 0x8e84, 0x01ff1f3f, 0x00000000, 493 0x8e84, 0x01ff1f3f, 0x00000000, 494 0x9060, 0x0000007f, 0x00000020, 495 0x9508, 0x00010000, 0x00010000, 496 0xac14, 0x000003ff, 0x00000003, 497 0xac14, 0x000003ff, 0x00000003, 498 0xac14, 0x000003ff, 0x00000003, 499 0xac10, 0xffffffff, 0x00000000, 500 0xac10, 0xffffffff, 0x00000000, 501 0xac10, 0xffffffff, 0x00000000, 502 0xac0c, 0xffffffff, 0x00001032, 503 0xac0c, 0xffffffff, 0x00001032, 504 0xac0c, 0xffffffff, 0x00001032, 505 0x88d4, 0x0000001f, 0x00000010, 506 0x88d4, 0x0000001f, 0x00000010, 507 0x88d4, 0x0000001f, 0x00000010, 508 0x15c0, 0x000c0fc0, 0x000c0400 509 }; 510 511 static const u32 oland_golden_rlc_registers[] = 512 { 513 0xc424, 0xffffffff, 0x00601005, 514 0xc47c, 0xffffffff, 0x10104040, 515 0xc488, 0xffffffff, 0x0100000a, 516 0xc314, 0xffffffff, 0x00000800, 517 0xc30c, 0xffffffff, 0x800000f4 518 }; 519 520 static const u32 oland_golden_registers[] = 521 { 522 0x9a10, 0x00010000, 0x00018208, 523 0x9830, 0xffffffff, 0x00000000, 524 0x9834, 0xf00fffff, 0x00000400, 525 0x9838, 0x0002021c, 0x00020200, 526 0xc78, 0x00000080, 0x00000000, 527 0xd030, 0x000300c0, 0x00800040, 528 0xd830, 0x000300c0, 0x00800040, 529 0x5bb0, 0x000000f0, 0x00000070, 530 0x5bc0, 0x00200000, 0x50100000, 531 0x7030, 0x31000311, 0x00000011, 532 0x2ae4, 0x00073ffe, 0x000022a2, 533 0x240c, 0x000007ff, 0x00000000, 534 0x8a14, 0xf000001f, 0x00000007, 535 0x8b24, 0xffffffff, 0x00ffffff, 536 0x8b10, 0x0000ff0f, 0x00000000, 537 0x28a4c, 0x07ffffff, 0x4e000000, 538 0x28350, 0x3f3f3fff, 0x00000082, 539 0x30, 0x000000ff, 0x0040, 540 0x34, 0x00000040, 0x00004040, 541 0x9100, 0x07ffffff, 0x03000000, 542 0x9060, 0x0000007f, 0x00000020, 543 0x9508, 0x00010000, 0x00010000, 544 0xac14, 0x000003ff, 0x000000f3, 545 0xac10, 0xffffffff, 0x00000000, 546 0xac0c, 0xffffffff, 0x00003210, 547 0x88d4, 0x0000001f, 0x00000010, 548 0x15c0, 0x000c0fc0, 0x000c0400 549 }; 550 551 static const u32 hainan_golden_registers[] = 552 { 553 0x9a10, 0x00010000, 0x00018208, 554 0x9830, 0xffffffff, 0x00000000, 555 0x9834, 0xf00fffff, 0x00000400, 556 0x9838, 0x0002021c, 0x00020200, 557 0xd0c0, 0xff000fff, 0x00000100, 558 0xd030, 0x000300c0, 0x00800040, 559 0xd8c0, 0xff000fff, 0x00000100, 560 0xd830, 0x000300c0, 0x00800040, 561 0x2ae4, 0x00073ffe, 0x000022a2, 562 0x240c, 0x000007ff, 0x00000000, 563 0x8a14, 0xf000001f, 0x00000007, 564 0x8b24, 0xffffffff, 0x00ffffff, 565 0x8b10, 0x0000ff0f, 0x00000000, 566 0x28a4c, 0x07ffffff, 0x4e000000, 567 0x28350, 0x3f3f3fff, 0x00000000, 568 0x30, 0x000000ff, 0x0040, 569 0x34, 0x00000040, 0x00004040, 570 0x9100, 0x03e00000, 0x03600000, 571 0x9060, 0x0000007f, 0x00000020, 572 0x9508, 0x00010000, 0x00010000, 573 0xac14, 0x000003ff, 0x000000f1, 574 0xac10, 0xffffffff, 0x00000000, 575 0xac0c, 0xffffffff, 0x00003210, 576 0x88d4, 0x0000001f, 0x00000010, 577 0x15c0, 0x000c0fc0, 0x000c0400 578 }; 579 580 static const u32 hainan_golden_registers2[] = 581 { 582 0x98f8, 0xffffffff, 0x02010001 583 }; 584 585 static const u32 tahiti_mgcg_cgcg_init[] = 586 { 587 0xc400, 0xffffffff, 0xfffffffc, 588 0x802c, 0xffffffff, 0xe0000000, 589 0x9a60, 0xffffffff, 0x00000100, 590 0x92a4, 0xffffffff, 0x00000100, 591 0xc164, 0xffffffff, 0x00000100, 592 0x9774, 0xffffffff, 0x00000100, 593 0x8984, 0xffffffff, 0x06000100, 594 0x8a18, 0xffffffff, 0x00000100, 595 0x92a0, 0xffffffff, 0x00000100, 596 0xc380, 0xffffffff, 0x00000100, 597 0x8b28, 0xffffffff, 0x00000100, 598 0x9144, 0xffffffff, 0x00000100, 599 0x8d88, 0xffffffff, 0x00000100, 600 0x8d8c, 0xffffffff, 0x00000100, 601 0x9030, 0xffffffff, 0x00000100, 602 0x9034, 0xffffffff, 0x00000100, 603 0x9038, 0xffffffff, 0x00000100, 604 0x903c, 0xffffffff, 0x00000100, 605 0xad80, 0xffffffff, 0x00000100, 606 0xac54, 0xffffffff, 0x00000100, 607 0x897c, 0xffffffff, 0x06000100, 608 0x9868, 0xffffffff, 0x00000100, 609 0x9510, 0xffffffff, 0x00000100, 610 0xaf04, 0xffffffff, 0x00000100, 611 0xae04, 0xffffffff, 0x00000100, 612 0x949c, 0xffffffff, 0x00000100, 613 0x802c, 0xffffffff, 0xe0000000, 614 0x9160, 0xffffffff, 0x00010000, 615 0x9164, 0xffffffff, 0x00030002, 616 0x9168, 0xffffffff, 0x00040007, 617 0x916c, 0xffffffff, 0x00060005, 618 0x9170, 0xffffffff, 0x00090008, 619 0x9174, 0xffffffff, 0x00020001, 620 0x9178, 0xffffffff, 0x00040003, 621 0x917c, 0xffffffff, 0x00000007, 622 0x9180, 0xffffffff, 0x00060005, 623 0x9184, 0xffffffff, 0x00090008, 624 0x9188, 0xffffffff, 0x00030002, 625 0x918c, 0xffffffff, 0x00050004, 626 0x9190, 0xffffffff, 0x00000008, 627 0x9194, 0xffffffff, 0x00070006, 628 0x9198, 0xffffffff, 0x000a0009, 629 0x919c, 0xffffffff, 0x00040003, 630 0x91a0, 0xffffffff, 0x00060005, 631 0x91a4, 0xffffffff, 0x00000009, 632 0x91a8, 0xffffffff, 0x00080007, 633 0x91ac, 0xffffffff, 0x000b000a, 634 0x91b0, 0xffffffff, 0x00050004, 635 0x91b4, 0xffffffff, 0x00070006, 636 0x91b8, 0xffffffff, 0x0008000b, 637 0x91bc, 0xffffffff, 0x000a0009, 638 0x91c0, 0xffffffff, 0x000d000c, 639 0x91c4, 0xffffffff, 0x00060005, 640 0x91c8, 0xffffffff, 0x00080007, 641 0x91cc, 0xffffffff, 0x0000000b, 642 0x91d0, 0xffffffff, 0x000a0009, 643 0x91d4, 0xffffffff, 0x000d000c, 644 0x91d8, 0xffffffff, 0x00070006, 645 0x91dc, 0xffffffff, 0x00090008, 646 0x91e0, 0xffffffff, 0x0000000c, 647 0x91e4, 0xffffffff, 0x000b000a, 648 0x91e8, 0xffffffff, 0x000e000d, 649 0x91ec, 0xffffffff, 0x00080007, 650 0x91f0, 0xffffffff, 0x000a0009, 651 0x91f4, 0xffffffff, 0x0000000d, 652 0x91f8, 0xffffffff, 0x000c000b, 653 0x91fc, 0xffffffff, 0x000f000e, 654 0x9200, 0xffffffff, 0x00090008, 655 0x9204, 0xffffffff, 0x000b000a, 656 0x9208, 0xffffffff, 0x000c000f, 657 0x920c, 0xffffffff, 0x000e000d, 658 0x9210, 0xffffffff, 0x00110010, 659 0x9214, 0xffffffff, 0x000a0009, 660 0x9218, 0xffffffff, 0x000c000b, 661 0x921c, 0xffffffff, 0x0000000f, 662 0x9220, 0xffffffff, 0x000e000d, 663 0x9224, 0xffffffff, 0x00110010, 664 0x9228, 0xffffffff, 0x000b000a, 665 0x922c, 0xffffffff, 0x000d000c, 666 0x9230, 0xffffffff, 0x00000010, 667 0x9234, 0xffffffff, 0x000f000e, 668 0x9238, 0xffffffff, 0x00120011, 669 0x923c, 0xffffffff, 0x000c000b, 670 0x9240, 0xffffffff, 0x000e000d, 671 0x9244, 0xffffffff, 0x00000011, 672 0x9248, 0xffffffff, 0x0010000f, 673 0x924c, 0xffffffff, 0x00130012, 674 0x9250, 0xffffffff, 0x000d000c, 675 0x9254, 0xffffffff, 0x000f000e, 676 0x9258, 0xffffffff, 0x00100013, 677 0x925c, 0xffffffff, 0x00120011, 678 0x9260, 0xffffffff, 0x00150014, 679 0x9264, 0xffffffff, 0x000e000d, 680 0x9268, 0xffffffff, 0x0010000f, 681 0x926c, 0xffffffff, 0x00000013, 682 0x9270, 0xffffffff, 0x00120011, 683 0x9274, 0xffffffff, 0x00150014, 684 0x9278, 0xffffffff, 0x000f000e, 685 0x927c, 0xffffffff, 0x00110010, 686 0x9280, 0xffffffff, 0x00000014, 687 0x9284, 0xffffffff, 0x00130012, 688 0x9288, 0xffffffff, 0x00160015, 689 0x928c, 0xffffffff, 0x0010000f, 690 0x9290, 0xffffffff, 0x00120011, 691 0x9294, 0xffffffff, 0x00000015, 692 0x9298, 0xffffffff, 0x00140013, 693 0x929c, 0xffffffff, 0x00170016, 694 0x9150, 0xffffffff, 0x96940200, 695 0x8708, 0xffffffff, 0x00900100, 696 0xc478, 0xffffffff, 0x00000080, 697 0xc404, 0xffffffff, 0x0020003f, 698 0x30, 0xffffffff, 0x0000001c, 699 0x34, 0x000f0000, 0x000f0000, 700 0x160c, 0xffffffff, 0x00000100, 701 0x1024, 0xffffffff, 0x00000100, 702 0x102c, 0x00000101, 0x00000000, 703 0x20a8, 0xffffffff, 0x00000104, 704 0x264c, 0x000c0000, 0x000c0000, 705 0x2648, 0x000c0000, 0x000c0000, 706 0x55e4, 0xff000fff, 0x00000100, 707 0x55e8, 0x00000001, 0x00000001, 708 0x2f50, 0x00000001, 0x00000001, 709 0x30cc, 0xc0000fff, 0x00000104, 710 0xc1e4, 0x00000001, 0x00000001, 711 0xd0c0, 0xfffffff0, 0x00000100, 712 0xd8c0, 0xfffffff0, 0x00000100 713 }; 714 715 static const u32 pitcairn_mgcg_cgcg_init[] = 716 { 717 0xc400, 0xffffffff, 0xfffffffc, 718 0x802c, 0xffffffff, 0xe0000000, 719 0x9a60, 0xffffffff, 0x00000100, 720 0x92a4, 0xffffffff, 0x00000100, 721 0xc164, 0xffffffff, 0x00000100, 722 0x9774, 0xffffffff, 0x00000100, 723 0x8984, 0xffffffff, 0x06000100, 724 0x8a18, 0xffffffff, 0x00000100, 725 0x92a0, 0xffffffff, 0x00000100, 726 0xc380, 0xffffffff, 0x00000100, 727 0x8b28, 0xffffffff, 0x00000100, 728 0x9144, 0xffffffff, 0x00000100, 729 0x8d88, 0xffffffff, 0x00000100, 730 0x8d8c, 0xffffffff, 0x00000100, 731 0x9030, 0xffffffff, 0x00000100, 732 0x9034, 0xffffffff, 0x00000100, 733 0x9038, 0xffffffff, 0x00000100, 734 0x903c, 0xffffffff, 0x00000100, 735 0xad80, 0xffffffff, 0x00000100, 736 0xac54, 0xffffffff, 0x00000100, 737 0x897c, 0xffffffff, 0x06000100, 738 0x9868, 0xffffffff, 0x00000100, 739 0x9510, 0xffffffff, 0x00000100, 740 0xaf04, 0xffffffff, 0x00000100, 741 0xae04, 0xffffffff, 0x00000100, 742 0x949c, 0xffffffff, 0x00000100, 743 0x802c, 0xffffffff, 0xe0000000, 744 0x9160, 0xffffffff, 0x00010000, 745 0x9164, 0xffffffff, 0x00030002, 746 0x9168, 0xffffffff, 0x00040007, 747 0x916c, 0xffffffff, 0x00060005, 748 0x9170, 0xffffffff, 0x00090008, 749 0x9174, 0xffffffff, 0x00020001, 750 0x9178, 0xffffffff, 0x00040003, 751 0x917c, 0xffffffff, 0x00000007, 752 0x9180, 0xffffffff, 0x00060005, 753 0x9184, 0xffffffff, 0x00090008, 754 0x9188, 0xffffffff, 0x00030002, 755 0x918c, 0xffffffff, 0x00050004, 756 0x9190, 0xffffffff, 0x00000008, 757 0x9194, 0xffffffff, 0x00070006, 758 0x9198, 0xffffffff, 0x000a0009, 759 0x919c, 0xffffffff, 0x00040003, 760 0x91a0, 0xffffffff, 0x00060005, 761 0x91a4, 0xffffffff, 0x00000009, 762 0x91a8, 0xffffffff, 0x00080007, 763 0x91ac, 0xffffffff, 0x000b000a, 764 0x91b0, 0xffffffff, 0x00050004, 765 0x91b4, 0xffffffff, 0x00070006, 766 0x91b8, 0xffffffff, 0x0008000b, 767 0x91bc, 0xffffffff, 0x000a0009, 768 0x91c0, 0xffffffff, 0x000d000c, 769 0x9200, 0xffffffff, 0x00090008, 770 0x9204, 0xffffffff, 0x000b000a, 771 0x9208, 0xffffffff, 0x000c000f, 772 0x920c, 0xffffffff, 0x000e000d, 773 0x9210, 0xffffffff, 0x00110010, 774 0x9214, 0xffffffff, 0x000a0009, 775 0x9218, 0xffffffff, 0x000c000b, 776 0x921c, 0xffffffff, 0x0000000f, 777 0x9220, 0xffffffff, 0x000e000d, 778 0x9224, 0xffffffff, 0x00110010, 779 0x9228, 0xffffffff, 0x000b000a, 780 0x922c, 0xffffffff, 0x000d000c, 781 0x9230, 0xffffffff, 0x00000010, 782 0x9234, 0xffffffff, 0x000f000e, 783 0x9238, 0xffffffff, 0x00120011, 784 0x923c, 0xffffffff, 0x000c000b, 785 0x9240, 0xffffffff, 0x000e000d, 786 0x9244, 0xffffffff, 0x00000011, 787 0x9248, 0xffffffff, 0x0010000f, 788 0x924c, 0xffffffff, 0x00130012, 789 0x9250, 0xffffffff, 0x000d000c, 790 0x9254, 0xffffffff, 0x000f000e, 791 0x9258, 0xffffffff, 0x00100013, 792 0x925c, 0xffffffff, 0x00120011, 793 0x9260, 0xffffffff, 0x00150014, 794 0x9150, 0xffffffff, 0x96940200, 795 0x8708, 0xffffffff, 0x00900100, 796 0xc478, 0xffffffff, 0x00000080, 797 0xc404, 0xffffffff, 0x0020003f, 798 0x30, 0xffffffff, 0x0000001c, 799 0x34, 0x000f0000, 0x000f0000, 800 0x160c, 0xffffffff, 0x00000100, 801 0x1024, 0xffffffff, 0x00000100, 802 0x102c, 0x00000101, 0x00000000, 803 0x20a8, 0xffffffff, 0x00000104, 804 0x55e4, 0xff000fff, 0x00000100, 805 0x55e8, 0x00000001, 0x00000001, 806 0x2f50, 0x00000001, 0x00000001, 807 0x30cc, 0xc0000fff, 0x00000104, 808 0xc1e4, 0x00000001, 0x00000001, 809 0xd0c0, 0xfffffff0, 0x00000100, 810 0xd8c0, 0xfffffff0, 0x00000100 811 }; 812 813 static const u32 verde_mgcg_cgcg_init[] = 814 { 815 0xc400, 0xffffffff, 0xfffffffc, 816 0x802c, 0xffffffff, 0xe0000000, 817 0x9a60, 0xffffffff, 0x00000100, 818 0x92a4, 0xffffffff, 0x00000100, 819 0xc164, 0xffffffff, 0x00000100, 820 0x9774, 0xffffffff, 0x00000100, 821 0x8984, 0xffffffff, 0x06000100, 822 0x8a18, 0xffffffff, 0x00000100, 823 0x92a0, 0xffffffff, 0x00000100, 824 0xc380, 0xffffffff, 0x00000100, 825 0x8b28, 0xffffffff, 0x00000100, 826 0x9144, 0xffffffff, 0x00000100, 827 0x8d88, 0xffffffff, 0x00000100, 828 0x8d8c, 0xffffffff, 0x00000100, 829 0x9030, 0xffffffff, 0x00000100, 830 0x9034, 0xffffffff, 0x00000100, 831 0x9038, 0xffffffff, 0x00000100, 832 0x903c, 0xffffffff, 0x00000100, 833 0xad80, 0xffffffff, 0x00000100, 834 0xac54, 0xffffffff, 0x00000100, 835 0x897c, 0xffffffff, 0x06000100, 836 0x9868, 0xffffffff, 0x00000100, 837 0x9510, 0xffffffff, 0x00000100, 838 0xaf04, 0xffffffff, 0x00000100, 839 0xae04, 0xffffffff, 0x00000100, 840 0x949c, 0xffffffff, 0x00000100, 841 0x802c, 0xffffffff, 0xe0000000, 842 0x9160, 0xffffffff, 0x00010000, 843 0x9164, 0xffffffff, 0x00030002, 844 0x9168, 0xffffffff, 0x00040007, 845 0x916c, 0xffffffff, 0x00060005, 846 0x9170, 0xffffffff, 0x00090008, 847 0x9174, 0xffffffff, 0x00020001, 848 0x9178, 0xffffffff, 0x00040003, 849 0x917c, 0xffffffff, 0x00000007, 850 0x9180, 0xffffffff, 0x00060005, 851 0x9184, 0xffffffff, 0x00090008, 852 0x9188, 0xffffffff, 0x00030002, 853 0x918c, 0xffffffff, 0x00050004, 854 0x9190, 0xffffffff, 0x00000008, 855 0x9194, 0xffffffff, 0x00070006, 856 0x9198, 0xffffffff, 0x000a0009, 857 0x919c, 0xffffffff, 0x00040003, 858 0x91a0, 0xffffffff, 0x00060005, 859 0x91a4, 0xffffffff, 0x00000009, 860 0x91a8, 0xffffffff, 0x00080007, 861 0x91ac, 0xffffffff, 0x000b000a, 862 0x91b0, 0xffffffff, 0x00050004, 863 0x91b4, 0xffffffff, 0x00070006, 864 0x91b8, 0xffffffff, 0x0008000b, 865 0x91bc, 0xffffffff, 0x000a0009, 866 0x91c0, 0xffffffff, 0x000d000c, 867 0x9200, 0xffffffff, 0x00090008, 868 0x9204, 0xffffffff, 0x000b000a, 869 0x9208, 0xffffffff, 0x000c000f, 870 0x920c, 0xffffffff, 0x000e000d, 871 0x9210, 0xffffffff, 0x00110010, 872 0x9214, 0xffffffff, 0x000a0009, 873 0x9218, 0xffffffff, 0x000c000b, 874 0x921c, 0xffffffff, 0x0000000f, 875 0x9220, 0xffffffff, 0x000e000d, 876 0x9224, 0xffffffff, 0x00110010, 877 0x9228, 0xffffffff, 0x000b000a, 878 0x922c, 0xffffffff, 0x000d000c, 879 0x9230, 0xffffffff, 0x00000010, 880 0x9234, 0xffffffff, 0x000f000e, 881 0x9238, 0xffffffff, 0x00120011, 882 0x923c, 0xffffffff, 0x000c000b, 883 0x9240, 0xffffffff, 0x000e000d, 884 0x9244, 0xffffffff, 0x00000011, 885 0x9248, 0xffffffff, 0x0010000f, 886 0x924c, 0xffffffff, 0x00130012, 887 0x9250, 0xffffffff, 0x000d000c, 888 0x9254, 0xffffffff, 0x000f000e, 889 0x9258, 0xffffffff, 0x00100013, 890 0x925c, 0xffffffff, 0x00120011, 891 0x9260, 0xffffffff, 0x00150014, 892 0x9150, 0xffffffff, 0x96940200, 893 0x8708, 0xffffffff, 0x00900100, 894 0xc478, 0xffffffff, 0x00000080, 895 0xc404, 0xffffffff, 0x0020003f, 896 0x30, 0xffffffff, 0x0000001c, 897 0x34, 0x000f0000, 0x000f0000, 898 0x160c, 0xffffffff, 0x00000100, 899 0x1024, 0xffffffff, 0x00000100, 900 0x102c, 0x00000101, 0x00000000, 901 0x20a8, 0xffffffff, 0x00000104, 902 0x264c, 0x000c0000, 0x000c0000, 903 0x2648, 0x000c0000, 0x000c0000, 904 0x55e4, 0xff000fff, 0x00000100, 905 0x55e8, 0x00000001, 0x00000001, 906 0x2f50, 0x00000001, 0x00000001, 907 0x30cc, 0xc0000fff, 0x00000104, 908 0xc1e4, 0x00000001, 0x00000001, 909 0xd0c0, 0xfffffff0, 0x00000100, 910 0xd8c0, 0xfffffff0, 0x00000100 911 }; 912 913 static const u32 oland_mgcg_cgcg_init[] = 914 { 915 0xc400, 0xffffffff, 0xfffffffc, 916 0x802c, 0xffffffff, 0xe0000000, 917 0x9a60, 0xffffffff, 0x00000100, 918 0x92a4, 0xffffffff, 0x00000100, 919 0xc164, 0xffffffff, 0x00000100, 920 0x9774, 0xffffffff, 0x00000100, 921 0x8984, 0xffffffff, 0x06000100, 922 0x8a18, 0xffffffff, 0x00000100, 923 0x92a0, 0xffffffff, 0x00000100, 924 0xc380, 0xffffffff, 0x00000100, 925 0x8b28, 0xffffffff, 0x00000100, 926 0x9144, 0xffffffff, 0x00000100, 927 0x8d88, 0xffffffff, 0x00000100, 928 0x8d8c, 0xffffffff, 0x00000100, 929 0x9030, 0xffffffff, 0x00000100, 930 0x9034, 0xffffffff, 0x00000100, 931 0x9038, 0xffffffff, 0x00000100, 932 0x903c, 0xffffffff, 0x00000100, 933 0xad80, 0xffffffff, 0x00000100, 934 0xac54, 0xffffffff, 0x00000100, 935 0x897c, 0xffffffff, 0x06000100, 936 0x9868, 0xffffffff, 0x00000100, 937 0x9510, 0xffffffff, 0x00000100, 938 0xaf04, 0xffffffff, 0x00000100, 939 0xae04, 0xffffffff, 0x00000100, 940 0x949c, 0xffffffff, 0x00000100, 941 0x802c, 0xffffffff, 0xe0000000, 942 0x9160, 0xffffffff, 0x00010000, 943 0x9164, 0xffffffff, 0x00030002, 944 0x9168, 0xffffffff, 0x00040007, 945 0x916c, 0xffffffff, 0x00060005, 946 0x9170, 0xffffffff, 0x00090008, 947 0x9174, 0xffffffff, 0x00020001, 948 0x9178, 0xffffffff, 0x00040003, 949 0x917c, 0xffffffff, 0x00000007, 950 0x9180, 0xffffffff, 0x00060005, 951 0x9184, 0xffffffff, 0x00090008, 952 0x9188, 0xffffffff, 0x00030002, 953 0x918c, 0xffffffff, 0x00050004, 954 0x9190, 0xffffffff, 0x00000008, 955 0x9194, 0xffffffff, 0x00070006, 956 0x9198, 0xffffffff, 0x000a0009, 957 0x919c, 0xffffffff, 0x00040003, 958 0x91a0, 0xffffffff, 0x00060005, 959 0x91a4, 0xffffffff, 0x00000009, 960 0x91a8, 0xffffffff, 0x00080007, 961 0x91ac, 0xffffffff, 0x000b000a, 962 0x91b0, 0xffffffff, 0x00050004, 963 0x91b4, 0xffffffff, 0x00070006, 964 0x91b8, 0xffffffff, 0x0008000b, 965 0x91bc, 0xffffffff, 0x000a0009, 966 0x91c0, 0xffffffff, 0x000d000c, 967 0x91c4, 0xffffffff, 0x00060005, 968 0x91c8, 0xffffffff, 0x00080007, 969 0x91cc, 0xffffffff, 0x0000000b, 970 0x91d0, 0xffffffff, 0x000a0009, 971 0x91d4, 0xffffffff, 0x000d000c, 972 0x9150, 0xffffffff, 0x96940200, 973 0x8708, 0xffffffff, 0x00900100, 974 0xc478, 0xffffffff, 0x00000080, 975 0xc404, 0xffffffff, 0x0020003f, 976 0x30, 0xffffffff, 0x0000001c, 977 0x34, 0x000f0000, 0x000f0000, 978 0x160c, 0xffffffff, 0x00000100, 979 0x1024, 0xffffffff, 0x00000100, 980 0x102c, 0x00000101, 0x00000000, 981 0x20a8, 0xffffffff, 0x00000104, 982 0x264c, 0x000c0000, 0x000c0000, 983 0x2648, 0x000c0000, 0x000c0000, 984 0x55e4, 0xff000fff, 0x00000100, 985 0x55e8, 0x00000001, 0x00000001, 986 0x2f50, 0x00000001, 0x00000001, 987 0x30cc, 0xc0000fff, 0x00000104, 988 0xc1e4, 0x00000001, 0x00000001, 989 0xd0c0, 0xfffffff0, 0x00000100, 990 0xd8c0, 0xfffffff0, 0x00000100 991 }; 992 993 static const u32 hainan_mgcg_cgcg_init[] = 994 { 995 0xc400, 0xffffffff, 0xfffffffc, 996 0x802c, 0xffffffff, 0xe0000000, 997 0x9a60, 0xffffffff, 0x00000100, 998 0x92a4, 0xffffffff, 0x00000100, 999 0xc164, 0xffffffff, 0x00000100, 1000 0x9774, 0xffffffff, 0x00000100, 1001 0x8984, 0xffffffff, 0x06000100, 1002 0x8a18, 0xffffffff, 0x00000100, 1003 0x92a0, 0xffffffff, 0x00000100, 1004 0xc380, 0xffffffff, 0x00000100, 1005 0x8b28, 0xffffffff, 0x00000100, 1006 0x9144, 0xffffffff, 0x00000100, 1007 0x8d88, 0xffffffff, 0x00000100, 1008 0x8d8c, 0xffffffff, 0x00000100, 1009 0x9030, 0xffffffff, 0x00000100, 1010 0x9034, 0xffffffff, 0x00000100, 1011 0x9038, 0xffffffff, 0x00000100, 1012 0x903c, 0xffffffff, 0x00000100, 1013 0xad80, 0xffffffff, 0x00000100, 1014 0xac54, 0xffffffff, 0x00000100, 1015 0x897c, 0xffffffff, 0x06000100, 1016 0x9868, 0xffffffff, 0x00000100, 1017 0x9510, 0xffffffff, 0x00000100, 1018 0xaf04, 0xffffffff, 0x00000100, 1019 0xae04, 0xffffffff, 0x00000100, 1020 0x949c, 0xffffffff, 0x00000100, 1021 0x802c, 0xffffffff, 0xe0000000, 1022 0x9160, 0xffffffff, 0x00010000, 1023 0x9164, 0xffffffff, 0x00030002, 1024 0x9168, 0xffffffff, 0x00040007, 1025 0x916c, 0xffffffff, 0x00060005, 1026 0x9170, 0xffffffff, 0x00090008, 1027 0x9174, 0xffffffff, 0x00020001, 1028 0x9178, 0xffffffff, 0x00040003, 1029 0x917c, 0xffffffff, 0x00000007, 1030 0x9180, 0xffffffff, 0x00060005, 1031 0x9184, 0xffffffff, 0x00090008, 1032 0x9188, 0xffffffff, 0x00030002, 1033 0x918c, 0xffffffff, 0x00050004, 1034 0x9190, 0xffffffff, 0x00000008, 1035 0x9194, 0xffffffff, 0x00070006, 1036 0x9198, 0xffffffff, 0x000a0009, 1037 0x919c, 0xffffffff, 0x00040003, 1038 0x91a0, 0xffffffff, 0x00060005, 1039 0x91a4, 0xffffffff, 0x00000009, 1040 0x91a8, 0xffffffff, 0x00080007, 1041 0x91ac, 0xffffffff, 0x000b000a, 1042 0x91b0, 0xffffffff, 0x00050004, 1043 0x91b4, 0xffffffff, 0x00070006, 1044 0x91b8, 0xffffffff, 0x0008000b, 1045 0x91bc, 0xffffffff, 0x000a0009, 1046 0x91c0, 0xffffffff, 0x000d000c, 1047 0x91c4, 0xffffffff, 0x00060005, 1048 0x91c8, 0xffffffff, 0x00080007, 1049 0x91cc, 0xffffffff, 0x0000000b, 1050 0x91d0, 0xffffffff, 0x000a0009, 1051 0x91d4, 0xffffffff, 0x000d000c, 1052 0x9150, 0xffffffff, 0x96940200, 1053 0x8708, 0xffffffff, 0x00900100, 1054 0xc478, 0xffffffff, 0x00000080, 1055 0xc404, 0xffffffff, 0x0020003f, 1056 0x30, 0xffffffff, 0x0000001c, 1057 0x34, 0x000f0000, 0x000f0000, 1058 0x160c, 0xffffffff, 0x00000100, 1059 0x1024, 0xffffffff, 0x00000100, 1060 0x20a8, 0xffffffff, 0x00000104, 1061 0x264c, 0x000c0000, 0x000c0000, 1062 0x2648, 0x000c0000, 0x000c0000, 1063 0x2f50, 0x00000001, 0x00000001, 1064 0x30cc, 0xc0000fff, 0x00000104, 1065 0xc1e4, 0x00000001, 0x00000001, 1066 0xd0c0, 0xfffffff0, 0x00000100, 1067 0xd8c0, 0xfffffff0, 0x00000100 1068 }; 1069 1070 static u32 verde_pg_init[] = 1071 { 1072 0x353c, 0xffffffff, 0x40000, 1073 0x3538, 0xffffffff, 0x200010ff, 1074 0x353c, 0xffffffff, 0x0, 1075 0x353c, 0xffffffff, 0x0, 1076 0x353c, 0xffffffff, 0x0, 1077 0x353c, 0xffffffff, 0x0, 1078 0x353c, 0xffffffff, 0x0, 1079 0x353c, 0xffffffff, 0x7007, 1080 0x3538, 0xffffffff, 0x300010ff, 1081 0x353c, 0xffffffff, 0x0, 1082 0x353c, 0xffffffff, 0x0, 1083 0x353c, 0xffffffff, 0x0, 1084 0x353c, 0xffffffff, 0x0, 1085 0x353c, 0xffffffff, 0x0, 1086 0x353c, 0xffffffff, 0x400000, 1087 0x3538, 0xffffffff, 0x100010ff, 1088 0x353c, 0xffffffff, 0x0, 1089 0x353c, 0xffffffff, 0x0, 1090 0x353c, 0xffffffff, 0x0, 1091 0x353c, 0xffffffff, 0x0, 1092 0x353c, 0xffffffff, 0x0, 1093 0x353c, 0xffffffff, 0x120200, 1094 0x3538, 0xffffffff, 0x500010ff, 1095 0x353c, 0xffffffff, 0x0, 1096 0x353c, 0xffffffff, 0x0, 1097 0x353c, 0xffffffff, 0x0, 1098 0x353c, 0xffffffff, 0x0, 1099 0x353c, 0xffffffff, 0x0, 1100 0x353c, 0xffffffff, 0x1e1e16, 1101 0x3538, 0xffffffff, 0x600010ff, 1102 0x353c, 0xffffffff, 0x0, 1103 0x353c, 0xffffffff, 0x0, 1104 0x353c, 0xffffffff, 0x0, 1105 0x353c, 0xffffffff, 0x0, 1106 0x353c, 0xffffffff, 0x0, 1107 0x353c, 0xffffffff, 0x171f1e, 1108 0x3538, 0xffffffff, 0x700010ff, 1109 0x353c, 0xffffffff, 0x0, 1110 0x353c, 0xffffffff, 0x0, 1111 0x353c, 0xffffffff, 0x0, 1112 0x353c, 0xffffffff, 0x0, 1113 0x353c, 0xffffffff, 0x0, 1114 0x353c, 0xffffffff, 0x0, 1115 0x3538, 0xffffffff, 0x9ff, 1116 0x3500, 0xffffffff, 0x0, 1117 0x3504, 0xffffffff, 0x10000800, 1118 0x3504, 0xffffffff, 0xf, 1119 0x3504, 0xffffffff, 0xf, 1120 0x3500, 0xffffffff, 0x4, 1121 0x3504, 0xffffffff, 0x1000051e, 1122 0x3504, 0xffffffff, 0xffff, 1123 0x3504, 0xffffffff, 0xffff, 1124 0x3500, 0xffffffff, 0x8, 1125 0x3504, 0xffffffff, 0x80500, 1126 0x3500, 0xffffffff, 0x12, 1127 0x3504, 0xffffffff, 0x9050c, 1128 0x3500, 0xffffffff, 0x1d, 1129 0x3504, 0xffffffff, 0xb052c, 1130 0x3500, 0xffffffff, 0x2a, 1131 0x3504, 0xffffffff, 0x1053e, 1132 0x3500, 0xffffffff, 0x2d, 1133 0x3504, 0xffffffff, 0x10546, 1134 0x3500, 0xffffffff, 0x30, 1135 0x3504, 0xffffffff, 0xa054e, 1136 0x3500, 0xffffffff, 0x3c, 1137 0x3504, 0xffffffff, 0x1055f, 1138 0x3500, 0xffffffff, 0x3f, 1139 0x3504, 0xffffffff, 0x10567, 1140 0x3500, 0xffffffff, 0x42, 1141 0x3504, 0xffffffff, 0x1056f, 1142 0x3500, 0xffffffff, 0x45, 1143 0x3504, 0xffffffff, 0x10572, 1144 0x3500, 0xffffffff, 0x48, 1145 0x3504, 0xffffffff, 0x20575, 1146 0x3500, 0xffffffff, 0x4c, 1147 0x3504, 0xffffffff, 0x190801, 1148 0x3500, 0xffffffff, 0x67, 1149 0x3504, 0xffffffff, 0x1082a, 1150 0x3500, 0xffffffff, 0x6a, 1151 0x3504, 0xffffffff, 0x1b082d, 1152 0x3500, 0xffffffff, 0x87, 1153 0x3504, 0xffffffff, 0x310851, 1154 0x3500, 0xffffffff, 0xba, 1155 0x3504, 0xffffffff, 0x891, 1156 0x3500, 0xffffffff, 0xbc, 1157 0x3504, 0xffffffff, 0x893, 1158 0x3500, 0xffffffff, 0xbe, 1159 0x3504, 0xffffffff, 0x20895, 1160 0x3500, 0xffffffff, 0xc2, 1161 0x3504, 0xffffffff, 0x20899, 1162 0x3500, 0xffffffff, 0xc6, 1163 0x3504, 0xffffffff, 0x2089d, 1164 0x3500, 0xffffffff, 0xca, 1165 0x3504, 0xffffffff, 0x8a1, 1166 0x3500, 0xffffffff, 0xcc, 1167 0x3504, 0xffffffff, 0x8a3, 1168 0x3500, 0xffffffff, 0xce, 1169 0x3504, 0xffffffff, 0x308a5, 1170 0x3500, 0xffffffff, 0xd3, 1171 0x3504, 0xffffffff, 0x6d08cd, 1172 0x3500, 0xffffffff, 0x142, 1173 0x3504, 0xffffffff, 0x2000095a, 1174 0x3504, 0xffffffff, 0x1, 1175 0x3500, 0xffffffff, 0x144, 1176 0x3504, 0xffffffff, 0x301f095b, 1177 0x3500, 0xffffffff, 0x165, 1178 0x3504, 0xffffffff, 0xc094d, 1179 0x3500, 0xffffffff, 0x173, 1180 0x3504, 0xffffffff, 0xf096d, 1181 0x3500, 0xffffffff, 0x184, 1182 0x3504, 0xffffffff, 0x15097f, 1183 0x3500, 0xffffffff, 0x19b, 1184 0x3504, 0xffffffff, 0xc0998, 1185 0x3500, 0xffffffff, 0x1a9, 1186 0x3504, 0xffffffff, 0x409a7, 1187 0x3500, 0xffffffff, 0x1af, 1188 0x3504, 0xffffffff, 0xcdc, 1189 0x3500, 0xffffffff, 0x1b1, 1190 0x3504, 0xffffffff, 0x800, 1191 0x3508, 0xffffffff, 0x6c9b2000, 1192 0x3510, 0xfc00, 0x2000, 1193 0x3544, 0xffffffff, 0xfc0, 1194 0x28d4, 0x00000100, 0x100 1195 }; 1196 1197 static void si_init_golden_registers(struct radeon_device *rdev) 1198 { 1199 switch (rdev->family) { 1200 case CHIP_TAHITI: 1201 radeon_program_register_sequence(rdev, 1202 tahiti_golden_registers, 1203 (const u32)ARRAY_SIZE(tahiti_golden_registers)); 1204 radeon_program_register_sequence(rdev, 1205 tahiti_golden_rlc_registers, 1206 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers)); 1207 radeon_program_register_sequence(rdev, 1208 tahiti_mgcg_cgcg_init, 1209 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init)); 1210 radeon_program_register_sequence(rdev, 1211 tahiti_golden_registers2, 1212 (const u32)ARRAY_SIZE(tahiti_golden_registers2)); 1213 break; 1214 case CHIP_PITCAIRN: 1215 radeon_program_register_sequence(rdev, 1216 pitcairn_golden_registers, 1217 (const u32)ARRAY_SIZE(pitcairn_golden_registers)); 1218 radeon_program_register_sequence(rdev, 1219 pitcairn_golden_rlc_registers, 1220 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers)); 1221 radeon_program_register_sequence(rdev, 1222 pitcairn_mgcg_cgcg_init, 1223 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init)); 1224 break; 1225 case CHIP_VERDE: 1226 radeon_program_register_sequence(rdev, 1227 verde_golden_registers, 1228 (const u32)ARRAY_SIZE(verde_golden_registers)); 1229 radeon_program_register_sequence(rdev, 1230 verde_golden_rlc_registers, 1231 (const u32)ARRAY_SIZE(verde_golden_rlc_registers)); 1232 radeon_program_register_sequence(rdev, 1233 verde_mgcg_cgcg_init, 1234 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init)); 1235 radeon_program_register_sequence(rdev, 1236 verde_pg_init, 1237 (const u32)ARRAY_SIZE(verde_pg_init)); 1238 break; 1239 case CHIP_OLAND: 1240 radeon_program_register_sequence(rdev, 1241 oland_golden_registers, 1242 (const u32)ARRAY_SIZE(oland_golden_registers)); 1243 radeon_program_register_sequence(rdev, 1244 oland_golden_rlc_registers, 1245 (const u32)ARRAY_SIZE(oland_golden_rlc_registers)); 1246 radeon_program_register_sequence(rdev, 1247 oland_mgcg_cgcg_init, 1248 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init)); 1249 break; 1250 case CHIP_HAINAN: 1251 radeon_program_register_sequence(rdev, 1252 hainan_golden_registers, 1253 (const u32)ARRAY_SIZE(hainan_golden_registers)); 1254 radeon_program_register_sequence(rdev, 1255 hainan_golden_registers2, 1256 (const u32)ARRAY_SIZE(hainan_golden_registers2)); 1257 radeon_program_register_sequence(rdev, 1258 hainan_mgcg_cgcg_init, 1259 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init)); 1260 break; 1261 default: 1262 break; 1263 } 1264 } 1265 1266 #define PCIE_BUS_CLK 10000 1267 #define TCLK (PCIE_BUS_CLK / 10) 1268 1269 /** 1270 * si_get_xclk - get the xclk 1271 * 1272 * @rdev: radeon_device pointer 1273 * 1274 * Returns the reference clock used by the gfx engine 1275 * (SI). 1276 */ 1277 u32 si_get_xclk(struct radeon_device *rdev) 1278 { 1279 u32 reference_clock = rdev->clock.spll.reference_freq; 1280 u32 tmp; 1281 1282 tmp = RREG32(CG_CLKPIN_CNTL_2); 1283 if (tmp & MUX_TCLK_TO_XCLK) 1284 return TCLK; 1285 1286 tmp = RREG32(CG_CLKPIN_CNTL); 1287 if (tmp & XTALIN_DIVIDE) 1288 return reference_clock / 4; 1289 1290 return reference_clock; 1291 } 1292 1293 /* get temperature in millidegrees */ 1294 int si_get_temp(struct radeon_device *rdev) 1295 { 1296 u32 temp; 1297 int actual_temp = 0; 1298 1299 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 1300 CTF_TEMP_SHIFT; 1301 1302 if (temp & 0x200) 1303 actual_temp = 255; 1304 else 1305 actual_temp = temp & 0x1ff; 1306 1307 actual_temp = (actual_temp * 1000); 1308 1309 return actual_temp; 1310 } 1311 1312 #define TAHITI_IO_MC_REGS_SIZE 36 1313 1314 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1315 {0x0000006f, 0x03044000}, 1316 {0x00000070, 0x0480c018}, 1317 {0x00000071, 0x00000040}, 1318 {0x00000072, 0x01000000}, 1319 {0x00000074, 0x000000ff}, 1320 {0x00000075, 0x00143400}, 1321 {0x00000076, 0x08ec0800}, 1322 {0x00000077, 0x040000cc}, 1323 {0x00000079, 0x00000000}, 1324 {0x0000007a, 0x21000409}, 1325 {0x0000007c, 0x00000000}, 1326 {0x0000007d, 0xe8000000}, 1327 {0x0000007e, 0x044408a8}, 1328 {0x0000007f, 0x00000003}, 1329 {0x00000080, 0x00000000}, 1330 {0x00000081, 0x01000000}, 1331 {0x00000082, 0x02000000}, 1332 {0x00000083, 0x00000000}, 1333 {0x00000084, 0xe3f3e4f4}, 1334 {0x00000085, 0x00052024}, 1335 {0x00000087, 0x00000000}, 1336 {0x00000088, 0x66036603}, 1337 {0x00000089, 0x01000000}, 1338 {0x0000008b, 0x1c0a0000}, 1339 {0x0000008c, 0xff010000}, 1340 {0x0000008e, 0xffffefff}, 1341 {0x0000008f, 0xfff3efff}, 1342 {0x00000090, 0xfff3efbf}, 1343 {0x00000094, 0x00101101}, 1344 {0x00000095, 0x00000fff}, 1345 {0x00000096, 0x00116fff}, 1346 {0x00000097, 0x60010000}, 1347 {0x00000098, 0x10010000}, 1348 {0x00000099, 0x00006000}, 1349 {0x0000009a, 0x00001000}, 1350 {0x0000009f, 0x00a77400} 1351 }; 1352 1353 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1354 {0x0000006f, 0x03044000}, 1355 {0x00000070, 0x0480c018}, 1356 {0x00000071, 0x00000040}, 1357 {0x00000072, 0x01000000}, 1358 {0x00000074, 0x000000ff}, 1359 {0x00000075, 0x00143400}, 1360 {0x00000076, 0x08ec0800}, 1361 {0x00000077, 0x040000cc}, 1362 {0x00000079, 0x00000000}, 1363 {0x0000007a, 0x21000409}, 1364 {0x0000007c, 0x00000000}, 1365 {0x0000007d, 0xe8000000}, 1366 {0x0000007e, 0x044408a8}, 1367 {0x0000007f, 0x00000003}, 1368 {0x00000080, 0x00000000}, 1369 {0x00000081, 0x01000000}, 1370 {0x00000082, 0x02000000}, 1371 {0x00000083, 0x00000000}, 1372 {0x00000084, 0xe3f3e4f4}, 1373 {0x00000085, 0x00052024}, 1374 {0x00000087, 0x00000000}, 1375 {0x00000088, 0x66036603}, 1376 {0x00000089, 0x01000000}, 1377 {0x0000008b, 0x1c0a0000}, 1378 {0x0000008c, 0xff010000}, 1379 {0x0000008e, 0xffffefff}, 1380 {0x0000008f, 0xfff3efff}, 1381 {0x00000090, 0xfff3efbf}, 1382 {0x00000094, 0x00101101}, 1383 {0x00000095, 0x00000fff}, 1384 {0x00000096, 0x00116fff}, 1385 {0x00000097, 0x60010000}, 1386 {0x00000098, 0x10010000}, 1387 {0x00000099, 0x00006000}, 1388 {0x0000009a, 0x00001000}, 1389 {0x0000009f, 0x00a47400} 1390 }; 1391 1392 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1393 {0x0000006f, 0x03044000}, 1394 {0x00000070, 0x0480c018}, 1395 {0x00000071, 0x00000040}, 1396 {0x00000072, 0x01000000}, 1397 {0x00000074, 0x000000ff}, 1398 {0x00000075, 0x00143400}, 1399 {0x00000076, 0x08ec0800}, 1400 {0x00000077, 0x040000cc}, 1401 {0x00000079, 0x00000000}, 1402 {0x0000007a, 0x21000409}, 1403 {0x0000007c, 0x00000000}, 1404 {0x0000007d, 0xe8000000}, 1405 {0x0000007e, 0x044408a8}, 1406 {0x0000007f, 0x00000003}, 1407 {0x00000080, 0x00000000}, 1408 {0x00000081, 0x01000000}, 1409 {0x00000082, 0x02000000}, 1410 {0x00000083, 0x00000000}, 1411 {0x00000084, 0xe3f3e4f4}, 1412 {0x00000085, 0x00052024}, 1413 {0x00000087, 0x00000000}, 1414 {0x00000088, 0x66036603}, 1415 {0x00000089, 0x01000000}, 1416 {0x0000008b, 0x1c0a0000}, 1417 {0x0000008c, 0xff010000}, 1418 {0x0000008e, 0xffffefff}, 1419 {0x0000008f, 0xfff3efff}, 1420 {0x00000090, 0xfff3efbf}, 1421 {0x00000094, 0x00101101}, 1422 {0x00000095, 0x00000fff}, 1423 {0x00000096, 0x00116fff}, 1424 {0x00000097, 0x60010000}, 1425 {0x00000098, 0x10010000}, 1426 {0x00000099, 0x00006000}, 1427 {0x0000009a, 0x00001000}, 1428 {0x0000009f, 0x00a37400} 1429 }; 1430 1431 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1432 {0x0000006f, 0x03044000}, 1433 {0x00000070, 0x0480c018}, 1434 {0x00000071, 0x00000040}, 1435 {0x00000072, 0x01000000}, 1436 {0x00000074, 0x000000ff}, 1437 {0x00000075, 0x00143400}, 1438 {0x00000076, 0x08ec0800}, 1439 {0x00000077, 0x040000cc}, 1440 {0x00000079, 0x00000000}, 1441 {0x0000007a, 0x21000409}, 1442 {0x0000007c, 0x00000000}, 1443 {0x0000007d, 0xe8000000}, 1444 {0x0000007e, 0x044408a8}, 1445 {0x0000007f, 0x00000003}, 1446 {0x00000080, 0x00000000}, 1447 {0x00000081, 0x01000000}, 1448 {0x00000082, 0x02000000}, 1449 {0x00000083, 0x00000000}, 1450 {0x00000084, 0xe3f3e4f4}, 1451 {0x00000085, 0x00052024}, 1452 {0x00000087, 0x00000000}, 1453 {0x00000088, 0x66036603}, 1454 {0x00000089, 0x01000000}, 1455 {0x0000008b, 0x1c0a0000}, 1456 {0x0000008c, 0xff010000}, 1457 {0x0000008e, 0xffffefff}, 1458 {0x0000008f, 0xfff3efff}, 1459 {0x00000090, 0xfff3efbf}, 1460 {0x00000094, 0x00101101}, 1461 {0x00000095, 0x00000fff}, 1462 {0x00000096, 0x00116fff}, 1463 {0x00000097, 0x60010000}, 1464 {0x00000098, 0x10010000}, 1465 {0x00000099, 0x00006000}, 1466 {0x0000009a, 0x00001000}, 1467 {0x0000009f, 0x00a17730} 1468 }; 1469 1470 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1471 {0x0000006f, 0x03044000}, 1472 {0x00000070, 0x0480c018}, 1473 {0x00000071, 0x00000040}, 1474 {0x00000072, 0x01000000}, 1475 {0x00000074, 0x000000ff}, 1476 {0x00000075, 0x00143400}, 1477 {0x00000076, 0x08ec0800}, 1478 {0x00000077, 0x040000cc}, 1479 {0x00000079, 0x00000000}, 1480 {0x0000007a, 0x21000409}, 1481 {0x0000007c, 0x00000000}, 1482 {0x0000007d, 0xe8000000}, 1483 {0x0000007e, 0x044408a8}, 1484 {0x0000007f, 0x00000003}, 1485 {0x00000080, 0x00000000}, 1486 {0x00000081, 0x01000000}, 1487 {0x00000082, 0x02000000}, 1488 {0x00000083, 0x00000000}, 1489 {0x00000084, 0xe3f3e4f4}, 1490 {0x00000085, 0x00052024}, 1491 {0x00000087, 0x00000000}, 1492 {0x00000088, 0x66036603}, 1493 {0x00000089, 0x01000000}, 1494 {0x0000008b, 0x1c0a0000}, 1495 {0x0000008c, 0xff010000}, 1496 {0x0000008e, 0xffffefff}, 1497 {0x0000008f, 0xfff3efff}, 1498 {0x00000090, 0xfff3efbf}, 1499 {0x00000094, 0x00101101}, 1500 {0x00000095, 0x00000fff}, 1501 {0x00000096, 0x00116fff}, 1502 {0x00000097, 0x60010000}, 1503 {0x00000098, 0x10010000}, 1504 {0x00000099, 0x00006000}, 1505 {0x0000009a, 0x00001000}, 1506 {0x0000009f, 0x00a07730} 1507 }; 1508 1509 /* ucode loading */ 1510 int si_mc_load_microcode(struct radeon_device *rdev) 1511 { 1512 const __be32 *fw_data = NULL; 1513 const __le32 *new_fw_data = NULL; 1514 u32 running, blackout = 0; 1515 u32 *io_mc_regs = NULL; 1516 const __le32 *new_io_mc_regs = NULL; 1517 int i, regs_size, ucode_size; 1518 1519 if (!rdev->mc_fw) 1520 return -EINVAL; 1521 1522 if (rdev->new_fw) { 1523 const struct mc_firmware_header_v1_0 *hdr = 1524 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; 1525 1526 radeon_ucode_print_mc_hdr(&hdr->header); 1527 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); 1528 new_io_mc_regs = (const __le32 *) 1529 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); 1530 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1531 new_fw_data = (const __le32 *) 1532 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1533 } else { 1534 ucode_size = rdev->mc_fw->size / 4; 1535 1536 switch (rdev->family) { 1537 case CHIP_TAHITI: 1538 io_mc_regs = (u32 *)&tahiti_io_mc_regs; 1539 regs_size = TAHITI_IO_MC_REGS_SIZE; 1540 break; 1541 case CHIP_PITCAIRN: 1542 io_mc_regs = (u32 *)&pitcairn_io_mc_regs; 1543 regs_size = TAHITI_IO_MC_REGS_SIZE; 1544 break; 1545 case CHIP_VERDE: 1546 default: 1547 io_mc_regs = (u32 *)&verde_io_mc_regs; 1548 regs_size = TAHITI_IO_MC_REGS_SIZE; 1549 break; 1550 case CHIP_OLAND: 1551 io_mc_regs = (u32 *)&oland_io_mc_regs; 1552 regs_size = TAHITI_IO_MC_REGS_SIZE; 1553 break; 1554 case CHIP_HAINAN: 1555 io_mc_regs = (u32 *)&hainan_io_mc_regs; 1556 regs_size = TAHITI_IO_MC_REGS_SIZE; 1557 break; 1558 } 1559 fw_data = (const __be32 *)rdev->mc_fw->data; 1560 } 1561 1562 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1563 1564 if (running == 0) { 1565 if (running) { 1566 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1567 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1568 } 1569 1570 /* reset the engine and set to writable */ 1571 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1572 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1573 1574 /* load mc io regs */ 1575 for (i = 0; i < regs_size; i++) { 1576 if (rdev->new_fw) { 1577 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++)); 1578 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++)); 1579 } else { 1580 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1581 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1582 } 1583 } 1584 /* load the MC ucode */ 1585 for (i = 0; i < ucode_size; i++) { 1586 if (rdev->new_fw) 1587 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++)); 1588 else 1589 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1590 } 1591 1592 /* put the engine back into the active state */ 1593 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1594 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1595 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1596 1597 /* wait for training to complete */ 1598 for (i = 0; i < rdev->usec_timeout; i++) { 1599 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1600 break; 1601 udelay(1); 1602 } 1603 for (i = 0; i < rdev->usec_timeout; i++) { 1604 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1605 break; 1606 udelay(1); 1607 } 1608 1609 if (running) 1610 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1611 } 1612 1613 return 0; 1614 } 1615 1616 static int si_init_microcode(struct radeon_device *rdev) 1617 { 1618 const char *chip_name; 1619 const char *new_chip_name; 1620 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size; 1621 size_t smc_req_size, mc2_req_size; 1622 char fw_name[30]; 1623 int err; 1624 int new_fw = 0; 1625 1626 DRM_DEBUG("\n"); 1627 1628 switch (rdev->family) { 1629 case CHIP_TAHITI: 1630 chip_name = "TAHITI"; 1631 new_chip_name = "tahiti"; 1632 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1633 me_req_size = SI_PM4_UCODE_SIZE * 4; 1634 ce_req_size = SI_CE_UCODE_SIZE * 4; 1635 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1636 mc_req_size = SI_MC_UCODE_SIZE * 4; 1637 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4; 1638 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4); 1639 break; 1640 case CHIP_PITCAIRN: 1641 chip_name = "PITCAIRN"; 1642 new_chip_name = "pitcairn"; 1643 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1644 me_req_size = SI_PM4_UCODE_SIZE * 4; 1645 ce_req_size = SI_CE_UCODE_SIZE * 4; 1646 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1647 mc_req_size = SI_MC_UCODE_SIZE * 4; 1648 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4; 1649 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4); 1650 break; 1651 case CHIP_VERDE: 1652 chip_name = "VERDE"; 1653 new_chip_name = "verde"; 1654 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1655 me_req_size = SI_PM4_UCODE_SIZE * 4; 1656 ce_req_size = SI_CE_UCODE_SIZE * 4; 1657 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1658 mc_req_size = SI_MC_UCODE_SIZE * 4; 1659 mc2_req_size = VERDE_MC_UCODE_SIZE * 4; 1660 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4); 1661 break; 1662 case CHIP_OLAND: 1663 chip_name = "OLAND"; 1664 new_chip_name = "oland"; 1665 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1666 me_req_size = SI_PM4_UCODE_SIZE * 4; 1667 ce_req_size = SI_CE_UCODE_SIZE * 4; 1668 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1669 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4; 1670 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4); 1671 break; 1672 case CHIP_HAINAN: 1673 chip_name = "HAINAN"; 1674 new_chip_name = "hainan"; 1675 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1676 me_req_size = SI_PM4_UCODE_SIZE * 4; 1677 ce_req_size = SI_CE_UCODE_SIZE * 4; 1678 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1679 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4; 1680 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4); 1681 break; 1682 default: BUG(); 1683 } 1684 1685 DRM_INFO("Loading %s Microcode\n", new_chip_name); 1686 1687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name); 1688 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1689 if (err) { 1690 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 1691 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1692 if (err) 1693 goto out; 1694 if (rdev->pfp_fw->size != pfp_req_size) { 1695 printk(KERN_ERR 1696 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1697 rdev->pfp_fw->size, fw_name); 1698 err = -EINVAL; 1699 goto out; 1700 } 1701 } else { 1702 err = radeon_ucode_validate(rdev->pfp_fw); 1703 if (err) { 1704 printk(KERN_ERR 1705 "si_cp: validation failed for firmware \"%s\"\n", 1706 fw_name); 1707 goto out; 1708 } else { 1709 new_fw++; 1710 } 1711 } 1712 1713 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name); 1714 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 1715 if (err) { 1716 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 1717 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 1718 if (err) 1719 goto out; 1720 if (rdev->me_fw->size != me_req_size) { 1721 printk(KERN_ERR 1722 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1723 rdev->me_fw->size, fw_name); 1724 err = -EINVAL; 1725 } 1726 } else { 1727 err = radeon_ucode_validate(rdev->me_fw); 1728 if (err) { 1729 printk(KERN_ERR 1730 "si_cp: validation failed for firmware \"%s\"\n", 1731 fw_name); 1732 goto out; 1733 } else { 1734 new_fw++; 1735 } 1736 } 1737 1738 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name); 1739 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 1740 if (err) { 1741 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 1742 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 1743 if (err) 1744 goto out; 1745 if (rdev->ce_fw->size != ce_req_size) { 1746 printk(KERN_ERR 1747 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1748 rdev->ce_fw->size, fw_name); 1749 err = -EINVAL; 1750 } 1751 } else { 1752 err = radeon_ucode_validate(rdev->ce_fw); 1753 if (err) { 1754 printk(KERN_ERR 1755 "si_cp: validation failed for firmware \"%s\"\n", 1756 fw_name); 1757 goto out; 1758 } else { 1759 new_fw++; 1760 } 1761 } 1762 1763 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name); 1764 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 1765 if (err) { 1766 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name); 1767 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 1768 if (err) 1769 goto out; 1770 if (rdev->rlc_fw->size != rlc_req_size) { 1771 printk(KERN_ERR 1772 "si_rlc: Bogus length %zu in firmware \"%s\"\n", 1773 rdev->rlc_fw->size, fw_name); 1774 err = -EINVAL; 1775 } 1776 } else { 1777 err = radeon_ucode_validate(rdev->rlc_fw); 1778 if (err) { 1779 printk(KERN_ERR 1780 "si_cp: validation failed for firmware \"%s\"\n", 1781 fw_name); 1782 goto out; 1783 } else { 1784 new_fw++; 1785 } 1786 } 1787 1788 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name); 1789 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 1790 if (err) { 1791 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name); 1792 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 1793 if (err) { 1794 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 1795 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 1796 if (err) 1797 goto out; 1798 } 1799 if ((rdev->mc_fw->size != mc_req_size) && 1800 (rdev->mc_fw->size != mc2_req_size)) { 1801 printk(KERN_ERR 1802 "si_mc: Bogus length %zu in firmware \"%s\"\n", 1803 rdev->mc_fw->size, fw_name); 1804 err = -EINVAL; 1805 } 1806 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); 1807 } else { 1808 err = radeon_ucode_validate(rdev->mc_fw); 1809 if (err) { 1810 printk(KERN_ERR 1811 "si_cp: validation failed for firmware \"%s\"\n", 1812 fw_name); 1813 goto out; 1814 } else { 1815 new_fw++; 1816 } 1817 } 1818 1819 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name); 1820 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 1821 if (err) { 1822 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 1823 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 1824 if (err) { 1825 printk(KERN_ERR 1826 "smc: error loading firmware \"%s\"\n", 1827 fw_name); 1828 release_firmware(rdev->smc_fw); 1829 rdev->smc_fw = NULL; 1830 err = 0; 1831 } else if (rdev->smc_fw->size != smc_req_size) { 1832 printk(KERN_ERR 1833 "si_smc: Bogus length %zu in firmware \"%s\"\n", 1834 rdev->smc_fw->size, fw_name); 1835 err = -EINVAL; 1836 } 1837 } else { 1838 err = radeon_ucode_validate(rdev->smc_fw); 1839 if (err) { 1840 printk(KERN_ERR 1841 "si_cp: validation failed for firmware \"%s\"\n", 1842 fw_name); 1843 goto out; 1844 } else { 1845 new_fw++; 1846 } 1847 } 1848 1849 if (new_fw == 0) { 1850 rdev->new_fw = false; 1851 } else if (new_fw < 6) { 1852 printk(KERN_ERR "si_fw: mixing new and old firmware!\n"); 1853 err = -EINVAL; 1854 } else { 1855 rdev->new_fw = true; 1856 } 1857 out: 1858 if (err) { 1859 if (err != -EINVAL) 1860 printk(KERN_ERR 1861 "si_cp: Failed to load firmware \"%s\"\n", 1862 fw_name); 1863 release_firmware(rdev->pfp_fw); 1864 rdev->pfp_fw = NULL; 1865 release_firmware(rdev->me_fw); 1866 rdev->me_fw = NULL; 1867 release_firmware(rdev->ce_fw); 1868 rdev->ce_fw = NULL; 1869 release_firmware(rdev->rlc_fw); 1870 rdev->rlc_fw = NULL; 1871 release_firmware(rdev->mc_fw); 1872 rdev->mc_fw = NULL; 1873 release_firmware(rdev->smc_fw); 1874 rdev->smc_fw = NULL; 1875 } 1876 return err; 1877 } 1878 1879 /* watermark setup */ 1880 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev, 1881 struct radeon_crtc *radeon_crtc, 1882 struct drm_display_mode *mode, 1883 struct drm_display_mode *other_mode) 1884 { 1885 u32 tmp, buffer_alloc, i; 1886 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1887 /* 1888 * Line Buffer Setup 1889 * There are 3 line buffers, each one shared by 2 display controllers. 1890 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1891 * the display controllers. The paritioning is done via one of four 1892 * preset allocations specified in bits 21:20: 1893 * 0 - half lb 1894 * 2 - whole lb, other crtc must be disabled 1895 */ 1896 /* this can get tricky if we have two large displays on a paired group 1897 * of crtcs. Ideally for multiple large displays we'd assign them to 1898 * non-linked crtcs for maximum line buffer allocation. 1899 */ 1900 if (radeon_crtc->base.enabled && mode) { 1901 if (other_mode) { 1902 tmp = 0; /* 1/2 */ 1903 buffer_alloc = 1; 1904 } else { 1905 tmp = 2; /* whole */ 1906 buffer_alloc = 2; 1907 } 1908 } else { 1909 tmp = 0; 1910 buffer_alloc = 0; 1911 } 1912 1913 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, 1914 DC_LB_MEMORY_CONFIG(tmp)); 1915 1916 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1917 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1918 for (i = 0; i < rdev->usec_timeout; i++) { 1919 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1920 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1921 break; 1922 udelay(1); 1923 } 1924 1925 if (radeon_crtc->base.enabled && mode) { 1926 switch (tmp) { 1927 case 0: 1928 default: 1929 return 4096 * 2; 1930 case 2: 1931 return 8192 * 2; 1932 } 1933 } 1934 1935 /* controller not enabled, so no lb used */ 1936 return 0; 1937 } 1938 1939 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev) 1940 { 1941 u32 tmp = RREG32(MC_SHARED_CHMAP); 1942 1943 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1944 case 0: 1945 default: 1946 return 1; 1947 case 1: 1948 return 2; 1949 case 2: 1950 return 4; 1951 case 3: 1952 return 8; 1953 case 4: 1954 return 3; 1955 case 5: 1956 return 6; 1957 case 6: 1958 return 10; 1959 case 7: 1960 return 12; 1961 case 8: 1962 return 16; 1963 } 1964 } 1965 1966 struct dce6_wm_params { 1967 u32 dram_channels; /* number of dram channels */ 1968 u32 yclk; /* bandwidth per dram data pin in kHz */ 1969 u32 sclk; /* engine clock in kHz */ 1970 u32 disp_clk; /* display clock in kHz */ 1971 u32 src_width; /* viewport width */ 1972 u32 active_time; /* active display time in ns */ 1973 u32 blank_time; /* blank time in ns */ 1974 bool interlaced; /* mode is interlaced */ 1975 fixed20_12 vsc; /* vertical scale ratio */ 1976 u32 num_heads; /* number of active crtcs */ 1977 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1978 u32 lb_size; /* line buffer allocated to pipe */ 1979 u32 vtaps; /* vertical scaler taps */ 1980 }; 1981 1982 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm) 1983 { 1984 /* Calculate raw DRAM Bandwidth */ 1985 fixed20_12 dram_efficiency; /* 0.7 */ 1986 fixed20_12 yclk, dram_channels, bandwidth; 1987 fixed20_12 a; 1988 1989 a.full = dfixed_const(1000); 1990 yclk.full = dfixed_const(wm->yclk); 1991 yclk.full = dfixed_div(yclk, a); 1992 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1993 a.full = dfixed_const(10); 1994 dram_efficiency.full = dfixed_const(7); 1995 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1996 bandwidth.full = dfixed_mul(dram_channels, yclk); 1997 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1998 1999 return dfixed_trunc(bandwidth); 2000 } 2001 2002 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm) 2003 { 2004 /* Calculate DRAM Bandwidth and the part allocated to display. */ 2005 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 2006 fixed20_12 yclk, dram_channels, bandwidth; 2007 fixed20_12 a; 2008 2009 a.full = dfixed_const(1000); 2010 yclk.full = dfixed_const(wm->yclk); 2011 yclk.full = dfixed_div(yclk, a); 2012 dram_channels.full = dfixed_const(wm->dram_channels * 4); 2013 a.full = dfixed_const(10); 2014 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 2015 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 2016 bandwidth.full = dfixed_mul(dram_channels, yclk); 2017 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 2018 2019 return dfixed_trunc(bandwidth); 2020 } 2021 2022 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm) 2023 { 2024 /* Calculate the display Data return Bandwidth */ 2025 fixed20_12 return_efficiency; /* 0.8 */ 2026 fixed20_12 sclk, bandwidth; 2027 fixed20_12 a; 2028 2029 a.full = dfixed_const(1000); 2030 sclk.full = dfixed_const(wm->sclk); 2031 sclk.full = dfixed_div(sclk, a); 2032 a.full = dfixed_const(10); 2033 return_efficiency.full = dfixed_const(8); 2034 return_efficiency.full = dfixed_div(return_efficiency, a); 2035 a.full = dfixed_const(32); 2036 bandwidth.full = dfixed_mul(a, sclk); 2037 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 2038 2039 return dfixed_trunc(bandwidth); 2040 } 2041 2042 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm) 2043 { 2044 return 32; 2045 } 2046 2047 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm) 2048 { 2049 /* Calculate the DMIF Request Bandwidth */ 2050 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 2051 fixed20_12 disp_clk, sclk, bandwidth; 2052 fixed20_12 a, b1, b2; 2053 u32 min_bandwidth; 2054 2055 a.full = dfixed_const(1000); 2056 disp_clk.full = dfixed_const(wm->disp_clk); 2057 disp_clk.full = dfixed_div(disp_clk, a); 2058 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2); 2059 b1.full = dfixed_mul(a, disp_clk); 2060 2061 a.full = dfixed_const(1000); 2062 sclk.full = dfixed_const(wm->sclk); 2063 sclk.full = dfixed_div(sclk, a); 2064 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm)); 2065 b2.full = dfixed_mul(a, sclk); 2066 2067 a.full = dfixed_const(10); 2068 disp_clk_request_efficiency.full = dfixed_const(8); 2069 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 2070 2071 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2)); 2072 2073 a.full = dfixed_const(min_bandwidth); 2074 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency); 2075 2076 return dfixed_trunc(bandwidth); 2077 } 2078 2079 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm) 2080 { 2081 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 2082 u32 dram_bandwidth = dce6_dram_bandwidth(wm); 2083 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm); 2084 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm); 2085 2086 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 2087 } 2088 2089 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm) 2090 { 2091 /* Calculate the display mode Average Bandwidth 2092 * DisplayMode should contain the source and destination dimensions, 2093 * timing, etc. 2094 */ 2095 fixed20_12 bpp; 2096 fixed20_12 line_time; 2097 fixed20_12 src_width; 2098 fixed20_12 bandwidth; 2099 fixed20_12 a; 2100 2101 a.full = dfixed_const(1000); 2102 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 2103 line_time.full = dfixed_div(line_time, a); 2104 bpp.full = dfixed_const(wm->bytes_per_pixel); 2105 src_width.full = dfixed_const(wm->src_width); 2106 bandwidth.full = dfixed_mul(src_width, bpp); 2107 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 2108 bandwidth.full = dfixed_div(bandwidth, line_time); 2109 2110 return dfixed_trunc(bandwidth); 2111 } 2112 2113 static u32 dce6_latency_watermark(struct dce6_wm_params *wm) 2114 { 2115 /* First calcualte the latency in ns */ 2116 u32 mc_latency = 2000; /* 2000 ns. */ 2117 u32 available_bandwidth = dce6_available_bandwidth(wm); 2118 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 2119 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 2120 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 2121 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 2122 (wm->num_heads * cursor_line_pair_return_time); 2123 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 2124 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 2125 u32 tmp, dmif_size = 12288; 2126 fixed20_12 a, b, c; 2127 2128 if (wm->num_heads == 0) 2129 return 0; 2130 2131 a.full = dfixed_const(2); 2132 b.full = dfixed_const(1); 2133 if ((wm->vsc.full > a.full) || 2134 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 2135 (wm->vtaps >= 5) || 2136 ((wm->vsc.full >= a.full) && wm->interlaced)) 2137 max_src_lines_per_dst_line = 4; 2138 else 2139 max_src_lines_per_dst_line = 2; 2140 2141 a.full = dfixed_const(available_bandwidth); 2142 b.full = dfixed_const(wm->num_heads); 2143 a.full = dfixed_div(a, b); 2144 2145 b.full = dfixed_const(mc_latency + 512); 2146 c.full = dfixed_const(wm->disp_clk); 2147 b.full = dfixed_div(b, c); 2148 2149 c.full = dfixed_const(dmif_size); 2150 b.full = dfixed_div(c, b); 2151 2152 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 2153 2154 b.full = dfixed_const(1000); 2155 c.full = dfixed_const(wm->disp_clk); 2156 b.full = dfixed_div(c, b); 2157 c.full = dfixed_const(wm->bytes_per_pixel); 2158 b.full = dfixed_mul(b, c); 2159 2160 lb_fill_bw = min(tmp, dfixed_trunc(b)); 2161 2162 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 2163 b.full = dfixed_const(1000); 2164 c.full = dfixed_const(lb_fill_bw); 2165 b.full = dfixed_div(c, b); 2166 a.full = dfixed_div(a, b); 2167 line_fill_time = dfixed_trunc(a); 2168 2169 if (line_fill_time < wm->active_time) 2170 return latency; 2171 else 2172 return latency + (line_fill_time - wm->active_time); 2173 2174 } 2175 2176 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm) 2177 { 2178 if (dce6_average_bandwidth(wm) <= 2179 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads)) 2180 return true; 2181 else 2182 return false; 2183 }; 2184 2185 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm) 2186 { 2187 if (dce6_average_bandwidth(wm) <= 2188 (dce6_available_bandwidth(wm) / wm->num_heads)) 2189 return true; 2190 else 2191 return false; 2192 }; 2193 2194 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm) 2195 { 2196 u32 lb_partitions = wm->lb_size / wm->src_width; 2197 u32 line_time = wm->active_time + wm->blank_time; 2198 u32 latency_tolerant_lines; 2199 u32 latency_hiding; 2200 fixed20_12 a; 2201 2202 a.full = dfixed_const(1); 2203 if (wm->vsc.full > a.full) 2204 latency_tolerant_lines = 1; 2205 else { 2206 if (lb_partitions <= (wm->vtaps + 1)) 2207 latency_tolerant_lines = 1; 2208 else 2209 latency_tolerant_lines = 2; 2210 } 2211 2212 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2213 2214 if (dce6_latency_watermark(wm) <= latency_hiding) 2215 return true; 2216 else 2217 return false; 2218 } 2219 2220 static void dce6_program_watermarks(struct radeon_device *rdev, 2221 struct radeon_crtc *radeon_crtc, 2222 u32 lb_size, u32 num_heads) 2223 { 2224 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2225 struct dce6_wm_params wm_low, wm_high; 2226 u32 dram_channels; 2227 u32 pixel_period; 2228 u32 line_time = 0; 2229 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2230 u32 priority_a_mark = 0, priority_b_mark = 0; 2231 u32 priority_a_cnt = PRIORITY_OFF; 2232 u32 priority_b_cnt = PRIORITY_OFF; 2233 u32 tmp, arb_control3; 2234 fixed20_12 a, b, c; 2235 2236 if (radeon_crtc->base.enabled && num_heads && mode) { 2237 pixel_period = 1000000 / (u32)mode->clock; 2238 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 2239 priority_a_cnt = 0; 2240 priority_b_cnt = 0; 2241 2242 if (rdev->family == CHIP_ARUBA) 2243 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2244 else 2245 dram_channels = si_get_number_of_dram_channels(rdev); 2246 2247 /* watermark for high clocks */ 2248 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2249 wm_high.yclk = 2250 radeon_dpm_get_mclk(rdev, false) * 10; 2251 wm_high.sclk = 2252 radeon_dpm_get_sclk(rdev, false) * 10; 2253 } else { 2254 wm_high.yclk = rdev->pm.current_mclk * 10; 2255 wm_high.sclk = rdev->pm.current_sclk * 10; 2256 } 2257 2258 wm_high.disp_clk = mode->clock; 2259 wm_high.src_width = mode->crtc_hdisplay; 2260 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 2261 wm_high.blank_time = line_time - wm_high.active_time; 2262 wm_high.interlaced = false; 2263 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2264 wm_high.interlaced = true; 2265 wm_high.vsc = radeon_crtc->vsc; 2266 wm_high.vtaps = 1; 2267 if (radeon_crtc->rmx_type != RMX_OFF) 2268 wm_high.vtaps = 2; 2269 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2270 wm_high.lb_size = lb_size; 2271 wm_high.dram_channels = dram_channels; 2272 wm_high.num_heads = num_heads; 2273 2274 /* watermark for low clocks */ 2275 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2276 wm_low.yclk = 2277 radeon_dpm_get_mclk(rdev, true) * 10; 2278 wm_low.sclk = 2279 radeon_dpm_get_sclk(rdev, true) * 10; 2280 } else { 2281 wm_low.yclk = rdev->pm.current_mclk * 10; 2282 wm_low.sclk = rdev->pm.current_sclk * 10; 2283 } 2284 2285 wm_low.disp_clk = mode->clock; 2286 wm_low.src_width = mode->crtc_hdisplay; 2287 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 2288 wm_low.blank_time = line_time - wm_low.active_time; 2289 wm_low.interlaced = false; 2290 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2291 wm_low.interlaced = true; 2292 wm_low.vsc = radeon_crtc->vsc; 2293 wm_low.vtaps = 1; 2294 if (radeon_crtc->rmx_type != RMX_OFF) 2295 wm_low.vtaps = 2; 2296 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2297 wm_low.lb_size = lb_size; 2298 wm_low.dram_channels = dram_channels; 2299 wm_low.num_heads = num_heads; 2300 2301 /* set for high clocks */ 2302 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535); 2303 /* set for low clocks */ 2304 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535); 2305 2306 /* possibly force display priority to high */ 2307 /* should really do this at mode validation time... */ 2308 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2309 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) || 2310 !dce6_check_latency_hiding(&wm_high) || 2311 (rdev->disp_priority == 2)) { 2312 DRM_DEBUG_KMS("force priority to high\n"); 2313 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2314 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2315 } 2316 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2317 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) || 2318 !dce6_check_latency_hiding(&wm_low) || 2319 (rdev->disp_priority == 2)) { 2320 DRM_DEBUG_KMS("force priority to high\n"); 2321 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2322 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2323 } 2324 2325 a.full = dfixed_const(1000); 2326 b.full = dfixed_const(mode->clock); 2327 b.full = dfixed_div(b, a); 2328 c.full = dfixed_const(latency_watermark_a); 2329 c.full = dfixed_mul(c, b); 2330 c.full = dfixed_mul(c, radeon_crtc->hsc); 2331 c.full = dfixed_div(c, a); 2332 a.full = dfixed_const(16); 2333 c.full = dfixed_div(c, a); 2334 priority_a_mark = dfixed_trunc(c); 2335 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2336 2337 a.full = dfixed_const(1000); 2338 b.full = dfixed_const(mode->clock); 2339 b.full = dfixed_div(b, a); 2340 c.full = dfixed_const(latency_watermark_b); 2341 c.full = dfixed_mul(c, b); 2342 c.full = dfixed_mul(c, radeon_crtc->hsc); 2343 c.full = dfixed_div(c, a); 2344 a.full = dfixed_const(16); 2345 c.full = dfixed_div(c, a); 2346 priority_b_mark = dfixed_trunc(c); 2347 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2348 } 2349 2350 /* select wm A */ 2351 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2352 tmp = arb_control3; 2353 tmp &= ~LATENCY_WATERMARK_MASK(3); 2354 tmp |= LATENCY_WATERMARK_MASK(1); 2355 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2356 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2357 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2358 LATENCY_HIGH_WATERMARK(line_time))); 2359 /* select wm B */ 2360 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2361 tmp &= ~LATENCY_WATERMARK_MASK(3); 2362 tmp |= LATENCY_WATERMARK_MASK(2); 2363 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2364 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2365 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2366 LATENCY_HIGH_WATERMARK(line_time))); 2367 /* restore original selection */ 2368 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3); 2369 2370 /* write the priority marks */ 2371 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2372 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2373 2374 /* save values for DPM */ 2375 radeon_crtc->line_time = line_time; 2376 radeon_crtc->wm_high = latency_watermark_a; 2377 radeon_crtc->wm_low = latency_watermark_b; 2378 } 2379 2380 void dce6_bandwidth_update(struct radeon_device *rdev) 2381 { 2382 struct drm_display_mode *mode0 = NULL; 2383 struct drm_display_mode *mode1 = NULL; 2384 u32 num_heads = 0, lb_size; 2385 int i; 2386 2387 radeon_update_display_priority(rdev); 2388 2389 for (i = 0; i < rdev->num_crtc; i++) { 2390 if (rdev->mode_info.crtcs[i]->base.enabled) 2391 num_heads++; 2392 } 2393 for (i = 0; i < rdev->num_crtc; i += 2) { 2394 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2395 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2396 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2397 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2398 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2399 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2400 } 2401 } 2402 2403 /* 2404 * Core functions 2405 */ 2406 static void si_tiling_mode_table_init(struct radeon_device *rdev) 2407 { 2408 const u32 num_tile_mode_states = 32; 2409 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2410 2411 switch (rdev->config.si.mem_row_size_in_kb) { 2412 case 1: 2413 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2414 break; 2415 case 2: 2416 default: 2417 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2418 break; 2419 case 4: 2420 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2421 break; 2422 } 2423 2424 if ((rdev->family == CHIP_TAHITI) || 2425 (rdev->family == CHIP_PITCAIRN)) { 2426 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2427 switch (reg_offset) { 2428 case 0: /* non-AA compressed depth or any compressed stencil */ 2429 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2430 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2431 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2432 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2433 NUM_BANKS(ADDR_SURF_16_BANK) | 2434 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2437 break; 2438 case 1: /* 2xAA/4xAA compressed depth only */ 2439 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2440 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2441 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2443 NUM_BANKS(ADDR_SURF_16_BANK) | 2444 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2447 break; 2448 case 2: /* 8xAA compressed depth only */ 2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2450 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2451 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2453 NUM_BANKS(ADDR_SURF_16_BANK) | 2454 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2455 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2456 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2457 break; 2458 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2459 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2460 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2461 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2462 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2463 NUM_BANKS(ADDR_SURF_16_BANK) | 2464 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2467 break; 2468 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2469 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2470 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2471 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2473 NUM_BANKS(ADDR_SURF_16_BANK) | 2474 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2475 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2476 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2477 break; 2478 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2479 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2480 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2481 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2482 TILE_SPLIT(split_equal_to_row_size) | 2483 NUM_BANKS(ADDR_SURF_16_BANK) | 2484 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2487 break; 2488 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2489 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2490 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2491 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2492 TILE_SPLIT(split_equal_to_row_size) | 2493 NUM_BANKS(ADDR_SURF_16_BANK) | 2494 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2495 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2496 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2497 break; 2498 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2499 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2500 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2501 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2502 TILE_SPLIT(split_equal_to_row_size) | 2503 NUM_BANKS(ADDR_SURF_16_BANK) | 2504 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2507 break; 2508 case 8: /* 1D and 1D Array Surfaces */ 2509 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2510 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2513 NUM_BANKS(ADDR_SURF_16_BANK) | 2514 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2515 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2516 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2517 break; 2518 case 9: /* Displayable maps. */ 2519 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2520 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2521 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2522 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2523 NUM_BANKS(ADDR_SURF_16_BANK) | 2524 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2525 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2526 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2527 break; 2528 case 10: /* Display 8bpp. */ 2529 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2530 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2531 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2532 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2533 NUM_BANKS(ADDR_SURF_16_BANK) | 2534 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2535 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2536 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2537 break; 2538 case 11: /* Display 16bpp. */ 2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2540 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2543 NUM_BANKS(ADDR_SURF_16_BANK) | 2544 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2545 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2546 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2547 break; 2548 case 12: /* Display 32bpp. */ 2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2550 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2552 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2553 NUM_BANKS(ADDR_SURF_16_BANK) | 2554 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2555 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2556 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2557 break; 2558 case 13: /* Thin. */ 2559 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2560 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2562 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2563 NUM_BANKS(ADDR_SURF_16_BANK) | 2564 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2565 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2566 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2567 break; 2568 case 14: /* Thin 8 bpp. */ 2569 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2570 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2571 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2572 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2573 NUM_BANKS(ADDR_SURF_16_BANK) | 2574 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2575 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2576 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2577 break; 2578 case 15: /* Thin 16 bpp. */ 2579 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2580 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2582 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2583 NUM_BANKS(ADDR_SURF_16_BANK) | 2584 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2585 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2586 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2587 break; 2588 case 16: /* Thin 32 bpp. */ 2589 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2590 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2592 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2593 NUM_BANKS(ADDR_SURF_16_BANK) | 2594 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2595 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2596 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2597 break; 2598 case 17: /* Thin 64 bpp. */ 2599 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2600 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2601 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2602 TILE_SPLIT(split_equal_to_row_size) | 2603 NUM_BANKS(ADDR_SURF_16_BANK) | 2604 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2605 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2606 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2607 break; 2608 case 21: /* 8 bpp PRT. */ 2609 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2610 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2611 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2612 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2613 NUM_BANKS(ADDR_SURF_16_BANK) | 2614 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2615 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2616 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2617 break; 2618 case 22: /* 16 bpp PRT */ 2619 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2620 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2621 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2622 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2623 NUM_BANKS(ADDR_SURF_16_BANK) | 2624 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2625 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2626 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2627 break; 2628 case 23: /* 32 bpp PRT */ 2629 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2630 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2631 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2632 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2633 NUM_BANKS(ADDR_SURF_16_BANK) | 2634 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2635 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2636 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2637 break; 2638 case 24: /* 64 bpp PRT */ 2639 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2640 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2642 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2643 NUM_BANKS(ADDR_SURF_16_BANK) | 2644 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2645 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2646 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2647 break; 2648 case 25: /* 128 bpp PRT */ 2649 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2650 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2651 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2652 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2653 NUM_BANKS(ADDR_SURF_8_BANK) | 2654 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2657 break; 2658 default: 2659 gb_tile_moden = 0; 2660 break; 2661 } 2662 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2664 } 2665 } else if ((rdev->family == CHIP_VERDE) || 2666 (rdev->family == CHIP_OLAND) || 2667 (rdev->family == CHIP_HAINAN)) { 2668 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2669 switch (reg_offset) { 2670 case 0: /* non-AA compressed depth or any compressed stencil */ 2671 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2672 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2673 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2674 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2675 NUM_BANKS(ADDR_SURF_16_BANK) | 2676 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2677 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2678 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2679 break; 2680 case 1: /* 2xAA/4xAA compressed depth only */ 2681 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2682 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2683 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2684 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2685 NUM_BANKS(ADDR_SURF_16_BANK) | 2686 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2689 break; 2690 case 2: /* 8xAA compressed depth only */ 2691 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2692 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2693 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2694 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2695 NUM_BANKS(ADDR_SURF_16_BANK) | 2696 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2699 break; 2700 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2701 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2702 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2703 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2704 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2705 NUM_BANKS(ADDR_SURF_16_BANK) | 2706 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2707 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2708 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2709 break; 2710 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2711 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2712 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2714 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2715 NUM_BANKS(ADDR_SURF_16_BANK) | 2716 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2719 break; 2720 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2721 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2722 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2723 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2724 TILE_SPLIT(split_equal_to_row_size) | 2725 NUM_BANKS(ADDR_SURF_16_BANK) | 2726 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2729 break; 2730 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2731 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2732 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2733 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2734 TILE_SPLIT(split_equal_to_row_size) | 2735 NUM_BANKS(ADDR_SURF_16_BANK) | 2736 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2737 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2738 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2739 break; 2740 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2741 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2742 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2743 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2744 TILE_SPLIT(split_equal_to_row_size) | 2745 NUM_BANKS(ADDR_SURF_16_BANK) | 2746 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2749 break; 2750 case 8: /* 1D and 1D Array Surfaces */ 2751 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2752 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2753 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2754 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2755 NUM_BANKS(ADDR_SURF_16_BANK) | 2756 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2759 break; 2760 case 9: /* Displayable maps. */ 2761 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2762 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2763 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2765 NUM_BANKS(ADDR_SURF_16_BANK) | 2766 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2767 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2768 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2769 break; 2770 case 10: /* Display 8bpp. */ 2771 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2772 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2773 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2774 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2775 NUM_BANKS(ADDR_SURF_16_BANK) | 2776 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2777 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2778 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2779 break; 2780 case 11: /* Display 16bpp. */ 2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2782 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2783 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2785 NUM_BANKS(ADDR_SURF_16_BANK) | 2786 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2787 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2788 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2789 break; 2790 case 12: /* Display 32bpp. */ 2791 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2792 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2793 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2794 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2795 NUM_BANKS(ADDR_SURF_16_BANK) | 2796 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2797 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2798 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2799 break; 2800 case 13: /* Thin. */ 2801 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2802 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2805 NUM_BANKS(ADDR_SURF_16_BANK) | 2806 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2809 break; 2810 case 14: /* Thin 8 bpp. */ 2811 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2812 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2814 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2815 NUM_BANKS(ADDR_SURF_16_BANK) | 2816 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2817 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2818 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2819 break; 2820 case 15: /* Thin 16 bpp. */ 2821 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2822 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2823 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2824 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2825 NUM_BANKS(ADDR_SURF_16_BANK) | 2826 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2827 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2828 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2829 break; 2830 case 16: /* Thin 32 bpp. */ 2831 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2832 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2833 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2834 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2835 NUM_BANKS(ADDR_SURF_16_BANK) | 2836 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2837 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2838 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2839 break; 2840 case 17: /* Thin 64 bpp. */ 2841 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2842 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2843 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2844 TILE_SPLIT(split_equal_to_row_size) | 2845 NUM_BANKS(ADDR_SURF_16_BANK) | 2846 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2847 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2848 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2849 break; 2850 case 21: /* 8 bpp PRT. */ 2851 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2852 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2853 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2854 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2855 NUM_BANKS(ADDR_SURF_16_BANK) | 2856 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2859 break; 2860 case 22: /* 16 bpp PRT */ 2861 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2862 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2863 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2864 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2865 NUM_BANKS(ADDR_SURF_16_BANK) | 2866 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2867 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2868 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2869 break; 2870 case 23: /* 32 bpp PRT */ 2871 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2872 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2873 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2874 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2875 NUM_BANKS(ADDR_SURF_16_BANK) | 2876 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2877 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2878 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2879 break; 2880 case 24: /* 64 bpp PRT */ 2881 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2882 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2883 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2884 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2885 NUM_BANKS(ADDR_SURF_16_BANK) | 2886 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2887 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2888 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2889 break; 2890 case 25: /* 128 bpp PRT */ 2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2892 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2893 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2895 NUM_BANKS(ADDR_SURF_8_BANK) | 2896 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2897 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2898 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2899 break; 2900 default: 2901 gb_tile_moden = 0; 2902 break; 2903 } 2904 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2905 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2906 } 2907 } else 2908 DRM_ERROR("unknown asic: 0x%x\n", rdev->family); 2909 } 2910 2911 static void si_select_se_sh(struct radeon_device *rdev, 2912 u32 se_num, u32 sh_num) 2913 { 2914 u32 data = INSTANCE_BROADCAST_WRITES; 2915 2916 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 2917 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 2918 else if (se_num == 0xffffffff) 2919 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 2920 else if (sh_num == 0xffffffff) 2921 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 2922 else 2923 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 2924 WREG32(GRBM_GFX_INDEX, data); 2925 } 2926 2927 static u32 si_create_bitmask(u32 bit_width) 2928 { 2929 u32 i, mask = 0; 2930 2931 for (i = 0; i < bit_width; i++) { 2932 mask <<= 1; 2933 mask |= 1; 2934 } 2935 return mask; 2936 } 2937 2938 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh) 2939 { 2940 u32 data, mask; 2941 2942 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 2943 if (data & 1) 2944 data &= INACTIVE_CUS_MASK; 2945 else 2946 data = 0; 2947 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG); 2948 2949 data >>= INACTIVE_CUS_SHIFT; 2950 2951 mask = si_create_bitmask(cu_per_sh); 2952 2953 return ~data & mask; 2954 } 2955 2956 static void si_setup_spi(struct radeon_device *rdev, 2957 u32 se_num, u32 sh_per_se, 2958 u32 cu_per_sh) 2959 { 2960 int i, j, k; 2961 u32 data, mask, active_cu; 2962 2963 for (i = 0; i < se_num; i++) { 2964 for (j = 0; j < sh_per_se; j++) { 2965 si_select_se_sh(rdev, i, j); 2966 data = RREG32(SPI_STATIC_THREAD_MGMT_3); 2967 active_cu = si_get_cu_enabled(rdev, cu_per_sh); 2968 2969 mask = 1; 2970 for (k = 0; k < 16; k++) { 2971 mask <<= k; 2972 if (active_cu & mask) { 2973 data &= ~mask; 2974 WREG32(SPI_STATIC_THREAD_MGMT_3, data); 2975 break; 2976 } 2977 } 2978 } 2979 } 2980 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2981 } 2982 2983 static u32 si_get_rb_disabled(struct radeon_device *rdev, 2984 u32 max_rb_num_per_se, 2985 u32 sh_per_se) 2986 { 2987 u32 data, mask; 2988 2989 data = RREG32(CC_RB_BACKEND_DISABLE); 2990 if (data & 1) 2991 data &= BACKEND_DISABLE_MASK; 2992 else 2993 data = 0; 2994 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 2995 2996 data >>= BACKEND_DISABLE_SHIFT; 2997 2998 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se); 2999 3000 return data & mask; 3001 } 3002 3003 static void si_setup_rb(struct radeon_device *rdev, 3004 u32 se_num, u32 sh_per_se, 3005 u32 max_rb_num_per_se) 3006 { 3007 int i, j; 3008 u32 data, mask; 3009 u32 disabled_rbs = 0; 3010 u32 enabled_rbs = 0; 3011 3012 for (i = 0; i < se_num; i++) { 3013 for (j = 0; j < sh_per_se; j++) { 3014 si_select_se_sh(rdev, i, j); 3015 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 3016 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH); 3017 } 3018 } 3019 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3020 3021 mask = 1; 3022 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 3023 if (!(disabled_rbs & mask)) 3024 enabled_rbs |= mask; 3025 mask <<= 1; 3026 } 3027 3028 rdev->config.si.backend_enable_mask = enabled_rbs; 3029 3030 for (i = 0; i < se_num; i++) { 3031 si_select_se_sh(rdev, i, 0xffffffff); 3032 data = 0; 3033 for (j = 0; j < sh_per_se; j++) { 3034 switch (enabled_rbs & 3) { 3035 case 1: 3036 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 3037 break; 3038 case 2: 3039 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 3040 break; 3041 case 3: 3042 default: 3043 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 3044 break; 3045 } 3046 enabled_rbs >>= 2; 3047 } 3048 WREG32(PA_SC_RASTER_CONFIG, data); 3049 } 3050 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 3051 } 3052 3053 static void si_gpu_init(struct radeon_device *rdev) 3054 { 3055 u32 gb_addr_config = 0; 3056 u32 mc_shared_chmap, mc_arb_ramcfg; 3057 u32 sx_debug_1; 3058 u32 hdp_host_path_cntl; 3059 u32 tmp; 3060 int i, j, k; 3061 3062 switch (rdev->family) { 3063 case CHIP_TAHITI: 3064 rdev->config.si.max_shader_engines = 2; 3065 rdev->config.si.max_tile_pipes = 12; 3066 rdev->config.si.max_cu_per_sh = 8; 3067 rdev->config.si.max_sh_per_se = 2; 3068 rdev->config.si.max_backends_per_se = 4; 3069 rdev->config.si.max_texture_channel_caches = 12; 3070 rdev->config.si.max_gprs = 256; 3071 rdev->config.si.max_gs_threads = 32; 3072 rdev->config.si.max_hw_contexts = 8; 3073 3074 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 3075 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 3076 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 3077 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 3078 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 3079 break; 3080 case CHIP_PITCAIRN: 3081 rdev->config.si.max_shader_engines = 2; 3082 rdev->config.si.max_tile_pipes = 8; 3083 rdev->config.si.max_cu_per_sh = 5; 3084 rdev->config.si.max_sh_per_se = 2; 3085 rdev->config.si.max_backends_per_se = 4; 3086 rdev->config.si.max_texture_channel_caches = 8; 3087 rdev->config.si.max_gprs = 256; 3088 rdev->config.si.max_gs_threads = 32; 3089 rdev->config.si.max_hw_contexts = 8; 3090 3091 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 3092 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 3093 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 3094 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 3095 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 3096 break; 3097 case CHIP_VERDE: 3098 default: 3099 rdev->config.si.max_shader_engines = 1; 3100 rdev->config.si.max_tile_pipes = 4; 3101 rdev->config.si.max_cu_per_sh = 5; 3102 rdev->config.si.max_sh_per_se = 2; 3103 rdev->config.si.max_backends_per_se = 4; 3104 rdev->config.si.max_texture_channel_caches = 4; 3105 rdev->config.si.max_gprs = 256; 3106 rdev->config.si.max_gs_threads = 32; 3107 rdev->config.si.max_hw_contexts = 8; 3108 3109 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 3110 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 3111 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 3112 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 3113 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 3114 break; 3115 case CHIP_OLAND: 3116 rdev->config.si.max_shader_engines = 1; 3117 rdev->config.si.max_tile_pipes = 4; 3118 rdev->config.si.max_cu_per_sh = 6; 3119 rdev->config.si.max_sh_per_se = 1; 3120 rdev->config.si.max_backends_per_se = 2; 3121 rdev->config.si.max_texture_channel_caches = 4; 3122 rdev->config.si.max_gprs = 256; 3123 rdev->config.si.max_gs_threads = 16; 3124 rdev->config.si.max_hw_contexts = 8; 3125 3126 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 3127 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 3128 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 3129 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 3130 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 3131 break; 3132 case CHIP_HAINAN: 3133 rdev->config.si.max_shader_engines = 1; 3134 rdev->config.si.max_tile_pipes = 4; 3135 rdev->config.si.max_cu_per_sh = 5; 3136 rdev->config.si.max_sh_per_se = 1; 3137 rdev->config.si.max_backends_per_se = 1; 3138 rdev->config.si.max_texture_channel_caches = 2; 3139 rdev->config.si.max_gprs = 256; 3140 rdev->config.si.max_gs_threads = 16; 3141 rdev->config.si.max_hw_contexts = 8; 3142 3143 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 3144 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 3145 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 3146 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 3147 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN; 3148 break; 3149 } 3150 3151 /* Initialize HDP */ 3152 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3153 WREG32((0x2c14 + j), 0x00000000); 3154 WREG32((0x2c18 + j), 0x00000000); 3155 WREG32((0x2c1c + j), 0x00000000); 3156 WREG32((0x2c20 + j), 0x00000000); 3157 WREG32((0x2c24 + j), 0x00000000); 3158 } 3159 3160 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3161 3162 evergreen_fix_pci_max_read_req_size(rdev); 3163 3164 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 3165 3166 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3167 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3168 3169 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes; 3170 rdev->config.si.mem_max_burst_length_bytes = 256; 3171 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3172 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3173 if (rdev->config.si.mem_row_size_in_kb > 4) 3174 rdev->config.si.mem_row_size_in_kb = 4; 3175 /* XXX use MC settings? */ 3176 rdev->config.si.shader_engine_tile_size = 32; 3177 rdev->config.si.num_gpus = 1; 3178 rdev->config.si.multi_gpu_tile_size = 64; 3179 3180 /* fix up row size */ 3181 gb_addr_config &= ~ROW_SIZE_MASK; 3182 switch (rdev->config.si.mem_row_size_in_kb) { 3183 case 1: 3184 default: 3185 gb_addr_config |= ROW_SIZE(0); 3186 break; 3187 case 2: 3188 gb_addr_config |= ROW_SIZE(1); 3189 break; 3190 case 4: 3191 gb_addr_config |= ROW_SIZE(2); 3192 break; 3193 } 3194 3195 /* setup tiling info dword. gb_addr_config is not adequate since it does 3196 * not have bank info, so create a custom tiling dword. 3197 * bits 3:0 num_pipes 3198 * bits 7:4 num_banks 3199 * bits 11:8 group_size 3200 * bits 15:12 row_size 3201 */ 3202 rdev->config.si.tile_config = 0; 3203 switch (rdev->config.si.num_tile_pipes) { 3204 case 1: 3205 rdev->config.si.tile_config |= (0 << 0); 3206 break; 3207 case 2: 3208 rdev->config.si.tile_config |= (1 << 0); 3209 break; 3210 case 4: 3211 rdev->config.si.tile_config |= (2 << 0); 3212 break; 3213 case 8: 3214 default: 3215 /* XXX what about 12? */ 3216 rdev->config.si.tile_config |= (3 << 0); 3217 break; 3218 } 3219 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3220 case 0: /* four banks */ 3221 rdev->config.si.tile_config |= 0 << 4; 3222 break; 3223 case 1: /* eight banks */ 3224 rdev->config.si.tile_config |= 1 << 4; 3225 break; 3226 case 2: /* sixteen banks */ 3227 default: 3228 rdev->config.si.tile_config |= 2 << 4; 3229 break; 3230 } 3231 rdev->config.si.tile_config |= 3232 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3233 rdev->config.si.tile_config |= 3234 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3235 3236 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3237 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3238 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3239 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3240 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); 3241 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); 3242 if (rdev->has_uvd) { 3243 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3244 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3245 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3246 } 3247 3248 si_tiling_mode_table_init(rdev); 3249 3250 si_setup_rb(rdev, rdev->config.si.max_shader_engines, 3251 rdev->config.si.max_sh_per_se, 3252 rdev->config.si.max_backends_per_se); 3253 3254 si_setup_spi(rdev, rdev->config.si.max_shader_engines, 3255 rdev->config.si.max_sh_per_se, 3256 rdev->config.si.max_cu_per_sh); 3257 3258 for (i = 0; i < rdev->config.si.max_shader_engines; i++) { 3259 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { 3260 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) { 3261 rdev->config.si.active_cus += 3262 hweight32(si_get_cu_active_bitmap(rdev, i, j)); 3263 } 3264 } 3265 } 3266 3267 /* set HW defaults for 3D engine */ 3268 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3269 ROQ_IB2_START(0x2b))); 3270 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3271 3272 sx_debug_1 = RREG32(SX_DEBUG_1); 3273 WREG32(SX_DEBUG_1, sx_debug_1); 3274 3275 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3276 3277 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) | 3278 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) | 3279 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) | 3280 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size))); 3281 3282 WREG32(VGT_NUM_INSTANCES, 1); 3283 3284 WREG32(CP_PERFMON_CNTL, 0); 3285 3286 WREG32(SQ_CONFIG, 0); 3287 3288 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3289 FORCE_EOV_MAX_REZ_CNT(255))); 3290 3291 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3292 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3293 3294 WREG32(VGT_GS_VERTEX_REUSE, 16); 3295 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3296 3297 WREG32(CB_PERFCOUNTER0_SELECT0, 0); 3298 WREG32(CB_PERFCOUNTER0_SELECT1, 0); 3299 WREG32(CB_PERFCOUNTER1_SELECT0, 0); 3300 WREG32(CB_PERFCOUNTER1_SELECT1, 0); 3301 WREG32(CB_PERFCOUNTER2_SELECT0, 0); 3302 WREG32(CB_PERFCOUNTER2_SELECT1, 0); 3303 WREG32(CB_PERFCOUNTER3_SELECT0, 0); 3304 WREG32(CB_PERFCOUNTER3_SELECT1, 0); 3305 3306 tmp = RREG32(HDP_MISC_CNTL); 3307 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3308 WREG32(HDP_MISC_CNTL, tmp); 3309 3310 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3311 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3312 3313 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3314 3315 udelay(50); 3316 } 3317 3318 /* 3319 * GPU scratch registers helpers function. 3320 */ 3321 static void si_scratch_init(struct radeon_device *rdev) 3322 { 3323 int i; 3324 3325 rdev->scratch.num_reg = 7; 3326 rdev->scratch.reg_base = SCRATCH_REG0; 3327 for (i = 0; i < rdev->scratch.num_reg; i++) { 3328 rdev->scratch.free[i] = true; 3329 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3330 } 3331 } 3332 3333 void si_fence_ring_emit(struct radeon_device *rdev, 3334 struct radeon_fence *fence) 3335 { 3336 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3337 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3338 3339 /* flush read cache over gart */ 3340 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3341 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3342 radeon_ring_write(ring, 0); 3343 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3344 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3345 PACKET3_TC_ACTION_ENA | 3346 PACKET3_SH_KCACHE_ACTION_ENA | 3347 PACKET3_SH_ICACHE_ACTION_ENA); 3348 radeon_ring_write(ring, 0xFFFFFFFF); 3349 radeon_ring_write(ring, 0); 3350 radeon_ring_write(ring, 10); /* poll interval */ 3351 /* EVENT_WRITE_EOP - flush caches, send int */ 3352 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3353 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5)); 3354 radeon_ring_write(ring, lower_32_bits(addr)); 3355 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); 3356 radeon_ring_write(ring, fence->seq); 3357 radeon_ring_write(ring, 0); 3358 } 3359 3360 /* 3361 * IB stuff 3362 */ 3363 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3364 { 3365 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3366 u32 header; 3367 3368 if (ib->is_const_ib) { 3369 /* set switch buffer packet before const IB */ 3370 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 3371 radeon_ring_write(ring, 0); 3372 3373 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 3374 } else { 3375 u32 next_rptr; 3376 if (ring->rptr_save_reg) { 3377 next_rptr = ring->wptr + 3 + 4 + 8; 3378 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3379 radeon_ring_write(ring, ((ring->rptr_save_reg - 3380 PACKET3_SET_CONFIG_REG_START) >> 2)); 3381 radeon_ring_write(ring, next_rptr); 3382 } else if (rdev->wb.enabled) { 3383 next_rptr = ring->wptr + 5 + 4 + 8; 3384 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3385 radeon_ring_write(ring, (1 << 8)); 3386 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3387 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); 3388 radeon_ring_write(ring, next_rptr); 3389 } 3390 3391 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 3392 } 3393 3394 radeon_ring_write(ring, header); 3395 radeon_ring_write(ring, 3396 #ifdef __BIG_ENDIAN 3397 (2 << 0) | 3398 #endif 3399 (ib->gpu_addr & 0xFFFFFFFC)); 3400 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 3401 radeon_ring_write(ring, ib->length_dw | 3402 (ib->vm ? (ib->vm->id << 24) : 0)); 3403 3404 if (!ib->is_const_ib) { 3405 /* flush read cache over gart for this vmid */ 3406 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3407 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3408 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); 3409 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3410 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3411 PACKET3_TC_ACTION_ENA | 3412 PACKET3_SH_KCACHE_ACTION_ENA | 3413 PACKET3_SH_ICACHE_ACTION_ENA); 3414 radeon_ring_write(ring, 0xFFFFFFFF); 3415 radeon_ring_write(ring, 0); 3416 radeon_ring_write(ring, 10); /* poll interval */ 3417 } 3418 } 3419 3420 /* 3421 * CP. 3422 */ 3423 static void si_cp_enable(struct radeon_device *rdev, bool enable) 3424 { 3425 if (enable) 3426 WREG32(CP_ME_CNTL, 0); 3427 else { 3428 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 3429 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 3430 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 3431 WREG32(SCRATCH_UMSK, 0); 3432 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3433 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3434 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3435 } 3436 udelay(50); 3437 } 3438 3439 static int si_cp_load_microcode(struct radeon_device *rdev) 3440 { 3441 int i; 3442 3443 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) 3444 return -EINVAL; 3445 3446 si_cp_enable(rdev, false); 3447 3448 if (rdev->new_fw) { 3449 const struct gfx_firmware_header_v1_0 *pfp_hdr = 3450 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; 3451 const struct gfx_firmware_header_v1_0 *ce_hdr = 3452 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; 3453 const struct gfx_firmware_header_v1_0 *me_hdr = 3454 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; 3455 const __le32 *fw_data; 3456 u32 fw_size; 3457 3458 radeon_ucode_print_gfx_hdr(&pfp_hdr->header); 3459 radeon_ucode_print_gfx_hdr(&ce_hdr->header); 3460 radeon_ucode_print_gfx_hdr(&me_hdr->header); 3461 3462 /* PFP */ 3463 fw_data = (const __le32 *) 3464 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3465 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3466 WREG32(CP_PFP_UCODE_ADDR, 0); 3467 for (i = 0; i < fw_size; i++) 3468 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3469 WREG32(CP_PFP_UCODE_ADDR, 0); 3470 3471 /* CE */ 3472 fw_data = (const __le32 *) 3473 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3474 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3475 WREG32(CP_CE_UCODE_ADDR, 0); 3476 for (i = 0; i < fw_size; i++) 3477 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3478 WREG32(CP_CE_UCODE_ADDR, 0); 3479 3480 /* ME */ 3481 fw_data = (const __be32 *) 3482 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3483 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3484 WREG32(CP_ME_RAM_WADDR, 0); 3485 for (i = 0; i < fw_size; i++) 3486 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3487 WREG32(CP_ME_RAM_WADDR, 0); 3488 } else { 3489 const __be32 *fw_data; 3490 3491 /* PFP */ 3492 fw_data = (const __be32 *)rdev->pfp_fw->data; 3493 WREG32(CP_PFP_UCODE_ADDR, 0); 3494 for (i = 0; i < SI_PFP_UCODE_SIZE; i++) 3495 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3496 WREG32(CP_PFP_UCODE_ADDR, 0); 3497 3498 /* CE */ 3499 fw_data = (const __be32 *)rdev->ce_fw->data; 3500 WREG32(CP_CE_UCODE_ADDR, 0); 3501 for (i = 0; i < SI_CE_UCODE_SIZE; i++) 3502 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 3503 WREG32(CP_CE_UCODE_ADDR, 0); 3504 3505 /* ME */ 3506 fw_data = (const __be32 *)rdev->me_fw->data; 3507 WREG32(CP_ME_RAM_WADDR, 0); 3508 for (i = 0; i < SI_PM4_UCODE_SIZE; i++) 3509 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3510 WREG32(CP_ME_RAM_WADDR, 0); 3511 } 3512 3513 WREG32(CP_PFP_UCODE_ADDR, 0); 3514 WREG32(CP_CE_UCODE_ADDR, 0); 3515 WREG32(CP_ME_RAM_WADDR, 0); 3516 WREG32(CP_ME_RAM_RADDR, 0); 3517 return 0; 3518 } 3519 3520 static int si_cp_start(struct radeon_device *rdev) 3521 { 3522 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3523 int r, i; 3524 3525 r = radeon_ring_lock(rdev, ring, 7 + 4); 3526 if (r) { 3527 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3528 return r; 3529 } 3530 /* init the CP */ 3531 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3532 radeon_ring_write(ring, 0x1); 3533 radeon_ring_write(ring, 0x0); 3534 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1); 3535 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3536 radeon_ring_write(ring, 0); 3537 radeon_ring_write(ring, 0); 3538 3539 /* init the CE partitions */ 3540 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3541 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3542 radeon_ring_write(ring, 0xc000); 3543 radeon_ring_write(ring, 0xe000); 3544 radeon_ring_unlock_commit(rdev, ring); 3545 3546 si_cp_enable(rdev, true); 3547 3548 r = radeon_ring_lock(rdev, ring, si_default_size + 10); 3549 if (r) { 3550 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3551 return r; 3552 } 3553 3554 /* setup clear context state */ 3555 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3556 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3557 3558 for (i = 0; i < si_default_size; i++) 3559 radeon_ring_write(ring, si_default_state[i]); 3560 3561 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3562 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3563 3564 /* set clear context state */ 3565 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3566 radeon_ring_write(ring, 0); 3567 3568 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 3569 radeon_ring_write(ring, 0x00000316); 3570 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3571 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 3572 3573 radeon_ring_unlock_commit(rdev, ring); 3574 3575 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) { 3576 ring = &rdev->ring[i]; 3577 r = radeon_ring_lock(rdev, ring, 2); 3578 3579 /* clear the compute context state */ 3580 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0)); 3581 radeon_ring_write(ring, 0); 3582 3583 radeon_ring_unlock_commit(rdev, ring); 3584 } 3585 3586 return 0; 3587 } 3588 3589 static void si_cp_fini(struct radeon_device *rdev) 3590 { 3591 struct radeon_ring *ring; 3592 si_cp_enable(rdev, false); 3593 3594 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3595 radeon_ring_fini(rdev, ring); 3596 radeon_scratch_free(rdev, ring->rptr_save_reg); 3597 3598 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3599 radeon_ring_fini(rdev, ring); 3600 radeon_scratch_free(rdev, ring->rptr_save_reg); 3601 3602 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3603 radeon_ring_fini(rdev, ring); 3604 radeon_scratch_free(rdev, ring->rptr_save_reg); 3605 } 3606 3607 static int si_cp_resume(struct radeon_device *rdev) 3608 { 3609 struct radeon_ring *ring; 3610 u32 tmp; 3611 u32 rb_bufsz; 3612 int r; 3613 3614 si_enable_gui_idle_interrupt(rdev, false); 3615 3616 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3617 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3618 3619 /* Set the write pointer delay */ 3620 WREG32(CP_RB_WPTR_DELAY, 0); 3621 3622 WREG32(CP_DEBUG, 0); 3623 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3624 3625 /* ring 0 - compute and gfx */ 3626 /* Set ring buffer size */ 3627 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3628 rb_bufsz = order_base_2(ring->ring_size / 8); 3629 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3630 #ifdef __BIG_ENDIAN 3631 tmp |= BUF_SWAP_32BIT; 3632 #endif 3633 WREG32(CP_RB0_CNTL, tmp); 3634 3635 /* Initialize the ring buffer's read and write pointers */ 3636 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 3637 ring->wptr = 0; 3638 WREG32(CP_RB0_WPTR, ring->wptr); 3639 3640 /* set the wb address whether it's enabled or not */ 3641 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 3642 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3643 3644 if (rdev->wb.enabled) 3645 WREG32(SCRATCH_UMSK, 0xff); 3646 else { 3647 tmp |= RB_NO_UPDATE; 3648 WREG32(SCRATCH_UMSK, 0); 3649 } 3650 3651 mdelay(1); 3652 WREG32(CP_RB0_CNTL, tmp); 3653 3654 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8); 3655 3656 /* ring1 - compute only */ 3657 /* Set ring buffer size */ 3658 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3659 rb_bufsz = order_base_2(ring->ring_size / 8); 3660 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3661 #ifdef __BIG_ENDIAN 3662 tmp |= BUF_SWAP_32BIT; 3663 #endif 3664 WREG32(CP_RB1_CNTL, tmp); 3665 3666 /* Initialize the ring buffer's read and write pointers */ 3667 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA); 3668 ring->wptr = 0; 3669 WREG32(CP_RB1_WPTR, ring->wptr); 3670 3671 /* set the wb address whether it's enabled or not */ 3672 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC); 3673 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF); 3674 3675 mdelay(1); 3676 WREG32(CP_RB1_CNTL, tmp); 3677 3678 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8); 3679 3680 /* ring2 - compute only */ 3681 /* Set ring buffer size */ 3682 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3683 rb_bufsz = order_base_2(ring->ring_size / 8); 3684 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3685 #ifdef __BIG_ENDIAN 3686 tmp |= BUF_SWAP_32BIT; 3687 #endif 3688 WREG32(CP_RB2_CNTL, tmp); 3689 3690 /* Initialize the ring buffer's read and write pointers */ 3691 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA); 3692 ring->wptr = 0; 3693 WREG32(CP_RB2_WPTR, ring->wptr); 3694 3695 /* set the wb address whether it's enabled or not */ 3696 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC); 3697 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF); 3698 3699 mdelay(1); 3700 WREG32(CP_RB2_CNTL, tmp); 3701 3702 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8); 3703 3704 /* start the rings */ 3705 si_cp_start(rdev); 3706 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 3707 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true; 3708 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true; 3709 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 3710 if (r) { 3711 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3712 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3713 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3714 return r; 3715 } 3716 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); 3717 if (r) { 3718 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3719 } 3720 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); 3721 if (r) { 3722 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3723 } 3724 3725 si_enable_gui_idle_interrupt(rdev, true); 3726 3727 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 3728 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 3729 3730 return 0; 3731 } 3732 3733 u32 si_gpu_check_soft_reset(struct radeon_device *rdev) 3734 { 3735 u32 reset_mask = 0; 3736 u32 tmp; 3737 3738 /* GRBM_STATUS */ 3739 tmp = RREG32(GRBM_STATUS); 3740 if (tmp & (PA_BUSY | SC_BUSY | 3741 BCI_BUSY | SX_BUSY | 3742 TA_BUSY | VGT_BUSY | 3743 DB_BUSY | CB_BUSY | 3744 GDS_BUSY | SPI_BUSY | 3745 IA_BUSY | IA_BUSY_NO_DMA)) 3746 reset_mask |= RADEON_RESET_GFX; 3747 3748 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3749 CP_BUSY | CP_COHERENCY_BUSY)) 3750 reset_mask |= RADEON_RESET_CP; 3751 3752 if (tmp & GRBM_EE_BUSY) 3753 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3754 3755 /* GRBM_STATUS2 */ 3756 tmp = RREG32(GRBM_STATUS2); 3757 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3758 reset_mask |= RADEON_RESET_RLC; 3759 3760 /* DMA_STATUS_REG 0 */ 3761 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); 3762 if (!(tmp & DMA_IDLE)) 3763 reset_mask |= RADEON_RESET_DMA; 3764 3765 /* DMA_STATUS_REG 1 */ 3766 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); 3767 if (!(tmp & DMA_IDLE)) 3768 reset_mask |= RADEON_RESET_DMA1; 3769 3770 /* SRBM_STATUS2 */ 3771 tmp = RREG32(SRBM_STATUS2); 3772 if (tmp & DMA_BUSY) 3773 reset_mask |= RADEON_RESET_DMA; 3774 3775 if (tmp & DMA1_BUSY) 3776 reset_mask |= RADEON_RESET_DMA1; 3777 3778 /* SRBM_STATUS */ 3779 tmp = RREG32(SRBM_STATUS); 3780 3781 if (tmp & IH_BUSY) 3782 reset_mask |= RADEON_RESET_IH; 3783 3784 if (tmp & SEM_BUSY) 3785 reset_mask |= RADEON_RESET_SEM; 3786 3787 if (tmp & GRBM_RQ_PENDING) 3788 reset_mask |= RADEON_RESET_GRBM; 3789 3790 if (tmp & VMC_BUSY) 3791 reset_mask |= RADEON_RESET_VMC; 3792 3793 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3794 MCC_BUSY | MCD_BUSY)) 3795 reset_mask |= RADEON_RESET_MC; 3796 3797 if (evergreen_is_display_hung(rdev)) 3798 reset_mask |= RADEON_RESET_DISPLAY; 3799 3800 /* VM_L2_STATUS */ 3801 tmp = RREG32(VM_L2_STATUS); 3802 if (tmp & L2_BUSY) 3803 reset_mask |= RADEON_RESET_VMC; 3804 3805 /* Skip MC reset as it's mostly likely not hung, just busy */ 3806 if (reset_mask & RADEON_RESET_MC) { 3807 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3808 reset_mask &= ~RADEON_RESET_MC; 3809 } 3810 3811 return reset_mask; 3812 } 3813 3814 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3815 { 3816 struct evergreen_mc_save save; 3817 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3818 u32 tmp; 3819 3820 if (reset_mask == 0) 3821 return; 3822 3823 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3824 3825 evergreen_print_gpu_status_regs(rdev); 3826 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 3827 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 3828 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 3829 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 3830 3831 /* disable PG/CG */ 3832 si_fini_pg(rdev); 3833 si_fini_cg(rdev); 3834 3835 /* stop the rlc */ 3836 si_rlc_stop(rdev); 3837 3838 /* Disable CP parsing/prefetching */ 3839 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 3840 3841 if (reset_mask & RADEON_RESET_DMA) { 3842 /* dma0 */ 3843 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 3844 tmp &= ~DMA_RB_ENABLE; 3845 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 3846 } 3847 if (reset_mask & RADEON_RESET_DMA1) { 3848 /* dma1 */ 3849 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 3850 tmp &= ~DMA_RB_ENABLE; 3851 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 3852 } 3853 3854 udelay(50); 3855 3856 evergreen_mc_stop(rdev, &save); 3857 if (evergreen_mc_wait_for_idle(rdev)) { 3858 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3859 } 3860 3861 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) { 3862 grbm_soft_reset = SOFT_RESET_CB | 3863 SOFT_RESET_DB | 3864 SOFT_RESET_GDS | 3865 SOFT_RESET_PA | 3866 SOFT_RESET_SC | 3867 SOFT_RESET_BCI | 3868 SOFT_RESET_SPI | 3869 SOFT_RESET_SX | 3870 SOFT_RESET_TC | 3871 SOFT_RESET_TA | 3872 SOFT_RESET_VGT | 3873 SOFT_RESET_IA; 3874 } 3875 3876 if (reset_mask & RADEON_RESET_CP) { 3877 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT; 3878 3879 srbm_soft_reset |= SOFT_RESET_GRBM; 3880 } 3881 3882 if (reset_mask & RADEON_RESET_DMA) 3883 srbm_soft_reset |= SOFT_RESET_DMA; 3884 3885 if (reset_mask & RADEON_RESET_DMA1) 3886 srbm_soft_reset |= SOFT_RESET_DMA1; 3887 3888 if (reset_mask & RADEON_RESET_DISPLAY) 3889 srbm_soft_reset |= SOFT_RESET_DC; 3890 3891 if (reset_mask & RADEON_RESET_RLC) 3892 grbm_soft_reset |= SOFT_RESET_RLC; 3893 3894 if (reset_mask & RADEON_RESET_SEM) 3895 srbm_soft_reset |= SOFT_RESET_SEM; 3896 3897 if (reset_mask & RADEON_RESET_IH) 3898 srbm_soft_reset |= SOFT_RESET_IH; 3899 3900 if (reset_mask & RADEON_RESET_GRBM) 3901 srbm_soft_reset |= SOFT_RESET_GRBM; 3902 3903 if (reset_mask & RADEON_RESET_VMC) 3904 srbm_soft_reset |= SOFT_RESET_VMC; 3905 3906 if (reset_mask & RADEON_RESET_MC) 3907 srbm_soft_reset |= SOFT_RESET_MC; 3908 3909 if (grbm_soft_reset) { 3910 tmp = RREG32(GRBM_SOFT_RESET); 3911 tmp |= grbm_soft_reset; 3912 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3913 WREG32(GRBM_SOFT_RESET, tmp); 3914 tmp = RREG32(GRBM_SOFT_RESET); 3915 3916 udelay(50); 3917 3918 tmp &= ~grbm_soft_reset; 3919 WREG32(GRBM_SOFT_RESET, tmp); 3920 tmp = RREG32(GRBM_SOFT_RESET); 3921 } 3922 3923 if (srbm_soft_reset) { 3924 tmp = RREG32(SRBM_SOFT_RESET); 3925 tmp |= srbm_soft_reset; 3926 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3927 WREG32(SRBM_SOFT_RESET, tmp); 3928 tmp = RREG32(SRBM_SOFT_RESET); 3929 3930 udelay(50); 3931 3932 tmp &= ~srbm_soft_reset; 3933 WREG32(SRBM_SOFT_RESET, tmp); 3934 tmp = RREG32(SRBM_SOFT_RESET); 3935 } 3936 3937 /* Wait a little for things to settle down */ 3938 udelay(50); 3939 3940 evergreen_mc_resume(rdev, &save); 3941 udelay(50); 3942 3943 evergreen_print_gpu_status_regs(rdev); 3944 } 3945 3946 static void si_set_clk_bypass_mode(struct radeon_device *rdev) 3947 { 3948 u32 tmp, i; 3949 3950 tmp = RREG32(CG_SPLL_FUNC_CNTL); 3951 tmp |= SPLL_BYPASS_EN; 3952 WREG32(CG_SPLL_FUNC_CNTL, tmp); 3953 3954 tmp = RREG32(CG_SPLL_FUNC_CNTL_2); 3955 tmp |= SPLL_CTLREQ_CHG; 3956 WREG32(CG_SPLL_FUNC_CNTL_2, tmp); 3957 3958 for (i = 0; i < rdev->usec_timeout; i++) { 3959 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS) 3960 break; 3961 udelay(1); 3962 } 3963 3964 tmp = RREG32(CG_SPLL_FUNC_CNTL_2); 3965 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE); 3966 WREG32(CG_SPLL_FUNC_CNTL_2, tmp); 3967 3968 tmp = RREG32(MPLL_CNTL_MODE); 3969 tmp &= ~MPLL_MCLK_SEL; 3970 WREG32(MPLL_CNTL_MODE, tmp); 3971 } 3972 3973 static void si_spll_powerdown(struct radeon_device *rdev) 3974 { 3975 u32 tmp; 3976 3977 tmp = RREG32(SPLL_CNTL_MODE); 3978 tmp |= SPLL_SW_DIR_CONTROL; 3979 WREG32(SPLL_CNTL_MODE, tmp); 3980 3981 tmp = RREG32(CG_SPLL_FUNC_CNTL); 3982 tmp |= SPLL_RESET; 3983 WREG32(CG_SPLL_FUNC_CNTL, tmp); 3984 3985 tmp = RREG32(CG_SPLL_FUNC_CNTL); 3986 tmp |= SPLL_SLEEP; 3987 WREG32(CG_SPLL_FUNC_CNTL, tmp); 3988 3989 tmp = RREG32(SPLL_CNTL_MODE); 3990 tmp &= ~SPLL_SW_DIR_CONTROL; 3991 WREG32(SPLL_CNTL_MODE, tmp); 3992 } 3993 3994 static void si_gpu_pci_config_reset(struct radeon_device *rdev) 3995 { 3996 struct evergreen_mc_save save; 3997 u32 tmp, i; 3998 3999 dev_info(rdev->dev, "GPU pci config reset\n"); 4000 4001 /* disable dpm? */ 4002 4003 /* disable cg/pg */ 4004 si_fini_pg(rdev); 4005 si_fini_cg(rdev); 4006 4007 /* Disable CP parsing/prefetching */ 4008 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 4009 /* dma0 */ 4010 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 4011 tmp &= ~DMA_RB_ENABLE; 4012 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 4013 /* dma1 */ 4014 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 4015 tmp &= ~DMA_RB_ENABLE; 4016 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 4017 /* XXX other engines? */ 4018 4019 /* halt the rlc, disable cp internal ints */ 4020 si_rlc_stop(rdev); 4021 4022 udelay(50); 4023 4024 /* disable mem access */ 4025 evergreen_mc_stop(rdev, &save); 4026 if (evergreen_mc_wait_for_idle(rdev)) { 4027 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 4028 } 4029 4030 /* set mclk/sclk to bypass */ 4031 si_set_clk_bypass_mode(rdev); 4032 /* powerdown spll */ 4033 si_spll_powerdown(rdev); 4034 /* disable BM */ 4035 pci_clear_master(rdev->pdev); 4036 /* reset */ 4037 radeon_pci_config_reset(rdev); 4038 /* wait for asic to come out of reset */ 4039 for (i = 0; i < rdev->usec_timeout; i++) { 4040 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 4041 break; 4042 udelay(1); 4043 } 4044 } 4045 4046 int si_asic_reset(struct radeon_device *rdev) 4047 { 4048 u32 reset_mask; 4049 4050 reset_mask = si_gpu_check_soft_reset(rdev); 4051 4052 if (reset_mask) 4053 r600_set_bios_scratch_engine_hung(rdev, true); 4054 4055 /* try soft reset */ 4056 si_gpu_soft_reset(rdev, reset_mask); 4057 4058 reset_mask = si_gpu_check_soft_reset(rdev); 4059 4060 /* try pci config reset */ 4061 if (reset_mask && radeon_hard_reset) 4062 si_gpu_pci_config_reset(rdev); 4063 4064 reset_mask = si_gpu_check_soft_reset(rdev); 4065 4066 if (!reset_mask) 4067 r600_set_bios_scratch_engine_hung(rdev, false); 4068 4069 return 0; 4070 } 4071 4072 /** 4073 * si_gfx_is_lockup - Check if the GFX engine is locked up 4074 * 4075 * @rdev: radeon_device pointer 4076 * @ring: radeon_ring structure holding ring information 4077 * 4078 * Check if the GFX engine is locked up. 4079 * Returns true if the engine appears to be locked up, false if not. 4080 */ 4081 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 4082 { 4083 u32 reset_mask = si_gpu_check_soft_reset(rdev); 4084 4085 if (!(reset_mask & (RADEON_RESET_GFX | 4086 RADEON_RESET_COMPUTE | 4087 RADEON_RESET_CP))) { 4088 radeon_ring_lockup_update(rdev, ring); 4089 return false; 4090 } 4091 return radeon_ring_test_lockup(rdev, ring); 4092 } 4093 4094 /* MC */ 4095 static void si_mc_program(struct radeon_device *rdev) 4096 { 4097 struct evergreen_mc_save save; 4098 u32 tmp; 4099 int i, j; 4100 4101 /* Initialize HDP */ 4102 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 4103 WREG32((0x2c14 + j), 0x00000000); 4104 WREG32((0x2c18 + j), 0x00000000); 4105 WREG32((0x2c1c + j), 0x00000000); 4106 WREG32((0x2c20 + j), 0x00000000); 4107 WREG32((0x2c24 + j), 0x00000000); 4108 } 4109 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 4110 4111 evergreen_mc_stop(rdev, &save); 4112 if (radeon_mc_wait_for_idle(rdev)) { 4113 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 4114 } 4115 if (!ASIC_IS_NODCE(rdev)) 4116 /* Lockout access through VGA aperture*/ 4117 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 4118 /* Update configuration */ 4119 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 4120 rdev->mc.vram_start >> 12); 4121 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 4122 rdev->mc.vram_end >> 12); 4123 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 4124 rdev->vram_scratch.gpu_addr >> 12); 4125 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 4126 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 4127 WREG32(MC_VM_FB_LOCATION, tmp); 4128 /* XXX double check these! */ 4129 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 4130 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 4131 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 4132 WREG32(MC_VM_AGP_BASE, 0); 4133 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 4134 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 4135 if (radeon_mc_wait_for_idle(rdev)) { 4136 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 4137 } 4138 evergreen_mc_resume(rdev, &save); 4139 if (!ASIC_IS_NODCE(rdev)) { 4140 /* we need to own VRAM, so turn off the VGA renderer here 4141 * to stop it overwriting our objects */ 4142 rv515_vga_render_disable(rdev); 4143 } 4144 } 4145 4146 void si_vram_gtt_location(struct radeon_device *rdev, 4147 struct radeon_mc *mc) 4148 { 4149 if (mc->mc_vram_size > 0xFFC0000000ULL) { 4150 /* leave room for at least 1024M GTT */ 4151 dev_warn(rdev->dev, "limiting VRAM\n"); 4152 mc->real_vram_size = 0xFFC0000000ULL; 4153 mc->mc_vram_size = 0xFFC0000000ULL; 4154 } 4155 radeon_vram_location(rdev, &rdev->mc, 0); 4156 rdev->mc.gtt_base_align = 0; 4157 radeon_gtt_location(rdev, mc); 4158 } 4159 4160 static int si_mc_init(struct radeon_device *rdev) 4161 { 4162 u32 tmp; 4163 int chansize, numchan; 4164 4165 /* Get VRAM informations */ 4166 rdev->mc.vram_is_ddr = true; 4167 tmp = RREG32(MC_ARB_RAMCFG); 4168 if (tmp & CHANSIZE_OVERRIDE) { 4169 chansize = 16; 4170 } else if (tmp & CHANSIZE_MASK) { 4171 chansize = 64; 4172 } else { 4173 chansize = 32; 4174 } 4175 tmp = RREG32(MC_SHARED_CHMAP); 4176 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 4177 case 0: 4178 default: 4179 numchan = 1; 4180 break; 4181 case 1: 4182 numchan = 2; 4183 break; 4184 case 2: 4185 numchan = 4; 4186 break; 4187 case 3: 4188 numchan = 8; 4189 break; 4190 case 4: 4191 numchan = 3; 4192 break; 4193 case 5: 4194 numchan = 6; 4195 break; 4196 case 6: 4197 numchan = 10; 4198 break; 4199 case 7: 4200 numchan = 12; 4201 break; 4202 case 8: 4203 numchan = 16; 4204 break; 4205 } 4206 rdev->mc.vram_width = numchan * chansize; 4207 /* Could aper size report 0 ? */ 4208 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 4209 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 4210 /* size in MB on si */ 4211 tmp = RREG32(CONFIG_MEMSIZE); 4212 /* some boards may have garbage in the upper 16 bits */ 4213 if (tmp & 0xffff0000) { 4214 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp); 4215 if (tmp & 0xffff) 4216 tmp &= 0xffff; 4217 } 4218 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL; 4219 rdev->mc.real_vram_size = rdev->mc.mc_vram_size; 4220 rdev->mc.visible_vram_size = rdev->mc.aper_size; 4221 si_vram_gtt_location(rdev, &rdev->mc); 4222 radeon_update_bandwidth_info(rdev); 4223 4224 return 0; 4225 } 4226 4227 /* 4228 * GART 4229 */ 4230 void si_pcie_gart_tlb_flush(struct radeon_device *rdev) 4231 { 4232 /* flush hdp cache */ 4233 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 4234 4235 /* bits 0-15 are the VM contexts0-15 */ 4236 WREG32(VM_INVALIDATE_REQUEST, 1); 4237 } 4238 4239 static int si_pcie_gart_enable(struct radeon_device *rdev) 4240 { 4241 int r, i; 4242 4243 if (rdev->gart.robj == NULL) { 4244 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 4245 return -EINVAL; 4246 } 4247 r = radeon_gart_table_vram_pin(rdev); 4248 if (r) 4249 return r; 4250 /* Setup TLB control */ 4251 WREG32(MC_VM_MX_L1_TLB_CNTL, 4252 (0xA << 7) | 4253 ENABLE_L1_TLB | 4254 ENABLE_L1_FRAGMENT_PROCESSING | 4255 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 4256 ENABLE_ADVANCED_DRIVER_MODEL | 4257 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 4258 /* Setup L2 cache */ 4259 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 4260 ENABLE_L2_FRAGMENT_PROCESSING | 4261 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 4262 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 4263 EFFECTIVE_L2_QUEUE_SIZE(7) | 4264 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 4265 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 4266 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 4267 BANK_SELECT(4) | 4268 L2_CACHE_BIGK_FRAGMENT_SIZE(4)); 4269 /* setup context0 */ 4270 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 4271 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 4272 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 4273 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 4274 (u32)(rdev->dummy_page.addr >> 12)); 4275 WREG32(VM_CONTEXT0_CNTL2, 0); 4276 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 4277 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 4278 4279 WREG32(0x15D4, 0); 4280 WREG32(0x15D8, 0); 4281 WREG32(0x15DC, 0); 4282 4283 /* empty context1-15 */ 4284 /* set vm size, must be a multiple of 4 */ 4285 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 4286 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 4287 /* Assign the pt base to something valid for now; the pts used for 4288 * the VMs are determined by the application and setup and assigned 4289 * on the fly in the vm part of radeon_gart.c 4290 */ 4291 for (i = 1; i < 16; i++) { 4292 if (i < 8) 4293 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 4294 rdev->gart.table_addr >> 12); 4295 else 4296 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 4297 rdev->gart.table_addr >> 12); 4298 } 4299 4300 /* enable context1-15 */ 4301 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 4302 (u32)(rdev->dummy_page.addr >> 12)); 4303 WREG32(VM_CONTEXT1_CNTL2, 4); 4304 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 4305 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) | 4306 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4307 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 4308 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4309 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 4310 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 4311 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 4312 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 4313 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 4314 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 4315 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 4316 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4317 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 4318 4319 si_pcie_gart_tlb_flush(rdev); 4320 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 4321 (unsigned)(rdev->mc.gtt_size >> 20), 4322 (unsigned long long)rdev->gart.table_addr); 4323 rdev->gart.ready = true; 4324 return 0; 4325 } 4326 4327 static void si_pcie_gart_disable(struct radeon_device *rdev) 4328 { 4329 /* Disable all tables */ 4330 WREG32(VM_CONTEXT0_CNTL, 0); 4331 WREG32(VM_CONTEXT1_CNTL, 0); 4332 /* Setup TLB control */ 4333 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 4334 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 4335 /* Setup L2 cache */ 4336 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 4337 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 4338 EFFECTIVE_L2_QUEUE_SIZE(7) | 4339 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 4340 WREG32(VM_L2_CNTL2, 0); 4341 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 4342 L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 4343 radeon_gart_table_vram_unpin(rdev); 4344 } 4345 4346 static void si_pcie_gart_fini(struct radeon_device *rdev) 4347 { 4348 si_pcie_gart_disable(rdev); 4349 radeon_gart_table_vram_free(rdev); 4350 radeon_gart_fini(rdev); 4351 } 4352 4353 /* vm parser */ 4354 static bool si_vm_reg_valid(u32 reg) 4355 { 4356 /* context regs are fine */ 4357 if (reg >= 0x28000) 4358 return true; 4359 4360 /* check config regs */ 4361 switch (reg) { 4362 case GRBM_GFX_INDEX: 4363 case CP_STRMOUT_CNTL: 4364 case VGT_VTX_VECT_EJECT_REG: 4365 case VGT_CACHE_INVALIDATION: 4366 case VGT_ESGS_RING_SIZE: 4367 case VGT_GSVS_RING_SIZE: 4368 case VGT_GS_VERTEX_REUSE: 4369 case VGT_PRIMITIVE_TYPE: 4370 case VGT_INDEX_TYPE: 4371 case VGT_NUM_INDICES: 4372 case VGT_NUM_INSTANCES: 4373 case VGT_TF_RING_SIZE: 4374 case VGT_HS_OFFCHIP_PARAM: 4375 case VGT_TF_MEMORY_BASE: 4376 case PA_CL_ENHANCE: 4377 case PA_SU_LINE_STIPPLE_VALUE: 4378 case PA_SC_LINE_STIPPLE_STATE: 4379 case PA_SC_ENHANCE: 4380 case SQC_CACHES: 4381 case SPI_STATIC_THREAD_MGMT_1: 4382 case SPI_STATIC_THREAD_MGMT_2: 4383 case SPI_STATIC_THREAD_MGMT_3: 4384 case SPI_PS_MAX_WAVE_ID: 4385 case SPI_CONFIG_CNTL: 4386 case SPI_CONFIG_CNTL_1: 4387 case TA_CNTL_AUX: 4388 return true; 4389 default: 4390 DRM_ERROR("Invalid register 0x%x in CS\n", reg); 4391 return false; 4392 } 4393 } 4394 4395 static int si_vm_packet3_ce_check(struct radeon_device *rdev, 4396 u32 *ib, struct radeon_cs_packet *pkt) 4397 { 4398 switch (pkt->opcode) { 4399 case PACKET3_NOP: 4400 case PACKET3_SET_BASE: 4401 case PACKET3_SET_CE_DE_COUNTERS: 4402 case PACKET3_LOAD_CONST_RAM: 4403 case PACKET3_WRITE_CONST_RAM: 4404 case PACKET3_WRITE_CONST_RAM_OFFSET: 4405 case PACKET3_DUMP_CONST_RAM: 4406 case PACKET3_INCREMENT_CE_COUNTER: 4407 case PACKET3_WAIT_ON_DE_COUNTER: 4408 case PACKET3_CE_WRITE: 4409 break; 4410 default: 4411 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode); 4412 return -EINVAL; 4413 } 4414 return 0; 4415 } 4416 4417 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx) 4418 { 4419 u32 start_reg, reg, i; 4420 u32 command = ib[idx + 4]; 4421 u32 info = ib[idx + 1]; 4422 u32 idx_value = ib[idx]; 4423 if (command & PACKET3_CP_DMA_CMD_SAS) { 4424 /* src address space is register */ 4425 if (((info & 0x60000000) >> 29) == 0) { 4426 start_reg = idx_value << 2; 4427 if (command & PACKET3_CP_DMA_CMD_SAIC) { 4428 reg = start_reg; 4429 if (!si_vm_reg_valid(reg)) { 4430 DRM_ERROR("CP DMA Bad SRC register\n"); 4431 return -EINVAL; 4432 } 4433 } else { 4434 for (i = 0; i < (command & 0x1fffff); i++) { 4435 reg = start_reg + (4 * i); 4436 if (!si_vm_reg_valid(reg)) { 4437 DRM_ERROR("CP DMA Bad SRC register\n"); 4438 return -EINVAL; 4439 } 4440 } 4441 } 4442 } 4443 } 4444 if (command & PACKET3_CP_DMA_CMD_DAS) { 4445 /* dst address space is register */ 4446 if (((info & 0x00300000) >> 20) == 0) { 4447 start_reg = ib[idx + 2]; 4448 if (command & PACKET3_CP_DMA_CMD_DAIC) { 4449 reg = start_reg; 4450 if (!si_vm_reg_valid(reg)) { 4451 DRM_ERROR("CP DMA Bad DST register\n"); 4452 return -EINVAL; 4453 } 4454 } else { 4455 for (i = 0; i < (command & 0x1fffff); i++) { 4456 reg = start_reg + (4 * i); 4457 if (!si_vm_reg_valid(reg)) { 4458 DRM_ERROR("CP DMA Bad DST register\n"); 4459 return -EINVAL; 4460 } 4461 } 4462 } 4463 } 4464 } 4465 return 0; 4466 } 4467 4468 static int si_vm_packet3_gfx_check(struct radeon_device *rdev, 4469 u32 *ib, struct radeon_cs_packet *pkt) 4470 { 4471 int r; 4472 u32 idx = pkt->idx + 1; 4473 u32 idx_value = ib[idx]; 4474 u32 start_reg, end_reg, reg, i; 4475 4476 switch (pkt->opcode) { 4477 case PACKET3_NOP: 4478 case PACKET3_SET_BASE: 4479 case PACKET3_CLEAR_STATE: 4480 case PACKET3_INDEX_BUFFER_SIZE: 4481 case PACKET3_DISPATCH_DIRECT: 4482 case PACKET3_DISPATCH_INDIRECT: 4483 case PACKET3_ALLOC_GDS: 4484 case PACKET3_WRITE_GDS_RAM: 4485 case PACKET3_ATOMIC_GDS: 4486 case PACKET3_ATOMIC: 4487 case PACKET3_OCCLUSION_QUERY: 4488 case PACKET3_SET_PREDICATION: 4489 case PACKET3_COND_EXEC: 4490 case PACKET3_PRED_EXEC: 4491 case PACKET3_DRAW_INDIRECT: 4492 case PACKET3_DRAW_INDEX_INDIRECT: 4493 case PACKET3_INDEX_BASE: 4494 case PACKET3_DRAW_INDEX_2: 4495 case PACKET3_CONTEXT_CONTROL: 4496 case PACKET3_INDEX_TYPE: 4497 case PACKET3_DRAW_INDIRECT_MULTI: 4498 case PACKET3_DRAW_INDEX_AUTO: 4499 case PACKET3_DRAW_INDEX_IMMD: 4500 case PACKET3_NUM_INSTANCES: 4501 case PACKET3_DRAW_INDEX_MULTI_AUTO: 4502 case PACKET3_STRMOUT_BUFFER_UPDATE: 4503 case PACKET3_DRAW_INDEX_OFFSET_2: 4504 case PACKET3_DRAW_INDEX_MULTI_ELEMENT: 4505 case PACKET3_DRAW_INDEX_INDIRECT_MULTI: 4506 case PACKET3_MPEG_INDEX: 4507 case PACKET3_WAIT_REG_MEM: 4508 case PACKET3_MEM_WRITE: 4509 case PACKET3_PFP_SYNC_ME: 4510 case PACKET3_SURFACE_SYNC: 4511 case PACKET3_EVENT_WRITE: 4512 case PACKET3_EVENT_WRITE_EOP: 4513 case PACKET3_EVENT_WRITE_EOS: 4514 case PACKET3_SET_CONTEXT_REG: 4515 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4516 case PACKET3_SET_SH_REG: 4517 case PACKET3_SET_SH_REG_OFFSET: 4518 case PACKET3_INCREMENT_DE_COUNTER: 4519 case PACKET3_WAIT_ON_CE_COUNTER: 4520 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4521 case PACKET3_ME_WRITE: 4522 break; 4523 case PACKET3_COPY_DATA: 4524 if ((idx_value & 0xf00) == 0) { 4525 reg = ib[idx + 3] * 4; 4526 if (!si_vm_reg_valid(reg)) 4527 return -EINVAL; 4528 } 4529 break; 4530 case PACKET3_WRITE_DATA: 4531 if ((idx_value & 0xf00) == 0) { 4532 start_reg = ib[idx + 1] * 4; 4533 if (idx_value & 0x10000) { 4534 if (!si_vm_reg_valid(start_reg)) 4535 return -EINVAL; 4536 } else { 4537 for (i = 0; i < (pkt->count - 2); i++) { 4538 reg = start_reg + (4 * i); 4539 if (!si_vm_reg_valid(reg)) 4540 return -EINVAL; 4541 } 4542 } 4543 } 4544 break; 4545 case PACKET3_COND_WRITE: 4546 if (idx_value & 0x100) { 4547 reg = ib[idx + 5] * 4; 4548 if (!si_vm_reg_valid(reg)) 4549 return -EINVAL; 4550 } 4551 break; 4552 case PACKET3_COPY_DW: 4553 if (idx_value & 0x2) { 4554 reg = ib[idx + 3] * 4; 4555 if (!si_vm_reg_valid(reg)) 4556 return -EINVAL; 4557 } 4558 break; 4559 case PACKET3_SET_CONFIG_REG: 4560 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START; 4561 end_reg = 4 * pkt->count + start_reg - 4; 4562 if ((start_reg < PACKET3_SET_CONFIG_REG_START) || 4563 (start_reg >= PACKET3_SET_CONFIG_REG_END) || 4564 (end_reg >= PACKET3_SET_CONFIG_REG_END)) { 4565 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); 4566 return -EINVAL; 4567 } 4568 for (i = 0; i < pkt->count; i++) { 4569 reg = start_reg + (4 * i); 4570 if (!si_vm_reg_valid(reg)) 4571 return -EINVAL; 4572 } 4573 break; 4574 case PACKET3_CP_DMA: 4575 r = si_vm_packet3_cp_dma_check(ib, idx); 4576 if (r) 4577 return r; 4578 break; 4579 default: 4580 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode); 4581 return -EINVAL; 4582 } 4583 return 0; 4584 } 4585 4586 static int si_vm_packet3_compute_check(struct radeon_device *rdev, 4587 u32 *ib, struct radeon_cs_packet *pkt) 4588 { 4589 int r; 4590 u32 idx = pkt->idx + 1; 4591 u32 idx_value = ib[idx]; 4592 u32 start_reg, reg, i; 4593 4594 switch (pkt->opcode) { 4595 case PACKET3_NOP: 4596 case PACKET3_SET_BASE: 4597 case PACKET3_CLEAR_STATE: 4598 case PACKET3_DISPATCH_DIRECT: 4599 case PACKET3_DISPATCH_INDIRECT: 4600 case PACKET3_ALLOC_GDS: 4601 case PACKET3_WRITE_GDS_RAM: 4602 case PACKET3_ATOMIC_GDS: 4603 case PACKET3_ATOMIC: 4604 case PACKET3_OCCLUSION_QUERY: 4605 case PACKET3_SET_PREDICATION: 4606 case PACKET3_COND_EXEC: 4607 case PACKET3_PRED_EXEC: 4608 case PACKET3_CONTEXT_CONTROL: 4609 case PACKET3_STRMOUT_BUFFER_UPDATE: 4610 case PACKET3_WAIT_REG_MEM: 4611 case PACKET3_MEM_WRITE: 4612 case PACKET3_PFP_SYNC_ME: 4613 case PACKET3_SURFACE_SYNC: 4614 case PACKET3_EVENT_WRITE: 4615 case PACKET3_EVENT_WRITE_EOP: 4616 case PACKET3_EVENT_WRITE_EOS: 4617 case PACKET3_SET_CONTEXT_REG: 4618 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4619 case PACKET3_SET_SH_REG: 4620 case PACKET3_SET_SH_REG_OFFSET: 4621 case PACKET3_INCREMENT_DE_COUNTER: 4622 case PACKET3_WAIT_ON_CE_COUNTER: 4623 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4624 case PACKET3_ME_WRITE: 4625 break; 4626 case PACKET3_COPY_DATA: 4627 if ((idx_value & 0xf00) == 0) { 4628 reg = ib[idx + 3] * 4; 4629 if (!si_vm_reg_valid(reg)) 4630 return -EINVAL; 4631 } 4632 break; 4633 case PACKET3_WRITE_DATA: 4634 if ((idx_value & 0xf00) == 0) { 4635 start_reg = ib[idx + 1] * 4; 4636 if (idx_value & 0x10000) { 4637 if (!si_vm_reg_valid(start_reg)) 4638 return -EINVAL; 4639 } else { 4640 for (i = 0; i < (pkt->count - 2); i++) { 4641 reg = start_reg + (4 * i); 4642 if (!si_vm_reg_valid(reg)) 4643 return -EINVAL; 4644 } 4645 } 4646 } 4647 break; 4648 case PACKET3_COND_WRITE: 4649 if (idx_value & 0x100) { 4650 reg = ib[idx + 5] * 4; 4651 if (!si_vm_reg_valid(reg)) 4652 return -EINVAL; 4653 } 4654 break; 4655 case PACKET3_COPY_DW: 4656 if (idx_value & 0x2) { 4657 reg = ib[idx + 3] * 4; 4658 if (!si_vm_reg_valid(reg)) 4659 return -EINVAL; 4660 } 4661 break; 4662 case PACKET3_CP_DMA: 4663 r = si_vm_packet3_cp_dma_check(ib, idx); 4664 if (r) 4665 return r; 4666 break; 4667 default: 4668 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode); 4669 return -EINVAL; 4670 } 4671 return 0; 4672 } 4673 4674 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 4675 { 4676 int ret = 0; 4677 u32 idx = 0; 4678 struct radeon_cs_packet pkt; 4679 4680 do { 4681 pkt.idx = idx; 4682 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]); 4683 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]); 4684 pkt.one_reg_wr = 0; 4685 switch (pkt.type) { 4686 case RADEON_PACKET_TYPE0: 4687 dev_err(rdev->dev, "Packet0 not allowed!\n"); 4688 ret = -EINVAL; 4689 break; 4690 case RADEON_PACKET_TYPE2: 4691 idx += 1; 4692 break; 4693 case RADEON_PACKET_TYPE3: 4694 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]); 4695 if (ib->is_const_ib) 4696 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt); 4697 else { 4698 switch (ib->ring) { 4699 case RADEON_RING_TYPE_GFX_INDEX: 4700 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt); 4701 break; 4702 case CAYMAN_RING_TYPE_CP1_INDEX: 4703 case CAYMAN_RING_TYPE_CP2_INDEX: 4704 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt); 4705 break; 4706 default: 4707 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring); 4708 ret = -EINVAL; 4709 break; 4710 } 4711 } 4712 idx += pkt.count + 2; 4713 break; 4714 default: 4715 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type); 4716 ret = -EINVAL; 4717 break; 4718 } 4719 if (ret) 4720 break; 4721 } while (idx < ib->length_dw); 4722 4723 return ret; 4724 } 4725 4726 /* 4727 * vm 4728 */ 4729 int si_vm_init(struct radeon_device *rdev) 4730 { 4731 /* number of VMs */ 4732 rdev->vm_manager.nvm = 16; 4733 /* base offset of vram pages */ 4734 rdev->vm_manager.vram_base_offset = 0; 4735 4736 return 0; 4737 } 4738 4739 void si_vm_fini(struct radeon_device *rdev) 4740 { 4741 } 4742 4743 /** 4744 * si_vm_decode_fault - print human readable fault info 4745 * 4746 * @rdev: radeon_device pointer 4747 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 4748 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 4749 * 4750 * Print human readable fault information (SI). 4751 */ 4752 static void si_vm_decode_fault(struct radeon_device *rdev, 4753 u32 status, u32 addr) 4754 { 4755 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 4756 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 4757 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 4758 char *block; 4759 4760 if (rdev->family == CHIP_TAHITI) { 4761 switch (mc_id) { 4762 case 160: 4763 case 144: 4764 case 96: 4765 case 80: 4766 case 224: 4767 case 208: 4768 case 32: 4769 case 16: 4770 block = "CB"; 4771 break; 4772 case 161: 4773 case 145: 4774 case 97: 4775 case 81: 4776 case 225: 4777 case 209: 4778 case 33: 4779 case 17: 4780 block = "CB_FMASK"; 4781 break; 4782 case 162: 4783 case 146: 4784 case 98: 4785 case 82: 4786 case 226: 4787 case 210: 4788 case 34: 4789 case 18: 4790 block = "CB_CMASK"; 4791 break; 4792 case 163: 4793 case 147: 4794 case 99: 4795 case 83: 4796 case 227: 4797 case 211: 4798 case 35: 4799 case 19: 4800 block = "CB_IMMED"; 4801 break; 4802 case 164: 4803 case 148: 4804 case 100: 4805 case 84: 4806 case 228: 4807 case 212: 4808 case 36: 4809 case 20: 4810 block = "DB"; 4811 break; 4812 case 165: 4813 case 149: 4814 case 101: 4815 case 85: 4816 case 229: 4817 case 213: 4818 case 37: 4819 case 21: 4820 block = "DB_HTILE"; 4821 break; 4822 case 167: 4823 case 151: 4824 case 103: 4825 case 87: 4826 case 231: 4827 case 215: 4828 case 39: 4829 case 23: 4830 block = "DB_STEN"; 4831 break; 4832 case 72: 4833 case 68: 4834 case 64: 4835 case 8: 4836 case 4: 4837 case 0: 4838 case 136: 4839 case 132: 4840 case 128: 4841 case 200: 4842 case 196: 4843 case 192: 4844 block = "TC"; 4845 break; 4846 case 112: 4847 case 48: 4848 block = "CP"; 4849 break; 4850 case 49: 4851 case 177: 4852 case 50: 4853 case 178: 4854 block = "SH"; 4855 break; 4856 case 53: 4857 case 190: 4858 block = "VGT"; 4859 break; 4860 case 117: 4861 block = "IH"; 4862 break; 4863 case 51: 4864 case 115: 4865 block = "RLC"; 4866 break; 4867 case 119: 4868 case 183: 4869 block = "DMA0"; 4870 break; 4871 case 61: 4872 block = "DMA1"; 4873 break; 4874 case 248: 4875 case 120: 4876 block = "HDP"; 4877 break; 4878 default: 4879 block = "unknown"; 4880 break; 4881 } 4882 } else { 4883 switch (mc_id) { 4884 case 32: 4885 case 16: 4886 case 96: 4887 case 80: 4888 case 160: 4889 case 144: 4890 case 224: 4891 case 208: 4892 block = "CB"; 4893 break; 4894 case 33: 4895 case 17: 4896 case 97: 4897 case 81: 4898 case 161: 4899 case 145: 4900 case 225: 4901 case 209: 4902 block = "CB_FMASK"; 4903 break; 4904 case 34: 4905 case 18: 4906 case 98: 4907 case 82: 4908 case 162: 4909 case 146: 4910 case 226: 4911 case 210: 4912 block = "CB_CMASK"; 4913 break; 4914 case 35: 4915 case 19: 4916 case 99: 4917 case 83: 4918 case 163: 4919 case 147: 4920 case 227: 4921 case 211: 4922 block = "CB_IMMED"; 4923 break; 4924 case 36: 4925 case 20: 4926 case 100: 4927 case 84: 4928 case 164: 4929 case 148: 4930 case 228: 4931 case 212: 4932 block = "DB"; 4933 break; 4934 case 37: 4935 case 21: 4936 case 101: 4937 case 85: 4938 case 165: 4939 case 149: 4940 case 229: 4941 case 213: 4942 block = "DB_HTILE"; 4943 break; 4944 case 39: 4945 case 23: 4946 case 103: 4947 case 87: 4948 case 167: 4949 case 151: 4950 case 231: 4951 case 215: 4952 block = "DB_STEN"; 4953 break; 4954 case 72: 4955 case 68: 4956 case 8: 4957 case 4: 4958 case 136: 4959 case 132: 4960 case 200: 4961 case 196: 4962 block = "TC"; 4963 break; 4964 case 112: 4965 case 48: 4966 block = "CP"; 4967 break; 4968 case 49: 4969 case 177: 4970 case 50: 4971 case 178: 4972 block = "SH"; 4973 break; 4974 case 53: 4975 block = "VGT"; 4976 break; 4977 case 117: 4978 block = "IH"; 4979 break; 4980 case 51: 4981 case 115: 4982 block = "RLC"; 4983 break; 4984 case 119: 4985 case 183: 4986 block = "DMA0"; 4987 break; 4988 case 61: 4989 block = "DMA1"; 4990 break; 4991 case 248: 4992 case 120: 4993 block = "HDP"; 4994 break; 4995 default: 4996 block = "unknown"; 4997 break; 4998 } 4999 } 5000 5001 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n", 5002 protections, vmid, addr, 5003 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 5004 block, mc_id); 5005 } 5006 5007 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 5008 { 5009 struct radeon_ring *ring = &rdev->ring[ridx]; 5010 5011 if (vm == NULL) 5012 return; 5013 5014 /* write new base address */ 5015 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5016 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5017 WRITE_DATA_DST_SEL(0))); 5018 5019 if (vm->id < 8) { 5020 radeon_ring_write(ring, 5021 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); 5022 } else { 5023 radeon_ring_write(ring, 5024 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); 5025 } 5026 radeon_ring_write(ring, 0); 5027 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 5028 5029 /* flush hdp cache */ 5030 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5031 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5032 WRITE_DATA_DST_SEL(0))); 5033 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2); 5034 radeon_ring_write(ring, 0); 5035 radeon_ring_write(ring, 0x1); 5036 5037 /* bits 0-15 are the VM contexts0-15 */ 5038 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5039 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5040 WRITE_DATA_DST_SEL(0))); 5041 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 5042 radeon_ring_write(ring, 0); 5043 radeon_ring_write(ring, 1 << vm->id); 5044 5045 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5046 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5047 radeon_ring_write(ring, 0x0); 5048 } 5049 5050 /* 5051 * Power and clock gating 5052 */ 5053 static void si_wait_for_rlc_serdes(struct radeon_device *rdev) 5054 { 5055 int i; 5056 5057 for (i = 0; i < rdev->usec_timeout; i++) { 5058 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0) 5059 break; 5060 udelay(1); 5061 } 5062 5063 for (i = 0; i < rdev->usec_timeout; i++) { 5064 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0) 5065 break; 5066 udelay(1); 5067 } 5068 } 5069 5070 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, 5071 bool enable) 5072 { 5073 u32 tmp = RREG32(CP_INT_CNTL_RING0); 5074 u32 mask; 5075 int i; 5076 5077 if (enable) 5078 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5079 else 5080 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5081 WREG32(CP_INT_CNTL_RING0, tmp); 5082 5083 if (!enable) { 5084 /* read a gfx register */ 5085 tmp = RREG32(DB_DEPTH_INFO); 5086 5087 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS; 5088 for (i = 0; i < rdev->usec_timeout; i++) { 5089 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS)) 5090 break; 5091 udelay(1); 5092 } 5093 } 5094 } 5095 5096 static void si_set_uvd_dcm(struct radeon_device *rdev, 5097 bool sw_mode) 5098 { 5099 u32 tmp, tmp2; 5100 5101 tmp = RREG32(UVD_CGC_CTRL); 5102 tmp &= ~(CLK_OD_MASK | CG_DT_MASK); 5103 tmp |= DCM | CG_DT(1) | CLK_OD(4); 5104 5105 if (sw_mode) { 5106 tmp &= ~0x7ffff800; 5107 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7); 5108 } else { 5109 tmp |= 0x7ffff800; 5110 tmp2 = 0; 5111 } 5112 5113 WREG32(UVD_CGC_CTRL, tmp); 5114 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2); 5115 } 5116 5117 void si_init_uvd_internal_cg(struct radeon_device *rdev) 5118 { 5119 bool hw_mode = true; 5120 5121 if (hw_mode) { 5122 si_set_uvd_dcm(rdev, false); 5123 } else { 5124 u32 tmp = RREG32(UVD_CGC_CTRL); 5125 tmp &= ~DCM; 5126 WREG32(UVD_CGC_CTRL, tmp); 5127 } 5128 } 5129 5130 static u32 si_halt_rlc(struct radeon_device *rdev) 5131 { 5132 u32 data, orig; 5133 5134 orig = data = RREG32(RLC_CNTL); 5135 5136 if (data & RLC_ENABLE) { 5137 data &= ~RLC_ENABLE; 5138 WREG32(RLC_CNTL, data); 5139 5140 si_wait_for_rlc_serdes(rdev); 5141 } 5142 5143 return orig; 5144 } 5145 5146 static void si_update_rlc(struct radeon_device *rdev, u32 rlc) 5147 { 5148 u32 tmp; 5149 5150 tmp = RREG32(RLC_CNTL); 5151 if (tmp != rlc) 5152 WREG32(RLC_CNTL, rlc); 5153 } 5154 5155 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable) 5156 { 5157 u32 data, orig; 5158 5159 orig = data = RREG32(DMA_PG); 5160 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA)) 5161 data |= PG_CNTL_ENABLE; 5162 else 5163 data &= ~PG_CNTL_ENABLE; 5164 if (orig != data) 5165 WREG32(DMA_PG, data); 5166 } 5167 5168 static void si_init_dma_pg(struct radeon_device *rdev) 5169 { 5170 u32 tmp; 5171 5172 WREG32(DMA_PGFSM_WRITE, 0x00002000); 5173 WREG32(DMA_PGFSM_CONFIG, 0x100010ff); 5174 5175 for (tmp = 0; tmp < 5; tmp++) 5176 WREG32(DMA_PGFSM_WRITE, 0); 5177 } 5178 5179 static void si_enable_gfx_cgpg(struct radeon_device *rdev, 5180 bool enable) 5181 { 5182 u32 tmp; 5183 5184 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 5185 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10); 5186 WREG32(RLC_TTOP_D, tmp); 5187 5188 tmp = RREG32(RLC_PG_CNTL); 5189 tmp |= GFX_PG_ENABLE; 5190 WREG32(RLC_PG_CNTL, tmp); 5191 5192 tmp = RREG32(RLC_AUTO_PG_CTRL); 5193 tmp |= AUTO_PG_EN; 5194 WREG32(RLC_AUTO_PG_CTRL, tmp); 5195 } else { 5196 tmp = RREG32(RLC_AUTO_PG_CTRL); 5197 tmp &= ~AUTO_PG_EN; 5198 WREG32(RLC_AUTO_PG_CTRL, tmp); 5199 5200 tmp = RREG32(DB_RENDER_CONTROL); 5201 } 5202 } 5203 5204 static void si_init_gfx_cgpg(struct radeon_device *rdev) 5205 { 5206 u32 tmp; 5207 5208 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5209 5210 tmp = RREG32(RLC_PG_CNTL); 5211 tmp |= GFX_PG_SRC; 5212 WREG32(RLC_PG_CNTL, tmp); 5213 5214 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5215 5216 tmp = RREG32(RLC_AUTO_PG_CTRL); 5217 5218 tmp &= ~GRBM_REG_SGIT_MASK; 5219 tmp |= GRBM_REG_SGIT(0x700); 5220 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK; 5221 WREG32(RLC_AUTO_PG_CTRL, tmp); 5222 } 5223 5224 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 5225 { 5226 u32 mask = 0, tmp, tmp1; 5227 int i; 5228 5229 si_select_se_sh(rdev, se, sh); 5230 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 5231 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 5232 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5233 5234 tmp &= 0xffff0000; 5235 5236 tmp |= tmp1; 5237 tmp >>= 16; 5238 5239 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) { 5240 mask <<= 1; 5241 mask |= 1; 5242 } 5243 5244 return (~tmp) & mask; 5245 } 5246 5247 static void si_init_ao_cu_mask(struct radeon_device *rdev) 5248 { 5249 u32 i, j, k, active_cu_number = 0; 5250 u32 mask, counter, cu_bitmap; 5251 u32 tmp = 0; 5252 5253 for (i = 0; i < rdev->config.si.max_shader_engines; i++) { 5254 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { 5255 mask = 1; 5256 cu_bitmap = 0; 5257 counter = 0; 5258 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) { 5259 if (si_get_cu_active_bitmap(rdev, i, j) & mask) { 5260 if (counter < 2) 5261 cu_bitmap |= mask; 5262 counter++; 5263 } 5264 mask <<= 1; 5265 } 5266 5267 active_cu_number += counter; 5268 tmp |= (cu_bitmap << (i * 16 + j * 8)); 5269 } 5270 } 5271 5272 WREG32(RLC_PG_AO_CU_MASK, tmp); 5273 5274 tmp = RREG32(RLC_MAX_PG_CU); 5275 tmp &= ~MAX_PU_CU_MASK; 5276 tmp |= MAX_PU_CU(active_cu_number); 5277 WREG32(RLC_MAX_PG_CU, tmp); 5278 } 5279 5280 static void si_enable_cgcg(struct radeon_device *rdev, 5281 bool enable) 5282 { 5283 u32 data, orig, tmp; 5284 5285 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 5286 5287 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 5288 si_enable_gui_idle_interrupt(rdev, true); 5289 5290 WREG32(RLC_GCPM_GENERAL_3, 0x00000080); 5291 5292 tmp = si_halt_rlc(rdev); 5293 5294 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5295 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5296 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff); 5297 5298 si_wait_for_rlc_serdes(rdev); 5299 5300 si_update_rlc(rdev, tmp); 5301 5302 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff); 5303 5304 data |= CGCG_EN | CGLS_EN; 5305 } else { 5306 si_enable_gui_idle_interrupt(rdev, false); 5307 5308 RREG32(CB_CGTT_SCLK_CTRL); 5309 RREG32(CB_CGTT_SCLK_CTRL); 5310 RREG32(CB_CGTT_SCLK_CTRL); 5311 RREG32(CB_CGTT_SCLK_CTRL); 5312 5313 data &= ~(CGCG_EN | CGLS_EN); 5314 } 5315 5316 if (orig != data) 5317 WREG32(RLC_CGCG_CGLS_CTRL, data); 5318 } 5319 5320 static void si_enable_mgcg(struct radeon_device *rdev, 5321 bool enable) 5322 { 5323 u32 data, orig, tmp = 0; 5324 5325 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 5326 orig = data = RREG32(CGTS_SM_CTRL_REG); 5327 data = 0x96940200; 5328 if (orig != data) 5329 WREG32(CGTS_SM_CTRL_REG, data); 5330 5331 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 5332 orig = data = RREG32(CP_MEM_SLP_CNTL); 5333 data |= CP_MEM_LS_EN; 5334 if (orig != data) 5335 WREG32(CP_MEM_SLP_CNTL, data); 5336 } 5337 5338 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5339 data &= 0xffffffc0; 5340 if (orig != data) 5341 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5342 5343 tmp = si_halt_rlc(rdev); 5344 5345 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5346 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5347 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff); 5348 5349 si_update_rlc(rdev, tmp); 5350 } else { 5351 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5352 data |= 0x00000003; 5353 if (orig != data) 5354 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5355 5356 data = RREG32(CP_MEM_SLP_CNTL); 5357 if (data & CP_MEM_LS_EN) { 5358 data &= ~CP_MEM_LS_EN; 5359 WREG32(CP_MEM_SLP_CNTL, data); 5360 } 5361 orig = data = RREG32(CGTS_SM_CTRL_REG); 5362 data |= LS_OVERRIDE | OVERRIDE; 5363 if (orig != data) 5364 WREG32(CGTS_SM_CTRL_REG, data); 5365 5366 tmp = si_halt_rlc(rdev); 5367 5368 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5369 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5370 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff); 5371 5372 si_update_rlc(rdev, tmp); 5373 } 5374 } 5375 5376 static void si_enable_uvd_mgcg(struct radeon_device *rdev, 5377 bool enable) 5378 { 5379 u32 orig, data, tmp; 5380 5381 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 5382 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5383 tmp |= 0x3fff; 5384 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5385 5386 orig = data = RREG32(UVD_CGC_CTRL); 5387 data |= DCM; 5388 if (orig != data) 5389 WREG32(UVD_CGC_CTRL, data); 5390 5391 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0); 5392 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0); 5393 } else { 5394 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5395 tmp &= ~0x3fff; 5396 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5397 5398 orig = data = RREG32(UVD_CGC_CTRL); 5399 data &= ~DCM; 5400 if (orig != data) 5401 WREG32(UVD_CGC_CTRL, data); 5402 5403 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff); 5404 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff); 5405 } 5406 } 5407 5408 static const u32 mc_cg_registers[] = 5409 { 5410 MC_HUB_MISC_HUB_CG, 5411 MC_HUB_MISC_SIP_CG, 5412 MC_HUB_MISC_VM_CG, 5413 MC_XPB_CLK_GAT, 5414 ATC_MISC_CG, 5415 MC_CITF_MISC_WR_CG, 5416 MC_CITF_MISC_RD_CG, 5417 MC_CITF_MISC_VM_CG, 5418 VM_L2_CG, 5419 }; 5420 5421 static void si_enable_mc_ls(struct radeon_device *rdev, 5422 bool enable) 5423 { 5424 int i; 5425 u32 orig, data; 5426 5427 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5428 orig = data = RREG32(mc_cg_registers[i]); 5429 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 5430 data |= MC_LS_ENABLE; 5431 else 5432 data &= ~MC_LS_ENABLE; 5433 if (data != orig) 5434 WREG32(mc_cg_registers[i], data); 5435 } 5436 } 5437 5438 static void si_enable_mc_mgcg(struct radeon_device *rdev, 5439 bool enable) 5440 { 5441 int i; 5442 u32 orig, data; 5443 5444 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5445 orig = data = RREG32(mc_cg_registers[i]); 5446 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 5447 data |= MC_CG_ENABLE; 5448 else 5449 data &= ~MC_CG_ENABLE; 5450 if (data != orig) 5451 WREG32(mc_cg_registers[i], data); 5452 } 5453 } 5454 5455 static void si_enable_dma_mgcg(struct radeon_device *rdev, 5456 bool enable) 5457 { 5458 u32 orig, data, offset; 5459 int i; 5460 5461 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 5462 for (i = 0; i < 2; i++) { 5463 if (i == 0) 5464 offset = DMA0_REGISTER_OFFSET; 5465 else 5466 offset = DMA1_REGISTER_OFFSET; 5467 orig = data = RREG32(DMA_POWER_CNTL + offset); 5468 data &= ~MEM_POWER_OVERRIDE; 5469 if (data != orig) 5470 WREG32(DMA_POWER_CNTL + offset, data); 5471 WREG32(DMA_CLK_CTRL + offset, 0x00000100); 5472 } 5473 } else { 5474 for (i = 0; i < 2; i++) { 5475 if (i == 0) 5476 offset = DMA0_REGISTER_OFFSET; 5477 else 5478 offset = DMA1_REGISTER_OFFSET; 5479 orig = data = RREG32(DMA_POWER_CNTL + offset); 5480 data |= MEM_POWER_OVERRIDE; 5481 if (data != orig) 5482 WREG32(DMA_POWER_CNTL + offset, data); 5483 5484 orig = data = RREG32(DMA_CLK_CTRL + offset); 5485 data = 0xff000000; 5486 if (data != orig) 5487 WREG32(DMA_CLK_CTRL + offset, data); 5488 } 5489 } 5490 } 5491 5492 static void si_enable_bif_mgls(struct radeon_device *rdev, 5493 bool enable) 5494 { 5495 u32 orig, data; 5496 5497 orig = data = RREG32_PCIE(PCIE_CNTL2); 5498 5499 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 5500 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 5501 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 5502 else 5503 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 5504 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 5505 5506 if (orig != data) 5507 WREG32_PCIE(PCIE_CNTL2, data); 5508 } 5509 5510 static void si_enable_hdp_mgcg(struct radeon_device *rdev, 5511 bool enable) 5512 { 5513 u32 orig, data; 5514 5515 orig = data = RREG32(HDP_HOST_PATH_CNTL); 5516 5517 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 5518 data &= ~CLOCK_GATING_DIS; 5519 else 5520 data |= CLOCK_GATING_DIS; 5521 5522 if (orig != data) 5523 WREG32(HDP_HOST_PATH_CNTL, data); 5524 } 5525 5526 static void si_enable_hdp_ls(struct radeon_device *rdev, 5527 bool enable) 5528 { 5529 u32 orig, data; 5530 5531 orig = data = RREG32(HDP_MEM_POWER_LS); 5532 5533 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 5534 data |= HDP_LS_ENABLE; 5535 else 5536 data &= ~HDP_LS_ENABLE; 5537 5538 if (orig != data) 5539 WREG32(HDP_MEM_POWER_LS, data); 5540 } 5541 5542 static void si_update_cg(struct radeon_device *rdev, 5543 u32 block, bool enable) 5544 { 5545 if (block & RADEON_CG_BLOCK_GFX) { 5546 si_enable_gui_idle_interrupt(rdev, false); 5547 /* order matters! */ 5548 if (enable) { 5549 si_enable_mgcg(rdev, true); 5550 si_enable_cgcg(rdev, true); 5551 } else { 5552 si_enable_cgcg(rdev, false); 5553 si_enable_mgcg(rdev, false); 5554 } 5555 si_enable_gui_idle_interrupt(rdev, true); 5556 } 5557 5558 if (block & RADEON_CG_BLOCK_MC) { 5559 si_enable_mc_mgcg(rdev, enable); 5560 si_enable_mc_ls(rdev, enable); 5561 } 5562 5563 if (block & RADEON_CG_BLOCK_SDMA) { 5564 si_enable_dma_mgcg(rdev, enable); 5565 } 5566 5567 if (block & RADEON_CG_BLOCK_BIF) { 5568 si_enable_bif_mgls(rdev, enable); 5569 } 5570 5571 if (block & RADEON_CG_BLOCK_UVD) { 5572 if (rdev->has_uvd) { 5573 si_enable_uvd_mgcg(rdev, enable); 5574 } 5575 } 5576 5577 if (block & RADEON_CG_BLOCK_HDP) { 5578 si_enable_hdp_mgcg(rdev, enable); 5579 si_enable_hdp_ls(rdev, enable); 5580 } 5581 } 5582 5583 static void si_init_cg(struct radeon_device *rdev) 5584 { 5585 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | 5586 RADEON_CG_BLOCK_MC | 5587 RADEON_CG_BLOCK_SDMA | 5588 RADEON_CG_BLOCK_BIF | 5589 RADEON_CG_BLOCK_HDP), true); 5590 if (rdev->has_uvd) { 5591 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true); 5592 si_init_uvd_internal_cg(rdev); 5593 } 5594 } 5595 5596 static void si_fini_cg(struct radeon_device *rdev) 5597 { 5598 if (rdev->has_uvd) { 5599 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false); 5600 } 5601 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | 5602 RADEON_CG_BLOCK_MC | 5603 RADEON_CG_BLOCK_SDMA | 5604 RADEON_CG_BLOCK_BIF | 5605 RADEON_CG_BLOCK_HDP), false); 5606 } 5607 5608 u32 si_get_csb_size(struct radeon_device *rdev) 5609 { 5610 u32 count = 0; 5611 const struct cs_section_def *sect = NULL; 5612 const struct cs_extent_def *ext = NULL; 5613 5614 if (rdev->rlc.cs_data == NULL) 5615 return 0; 5616 5617 /* begin clear state */ 5618 count += 2; 5619 /* context control state */ 5620 count += 3; 5621 5622 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 5623 for (ext = sect->section; ext->extent != NULL; ++ext) { 5624 if (sect->id == SECT_CONTEXT) 5625 count += 2 + ext->reg_count; 5626 else 5627 return 0; 5628 } 5629 } 5630 /* pa_sc_raster_config */ 5631 count += 3; 5632 /* end clear state */ 5633 count += 2; 5634 /* clear state */ 5635 count += 2; 5636 5637 return count; 5638 } 5639 5640 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 5641 { 5642 u32 count = 0, i; 5643 const struct cs_section_def *sect = NULL; 5644 const struct cs_extent_def *ext = NULL; 5645 5646 if (rdev->rlc.cs_data == NULL) 5647 return; 5648 if (buffer == NULL) 5649 return; 5650 5651 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 5652 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 5653 5654 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5655 buffer[count++] = cpu_to_le32(0x80000000); 5656 buffer[count++] = cpu_to_le32(0x80000000); 5657 5658 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 5659 for (ext = sect->section; ext->extent != NULL; ++ext) { 5660 if (sect->id == SECT_CONTEXT) { 5661 buffer[count++] = 5662 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 5663 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 5664 for (i = 0; i < ext->reg_count; i++) 5665 buffer[count++] = cpu_to_le32(ext->extent[i]); 5666 } else { 5667 return; 5668 } 5669 } 5670 } 5671 5672 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 5673 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 5674 switch (rdev->family) { 5675 case CHIP_TAHITI: 5676 case CHIP_PITCAIRN: 5677 buffer[count++] = cpu_to_le32(0x2a00126a); 5678 break; 5679 case CHIP_VERDE: 5680 buffer[count++] = cpu_to_le32(0x0000124a); 5681 break; 5682 case CHIP_OLAND: 5683 buffer[count++] = cpu_to_le32(0x00000082); 5684 break; 5685 case CHIP_HAINAN: 5686 buffer[count++] = cpu_to_le32(0x00000000); 5687 break; 5688 default: 5689 buffer[count++] = cpu_to_le32(0x00000000); 5690 break; 5691 } 5692 5693 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 5694 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 5695 5696 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 5697 buffer[count++] = cpu_to_le32(0); 5698 } 5699 5700 static void si_init_pg(struct radeon_device *rdev) 5701 { 5702 if (rdev->pg_flags) { 5703 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) { 5704 si_init_dma_pg(rdev); 5705 } 5706 si_init_ao_cu_mask(rdev); 5707 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 5708 si_init_gfx_cgpg(rdev); 5709 } else { 5710 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5711 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5712 } 5713 si_enable_dma_pg(rdev, true); 5714 si_enable_gfx_cgpg(rdev, true); 5715 } else { 5716 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5717 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5718 } 5719 } 5720 5721 static void si_fini_pg(struct radeon_device *rdev) 5722 { 5723 if (rdev->pg_flags) { 5724 si_enable_dma_pg(rdev, false); 5725 si_enable_gfx_cgpg(rdev, false); 5726 } 5727 } 5728 5729 /* 5730 * RLC 5731 */ 5732 void si_rlc_reset(struct radeon_device *rdev) 5733 { 5734 u32 tmp = RREG32(GRBM_SOFT_RESET); 5735 5736 tmp |= SOFT_RESET_RLC; 5737 WREG32(GRBM_SOFT_RESET, tmp); 5738 udelay(50); 5739 tmp &= ~SOFT_RESET_RLC; 5740 WREG32(GRBM_SOFT_RESET, tmp); 5741 udelay(50); 5742 } 5743 5744 static void si_rlc_stop(struct radeon_device *rdev) 5745 { 5746 WREG32(RLC_CNTL, 0); 5747 5748 si_enable_gui_idle_interrupt(rdev, false); 5749 5750 si_wait_for_rlc_serdes(rdev); 5751 } 5752 5753 static void si_rlc_start(struct radeon_device *rdev) 5754 { 5755 WREG32(RLC_CNTL, RLC_ENABLE); 5756 5757 si_enable_gui_idle_interrupt(rdev, true); 5758 5759 udelay(50); 5760 } 5761 5762 static bool si_lbpw_supported(struct radeon_device *rdev) 5763 { 5764 u32 tmp; 5765 5766 /* Enable LBPW only for DDR3 */ 5767 tmp = RREG32(MC_SEQ_MISC0); 5768 if ((tmp & 0xF0000000) == 0xB0000000) 5769 return true; 5770 return false; 5771 } 5772 5773 static void si_enable_lbpw(struct radeon_device *rdev, bool enable) 5774 { 5775 u32 tmp; 5776 5777 tmp = RREG32(RLC_LB_CNTL); 5778 if (enable) 5779 tmp |= LOAD_BALANCE_ENABLE; 5780 else 5781 tmp &= ~LOAD_BALANCE_ENABLE; 5782 WREG32(RLC_LB_CNTL, tmp); 5783 5784 if (!enable) { 5785 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5786 WREG32(SPI_LB_CU_MASK, 0x00ff); 5787 } 5788 } 5789 5790 static int si_rlc_resume(struct radeon_device *rdev) 5791 { 5792 u32 i; 5793 5794 if (!rdev->rlc_fw) 5795 return -EINVAL; 5796 5797 si_rlc_stop(rdev); 5798 5799 si_rlc_reset(rdev); 5800 5801 si_init_pg(rdev); 5802 5803 si_init_cg(rdev); 5804 5805 WREG32(RLC_RL_BASE, 0); 5806 WREG32(RLC_RL_SIZE, 0); 5807 WREG32(RLC_LB_CNTL, 0); 5808 WREG32(RLC_LB_CNTR_MAX, 0xffffffff); 5809 WREG32(RLC_LB_CNTR_INIT, 0); 5810 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 5811 5812 WREG32(RLC_MC_CNTL, 0); 5813 WREG32(RLC_UCODE_CNTL, 0); 5814 5815 if (rdev->new_fw) { 5816 const struct rlc_firmware_header_v1_0 *hdr = 5817 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; 5818 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 5819 const __le32 *fw_data = (const __le32 *) 5820 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 5821 5822 radeon_ucode_print_rlc_hdr(&hdr->header); 5823 5824 for (i = 0; i < fw_size; i++) { 5825 WREG32(RLC_UCODE_ADDR, i); 5826 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++)); 5827 } 5828 } else { 5829 const __be32 *fw_data = 5830 (const __be32 *)rdev->rlc_fw->data; 5831 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) { 5832 WREG32(RLC_UCODE_ADDR, i); 5833 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 5834 } 5835 } 5836 WREG32(RLC_UCODE_ADDR, 0); 5837 5838 si_enable_lbpw(rdev, si_lbpw_supported(rdev)); 5839 5840 si_rlc_start(rdev); 5841 5842 return 0; 5843 } 5844 5845 static void si_enable_interrupts(struct radeon_device *rdev) 5846 { 5847 u32 ih_cntl = RREG32(IH_CNTL); 5848 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5849 5850 ih_cntl |= ENABLE_INTR; 5851 ih_rb_cntl |= IH_RB_ENABLE; 5852 WREG32(IH_CNTL, ih_cntl); 5853 WREG32(IH_RB_CNTL, ih_rb_cntl); 5854 rdev->ih.enabled = true; 5855 } 5856 5857 static void si_disable_interrupts(struct radeon_device *rdev) 5858 { 5859 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5860 u32 ih_cntl = RREG32(IH_CNTL); 5861 5862 ih_rb_cntl &= ~IH_RB_ENABLE; 5863 ih_cntl &= ~ENABLE_INTR; 5864 WREG32(IH_RB_CNTL, ih_rb_cntl); 5865 WREG32(IH_CNTL, ih_cntl); 5866 /* set rptr, wptr to 0 */ 5867 WREG32(IH_RB_RPTR, 0); 5868 WREG32(IH_RB_WPTR, 0); 5869 rdev->ih.enabled = false; 5870 rdev->ih.rptr = 0; 5871 } 5872 5873 static void si_disable_interrupt_state(struct radeon_device *rdev) 5874 { 5875 u32 tmp; 5876 5877 tmp = RREG32(CP_INT_CNTL_RING0) & 5878 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5879 WREG32(CP_INT_CNTL_RING0, tmp); 5880 WREG32(CP_INT_CNTL_RING1, 0); 5881 WREG32(CP_INT_CNTL_RING2, 0); 5882 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5883 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp); 5884 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5885 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp); 5886 WREG32(GRBM_INT_CNTL, 0); 5887 if (rdev->num_crtc >= 2) { 5888 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5889 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5890 } 5891 if (rdev->num_crtc >= 4) { 5892 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5893 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5894 } 5895 if (rdev->num_crtc >= 6) { 5896 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5897 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5898 } 5899 5900 if (rdev->num_crtc >= 2) { 5901 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5902 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5903 } 5904 if (rdev->num_crtc >= 4) { 5905 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5906 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5907 } 5908 if (rdev->num_crtc >= 6) { 5909 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5910 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5911 } 5912 5913 if (!ASIC_IS_NODCE(rdev)) { 5914 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 5915 5916 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5917 WREG32(DC_HPD1_INT_CONTROL, tmp); 5918 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5919 WREG32(DC_HPD2_INT_CONTROL, tmp); 5920 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5921 WREG32(DC_HPD3_INT_CONTROL, tmp); 5922 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5923 WREG32(DC_HPD4_INT_CONTROL, tmp); 5924 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5925 WREG32(DC_HPD5_INT_CONTROL, tmp); 5926 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5927 WREG32(DC_HPD6_INT_CONTROL, tmp); 5928 } 5929 } 5930 5931 static int si_irq_init(struct radeon_device *rdev) 5932 { 5933 int ret = 0; 5934 int rb_bufsz; 5935 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 5936 5937 /* allocate ring */ 5938 ret = r600_ih_ring_alloc(rdev); 5939 if (ret) 5940 return ret; 5941 5942 /* disable irqs */ 5943 si_disable_interrupts(rdev); 5944 5945 /* init rlc */ 5946 ret = si_rlc_resume(rdev); 5947 if (ret) { 5948 r600_ih_ring_fini(rdev); 5949 return ret; 5950 } 5951 5952 /* setup interrupt control */ 5953 /* set dummy read address to ring address */ 5954 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 5955 interrupt_cntl = RREG32(INTERRUPT_CNTL); 5956 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 5957 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 5958 */ 5959 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 5960 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 5961 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 5962 WREG32(INTERRUPT_CNTL, interrupt_cntl); 5963 5964 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 5965 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 5966 5967 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 5968 IH_WPTR_OVERFLOW_CLEAR | 5969 (rb_bufsz << 1)); 5970 5971 if (rdev->wb.enabled) 5972 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 5973 5974 /* set the writeback address whether it's enabled or not */ 5975 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 5976 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 5977 5978 WREG32(IH_RB_CNTL, ih_rb_cntl); 5979 5980 /* set rptr, wptr to 0 */ 5981 WREG32(IH_RB_RPTR, 0); 5982 WREG32(IH_RB_WPTR, 0); 5983 5984 /* Default settings for IH_CNTL (disabled at first) */ 5985 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 5986 /* RPTR_REARM only works if msi's are enabled */ 5987 if (rdev->msi_enabled) 5988 ih_cntl |= RPTR_REARM; 5989 WREG32(IH_CNTL, ih_cntl); 5990 5991 /* force the active interrupt state to all disabled */ 5992 si_disable_interrupt_state(rdev); 5993 5994 pci_set_master(rdev->pdev); 5995 5996 /* enable irqs */ 5997 si_enable_interrupts(rdev); 5998 5999 return ret; 6000 } 6001 6002 int si_irq_set(struct radeon_device *rdev) 6003 { 6004 u32 cp_int_cntl; 6005 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 6006 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 6007 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0; 6008 u32 grbm_int_cntl = 0; 6009 u32 dma_cntl, dma_cntl1; 6010 u32 thermal_int = 0; 6011 6012 if (!rdev->irq.installed) { 6013 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 6014 return -EINVAL; 6015 } 6016 /* don't enable anything if the ih is disabled */ 6017 if (!rdev->ih.enabled) { 6018 si_disable_interrupts(rdev); 6019 /* force the active interrupt state to all disabled */ 6020 si_disable_interrupt_state(rdev); 6021 return 0; 6022 } 6023 6024 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 6025 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 6026 6027 if (!ASIC_IS_NODCE(rdev)) { 6028 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 6029 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 6030 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 6031 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 6032 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 6033 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 6034 } 6035 6036 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 6037 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 6038 6039 thermal_int = RREG32(CG_THERMAL_INT) & 6040 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 6041 6042 /* enable CP interrupts on all rings */ 6043 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 6044 DRM_DEBUG("si_irq_set: sw int gfx\n"); 6045 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 6046 } 6047 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 6048 DRM_DEBUG("si_irq_set: sw int cp1\n"); 6049 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 6050 } 6051 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 6052 DRM_DEBUG("si_irq_set: sw int cp2\n"); 6053 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 6054 } 6055 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 6056 DRM_DEBUG("si_irq_set: sw int dma\n"); 6057 dma_cntl |= TRAP_ENABLE; 6058 } 6059 6060 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 6061 DRM_DEBUG("si_irq_set: sw int dma1\n"); 6062 dma_cntl1 |= TRAP_ENABLE; 6063 } 6064 if (rdev->irq.crtc_vblank_int[0] || 6065 atomic_read(&rdev->irq.pflip[0])) { 6066 DRM_DEBUG("si_irq_set: vblank 0\n"); 6067 crtc1 |= VBLANK_INT_MASK; 6068 } 6069 if (rdev->irq.crtc_vblank_int[1] || 6070 atomic_read(&rdev->irq.pflip[1])) { 6071 DRM_DEBUG("si_irq_set: vblank 1\n"); 6072 crtc2 |= VBLANK_INT_MASK; 6073 } 6074 if (rdev->irq.crtc_vblank_int[2] || 6075 atomic_read(&rdev->irq.pflip[2])) { 6076 DRM_DEBUG("si_irq_set: vblank 2\n"); 6077 crtc3 |= VBLANK_INT_MASK; 6078 } 6079 if (rdev->irq.crtc_vblank_int[3] || 6080 atomic_read(&rdev->irq.pflip[3])) { 6081 DRM_DEBUG("si_irq_set: vblank 3\n"); 6082 crtc4 |= VBLANK_INT_MASK; 6083 } 6084 if (rdev->irq.crtc_vblank_int[4] || 6085 atomic_read(&rdev->irq.pflip[4])) { 6086 DRM_DEBUG("si_irq_set: vblank 4\n"); 6087 crtc5 |= VBLANK_INT_MASK; 6088 } 6089 if (rdev->irq.crtc_vblank_int[5] || 6090 atomic_read(&rdev->irq.pflip[5])) { 6091 DRM_DEBUG("si_irq_set: vblank 5\n"); 6092 crtc6 |= VBLANK_INT_MASK; 6093 } 6094 if (rdev->irq.hpd[0]) { 6095 DRM_DEBUG("si_irq_set: hpd 1\n"); 6096 hpd1 |= DC_HPDx_INT_EN; 6097 } 6098 if (rdev->irq.hpd[1]) { 6099 DRM_DEBUG("si_irq_set: hpd 2\n"); 6100 hpd2 |= DC_HPDx_INT_EN; 6101 } 6102 if (rdev->irq.hpd[2]) { 6103 DRM_DEBUG("si_irq_set: hpd 3\n"); 6104 hpd3 |= DC_HPDx_INT_EN; 6105 } 6106 if (rdev->irq.hpd[3]) { 6107 DRM_DEBUG("si_irq_set: hpd 4\n"); 6108 hpd4 |= DC_HPDx_INT_EN; 6109 } 6110 if (rdev->irq.hpd[4]) { 6111 DRM_DEBUG("si_irq_set: hpd 5\n"); 6112 hpd5 |= DC_HPDx_INT_EN; 6113 } 6114 if (rdev->irq.hpd[5]) { 6115 DRM_DEBUG("si_irq_set: hpd 6\n"); 6116 hpd6 |= DC_HPDx_INT_EN; 6117 } 6118 6119 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 6120 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1); 6121 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2); 6122 6123 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl); 6124 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1); 6125 6126 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 6127 6128 if (rdev->irq.dpm_thermal) { 6129 DRM_DEBUG("dpm thermal\n"); 6130 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 6131 } 6132 6133 if (rdev->num_crtc >= 2) { 6134 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 6135 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 6136 } 6137 if (rdev->num_crtc >= 4) { 6138 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 6139 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 6140 } 6141 if (rdev->num_crtc >= 6) { 6142 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 6143 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 6144 } 6145 6146 if (rdev->num_crtc >= 2) { 6147 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 6148 GRPH_PFLIP_INT_MASK); 6149 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 6150 GRPH_PFLIP_INT_MASK); 6151 } 6152 if (rdev->num_crtc >= 4) { 6153 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 6154 GRPH_PFLIP_INT_MASK); 6155 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 6156 GRPH_PFLIP_INT_MASK); 6157 } 6158 if (rdev->num_crtc >= 6) { 6159 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 6160 GRPH_PFLIP_INT_MASK); 6161 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 6162 GRPH_PFLIP_INT_MASK); 6163 } 6164 6165 if (!ASIC_IS_NODCE(rdev)) { 6166 WREG32(DC_HPD1_INT_CONTROL, hpd1); 6167 WREG32(DC_HPD2_INT_CONTROL, hpd2); 6168 WREG32(DC_HPD3_INT_CONTROL, hpd3); 6169 WREG32(DC_HPD4_INT_CONTROL, hpd4); 6170 WREG32(DC_HPD5_INT_CONTROL, hpd5); 6171 WREG32(DC_HPD6_INT_CONTROL, hpd6); 6172 } 6173 6174 WREG32(CG_THERMAL_INT, thermal_int); 6175 6176 return 0; 6177 } 6178 6179 static inline void si_irq_ack(struct radeon_device *rdev) 6180 { 6181 u32 tmp; 6182 6183 if (ASIC_IS_NODCE(rdev)) 6184 return; 6185 6186 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 6187 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 6188 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 6189 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 6190 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 6191 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 6192 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 6193 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 6194 if (rdev->num_crtc >= 4) { 6195 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 6196 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 6197 } 6198 if (rdev->num_crtc >= 6) { 6199 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 6200 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 6201 } 6202 6203 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 6204 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 6205 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 6206 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 6207 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 6208 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 6209 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 6210 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 6211 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 6212 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 6213 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 6214 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 6215 6216 if (rdev->num_crtc >= 4) { 6217 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 6218 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 6219 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 6220 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 6221 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 6222 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 6223 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 6224 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 6225 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 6226 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 6227 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 6228 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 6229 } 6230 6231 if (rdev->num_crtc >= 6) { 6232 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 6233 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 6234 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 6235 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 6236 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 6237 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 6238 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 6239 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 6240 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 6241 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 6242 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 6243 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 6244 } 6245 6246 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 6247 tmp = RREG32(DC_HPD1_INT_CONTROL); 6248 tmp |= DC_HPDx_INT_ACK; 6249 WREG32(DC_HPD1_INT_CONTROL, tmp); 6250 } 6251 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 6252 tmp = RREG32(DC_HPD2_INT_CONTROL); 6253 tmp |= DC_HPDx_INT_ACK; 6254 WREG32(DC_HPD2_INT_CONTROL, tmp); 6255 } 6256 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 6257 tmp = RREG32(DC_HPD3_INT_CONTROL); 6258 tmp |= DC_HPDx_INT_ACK; 6259 WREG32(DC_HPD3_INT_CONTROL, tmp); 6260 } 6261 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 6262 tmp = RREG32(DC_HPD4_INT_CONTROL); 6263 tmp |= DC_HPDx_INT_ACK; 6264 WREG32(DC_HPD4_INT_CONTROL, tmp); 6265 } 6266 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 6267 tmp = RREG32(DC_HPD5_INT_CONTROL); 6268 tmp |= DC_HPDx_INT_ACK; 6269 WREG32(DC_HPD5_INT_CONTROL, tmp); 6270 } 6271 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 6272 tmp = RREG32(DC_HPD5_INT_CONTROL); 6273 tmp |= DC_HPDx_INT_ACK; 6274 WREG32(DC_HPD6_INT_CONTROL, tmp); 6275 } 6276 } 6277 6278 static void si_irq_disable(struct radeon_device *rdev) 6279 { 6280 si_disable_interrupts(rdev); 6281 /* Wait and acknowledge irq */ 6282 mdelay(1); 6283 si_irq_ack(rdev); 6284 si_disable_interrupt_state(rdev); 6285 } 6286 6287 static void si_irq_suspend(struct radeon_device *rdev) 6288 { 6289 si_irq_disable(rdev); 6290 si_rlc_stop(rdev); 6291 } 6292 6293 static void si_irq_fini(struct radeon_device *rdev) 6294 { 6295 si_irq_suspend(rdev); 6296 r600_ih_ring_fini(rdev); 6297 } 6298 6299 static inline u32 si_get_ih_wptr(struct radeon_device *rdev) 6300 { 6301 u32 wptr, tmp; 6302 6303 if (rdev->wb.enabled) 6304 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 6305 else 6306 wptr = RREG32(IH_RB_WPTR); 6307 6308 if (wptr & RB_OVERFLOW) { 6309 /* When a ring buffer overflow happen start parsing interrupt 6310 * from the last not overwritten vector (wptr + 16). Hopefully 6311 * this should allow us to catchup. 6312 */ 6313 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 6314 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 6315 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 6316 tmp = RREG32(IH_RB_CNTL); 6317 tmp |= IH_WPTR_OVERFLOW_CLEAR; 6318 WREG32(IH_RB_CNTL, tmp); 6319 wptr &= ~RB_OVERFLOW; 6320 } 6321 return (wptr & rdev->ih.ptr_mask); 6322 } 6323 6324 /* SI IV Ring 6325 * Each IV ring entry is 128 bits: 6326 * [7:0] - interrupt source id 6327 * [31:8] - reserved 6328 * [59:32] - interrupt source data 6329 * [63:60] - reserved 6330 * [71:64] - RINGID 6331 * [79:72] - VMID 6332 * [127:80] - reserved 6333 */ 6334 int si_irq_process(struct radeon_device *rdev) 6335 { 6336 u32 wptr; 6337 u32 rptr; 6338 u32 src_id, src_data, ring_id; 6339 u32 ring_index; 6340 bool queue_hotplug = false; 6341 bool queue_thermal = false; 6342 u32 status, addr; 6343 6344 if (!rdev->ih.enabled || rdev->shutdown) 6345 return IRQ_NONE; 6346 6347 wptr = si_get_ih_wptr(rdev); 6348 6349 restart_ih: 6350 /* is somebody else already processing irqs? */ 6351 if (atomic_xchg(&rdev->ih.lock, 1)) 6352 return IRQ_NONE; 6353 6354 rptr = rdev->ih.rptr; 6355 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 6356 6357 /* Order reading of wptr vs. reading of IH ring data */ 6358 rmb(); 6359 6360 /* display interrupts */ 6361 si_irq_ack(rdev); 6362 6363 while (rptr != wptr) { 6364 /* wptr/rptr are in bytes! */ 6365 ring_index = rptr / 4; 6366 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 6367 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 6368 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 6369 6370 switch (src_id) { 6371 case 1: /* D1 vblank/vline */ 6372 switch (src_data) { 6373 case 0: /* D1 vblank */ 6374 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 6375 if (rdev->irq.crtc_vblank_int[0]) { 6376 drm_handle_vblank(rdev->ddev, 0); 6377 rdev->pm.vblank_sync = true; 6378 wake_up(&rdev->irq.vblank_queue); 6379 } 6380 if (atomic_read(&rdev->irq.pflip[0])) 6381 radeon_crtc_handle_vblank(rdev, 0); 6382 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 6383 DRM_DEBUG("IH: D1 vblank\n"); 6384 } 6385 break; 6386 case 1: /* D1 vline */ 6387 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 6388 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 6389 DRM_DEBUG("IH: D1 vline\n"); 6390 } 6391 break; 6392 default: 6393 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6394 break; 6395 } 6396 break; 6397 case 2: /* D2 vblank/vline */ 6398 switch (src_data) { 6399 case 0: /* D2 vblank */ 6400 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 6401 if (rdev->irq.crtc_vblank_int[1]) { 6402 drm_handle_vblank(rdev->ddev, 1); 6403 rdev->pm.vblank_sync = true; 6404 wake_up(&rdev->irq.vblank_queue); 6405 } 6406 if (atomic_read(&rdev->irq.pflip[1])) 6407 radeon_crtc_handle_vblank(rdev, 1); 6408 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 6409 DRM_DEBUG("IH: D2 vblank\n"); 6410 } 6411 break; 6412 case 1: /* D2 vline */ 6413 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 6414 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 6415 DRM_DEBUG("IH: D2 vline\n"); 6416 } 6417 break; 6418 default: 6419 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6420 break; 6421 } 6422 break; 6423 case 3: /* D3 vblank/vline */ 6424 switch (src_data) { 6425 case 0: /* D3 vblank */ 6426 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 6427 if (rdev->irq.crtc_vblank_int[2]) { 6428 drm_handle_vblank(rdev->ddev, 2); 6429 rdev->pm.vblank_sync = true; 6430 wake_up(&rdev->irq.vblank_queue); 6431 } 6432 if (atomic_read(&rdev->irq.pflip[2])) 6433 radeon_crtc_handle_vblank(rdev, 2); 6434 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 6435 DRM_DEBUG("IH: D3 vblank\n"); 6436 } 6437 break; 6438 case 1: /* D3 vline */ 6439 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 6440 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 6441 DRM_DEBUG("IH: D3 vline\n"); 6442 } 6443 break; 6444 default: 6445 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6446 break; 6447 } 6448 break; 6449 case 4: /* D4 vblank/vline */ 6450 switch (src_data) { 6451 case 0: /* D4 vblank */ 6452 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 6453 if (rdev->irq.crtc_vblank_int[3]) { 6454 drm_handle_vblank(rdev->ddev, 3); 6455 rdev->pm.vblank_sync = true; 6456 wake_up(&rdev->irq.vblank_queue); 6457 } 6458 if (atomic_read(&rdev->irq.pflip[3])) 6459 radeon_crtc_handle_vblank(rdev, 3); 6460 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 6461 DRM_DEBUG("IH: D4 vblank\n"); 6462 } 6463 break; 6464 case 1: /* D4 vline */ 6465 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 6466 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 6467 DRM_DEBUG("IH: D4 vline\n"); 6468 } 6469 break; 6470 default: 6471 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6472 break; 6473 } 6474 break; 6475 case 5: /* D5 vblank/vline */ 6476 switch (src_data) { 6477 case 0: /* D5 vblank */ 6478 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 6479 if (rdev->irq.crtc_vblank_int[4]) { 6480 drm_handle_vblank(rdev->ddev, 4); 6481 rdev->pm.vblank_sync = true; 6482 wake_up(&rdev->irq.vblank_queue); 6483 } 6484 if (atomic_read(&rdev->irq.pflip[4])) 6485 radeon_crtc_handle_vblank(rdev, 4); 6486 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 6487 DRM_DEBUG("IH: D5 vblank\n"); 6488 } 6489 break; 6490 case 1: /* D5 vline */ 6491 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 6492 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 6493 DRM_DEBUG("IH: D5 vline\n"); 6494 } 6495 break; 6496 default: 6497 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6498 break; 6499 } 6500 break; 6501 case 6: /* D6 vblank/vline */ 6502 switch (src_data) { 6503 case 0: /* D6 vblank */ 6504 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 6505 if (rdev->irq.crtc_vblank_int[5]) { 6506 drm_handle_vblank(rdev->ddev, 5); 6507 rdev->pm.vblank_sync = true; 6508 wake_up(&rdev->irq.vblank_queue); 6509 } 6510 if (atomic_read(&rdev->irq.pflip[5])) 6511 radeon_crtc_handle_vblank(rdev, 5); 6512 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 6513 DRM_DEBUG("IH: D6 vblank\n"); 6514 } 6515 break; 6516 case 1: /* D6 vline */ 6517 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 6518 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 6519 DRM_DEBUG("IH: D6 vline\n"); 6520 } 6521 break; 6522 default: 6523 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6524 break; 6525 } 6526 break; 6527 case 8: /* D1 page flip */ 6528 case 10: /* D2 page flip */ 6529 case 12: /* D3 page flip */ 6530 case 14: /* D4 page flip */ 6531 case 16: /* D5 page flip */ 6532 case 18: /* D6 page flip */ 6533 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 6534 if (radeon_use_pflipirq > 0) 6535 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 6536 break; 6537 case 42: /* HPD hotplug */ 6538 switch (src_data) { 6539 case 0: 6540 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 6541 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 6542 queue_hotplug = true; 6543 DRM_DEBUG("IH: HPD1\n"); 6544 } 6545 break; 6546 case 1: 6547 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 6548 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 6549 queue_hotplug = true; 6550 DRM_DEBUG("IH: HPD2\n"); 6551 } 6552 break; 6553 case 2: 6554 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 6555 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 6556 queue_hotplug = true; 6557 DRM_DEBUG("IH: HPD3\n"); 6558 } 6559 break; 6560 case 3: 6561 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 6562 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 6563 queue_hotplug = true; 6564 DRM_DEBUG("IH: HPD4\n"); 6565 } 6566 break; 6567 case 4: 6568 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 6569 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 6570 queue_hotplug = true; 6571 DRM_DEBUG("IH: HPD5\n"); 6572 } 6573 break; 6574 case 5: 6575 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 6576 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 6577 queue_hotplug = true; 6578 DRM_DEBUG("IH: HPD6\n"); 6579 } 6580 break; 6581 default: 6582 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6583 break; 6584 } 6585 break; 6586 case 124: /* UVD */ 6587 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 6588 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 6589 break; 6590 case 146: 6591 case 147: 6592 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 6593 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 6594 /* reset addr and status */ 6595 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 6596 if (addr == 0x0 && status == 0x0) 6597 break; 6598 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 6599 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 6600 addr); 6601 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 6602 status); 6603 si_vm_decode_fault(rdev, status, addr); 6604 break; 6605 case 176: /* RINGID0 CP_INT */ 6606 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6607 break; 6608 case 177: /* RINGID1 CP_INT */ 6609 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6610 break; 6611 case 178: /* RINGID2 CP_INT */ 6612 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6613 break; 6614 case 181: /* CP EOP event */ 6615 DRM_DEBUG("IH: CP EOP\n"); 6616 switch (ring_id) { 6617 case 0: 6618 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6619 break; 6620 case 1: 6621 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6622 break; 6623 case 2: 6624 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6625 break; 6626 } 6627 break; 6628 case 224: /* DMA trap event */ 6629 DRM_DEBUG("IH: DMA trap\n"); 6630 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 6631 break; 6632 case 230: /* thermal low to high */ 6633 DRM_DEBUG("IH: thermal low to high\n"); 6634 rdev->pm.dpm.thermal.high_to_low = false; 6635 queue_thermal = true; 6636 break; 6637 case 231: /* thermal high to low */ 6638 DRM_DEBUG("IH: thermal high to low\n"); 6639 rdev->pm.dpm.thermal.high_to_low = true; 6640 queue_thermal = true; 6641 break; 6642 case 233: /* GUI IDLE */ 6643 DRM_DEBUG("IH: GUI idle\n"); 6644 break; 6645 case 244: /* DMA trap event */ 6646 DRM_DEBUG("IH: DMA1 trap\n"); 6647 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6648 break; 6649 default: 6650 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6651 break; 6652 } 6653 6654 /* wptr/rptr are in bytes! */ 6655 rptr += 16; 6656 rptr &= rdev->ih.ptr_mask; 6657 } 6658 if (queue_hotplug) 6659 schedule_work(&rdev->hotplug_work); 6660 if (queue_thermal && rdev->pm.dpm_enabled) 6661 schedule_work(&rdev->pm.dpm.thermal.work); 6662 rdev->ih.rptr = rptr; 6663 WREG32(IH_RB_RPTR, rdev->ih.rptr); 6664 atomic_set(&rdev->ih.lock, 0); 6665 6666 /* make sure wptr hasn't changed while processing */ 6667 wptr = si_get_ih_wptr(rdev); 6668 if (wptr != rptr) 6669 goto restart_ih; 6670 6671 return IRQ_HANDLED; 6672 } 6673 6674 /* 6675 * startup/shutdown callbacks 6676 */ 6677 static int si_startup(struct radeon_device *rdev) 6678 { 6679 struct radeon_ring *ring; 6680 int r; 6681 6682 /* enable pcie gen2/3 link */ 6683 si_pcie_gen3_enable(rdev); 6684 /* enable aspm */ 6685 si_program_aspm(rdev); 6686 6687 /* scratch needs to be initialized before MC */ 6688 r = r600_vram_scratch_init(rdev); 6689 if (r) 6690 return r; 6691 6692 si_mc_program(rdev); 6693 6694 if (!rdev->pm.dpm_enabled) { 6695 r = si_mc_load_microcode(rdev); 6696 if (r) { 6697 DRM_ERROR("Failed to load MC firmware!\n"); 6698 return r; 6699 } 6700 } 6701 6702 r = si_pcie_gart_enable(rdev); 6703 if (r) 6704 return r; 6705 si_gpu_init(rdev); 6706 6707 /* allocate rlc buffers */ 6708 if (rdev->family == CHIP_VERDE) { 6709 rdev->rlc.reg_list = verde_rlc_save_restore_register_list; 6710 rdev->rlc.reg_list_size = 6711 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list); 6712 } 6713 rdev->rlc.cs_data = si_cs_data; 6714 r = sumo_rlc_init(rdev); 6715 if (r) { 6716 DRM_ERROR("Failed to init rlc BOs!\n"); 6717 return r; 6718 } 6719 6720 /* allocate wb buffer */ 6721 r = radeon_wb_init(rdev); 6722 if (r) 6723 return r; 6724 6725 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 6726 if (r) { 6727 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6728 return r; 6729 } 6730 6731 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6732 if (r) { 6733 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6734 return r; 6735 } 6736 6737 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6738 if (r) { 6739 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6740 return r; 6741 } 6742 6743 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 6744 if (r) { 6745 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6746 return r; 6747 } 6748 6749 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6750 if (r) { 6751 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6752 return r; 6753 } 6754 6755 if (rdev->has_uvd) { 6756 r = uvd_v2_2_resume(rdev); 6757 if (!r) { 6758 r = radeon_fence_driver_start_ring(rdev, 6759 R600_RING_TYPE_UVD_INDEX); 6760 if (r) 6761 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 6762 } 6763 if (r) 6764 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 6765 } 6766 6767 /* Enable IRQ */ 6768 if (!rdev->irq.installed) { 6769 r = radeon_irq_kms_init(rdev); 6770 if (r) 6771 return r; 6772 } 6773 6774 r = si_irq_init(rdev); 6775 if (r) { 6776 DRM_ERROR("radeon: IH init failed (%d).\n", r); 6777 radeon_irq_kms_fini(rdev); 6778 return r; 6779 } 6780 si_irq_set(rdev); 6781 6782 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6783 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 6784 RADEON_CP_PACKET2); 6785 if (r) 6786 return r; 6787 6788 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6789 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 6790 RADEON_CP_PACKET2); 6791 if (r) 6792 return r; 6793 6794 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6795 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 6796 RADEON_CP_PACKET2); 6797 if (r) 6798 return r; 6799 6800 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6801 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 6802 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6803 if (r) 6804 return r; 6805 6806 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6807 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 6808 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6809 if (r) 6810 return r; 6811 6812 r = si_cp_load_microcode(rdev); 6813 if (r) 6814 return r; 6815 r = si_cp_resume(rdev); 6816 if (r) 6817 return r; 6818 6819 r = cayman_dma_resume(rdev); 6820 if (r) 6821 return r; 6822 6823 if (rdev->has_uvd) { 6824 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6825 if (ring->ring_size) { 6826 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 6827 RADEON_CP_PACKET2); 6828 if (!r) 6829 r = uvd_v1_0_init(rdev); 6830 if (r) 6831 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 6832 } 6833 } 6834 6835 r = radeon_ib_pool_init(rdev); 6836 if (r) { 6837 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 6838 return r; 6839 } 6840 6841 r = radeon_vm_manager_init(rdev); 6842 if (r) { 6843 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 6844 return r; 6845 } 6846 6847 r = dce6_audio_init(rdev); 6848 if (r) 6849 return r; 6850 6851 return 0; 6852 } 6853 6854 int si_resume(struct radeon_device *rdev) 6855 { 6856 int r; 6857 6858 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 6859 * posting will perform necessary task to bring back GPU into good 6860 * shape. 6861 */ 6862 /* post card */ 6863 atom_asic_init(rdev->mode_info.atom_context); 6864 6865 /* init golden registers */ 6866 si_init_golden_registers(rdev); 6867 6868 if (rdev->pm.pm_method == PM_METHOD_DPM) 6869 radeon_pm_resume(rdev); 6870 6871 rdev->accel_working = true; 6872 r = si_startup(rdev); 6873 if (r) { 6874 DRM_ERROR("si startup failed on resume\n"); 6875 rdev->accel_working = false; 6876 return r; 6877 } 6878 6879 return r; 6880 6881 } 6882 6883 int si_suspend(struct radeon_device *rdev) 6884 { 6885 radeon_pm_suspend(rdev); 6886 dce6_audio_fini(rdev); 6887 radeon_vm_manager_fini(rdev); 6888 si_cp_enable(rdev, false); 6889 cayman_dma_stop(rdev); 6890 if (rdev->has_uvd) { 6891 uvd_v1_0_fini(rdev); 6892 radeon_uvd_suspend(rdev); 6893 } 6894 si_fini_pg(rdev); 6895 si_fini_cg(rdev); 6896 si_irq_suspend(rdev); 6897 radeon_wb_disable(rdev); 6898 si_pcie_gart_disable(rdev); 6899 return 0; 6900 } 6901 6902 /* Plan is to move initialization in that function and use 6903 * helper function so that radeon_device_init pretty much 6904 * do nothing more than calling asic specific function. This 6905 * should also allow to remove a bunch of callback function 6906 * like vram_info. 6907 */ 6908 int si_init(struct radeon_device *rdev) 6909 { 6910 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6911 int r; 6912 6913 /* Read BIOS */ 6914 if (!radeon_get_bios(rdev)) { 6915 if (ASIC_IS_AVIVO(rdev)) 6916 return -EINVAL; 6917 } 6918 /* Must be an ATOMBIOS */ 6919 if (!rdev->is_atom_bios) { 6920 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 6921 return -EINVAL; 6922 } 6923 r = radeon_atombios_init(rdev); 6924 if (r) 6925 return r; 6926 6927 /* Post card if necessary */ 6928 if (!radeon_card_posted(rdev)) { 6929 if (!rdev->bios) { 6930 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 6931 return -EINVAL; 6932 } 6933 DRM_INFO("GPU not posted. posting now...\n"); 6934 atom_asic_init(rdev->mode_info.atom_context); 6935 } 6936 /* init golden registers */ 6937 si_init_golden_registers(rdev); 6938 /* Initialize scratch registers */ 6939 si_scratch_init(rdev); 6940 /* Initialize surface registers */ 6941 radeon_surface_init(rdev); 6942 /* Initialize clocks */ 6943 radeon_get_clock_info(rdev->ddev); 6944 6945 /* Fence driver */ 6946 r = radeon_fence_driver_init(rdev); 6947 if (r) 6948 return r; 6949 6950 /* initialize memory controller */ 6951 r = si_mc_init(rdev); 6952 if (r) 6953 return r; 6954 /* Memory manager */ 6955 r = radeon_bo_init(rdev); 6956 if (r) 6957 return r; 6958 6959 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 6960 !rdev->rlc_fw || !rdev->mc_fw) { 6961 r = si_init_microcode(rdev); 6962 if (r) { 6963 DRM_ERROR("Failed to load firmware!\n"); 6964 return r; 6965 } 6966 } 6967 6968 /* Initialize power management */ 6969 radeon_pm_init(rdev); 6970 6971 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6972 ring->ring_obj = NULL; 6973 r600_ring_init(rdev, ring, 1024 * 1024); 6974 6975 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6976 ring->ring_obj = NULL; 6977 r600_ring_init(rdev, ring, 1024 * 1024); 6978 6979 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6980 ring->ring_obj = NULL; 6981 r600_ring_init(rdev, ring, 1024 * 1024); 6982 6983 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6984 ring->ring_obj = NULL; 6985 r600_ring_init(rdev, ring, 64 * 1024); 6986 6987 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6988 ring->ring_obj = NULL; 6989 r600_ring_init(rdev, ring, 64 * 1024); 6990 6991 if (rdev->has_uvd) { 6992 r = radeon_uvd_init(rdev); 6993 if (!r) { 6994 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6995 ring->ring_obj = NULL; 6996 r600_ring_init(rdev, ring, 4096); 6997 } 6998 } 6999 7000 rdev->ih.ring_obj = NULL; 7001 r600_ih_ring_init(rdev, 64 * 1024); 7002 7003 r = r600_pcie_gart_init(rdev); 7004 if (r) 7005 return r; 7006 7007 rdev->accel_working = true; 7008 r = si_startup(rdev); 7009 if (r) { 7010 dev_err(rdev->dev, "disabling GPU acceleration\n"); 7011 si_cp_fini(rdev); 7012 cayman_dma_fini(rdev); 7013 si_irq_fini(rdev); 7014 sumo_rlc_fini(rdev); 7015 radeon_wb_fini(rdev); 7016 radeon_ib_pool_fini(rdev); 7017 radeon_vm_manager_fini(rdev); 7018 radeon_irq_kms_fini(rdev); 7019 si_pcie_gart_fini(rdev); 7020 rdev->accel_working = false; 7021 } 7022 7023 /* Don't start up if the MC ucode is missing. 7024 * The default clocks and voltages before the MC ucode 7025 * is loaded are not suffient for advanced operations. 7026 */ 7027 if (!rdev->mc_fw) { 7028 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 7029 return -EINVAL; 7030 } 7031 7032 return 0; 7033 } 7034 7035 void si_fini(struct radeon_device *rdev) 7036 { 7037 radeon_pm_fini(rdev); 7038 si_cp_fini(rdev); 7039 cayman_dma_fini(rdev); 7040 si_fini_pg(rdev); 7041 si_fini_cg(rdev); 7042 si_irq_fini(rdev); 7043 sumo_rlc_fini(rdev); 7044 radeon_wb_fini(rdev); 7045 radeon_vm_manager_fini(rdev); 7046 radeon_ib_pool_fini(rdev); 7047 radeon_irq_kms_fini(rdev); 7048 if (rdev->has_uvd) { 7049 uvd_v1_0_fini(rdev); 7050 radeon_uvd_fini(rdev); 7051 } 7052 si_pcie_gart_fini(rdev); 7053 r600_vram_scratch_fini(rdev); 7054 radeon_gem_fini(rdev); 7055 radeon_fence_driver_fini(rdev); 7056 radeon_bo_fini(rdev); 7057 radeon_atombios_fini(rdev); 7058 kfree(rdev->bios); 7059 rdev->bios = NULL; 7060 } 7061 7062 /** 7063 * si_get_gpu_clock_counter - return GPU clock counter snapshot 7064 * 7065 * @rdev: radeon_device pointer 7066 * 7067 * Fetches a GPU clock counter snapshot (SI). 7068 * Returns the 64 bit clock counter snapshot. 7069 */ 7070 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev) 7071 { 7072 uint64_t clock; 7073 7074 mutex_lock(&rdev->gpu_clock_mutex); 7075 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 7076 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 7077 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 7078 mutex_unlock(&rdev->gpu_clock_mutex); 7079 return clock; 7080 } 7081 7082 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 7083 { 7084 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 7085 int r; 7086 7087 /* bypass vclk and dclk with bclk */ 7088 WREG32_P(CG_UPLL_FUNC_CNTL_2, 7089 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 7090 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 7091 7092 /* put PLL in bypass mode */ 7093 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 7094 7095 if (!vclk || !dclk) { 7096 /* keep the Bypass mode, put PLL to sleep */ 7097 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 7098 return 0; 7099 } 7100 7101 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 7102 16384, 0x03FFFFFF, 0, 128, 5, 7103 &fb_div, &vclk_div, &dclk_div); 7104 if (r) 7105 return r; 7106 7107 /* set RESET_ANTI_MUX to 0 */ 7108 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK); 7109 7110 /* set VCO_MODE to 1 */ 7111 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 7112 7113 /* toggle UPLL_SLEEP to 1 then back to 0 */ 7114 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 7115 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 7116 7117 /* deassert UPLL_RESET */ 7118 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 7119 7120 mdelay(1); 7121 7122 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 7123 if (r) 7124 return r; 7125 7126 /* assert UPLL_RESET again */ 7127 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 7128 7129 /* disable spread spectrum. */ 7130 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 7131 7132 /* set feedback divider */ 7133 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 7134 7135 /* set ref divider to 0 */ 7136 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 7137 7138 if (fb_div < 307200) 7139 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 7140 else 7141 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 7142 7143 /* set PDIV_A and PDIV_B */ 7144 WREG32_P(CG_UPLL_FUNC_CNTL_2, 7145 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 7146 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 7147 7148 /* give the PLL some time to settle */ 7149 mdelay(15); 7150 7151 /* deassert PLL_RESET */ 7152 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 7153 7154 mdelay(15); 7155 7156 /* switch from bypass mode to normal mode */ 7157 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 7158 7159 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 7160 if (r) 7161 return r; 7162 7163 /* switch VCLK and DCLK selection */ 7164 WREG32_P(CG_UPLL_FUNC_CNTL_2, 7165 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 7166 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 7167 7168 mdelay(100); 7169 7170 return 0; 7171 } 7172 7173 static void si_pcie_gen3_enable(struct radeon_device *rdev) 7174 { 7175 struct pci_dev *root = rdev->pdev->bus->self; 7176 int bridge_pos, gpu_pos; 7177 u32 speed_cntl, mask, current_data_rate; 7178 int ret, i; 7179 u16 tmp16; 7180 7181 if (radeon_pcie_gen2 == 0) 7182 return; 7183 7184 if (rdev->flags & RADEON_IS_IGP) 7185 return; 7186 7187 if (!(rdev->flags & RADEON_IS_PCIE)) 7188 return; 7189 7190 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 7191 if (ret != 0) 7192 return; 7193 7194 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 7195 return; 7196 7197 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 7198 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 7199 LC_CURRENT_DATA_RATE_SHIFT; 7200 if (mask & DRM_PCIE_SPEED_80) { 7201 if (current_data_rate == 2) { 7202 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 7203 return; 7204 } 7205 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 7206 } else if (mask & DRM_PCIE_SPEED_50) { 7207 if (current_data_rate == 1) { 7208 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 7209 return; 7210 } 7211 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 7212 } 7213 7214 bridge_pos = pci_pcie_cap(root); 7215 if (!bridge_pos) 7216 return; 7217 7218 gpu_pos = pci_pcie_cap(rdev->pdev); 7219 if (!gpu_pos) 7220 return; 7221 7222 if (mask & DRM_PCIE_SPEED_80) { 7223 /* re-try equalization if gen3 is not already enabled */ 7224 if (current_data_rate != 2) { 7225 u16 bridge_cfg, gpu_cfg; 7226 u16 bridge_cfg2, gpu_cfg2; 7227 u32 max_lw, current_lw, tmp; 7228 7229 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 7230 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 7231 7232 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 7233 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 7234 7235 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 7236 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 7237 7238 tmp = RREG32_PCIE(PCIE_LC_STATUS1); 7239 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 7240 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 7241 7242 if (current_lw < max_lw) { 7243 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 7244 if (tmp & LC_RENEGOTIATION_SUPPORT) { 7245 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 7246 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 7247 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 7248 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 7249 } 7250 } 7251 7252 for (i = 0; i < 10; i++) { 7253 /* check status */ 7254 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 7255 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 7256 break; 7257 7258 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 7259 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 7260 7261 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 7262 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 7263 7264 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7265 tmp |= LC_SET_QUIESCE; 7266 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7267 7268 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7269 tmp |= LC_REDO_EQ; 7270 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7271 7272 mdelay(100); 7273 7274 /* linkctl */ 7275 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 7276 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 7277 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 7278 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 7279 7280 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 7281 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 7282 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 7283 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 7284 7285 /* linkctl2 */ 7286 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 7287 tmp16 &= ~((1 << 4) | (7 << 9)); 7288 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 7289 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 7290 7291 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 7292 tmp16 &= ~((1 << 4) | (7 << 9)); 7293 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 7294 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 7295 7296 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7297 tmp &= ~LC_SET_QUIESCE; 7298 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7299 } 7300 } 7301 } 7302 7303 /* set the link speed */ 7304 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 7305 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 7306 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 7307 7308 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 7309 tmp16 &= ~0xf; 7310 if (mask & DRM_PCIE_SPEED_80) 7311 tmp16 |= 3; /* gen3 */ 7312 else if (mask & DRM_PCIE_SPEED_50) 7313 tmp16 |= 2; /* gen2 */ 7314 else 7315 tmp16 |= 1; /* gen1 */ 7316 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 7317 7318 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 7319 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 7320 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 7321 7322 for (i = 0; i < rdev->usec_timeout; i++) { 7323 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 7324 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 7325 break; 7326 udelay(1); 7327 } 7328 } 7329 7330 static void si_program_aspm(struct radeon_device *rdev) 7331 { 7332 u32 data, orig; 7333 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 7334 bool disable_clkreq = false; 7335 7336 if (radeon_aspm == 0) 7337 return; 7338 7339 if (!(rdev->flags & RADEON_IS_PCIE)) 7340 return; 7341 7342 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7343 data &= ~LC_XMIT_N_FTS_MASK; 7344 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 7345 if (orig != data) 7346 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 7347 7348 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 7349 data |= LC_GO_TO_RECOVERY; 7350 if (orig != data) 7351 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 7352 7353 orig = data = RREG32_PCIE(PCIE_P_CNTL); 7354 data |= P_IGNORE_EDB_ERR; 7355 if (orig != data) 7356 WREG32_PCIE(PCIE_P_CNTL, data); 7357 7358 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7359 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 7360 data |= LC_PMI_TO_L1_DIS; 7361 if (!disable_l0s) 7362 data |= LC_L0S_INACTIVITY(7); 7363 7364 if (!disable_l1) { 7365 data |= LC_L1_INACTIVITY(7); 7366 data &= ~LC_PMI_TO_L1_DIS; 7367 if (orig != data) 7368 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7369 7370 if (!disable_plloff_in_l1) { 7371 bool clk_req_support; 7372 7373 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7374 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7375 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7376 if (orig != data) 7377 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7378 7379 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7380 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7381 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7382 if (orig != data) 7383 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7384 7385 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7386 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7387 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7388 if (orig != data) 7389 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7390 7391 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7392 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7393 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7394 if (orig != data) 7395 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7396 7397 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) { 7398 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7399 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7400 if (orig != data) 7401 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7402 7403 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7404 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7405 if (orig != data) 7406 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7407 7408 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2); 7409 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7410 if (orig != data) 7411 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data); 7412 7413 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3); 7414 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7415 if (orig != data) 7416 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data); 7417 7418 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7419 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7420 if (orig != data) 7421 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7422 7423 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7424 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7425 if (orig != data) 7426 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7427 7428 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2); 7429 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7430 if (orig != data) 7431 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data); 7432 7433 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3); 7434 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7435 if (orig != data) 7436 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data); 7437 } 7438 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 7439 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 7440 data |= LC_DYN_LANES_PWR_STATE(3); 7441 if (orig != data) 7442 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 7443 7444 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL); 7445 data &= ~LS2_EXIT_TIME_MASK; 7446 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7447 data |= LS2_EXIT_TIME(5); 7448 if (orig != data) 7449 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 7450 7451 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL); 7452 data &= ~LS2_EXIT_TIME_MASK; 7453 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7454 data |= LS2_EXIT_TIME(5); 7455 if (orig != data) 7456 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 7457 7458 if (!disable_clkreq) { 7459 struct pci_dev *root = rdev->pdev->bus->self; 7460 u32 lnkcap; 7461 7462 clk_req_support = false; 7463 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 7464 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 7465 clk_req_support = true; 7466 } else { 7467 clk_req_support = false; 7468 } 7469 7470 if (clk_req_support) { 7471 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 7472 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 7473 if (orig != data) 7474 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 7475 7476 orig = data = RREG32(THM_CLK_CNTL); 7477 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 7478 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 7479 if (orig != data) 7480 WREG32(THM_CLK_CNTL, data); 7481 7482 orig = data = RREG32(MISC_CLK_CNTL); 7483 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 7484 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 7485 if (orig != data) 7486 WREG32(MISC_CLK_CNTL, data); 7487 7488 orig = data = RREG32(CG_CLKPIN_CNTL); 7489 data &= ~BCLK_AS_XCLK; 7490 if (orig != data) 7491 WREG32(CG_CLKPIN_CNTL, data); 7492 7493 orig = data = RREG32(CG_CLKPIN_CNTL_2); 7494 data &= ~FORCE_BIF_REFCLK_EN; 7495 if (orig != data) 7496 WREG32(CG_CLKPIN_CNTL_2, data); 7497 7498 orig = data = RREG32(MPLL_BYPASSCLK_SEL); 7499 data &= ~MPLL_CLKOUT_SEL_MASK; 7500 data |= MPLL_CLKOUT_SEL(4); 7501 if (orig != data) 7502 WREG32(MPLL_BYPASSCLK_SEL, data); 7503 7504 orig = data = RREG32(SPLL_CNTL_MODE); 7505 data &= ~SPLL_REFCLK_SEL_MASK; 7506 if (orig != data) 7507 WREG32(SPLL_CNTL_MODE, data); 7508 } 7509 } 7510 } else { 7511 if (orig != data) 7512 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7513 } 7514 7515 orig = data = RREG32_PCIE(PCIE_CNTL2); 7516 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 7517 if (orig != data) 7518 WREG32_PCIE(PCIE_CNTL2, data); 7519 7520 if (!disable_l0s) { 7521 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7522 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 7523 data = RREG32_PCIE(PCIE_LC_STATUS1); 7524 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 7525 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7526 data &= ~LC_L0S_INACTIVITY_MASK; 7527 if (orig != data) 7528 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7529 } 7530 } 7531 } 7532 } 7533