1 /* 2 * Copyright 2011 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include <drm/drmP.h> 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include <drm/radeon_drm.h> 31 #include "sid.h" 32 #include "atom.h" 33 #include "si_blit_shaders.h" 34 #include "clearstate_si.h" 35 #include "radeon_ucode.h" 36 37 38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin"); 39 MODULE_FIRMWARE("radeon/TAHITI_me.bin"); 40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin"); 41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin"); 42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin"); 43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin"); 44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin"); 45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin"); 46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin"); 47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin"); 48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin"); 49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin"); 50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin"); 51 MODULE_FIRMWARE("radeon/VERDE_me.bin"); 52 MODULE_FIRMWARE("radeon/VERDE_ce.bin"); 53 MODULE_FIRMWARE("radeon/VERDE_mc.bin"); 54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin"); 55 MODULE_FIRMWARE("radeon/VERDE_smc.bin"); 56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin"); 57 MODULE_FIRMWARE("radeon/OLAND_me.bin"); 58 MODULE_FIRMWARE("radeon/OLAND_ce.bin"); 59 MODULE_FIRMWARE("radeon/OLAND_mc.bin"); 60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin"); 61 MODULE_FIRMWARE("radeon/OLAND_smc.bin"); 62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin"); 63 MODULE_FIRMWARE("radeon/HAINAN_me.bin"); 64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin"); 65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin"); 66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin"); 67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin"); 68 69 static void si_pcie_gen3_enable(struct radeon_device *rdev); 70 static void si_program_aspm(struct radeon_device *rdev); 71 extern void sumo_rlc_fini(struct radeon_device *rdev); 72 extern int sumo_rlc_init(struct radeon_device *rdev); 73 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 74 extern void r600_ih_ring_fini(struct radeon_device *rdev); 75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev); 76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev); 79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev); 80 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 81 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, 82 bool enable); 83 static void si_init_pg(struct radeon_device *rdev); 84 static void si_init_cg(struct radeon_device *rdev); 85 static void si_fini_pg(struct radeon_device *rdev); 86 static void si_fini_cg(struct radeon_device *rdev); 87 static void si_rlc_stop(struct radeon_device *rdev); 88 89 static const u32 verde_rlc_save_restore_register_list[] = 90 { 91 (0x8000 << 16) | (0x98f4 >> 2), 92 0x00000000, 93 (0x8040 << 16) | (0x98f4 >> 2), 94 0x00000000, 95 (0x8000 << 16) | (0xe80 >> 2), 96 0x00000000, 97 (0x8040 << 16) | (0xe80 >> 2), 98 0x00000000, 99 (0x8000 << 16) | (0x89bc >> 2), 100 0x00000000, 101 (0x8040 << 16) | (0x89bc >> 2), 102 0x00000000, 103 (0x8000 << 16) | (0x8c1c >> 2), 104 0x00000000, 105 (0x8040 << 16) | (0x8c1c >> 2), 106 0x00000000, 107 (0x9c00 << 16) | (0x98f0 >> 2), 108 0x00000000, 109 (0x9c00 << 16) | (0xe7c >> 2), 110 0x00000000, 111 (0x8000 << 16) | (0x9148 >> 2), 112 0x00000000, 113 (0x8040 << 16) | (0x9148 >> 2), 114 0x00000000, 115 (0x9c00 << 16) | (0x9150 >> 2), 116 0x00000000, 117 (0x9c00 << 16) | (0x897c >> 2), 118 0x00000000, 119 (0x9c00 << 16) | (0x8d8c >> 2), 120 0x00000000, 121 (0x9c00 << 16) | (0xac54 >> 2), 122 0X00000000, 123 0x3, 124 (0x9c00 << 16) | (0x98f8 >> 2), 125 0x00000000, 126 (0x9c00 << 16) | (0x9910 >> 2), 127 0x00000000, 128 (0x9c00 << 16) | (0x9914 >> 2), 129 0x00000000, 130 (0x9c00 << 16) | (0x9918 >> 2), 131 0x00000000, 132 (0x9c00 << 16) | (0x991c >> 2), 133 0x00000000, 134 (0x9c00 << 16) | (0x9920 >> 2), 135 0x00000000, 136 (0x9c00 << 16) | (0x9924 >> 2), 137 0x00000000, 138 (0x9c00 << 16) | (0x9928 >> 2), 139 0x00000000, 140 (0x9c00 << 16) | (0x992c >> 2), 141 0x00000000, 142 (0x9c00 << 16) | (0x9930 >> 2), 143 0x00000000, 144 (0x9c00 << 16) | (0x9934 >> 2), 145 0x00000000, 146 (0x9c00 << 16) | (0x9938 >> 2), 147 0x00000000, 148 (0x9c00 << 16) | (0x993c >> 2), 149 0x00000000, 150 (0x9c00 << 16) | (0x9940 >> 2), 151 0x00000000, 152 (0x9c00 << 16) | (0x9944 >> 2), 153 0x00000000, 154 (0x9c00 << 16) | (0x9948 >> 2), 155 0x00000000, 156 (0x9c00 << 16) | (0x994c >> 2), 157 0x00000000, 158 (0x9c00 << 16) | (0x9950 >> 2), 159 0x00000000, 160 (0x9c00 << 16) | (0x9954 >> 2), 161 0x00000000, 162 (0x9c00 << 16) | (0x9958 >> 2), 163 0x00000000, 164 (0x9c00 << 16) | (0x995c >> 2), 165 0x00000000, 166 (0x9c00 << 16) | (0x9960 >> 2), 167 0x00000000, 168 (0x9c00 << 16) | (0x9964 >> 2), 169 0x00000000, 170 (0x9c00 << 16) | (0x9968 >> 2), 171 0x00000000, 172 (0x9c00 << 16) | (0x996c >> 2), 173 0x00000000, 174 (0x9c00 << 16) | (0x9970 >> 2), 175 0x00000000, 176 (0x9c00 << 16) | (0x9974 >> 2), 177 0x00000000, 178 (0x9c00 << 16) | (0x9978 >> 2), 179 0x00000000, 180 (0x9c00 << 16) | (0x997c >> 2), 181 0x00000000, 182 (0x9c00 << 16) | (0x9980 >> 2), 183 0x00000000, 184 (0x9c00 << 16) | (0x9984 >> 2), 185 0x00000000, 186 (0x9c00 << 16) | (0x9988 >> 2), 187 0x00000000, 188 (0x9c00 << 16) | (0x998c >> 2), 189 0x00000000, 190 (0x9c00 << 16) | (0x8c00 >> 2), 191 0x00000000, 192 (0x9c00 << 16) | (0x8c14 >> 2), 193 0x00000000, 194 (0x9c00 << 16) | (0x8c04 >> 2), 195 0x00000000, 196 (0x9c00 << 16) | (0x8c08 >> 2), 197 0x00000000, 198 (0x8000 << 16) | (0x9b7c >> 2), 199 0x00000000, 200 (0x8040 << 16) | (0x9b7c >> 2), 201 0x00000000, 202 (0x8000 << 16) | (0xe84 >> 2), 203 0x00000000, 204 (0x8040 << 16) | (0xe84 >> 2), 205 0x00000000, 206 (0x8000 << 16) | (0x89c0 >> 2), 207 0x00000000, 208 (0x8040 << 16) | (0x89c0 >> 2), 209 0x00000000, 210 (0x8000 << 16) | (0x914c >> 2), 211 0x00000000, 212 (0x8040 << 16) | (0x914c >> 2), 213 0x00000000, 214 (0x8000 << 16) | (0x8c20 >> 2), 215 0x00000000, 216 (0x8040 << 16) | (0x8c20 >> 2), 217 0x00000000, 218 (0x8000 << 16) | (0x9354 >> 2), 219 0x00000000, 220 (0x8040 << 16) | (0x9354 >> 2), 221 0x00000000, 222 (0x9c00 << 16) | (0x9060 >> 2), 223 0x00000000, 224 (0x9c00 << 16) | (0x9364 >> 2), 225 0x00000000, 226 (0x9c00 << 16) | (0x9100 >> 2), 227 0x00000000, 228 (0x9c00 << 16) | (0x913c >> 2), 229 0x00000000, 230 (0x8000 << 16) | (0x90e0 >> 2), 231 0x00000000, 232 (0x8000 << 16) | (0x90e4 >> 2), 233 0x00000000, 234 (0x8000 << 16) | (0x90e8 >> 2), 235 0x00000000, 236 (0x8040 << 16) | (0x90e0 >> 2), 237 0x00000000, 238 (0x8040 << 16) | (0x90e4 >> 2), 239 0x00000000, 240 (0x8040 << 16) | (0x90e8 >> 2), 241 0x00000000, 242 (0x9c00 << 16) | (0x8bcc >> 2), 243 0x00000000, 244 (0x9c00 << 16) | (0x8b24 >> 2), 245 0x00000000, 246 (0x9c00 << 16) | (0x88c4 >> 2), 247 0x00000000, 248 (0x9c00 << 16) | (0x8e50 >> 2), 249 0x00000000, 250 (0x9c00 << 16) | (0x8c0c >> 2), 251 0x00000000, 252 (0x9c00 << 16) | (0x8e58 >> 2), 253 0x00000000, 254 (0x9c00 << 16) | (0x8e5c >> 2), 255 0x00000000, 256 (0x9c00 << 16) | (0x9508 >> 2), 257 0x00000000, 258 (0x9c00 << 16) | (0x950c >> 2), 259 0x00000000, 260 (0x9c00 << 16) | (0x9494 >> 2), 261 0x00000000, 262 (0x9c00 << 16) | (0xac0c >> 2), 263 0x00000000, 264 (0x9c00 << 16) | (0xac10 >> 2), 265 0x00000000, 266 (0x9c00 << 16) | (0xac14 >> 2), 267 0x00000000, 268 (0x9c00 << 16) | (0xae00 >> 2), 269 0x00000000, 270 (0x9c00 << 16) | (0xac08 >> 2), 271 0x00000000, 272 (0x9c00 << 16) | (0x88d4 >> 2), 273 0x00000000, 274 (0x9c00 << 16) | (0x88c8 >> 2), 275 0x00000000, 276 (0x9c00 << 16) | (0x88cc >> 2), 277 0x00000000, 278 (0x9c00 << 16) | (0x89b0 >> 2), 279 0x00000000, 280 (0x9c00 << 16) | (0x8b10 >> 2), 281 0x00000000, 282 (0x9c00 << 16) | (0x8a14 >> 2), 283 0x00000000, 284 (0x9c00 << 16) | (0x9830 >> 2), 285 0x00000000, 286 (0x9c00 << 16) | (0x9834 >> 2), 287 0x00000000, 288 (0x9c00 << 16) | (0x9838 >> 2), 289 0x00000000, 290 (0x9c00 << 16) | (0x9a10 >> 2), 291 0x00000000, 292 (0x8000 << 16) | (0x9870 >> 2), 293 0x00000000, 294 (0x8000 << 16) | (0x9874 >> 2), 295 0x00000000, 296 (0x8001 << 16) | (0x9870 >> 2), 297 0x00000000, 298 (0x8001 << 16) | (0x9874 >> 2), 299 0x00000000, 300 (0x8040 << 16) | (0x9870 >> 2), 301 0x00000000, 302 (0x8040 << 16) | (0x9874 >> 2), 303 0x00000000, 304 (0x8041 << 16) | (0x9870 >> 2), 305 0x00000000, 306 (0x8041 << 16) | (0x9874 >> 2), 307 0x00000000, 308 0x00000000 309 }; 310 311 static const u32 tahiti_golden_rlc_registers[] = 312 { 313 0xc424, 0xffffffff, 0x00601005, 314 0xc47c, 0xffffffff, 0x10104040, 315 0xc488, 0xffffffff, 0x0100000a, 316 0xc314, 0xffffffff, 0x00000800, 317 0xc30c, 0xffffffff, 0x800000f4, 318 0xf4a8, 0xffffffff, 0x00000000 319 }; 320 321 static const u32 tahiti_golden_registers[] = 322 { 323 0x9a10, 0x00010000, 0x00018208, 324 0x9830, 0xffffffff, 0x00000000, 325 0x9834, 0xf00fffff, 0x00000400, 326 0x9838, 0x0002021c, 0x00020200, 327 0xc78, 0x00000080, 0x00000000, 328 0xd030, 0x000300c0, 0x00800040, 329 0xd830, 0x000300c0, 0x00800040, 330 0x5bb0, 0x000000f0, 0x00000070, 331 0x5bc0, 0x00200000, 0x50100000, 332 0x7030, 0x31000311, 0x00000011, 333 0x277c, 0x00000003, 0x000007ff, 334 0x240c, 0x000007ff, 0x00000000, 335 0x8a14, 0xf000001f, 0x00000007, 336 0x8b24, 0xffffffff, 0x00ffffff, 337 0x8b10, 0x0000ff0f, 0x00000000, 338 0x28a4c, 0x07ffffff, 0x4e000000, 339 0x28350, 0x3f3f3fff, 0x2a00126a, 340 0x30, 0x000000ff, 0x0040, 341 0x34, 0x00000040, 0x00004040, 342 0x9100, 0x07ffffff, 0x03000000, 343 0x8e88, 0x01ff1f3f, 0x00000000, 344 0x8e84, 0x01ff1f3f, 0x00000000, 345 0x9060, 0x0000007f, 0x00000020, 346 0x9508, 0x00010000, 0x00010000, 347 0xac14, 0x00000200, 0x000002fb, 348 0xac10, 0xffffffff, 0x0000543b, 349 0xac0c, 0xffffffff, 0xa9210876, 350 0x88d0, 0xffffffff, 0x000fff40, 351 0x88d4, 0x0000001f, 0x00000010, 352 0x1410, 0x20000000, 0x20fffed8, 353 0x15c0, 0x000c0fc0, 0x000c0400 354 }; 355 356 static const u32 tahiti_golden_registers2[] = 357 { 358 0xc64, 0x00000001, 0x00000001 359 }; 360 361 static const u32 pitcairn_golden_rlc_registers[] = 362 { 363 0xc424, 0xffffffff, 0x00601004, 364 0xc47c, 0xffffffff, 0x10102020, 365 0xc488, 0xffffffff, 0x01000020, 366 0xc314, 0xffffffff, 0x00000800, 367 0xc30c, 0xffffffff, 0x800000a4 368 }; 369 370 static const u32 pitcairn_golden_registers[] = 371 { 372 0x9a10, 0x00010000, 0x00018208, 373 0x9830, 0xffffffff, 0x00000000, 374 0x9834, 0xf00fffff, 0x00000400, 375 0x9838, 0x0002021c, 0x00020200, 376 0xc78, 0x00000080, 0x00000000, 377 0xd030, 0x000300c0, 0x00800040, 378 0xd830, 0x000300c0, 0x00800040, 379 0x5bb0, 0x000000f0, 0x00000070, 380 0x5bc0, 0x00200000, 0x50100000, 381 0x7030, 0x31000311, 0x00000011, 382 0x2ae4, 0x00073ffe, 0x000022a2, 383 0x240c, 0x000007ff, 0x00000000, 384 0x8a14, 0xf000001f, 0x00000007, 385 0x8b24, 0xffffffff, 0x00ffffff, 386 0x8b10, 0x0000ff0f, 0x00000000, 387 0x28a4c, 0x07ffffff, 0x4e000000, 388 0x28350, 0x3f3f3fff, 0x2a00126a, 389 0x30, 0x000000ff, 0x0040, 390 0x34, 0x00000040, 0x00004040, 391 0x9100, 0x07ffffff, 0x03000000, 392 0x9060, 0x0000007f, 0x00000020, 393 0x9508, 0x00010000, 0x00010000, 394 0xac14, 0x000003ff, 0x000000f7, 395 0xac10, 0xffffffff, 0x00000000, 396 0xac0c, 0xffffffff, 0x32761054, 397 0x88d4, 0x0000001f, 0x00000010, 398 0x15c0, 0x000c0fc0, 0x000c0400 399 }; 400 401 static const u32 verde_golden_rlc_registers[] = 402 { 403 0xc424, 0xffffffff, 0x033f1005, 404 0xc47c, 0xffffffff, 0x10808020, 405 0xc488, 0xffffffff, 0x00800008, 406 0xc314, 0xffffffff, 0x00001000, 407 0xc30c, 0xffffffff, 0x80010014 408 }; 409 410 static const u32 verde_golden_registers[] = 411 { 412 0x9a10, 0x00010000, 0x00018208, 413 0x9830, 0xffffffff, 0x00000000, 414 0x9834, 0xf00fffff, 0x00000400, 415 0x9838, 0x0002021c, 0x00020200, 416 0xc78, 0x00000080, 0x00000000, 417 0xd030, 0x000300c0, 0x00800040, 418 0xd030, 0x000300c0, 0x00800040, 419 0xd830, 0x000300c0, 0x00800040, 420 0xd830, 0x000300c0, 0x00800040, 421 0x5bb0, 0x000000f0, 0x00000070, 422 0x5bc0, 0x00200000, 0x50100000, 423 0x7030, 0x31000311, 0x00000011, 424 0x2ae4, 0x00073ffe, 0x000022a2, 425 0x2ae4, 0x00073ffe, 0x000022a2, 426 0x2ae4, 0x00073ffe, 0x000022a2, 427 0x240c, 0x000007ff, 0x00000000, 428 0x240c, 0x000007ff, 0x00000000, 429 0x240c, 0x000007ff, 0x00000000, 430 0x8a14, 0xf000001f, 0x00000007, 431 0x8a14, 0xf000001f, 0x00000007, 432 0x8a14, 0xf000001f, 0x00000007, 433 0x8b24, 0xffffffff, 0x00ffffff, 434 0x8b10, 0x0000ff0f, 0x00000000, 435 0x28a4c, 0x07ffffff, 0x4e000000, 436 0x28350, 0x3f3f3fff, 0x0000124a, 437 0x28350, 0x3f3f3fff, 0x0000124a, 438 0x28350, 0x3f3f3fff, 0x0000124a, 439 0x30, 0x000000ff, 0x0040, 440 0x34, 0x00000040, 0x00004040, 441 0x9100, 0x07ffffff, 0x03000000, 442 0x9100, 0x07ffffff, 0x03000000, 443 0x8e88, 0x01ff1f3f, 0x00000000, 444 0x8e88, 0x01ff1f3f, 0x00000000, 445 0x8e88, 0x01ff1f3f, 0x00000000, 446 0x8e84, 0x01ff1f3f, 0x00000000, 447 0x8e84, 0x01ff1f3f, 0x00000000, 448 0x8e84, 0x01ff1f3f, 0x00000000, 449 0x9060, 0x0000007f, 0x00000020, 450 0x9508, 0x00010000, 0x00010000, 451 0xac14, 0x000003ff, 0x00000003, 452 0xac14, 0x000003ff, 0x00000003, 453 0xac14, 0x000003ff, 0x00000003, 454 0xac10, 0xffffffff, 0x00000000, 455 0xac10, 0xffffffff, 0x00000000, 456 0xac10, 0xffffffff, 0x00000000, 457 0xac0c, 0xffffffff, 0x00001032, 458 0xac0c, 0xffffffff, 0x00001032, 459 0xac0c, 0xffffffff, 0x00001032, 460 0x88d4, 0x0000001f, 0x00000010, 461 0x88d4, 0x0000001f, 0x00000010, 462 0x88d4, 0x0000001f, 0x00000010, 463 0x15c0, 0x000c0fc0, 0x000c0400 464 }; 465 466 static const u32 oland_golden_rlc_registers[] = 467 { 468 0xc424, 0xffffffff, 0x00601005, 469 0xc47c, 0xffffffff, 0x10104040, 470 0xc488, 0xffffffff, 0x0100000a, 471 0xc314, 0xffffffff, 0x00000800, 472 0xc30c, 0xffffffff, 0x800000f4 473 }; 474 475 static const u32 oland_golden_registers[] = 476 { 477 0x9a10, 0x00010000, 0x00018208, 478 0x9830, 0xffffffff, 0x00000000, 479 0x9834, 0xf00fffff, 0x00000400, 480 0x9838, 0x0002021c, 0x00020200, 481 0xc78, 0x00000080, 0x00000000, 482 0xd030, 0x000300c0, 0x00800040, 483 0xd830, 0x000300c0, 0x00800040, 484 0x5bb0, 0x000000f0, 0x00000070, 485 0x5bc0, 0x00200000, 0x50100000, 486 0x7030, 0x31000311, 0x00000011, 487 0x2ae4, 0x00073ffe, 0x000022a2, 488 0x240c, 0x000007ff, 0x00000000, 489 0x8a14, 0xf000001f, 0x00000007, 490 0x8b24, 0xffffffff, 0x00ffffff, 491 0x8b10, 0x0000ff0f, 0x00000000, 492 0x28a4c, 0x07ffffff, 0x4e000000, 493 0x28350, 0x3f3f3fff, 0x00000082, 494 0x30, 0x000000ff, 0x0040, 495 0x34, 0x00000040, 0x00004040, 496 0x9100, 0x07ffffff, 0x03000000, 497 0x9060, 0x0000007f, 0x00000020, 498 0x9508, 0x00010000, 0x00010000, 499 0xac14, 0x000003ff, 0x000000f3, 500 0xac10, 0xffffffff, 0x00000000, 501 0xac0c, 0xffffffff, 0x00003210, 502 0x88d4, 0x0000001f, 0x00000010, 503 0x15c0, 0x000c0fc0, 0x000c0400 504 }; 505 506 static const u32 hainan_golden_registers[] = 507 { 508 0x9a10, 0x00010000, 0x00018208, 509 0x9830, 0xffffffff, 0x00000000, 510 0x9834, 0xf00fffff, 0x00000400, 511 0x9838, 0x0002021c, 0x00020200, 512 0xd0c0, 0xff000fff, 0x00000100, 513 0xd030, 0x000300c0, 0x00800040, 514 0xd8c0, 0xff000fff, 0x00000100, 515 0xd830, 0x000300c0, 0x00800040, 516 0x2ae4, 0x00073ffe, 0x000022a2, 517 0x240c, 0x000007ff, 0x00000000, 518 0x8a14, 0xf000001f, 0x00000007, 519 0x8b24, 0xffffffff, 0x00ffffff, 520 0x8b10, 0x0000ff0f, 0x00000000, 521 0x28a4c, 0x07ffffff, 0x4e000000, 522 0x28350, 0x3f3f3fff, 0x00000000, 523 0x30, 0x000000ff, 0x0040, 524 0x34, 0x00000040, 0x00004040, 525 0x9100, 0x03e00000, 0x03600000, 526 0x9060, 0x0000007f, 0x00000020, 527 0x9508, 0x00010000, 0x00010000, 528 0xac14, 0x000003ff, 0x000000f1, 529 0xac10, 0xffffffff, 0x00000000, 530 0xac0c, 0xffffffff, 0x00003210, 531 0x88d4, 0x0000001f, 0x00000010, 532 0x15c0, 0x000c0fc0, 0x000c0400 533 }; 534 535 static const u32 hainan_golden_registers2[] = 536 { 537 0x98f8, 0xffffffff, 0x02010001 538 }; 539 540 static const u32 tahiti_mgcg_cgcg_init[] = 541 { 542 0xc400, 0xffffffff, 0xfffffffc, 543 0x802c, 0xffffffff, 0xe0000000, 544 0x9a60, 0xffffffff, 0x00000100, 545 0x92a4, 0xffffffff, 0x00000100, 546 0xc164, 0xffffffff, 0x00000100, 547 0x9774, 0xffffffff, 0x00000100, 548 0x8984, 0xffffffff, 0x06000100, 549 0x8a18, 0xffffffff, 0x00000100, 550 0x92a0, 0xffffffff, 0x00000100, 551 0xc380, 0xffffffff, 0x00000100, 552 0x8b28, 0xffffffff, 0x00000100, 553 0x9144, 0xffffffff, 0x00000100, 554 0x8d88, 0xffffffff, 0x00000100, 555 0x8d8c, 0xffffffff, 0x00000100, 556 0x9030, 0xffffffff, 0x00000100, 557 0x9034, 0xffffffff, 0x00000100, 558 0x9038, 0xffffffff, 0x00000100, 559 0x903c, 0xffffffff, 0x00000100, 560 0xad80, 0xffffffff, 0x00000100, 561 0xac54, 0xffffffff, 0x00000100, 562 0x897c, 0xffffffff, 0x06000100, 563 0x9868, 0xffffffff, 0x00000100, 564 0x9510, 0xffffffff, 0x00000100, 565 0xaf04, 0xffffffff, 0x00000100, 566 0xae04, 0xffffffff, 0x00000100, 567 0x949c, 0xffffffff, 0x00000100, 568 0x802c, 0xffffffff, 0xe0000000, 569 0x9160, 0xffffffff, 0x00010000, 570 0x9164, 0xffffffff, 0x00030002, 571 0x9168, 0xffffffff, 0x00040007, 572 0x916c, 0xffffffff, 0x00060005, 573 0x9170, 0xffffffff, 0x00090008, 574 0x9174, 0xffffffff, 0x00020001, 575 0x9178, 0xffffffff, 0x00040003, 576 0x917c, 0xffffffff, 0x00000007, 577 0x9180, 0xffffffff, 0x00060005, 578 0x9184, 0xffffffff, 0x00090008, 579 0x9188, 0xffffffff, 0x00030002, 580 0x918c, 0xffffffff, 0x00050004, 581 0x9190, 0xffffffff, 0x00000008, 582 0x9194, 0xffffffff, 0x00070006, 583 0x9198, 0xffffffff, 0x000a0009, 584 0x919c, 0xffffffff, 0x00040003, 585 0x91a0, 0xffffffff, 0x00060005, 586 0x91a4, 0xffffffff, 0x00000009, 587 0x91a8, 0xffffffff, 0x00080007, 588 0x91ac, 0xffffffff, 0x000b000a, 589 0x91b0, 0xffffffff, 0x00050004, 590 0x91b4, 0xffffffff, 0x00070006, 591 0x91b8, 0xffffffff, 0x0008000b, 592 0x91bc, 0xffffffff, 0x000a0009, 593 0x91c0, 0xffffffff, 0x000d000c, 594 0x91c4, 0xffffffff, 0x00060005, 595 0x91c8, 0xffffffff, 0x00080007, 596 0x91cc, 0xffffffff, 0x0000000b, 597 0x91d0, 0xffffffff, 0x000a0009, 598 0x91d4, 0xffffffff, 0x000d000c, 599 0x91d8, 0xffffffff, 0x00070006, 600 0x91dc, 0xffffffff, 0x00090008, 601 0x91e0, 0xffffffff, 0x0000000c, 602 0x91e4, 0xffffffff, 0x000b000a, 603 0x91e8, 0xffffffff, 0x000e000d, 604 0x91ec, 0xffffffff, 0x00080007, 605 0x91f0, 0xffffffff, 0x000a0009, 606 0x91f4, 0xffffffff, 0x0000000d, 607 0x91f8, 0xffffffff, 0x000c000b, 608 0x91fc, 0xffffffff, 0x000f000e, 609 0x9200, 0xffffffff, 0x00090008, 610 0x9204, 0xffffffff, 0x000b000a, 611 0x9208, 0xffffffff, 0x000c000f, 612 0x920c, 0xffffffff, 0x000e000d, 613 0x9210, 0xffffffff, 0x00110010, 614 0x9214, 0xffffffff, 0x000a0009, 615 0x9218, 0xffffffff, 0x000c000b, 616 0x921c, 0xffffffff, 0x0000000f, 617 0x9220, 0xffffffff, 0x000e000d, 618 0x9224, 0xffffffff, 0x00110010, 619 0x9228, 0xffffffff, 0x000b000a, 620 0x922c, 0xffffffff, 0x000d000c, 621 0x9230, 0xffffffff, 0x00000010, 622 0x9234, 0xffffffff, 0x000f000e, 623 0x9238, 0xffffffff, 0x00120011, 624 0x923c, 0xffffffff, 0x000c000b, 625 0x9240, 0xffffffff, 0x000e000d, 626 0x9244, 0xffffffff, 0x00000011, 627 0x9248, 0xffffffff, 0x0010000f, 628 0x924c, 0xffffffff, 0x00130012, 629 0x9250, 0xffffffff, 0x000d000c, 630 0x9254, 0xffffffff, 0x000f000e, 631 0x9258, 0xffffffff, 0x00100013, 632 0x925c, 0xffffffff, 0x00120011, 633 0x9260, 0xffffffff, 0x00150014, 634 0x9264, 0xffffffff, 0x000e000d, 635 0x9268, 0xffffffff, 0x0010000f, 636 0x926c, 0xffffffff, 0x00000013, 637 0x9270, 0xffffffff, 0x00120011, 638 0x9274, 0xffffffff, 0x00150014, 639 0x9278, 0xffffffff, 0x000f000e, 640 0x927c, 0xffffffff, 0x00110010, 641 0x9280, 0xffffffff, 0x00000014, 642 0x9284, 0xffffffff, 0x00130012, 643 0x9288, 0xffffffff, 0x00160015, 644 0x928c, 0xffffffff, 0x0010000f, 645 0x9290, 0xffffffff, 0x00120011, 646 0x9294, 0xffffffff, 0x00000015, 647 0x9298, 0xffffffff, 0x00140013, 648 0x929c, 0xffffffff, 0x00170016, 649 0x9150, 0xffffffff, 0x96940200, 650 0x8708, 0xffffffff, 0x00900100, 651 0xc478, 0xffffffff, 0x00000080, 652 0xc404, 0xffffffff, 0x0020003f, 653 0x30, 0xffffffff, 0x0000001c, 654 0x34, 0x000f0000, 0x000f0000, 655 0x160c, 0xffffffff, 0x00000100, 656 0x1024, 0xffffffff, 0x00000100, 657 0x102c, 0x00000101, 0x00000000, 658 0x20a8, 0xffffffff, 0x00000104, 659 0x264c, 0x000c0000, 0x000c0000, 660 0x2648, 0x000c0000, 0x000c0000, 661 0x55e4, 0xff000fff, 0x00000100, 662 0x55e8, 0x00000001, 0x00000001, 663 0x2f50, 0x00000001, 0x00000001, 664 0x30cc, 0xc0000fff, 0x00000104, 665 0xc1e4, 0x00000001, 0x00000001, 666 0xd0c0, 0xfffffff0, 0x00000100, 667 0xd8c0, 0xfffffff0, 0x00000100 668 }; 669 670 static const u32 pitcairn_mgcg_cgcg_init[] = 671 { 672 0xc400, 0xffffffff, 0xfffffffc, 673 0x802c, 0xffffffff, 0xe0000000, 674 0x9a60, 0xffffffff, 0x00000100, 675 0x92a4, 0xffffffff, 0x00000100, 676 0xc164, 0xffffffff, 0x00000100, 677 0x9774, 0xffffffff, 0x00000100, 678 0x8984, 0xffffffff, 0x06000100, 679 0x8a18, 0xffffffff, 0x00000100, 680 0x92a0, 0xffffffff, 0x00000100, 681 0xc380, 0xffffffff, 0x00000100, 682 0x8b28, 0xffffffff, 0x00000100, 683 0x9144, 0xffffffff, 0x00000100, 684 0x8d88, 0xffffffff, 0x00000100, 685 0x8d8c, 0xffffffff, 0x00000100, 686 0x9030, 0xffffffff, 0x00000100, 687 0x9034, 0xffffffff, 0x00000100, 688 0x9038, 0xffffffff, 0x00000100, 689 0x903c, 0xffffffff, 0x00000100, 690 0xad80, 0xffffffff, 0x00000100, 691 0xac54, 0xffffffff, 0x00000100, 692 0x897c, 0xffffffff, 0x06000100, 693 0x9868, 0xffffffff, 0x00000100, 694 0x9510, 0xffffffff, 0x00000100, 695 0xaf04, 0xffffffff, 0x00000100, 696 0xae04, 0xffffffff, 0x00000100, 697 0x949c, 0xffffffff, 0x00000100, 698 0x802c, 0xffffffff, 0xe0000000, 699 0x9160, 0xffffffff, 0x00010000, 700 0x9164, 0xffffffff, 0x00030002, 701 0x9168, 0xffffffff, 0x00040007, 702 0x916c, 0xffffffff, 0x00060005, 703 0x9170, 0xffffffff, 0x00090008, 704 0x9174, 0xffffffff, 0x00020001, 705 0x9178, 0xffffffff, 0x00040003, 706 0x917c, 0xffffffff, 0x00000007, 707 0x9180, 0xffffffff, 0x00060005, 708 0x9184, 0xffffffff, 0x00090008, 709 0x9188, 0xffffffff, 0x00030002, 710 0x918c, 0xffffffff, 0x00050004, 711 0x9190, 0xffffffff, 0x00000008, 712 0x9194, 0xffffffff, 0x00070006, 713 0x9198, 0xffffffff, 0x000a0009, 714 0x919c, 0xffffffff, 0x00040003, 715 0x91a0, 0xffffffff, 0x00060005, 716 0x91a4, 0xffffffff, 0x00000009, 717 0x91a8, 0xffffffff, 0x00080007, 718 0x91ac, 0xffffffff, 0x000b000a, 719 0x91b0, 0xffffffff, 0x00050004, 720 0x91b4, 0xffffffff, 0x00070006, 721 0x91b8, 0xffffffff, 0x0008000b, 722 0x91bc, 0xffffffff, 0x000a0009, 723 0x91c0, 0xffffffff, 0x000d000c, 724 0x9200, 0xffffffff, 0x00090008, 725 0x9204, 0xffffffff, 0x000b000a, 726 0x9208, 0xffffffff, 0x000c000f, 727 0x920c, 0xffffffff, 0x000e000d, 728 0x9210, 0xffffffff, 0x00110010, 729 0x9214, 0xffffffff, 0x000a0009, 730 0x9218, 0xffffffff, 0x000c000b, 731 0x921c, 0xffffffff, 0x0000000f, 732 0x9220, 0xffffffff, 0x000e000d, 733 0x9224, 0xffffffff, 0x00110010, 734 0x9228, 0xffffffff, 0x000b000a, 735 0x922c, 0xffffffff, 0x000d000c, 736 0x9230, 0xffffffff, 0x00000010, 737 0x9234, 0xffffffff, 0x000f000e, 738 0x9238, 0xffffffff, 0x00120011, 739 0x923c, 0xffffffff, 0x000c000b, 740 0x9240, 0xffffffff, 0x000e000d, 741 0x9244, 0xffffffff, 0x00000011, 742 0x9248, 0xffffffff, 0x0010000f, 743 0x924c, 0xffffffff, 0x00130012, 744 0x9250, 0xffffffff, 0x000d000c, 745 0x9254, 0xffffffff, 0x000f000e, 746 0x9258, 0xffffffff, 0x00100013, 747 0x925c, 0xffffffff, 0x00120011, 748 0x9260, 0xffffffff, 0x00150014, 749 0x9150, 0xffffffff, 0x96940200, 750 0x8708, 0xffffffff, 0x00900100, 751 0xc478, 0xffffffff, 0x00000080, 752 0xc404, 0xffffffff, 0x0020003f, 753 0x30, 0xffffffff, 0x0000001c, 754 0x34, 0x000f0000, 0x000f0000, 755 0x160c, 0xffffffff, 0x00000100, 756 0x1024, 0xffffffff, 0x00000100, 757 0x102c, 0x00000101, 0x00000000, 758 0x20a8, 0xffffffff, 0x00000104, 759 0x55e4, 0xff000fff, 0x00000100, 760 0x55e8, 0x00000001, 0x00000001, 761 0x2f50, 0x00000001, 0x00000001, 762 0x30cc, 0xc0000fff, 0x00000104, 763 0xc1e4, 0x00000001, 0x00000001, 764 0xd0c0, 0xfffffff0, 0x00000100, 765 0xd8c0, 0xfffffff0, 0x00000100 766 }; 767 768 static const u32 verde_mgcg_cgcg_init[] = 769 { 770 0xc400, 0xffffffff, 0xfffffffc, 771 0x802c, 0xffffffff, 0xe0000000, 772 0x9a60, 0xffffffff, 0x00000100, 773 0x92a4, 0xffffffff, 0x00000100, 774 0xc164, 0xffffffff, 0x00000100, 775 0x9774, 0xffffffff, 0x00000100, 776 0x8984, 0xffffffff, 0x06000100, 777 0x8a18, 0xffffffff, 0x00000100, 778 0x92a0, 0xffffffff, 0x00000100, 779 0xc380, 0xffffffff, 0x00000100, 780 0x8b28, 0xffffffff, 0x00000100, 781 0x9144, 0xffffffff, 0x00000100, 782 0x8d88, 0xffffffff, 0x00000100, 783 0x8d8c, 0xffffffff, 0x00000100, 784 0x9030, 0xffffffff, 0x00000100, 785 0x9034, 0xffffffff, 0x00000100, 786 0x9038, 0xffffffff, 0x00000100, 787 0x903c, 0xffffffff, 0x00000100, 788 0xad80, 0xffffffff, 0x00000100, 789 0xac54, 0xffffffff, 0x00000100, 790 0x897c, 0xffffffff, 0x06000100, 791 0x9868, 0xffffffff, 0x00000100, 792 0x9510, 0xffffffff, 0x00000100, 793 0xaf04, 0xffffffff, 0x00000100, 794 0xae04, 0xffffffff, 0x00000100, 795 0x949c, 0xffffffff, 0x00000100, 796 0x802c, 0xffffffff, 0xe0000000, 797 0x9160, 0xffffffff, 0x00010000, 798 0x9164, 0xffffffff, 0x00030002, 799 0x9168, 0xffffffff, 0x00040007, 800 0x916c, 0xffffffff, 0x00060005, 801 0x9170, 0xffffffff, 0x00090008, 802 0x9174, 0xffffffff, 0x00020001, 803 0x9178, 0xffffffff, 0x00040003, 804 0x917c, 0xffffffff, 0x00000007, 805 0x9180, 0xffffffff, 0x00060005, 806 0x9184, 0xffffffff, 0x00090008, 807 0x9188, 0xffffffff, 0x00030002, 808 0x918c, 0xffffffff, 0x00050004, 809 0x9190, 0xffffffff, 0x00000008, 810 0x9194, 0xffffffff, 0x00070006, 811 0x9198, 0xffffffff, 0x000a0009, 812 0x919c, 0xffffffff, 0x00040003, 813 0x91a0, 0xffffffff, 0x00060005, 814 0x91a4, 0xffffffff, 0x00000009, 815 0x91a8, 0xffffffff, 0x00080007, 816 0x91ac, 0xffffffff, 0x000b000a, 817 0x91b0, 0xffffffff, 0x00050004, 818 0x91b4, 0xffffffff, 0x00070006, 819 0x91b8, 0xffffffff, 0x0008000b, 820 0x91bc, 0xffffffff, 0x000a0009, 821 0x91c0, 0xffffffff, 0x000d000c, 822 0x9200, 0xffffffff, 0x00090008, 823 0x9204, 0xffffffff, 0x000b000a, 824 0x9208, 0xffffffff, 0x000c000f, 825 0x920c, 0xffffffff, 0x000e000d, 826 0x9210, 0xffffffff, 0x00110010, 827 0x9214, 0xffffffff, 0x000a0009, 828 0x9218, 0xffffffff, 0x000c000b, 829 0x921c, 0xffffffff, 0x0000000f, 830 0x9220, 0xffffffff, 0x000e000d, 831 0x9224, 0xffffffff, 0x00110010, 832 0x9228, 0xffffffff, 0x000b000a, 833 0x922c, 0xffffffff, 0x000d000c, 834 0x9230, 0xffffffff, 0x00000010, 835 0x9234, 0xffffffff, 0x000f000e, 836 0x9238, 0xffffffff, 0x00120011, 837 0x923c, 0xffffffff, 0x000c000b, 838 0x9240, 0xffffffff, 0x000e000d, 839 0x9244, 0xffffffff, 0x00000011, 840 0x9248, 0xffffffff, 0x0010000f, 841 0x924c, 0xffffffff, 0x00130012, 842 0x9250, 0xffffffff, 0x000d000c, 843 0x9254, 0xffffffff, 0x000f000e, 844 0x9258, 0xffffffff, 0x00100013, 845 0x925c, 0xffffffff, 0x00120011, 846 0x9260, 0xffffffff, 0x00150014, 847 0x9150, 0xffffffff, 0x96940200, 848 0x8708, 0xffffffff, 0x00900100, 849 0xc478, 0xffffffff, 0x00000080, 850 0xc404, 0xffffffff, 0x0020003f, 851 0x30, 0xffffffff, 0x0000001c, 852 0x34, 0x000f0000, 0x000f0000, 853 0x160c, 0xffffffff, 0x00000100, 854 0x1024, 0xffffffff, 0x00000100, 855 0x102c, 0x00000101, 0x00000000, 856 0x20a8, 0xffffffff, 0x00000104, 857 0x264c, 0x000c0000, 0x000c0000, 858 0x2648, 0x000c0000, 0x000c0000, 859 0x55e4, 0xff000fff, 0x00000100, 860 0x55e8, 0x00000001, 0x00000001, 861 0x2f50, 0x00000001, 0x00000001, 862 0x30cc, 0xc0000fff, 0x00000104, 863 0xc1e4, 0x00000001, 0x00000001, 864 0xd0c0, 0xfffffff0, 0x00000100, 865 0xd8c0, 0xfffffff0, 0x00000100 866 }; 867 868 static const u32 oland_mgcg_cgcg_init[] = 869 { 870 0xc400, 0xffffffff, 0xfffffffc, 871 0x802c, 0xffffffff, 0xe0000000, 872 0x9a60, 0xffffffff, 0x00000100, 873 0x92a4, 0xffffffff, 0x00000100, 874 0xc164, 0xffffffff, 0x00000100, 875 0x9774, 0xffffffff, 0x00000100, 876 0x8984, 0xffffffff, 0x06000100, 877 0x8a18, 0xffffffff, 0x00000100, 878 0x92a0, 0xffffffff, 0x00000100, 879 0xc380, 0xffffffff, 0x00000100, 880 0x8b28, 0xffffffff, 0x00000100, 881 0x9144, 0xffffffff, 0x00000100, 882 0x8d88, 0xffffffff, 0x00000100, 883 0x8d8c, 0xffffffff, 0x00000100, 884 0x9030, 0xffffffff, 0x00000100, 885 0x9034, 0xffffffff, 0x00000100, 886 0x9038, 0xffffffff, 0x00000100, 887 0x903c, 0xffffffff, 0x00000100, 888 0xad80, 0xffffffff, 0x00000100, 889 0xac54, 0xffffffff, 0x00000100, 890 0x897c, 0xffffffff, 0x06000100, 891 0x9868, 0xffffffff, 0x00000100, 892 0x9510, 0xffffffff, 0x00000100, 893 0xaf04, 0xffffffff, 0x00000100, 894 0xae04, 0xffffffff, 0x00000100, 895 0x949c, 0xffffffff, 0x00000100, 896 0x802c, 0xffffffff, 0xe0000000, 897 0x9160, 0xffffffff, 0x00010000, 898 0x9164, 0xffffffff, 0x00030002, 899 0x9168, 0xffffffff, 0x00040007, 900 0x916c, 0xffffffff, 0x00060005, 901 0x9170, 0xffffffff, 0x00090008, 902 0x9174, 0xffffffff, 0x00020001, 903 0x9178, 0xffffffff, 0x00040003, 904 0x917c, 0xffffffff, 0x00000007, 905 0x9180, 0xffffffff, 0x00060005, 906 0x9184, 0xffffffff, 0x00090008, 907 0x9188, 0xffffffff, 0x00030002, 908 0x918c, 0xffffffff, 0x00050004, 909 0x9190, 0xffffffff, 0x00000008, 910 0x9194, 0xffffffff, 0x00070006, 911 0x9198, 0xffffffff, 0x000a0009, 912 0x919c, 0xffffffff, 0x00040003, 913 0x91a0, 0xffffffff, 0x00060005, 914 0x91a4, 0xffffffff, 0x00000009, 915 0x91a8, 0xffffffff, 0x00080007, 916 0x91ac, 0xffffffff, 0x000b000a, 917 0x91b0, 0xffffffff, 0x00050004, 918 0x91b4, 0xffffffff, 0x00070006, 919 0x91b8, 0xffffffff, 0x0008000b, 920 0x91bc, 0xffffffff, 0x000a0009, 921 0x91c0, 0xffffffff, 0x000d000c, 922 0x91c4, 0xffffffff, 0x00060005, 923 0x91c8, 0xffffffff, 0x00080007, 924 0x91cc, 0xffffffff, 0x0000000b, 925 0x91d0, 0xffffffff, 0x000a0009, 926 0x91d4, 0xffffffff, 0x000d000c, 927 0x9150, 0xffffffff, 0x96940200, 928 0x8708, 0xffffffff, 0x00900100, 929 0xc478, 0xffffffff, 0x00000080, 930 0xc404, 0xffffffff, 0x0020003f, 931 0x30, 0xffffffff, 0x0000001c, 932 0x34, 0x000f0000, 0x000f0000, 933 0x160c, 0xffffffff, 0x00000100, 934 0x1024, 0xffffffff, 0x00000100, 935 0x102c, 0x00000101, 0x00000000, 936 0x20a8, 0xffffffff, 0x00000104, 937 0x264c, 0x000c0000, 0x000c0000, 938 0x2648, 0x000c0000, 0x000c0000, 939 0x55e4, 0xff000fff, 0x00000100, 940 0x55e8, 0x00000001, 0x00000001, 941 0x2f50, 0x00000001, 0x00000001, 942 0x30cc, 0xc0000fff, 0x00000104, 943 0xc1e4, 0x00000001, 0x00000001, 944 0xd0c0, 0xfffffff0, 0x00000100, 945 0xd8c0, 0xfffffff0, 0x00000100 946 }; 947 948 static const u32 hainan_mgcg_cgcg_init[] = 949 { 950 0xc400, 0xffffffff, 0xfffffffc, 951 0x802c, 0xffffffff, 0xe0000000, 952 0x9a60, 0xffffffff, 0x00000100, 953 0x92a4, 0xffffffff, 0x00000100, 954 0xc164, 0xffffffff, 0x00000100, 955 0x9774, 0xffffffff, 0x00000100, 956 0x8984, 0xffffffff, 0x06000100, 957 0x8a18, 0xffffffff, 0x00000100, 958 0x92a0, 0xffffffff, 0x00000100, 959 0xc380, 0xffffffff, 0x00000100, 960 0x8b28, 0xffffffff, 0x00000100, 961 0x9144, 0xffffffff, 0x00000100, 962 0x8d88, 0xffffffff, 0x00000100, 963 0x8d8c, 0xffffffff, 0x00000100, 964 0x9030, 0xffffffff, 0x00000100, 965 0x9034, 0xffffffff, 0x00000100, 966 0x9038, 0xffffffff, 0x00000100, 967 0x903c, 0xffffffff, 0x00000100, 968 0xad80, 0xffffffff, 0x00000100, 969 0xac54, 0xffffffff, 0x00000100, 970 0x897c, 0xffffffff, 0x06000100, 971 0x9868, 0xffffffff, 0x00000100, 972 0x9510, 0xffffffff, 0x00000100, 973 0xaf04, 0xffffffff, 0x00000100, 974 0xae04, 0xffffffff, 0x00000100, 975 0x949c, 0xffffffff, 0x00000100, 976 0x802c, 0xffffffff, 0xe0000000, 977 0x9160, 0xffffffff, 0x00010000, 978 0x9164, 0xffffffff, 0x00030002, 979 0x9168, 0xffffffff, 0x00040007, 980 0x916c, 0xffffffff, 0x00060005, 981 0x9170, 0xffffffff, 0x00090008, 982 0x9174, 0xffffffff, 0x00020001, 983 0x9178, 0xffffffff, 0x00040003, 984 0x917c, 0xffffffff, 0x00000007, 985 0x9180, 0xffffffff, 0x00060005, 986 0x9184, 0xffffffff, 0x00090008, 987 0x9188, 0xffffffff, 0x00030002, 988 0x918c, 0xffffffff, 0x00050004, 989 0x9190, 0xffffffff, 0x00000008, 990 0x9194, 0xffffffff, 0x00070006, 991 0x9198, 0xffffffff, 0x000a0009, 992 0x919c, 0xffffffff, 0x00040003, 993 0x91a0, 0xffffffff, 0x00060005, 994 0x91a4, 0xffffffff, 0x00000009, 995 0x91a8, 0xffffffff, 0x00080007, 996 0x91ac, 0xffffffff, 0x000b000a, 997 0x91b0, 0xffffffff, 0x00050004, 998 0x91b4, 0xffffffff, 0x00070006, 999 0x91b8, 0xffffffff, 0x0008000b, 1000 0x91bc, 0xffffffff, 0x000a0009, 1001 0x91c0, 0xffffffff, 0x000d000c, 1002 0x91c4, 0xffffffff, 0x00060005, 1003 0x91c8, 0xffffffff, 0x00080007, 1004 0x91cc, 0xffffffff, 0x0000000b, 1005 0x91d0, 0xffffffff, 0x000a0009, 1006 0x91d4, 0xffffffff, 0x000d000c, 1007 0x9150, 0xffffffff, 0x96940200, 1008 0x8708, 0xffffffff, 0x00900100, 1009 0xc478, 0xffffffff, 0x00000080, 1010 0xc404, 0xffffffff, 0x0020003f, 1011 0x30, 0xffffffff, 0x0000001c, 1012 0x34, 0x000f0000, 0x000f0000, 1013 0x160c, 0xffffffff, 0x00000100, 1014 0x1024, 0xffffffff, 0x00000100, 1015 0x20a8, 0xffffffff, 0x00000104, 1016 0x264c, 0x000c0000, 0x000c0000, 1017 0x2648, 0x000c0000, 0x000c0000, 1018 0x2f50, 0x00000001, 0x00000001, 1019 0x30cc, 0xc0000fff, 0x00000104, 1020 0xc1e4, 0x00000001, 0x00000001, 1021 0xd0c0, 0xfffffff0, 0x00000100, 1022 0xd8c0, 0xfffffff0, 0x00000100 1023 }; 1024 1025 static u32 verde_pg_init[] = 1026 { 1027 0x353c, 0xffffffff, 0x40000, 1028 0x3538, 0xffffffff, 0x200010ff, 1029 0x353c, 0xffffffff, 0x0, 1030 0x353c, 0xffffffff, 0x0, 1031 0x353c, 0xffffffff, 0x0, 1032 0x353c, 0xffffffff, 0x0, 1033 0x353c, 0xffffffff, 0x0, 1034 0x353c, 0xffffffff, 0x7007, 1035 0x3538, 0xffffffff, 0x300010ff, 1036 0x353c, 0xffffffff, 0x0, 1037 0x353c, 0xffffffff, 0x0, 1038 0x353c, 0xffffffff, 0x0, 1039 0x353c, 0xffffffff, 0x0, 1040 0x353c, 0xffffffff, 0x0, 1041 0x353c, 0xffffffff, 0x400000, 1042 0x3538, 0xffffffff, 0x100010ff, 1043 0x353c, 0xffffffff, 0x0, 1044 0x353c, 0xffffffff, 0x0, 1045 0x353c, 0xffffffff, 0x0, 1046 0x353c, 0xffffffff, 0x0, 1047 0x353c, 0xffffffff, 0x0, 1048 0x353c, 0xffffffff, 0x120200, 1049 0x3538, 0xffffffff, 0x500010ff, 1050 0x353c, 0xffffffff, 0x0, 1051 0x353c, 0xffffffff, 0x0, 1052 0x353c, 0xffffffff, 0x0, 1053 0x353c, 0xffffffff, 0x0, 1054 0x353c, 0xffffffff, 0x0, 1055 0x353c, 0xffffffff, 0x1e1e16, 1056 0x3538, 0xffffffff, 0x600010ff, 1057 0x353c, 0xffffffff, 0x0, 1058 0x353c, 0xffffffff, 0x0, 1059 0x353c, 0xffffffff, 0x0, 1060 0x353c, 0xffffffff, 0x0, 1061 0x353c, 0xffffffff, 0x0, 1062 0x353c, 0xffffffff, 0x171f1e, 1063 0x3538, 0xffffffff, 0x700010ff, 1064 0x353c, 0xffffffff, 0x0, 1065 0x353c, 0xffffffff, 0x0, 1066 0x353c, 0xffffffff, 0x0, 1067 0x353c, 0xffffffff, 0x0, 1068 0x353c, 0xffffffff, 0x0, 1069 0x353c, 0xffffffff, 0x0, 1070 0x3538, 0xffffffff, 0x9ff, 1071 0x3500, 0xffffffff, 0x0, 1072 0x3504, 0xffffffff, 0x10000800, 1073 0x3504, 0xffffffff, 0xf, 1074 0x3504, 0xffffffff, 0xf, 1075 0x3500, 0xffffffff, 0x4, 1076 0x3504, 0xffffffff, 0x1000051e, 1077 0x3504, 0xffffffff, 0xffff, 1078 0x3504, 0xffffffff, 0xffff, 1079 0x3500, 0xffffffff, 0x8, 1080 0x3504, 0xffffffff, 0x80500, 1081 0x3500, 0xffffffff, 0x12, 1082 0x3504, 0xffffffff, 0x9050c, 1083 0x3500, 0xffffffff, 0x1d, 1084 0x3504, 0xffffffff, 0xb052c, 1085 0x3500, 0xffffffff, 0x2a, 1086 0x3504, 0xffffffff, 0x1053e, 1087 0x3500, 0xffffffff, 0x2d, 1088 0x3504, 0xffffffff, 0x10546, 1089 0x3500, 0xffffffff, 0x30, 1090 0x3504, 0xffffffff, 0xa054e, 1091 0x3500, 0xffffffff, 0x3c, 1092 0x3504, 0xffffffff, 0x1055f, 1093 0x3500, 0xffffffff, 0x3f, 1094 0x3504, 0xffffffff, 0x10567, 1095 0x3500, 0xffffffff, 0x42, 1096 0x3504, 0xffffffff, 0x1056f, 1097 0x3500, 0xffffffff, 0x45, 1098 0x3504, 0xffffffff, 0x10572, 1099 0x3500, 0xffffffff, 0x48, 1100 0x3504, 0xffffffff, 0x20575, 1101 0x3500, 0xffffffff, 0x4c, 1102 0x3504, 0xffffffff, 0x190801, 1103 0x3500, 0xffffffff, 0x67, 1104 0x3504, 0xffffffff, 0x1082a, 1105 0x3500, 0xffffffff, 0x6a, 1106 0x3504, 0xffffffff, 0x1b082d, 1107 0x3500, 0xffffffff, 0x87, 1108 0x3504, 0xffffffff, 0x310851, 1109 0x3500, 0xffffffff, 0xba, 1110 0x3504, 0xffffffff, 0x891, 1111 0x3500, 0xffffffff, 0xbc, 1112 0x3504, 0xffffffff, 0x893, 1113 0x3500, 0xffffffff, 0xbe, 1114 0x3504, 0xffffffff, 0x20895, 1115 0x3500, 0xffffffff, 0xc2, 1116 0x3504, 0xffffffff, 0x20899, 1117 0x3500, 0xffffffff, 0xc6, 1118 0x3504, 0xffffffff, 0x2089d, 1119 0x3500, 0xffffffff, 0xca, 1120 0x3504, 0xffffffff, 0x8a1, 1121 0x3500, 0xffffffff, 0xcc, 1122 0x3504, 0xffffffff, 0x8a3, 1123 0x3500, 0xffffffff, 0xce, 1124 0x3504, 0xffffffff, 0x308a5, 1125 0x3500, 0xffffffff, 0xd3, 1126 0x3504, 0xffffffff, 0x6d08cd, 1127 0x3500, 0xffffffff, 0x142, 1128 0x3504, 0xffffffff, 0x2000095a, 1129 0x3504, 0xffffffff, 0x1, 1130 0x3500, 0xffffffff, 0x144, 1131 0x3504, 0xffffffff, 0x301f095b, 1132 0x3500, 0xffffffff, 0x165, 1133 0x3504, 0xffffffff, 0xc094d, 1134 0x3500, 0xffffffff, 0x173, 1135 0x3504, 0xffffffff, 0xf096d, 1136 0x3500, 0xffffffff, 0x184, 1137 0x3504, 0xffffffff, 0x15097f, 1138 0x3500, 0xffffffff, 0x19b, 1139 0x3504, 0xffffffff, 0xc0998, 1140 0x3500, 0xffffffff, 0x1a9, 1141 0x3504, 0xffffffff, 0x409a7, 1142 0x3500, 0xffffffff, 0x1af, 1143 0x3504, 0xffffffff, 0xcdc, 1144 0x3500, 0xffffffff, 0x1b1, 1145 0x3504, 0xffffffff, 0x800, 1146 0x3508, 0xffffffff, 0x6c9b2000, 1147 0x3510, 0xfc00, 0x2000, 1148 0x3544, 0xffffffff, 0xfc0, 1149 0x28d4, 0x00000100, 0x100 1150 }; 1151 1152 static void si_init_golden_registers(struct radeon_device *rdev) 1153 { 1154 switch (rdev->family) { 1155 case CHIP_TAHITI: 1156 radeon_program_register_sequence(rdev, 1157 tahiti_golden_registers, 1158 (const u32)ARRAY_SIZE(tahiti_golden_registers)); 1159 radeon_program_register_sequence(rdev, 1160 tahiti_golden_rlc_registers, 1161 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers)); 1162 radeon_program_register_sequence(rdev, 1163 tahiti_mgcg_cgcg_init, 1164 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init)); 1165 radeon_program_register_sequence(rdev, 1166 tahiti_golden_registers2, 1167 (const u32)ARRAY_SIZE(tahiti_golden_registers2)); 1168 break; 1169 case CHIP_PITCAIRN: 1170 radeon_program_register_sequence(rdev, 1171 pitcairn_golden_registers, 1172 (const u32)ARRAY_SIZE(pitcairn_golden_registers)); 1173 radeon_program_register_sequence(rdev, 1174 pitcairn_golden_rlc_registers, 1175 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers)); 1176 radeon_program_register_sequence(rdev, 1177 pitcairn_mgcg_cgcg_init, 1178 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init)); 1179 break; 1180 case CHIP_VERDE: 1181 radeon_program_register_sequence(rdev, 1182 verde_golden_registers, 1183 (const u32)ARRAY_SIZE(verde_golden_registers)); 1184 radeon_program_register_sequence(rdev, 1185 verde_golden_rlc_registers, 1186 (const u32)ARRAY_SIZE(verde_golden_rlc_registers)); 1187 radeon_program_register_sequence(rdev, 1188 verde_mgcg_cgcg_init, 1189 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init)); 1190 radeon_program_register_sequence(rdev, 1191 verde_pg_init, 1192 (const u32)ARRAY_SIZE(verde_pg_init)); 1193 break; 1194 case CHIP_OLAND: 1195 radeon_program_register_sequence(rdev, 1196 oland_golden_registers, 1197 (const u32)ARRAY_SIZE(oland_golden_registers)); 1198 radeon_program_register_sequence(rdev, 1199 oland_golden_rlc_registers, 1200 (const u32)ARRAY_SIZE(oland_golden_rlc_registers)); 1201 radeon_program_register_sequence(rdev, 1202 oland_mgcg_cgcg_init, 1203 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init)); 1204 break; 1205 case CHIP_HAINAN: 1206 radeon_program_register_sequence(rdev, 1207 hainan_golden_registers, 1208 (const u32)ARRAY_SIZE(hainan_golden_registers)); 1209 radeon_program_register_sequence(rdev, 1210 hainan_golden_registers2, 1211 (const u32)ARRAY_SIZE(hainan_golden_registers2)); 1212 radeon_program_register_sequence(rdev, 1213 hainan_mgcg_cgcg_init, 1214 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init)); 1215 break; 1216 default: 1217 break; 1218 } 1219 } 1220 1221 #define PCIE_BUS_CLK 10000 1222 #define TCLK (PCIE_BUS_CLK / 10) 1223 1224 /** 1225 * si_get_xclk - get the xclk 1226 * 1227 * @rdev: radeon_device pointer 1228 * 1229 * Returns the reference clock used by the gfx engine 1230 * (SI). 1231 */ 1232 u32 si_get_xclk(struct radeon_device *rdev) 1233 { 1234 u32 reference_clock = rdev->clock.spll.reference_freq; 1235 u32 tmp; 1236 1237 tmp = RREG32(CG_CLKPIN_CNTL_2); 1238 if (tmp & MUX_TCLK_TO_XCLK) 1239 return TCLK; 1240 1241 tmp = RREG32(CG_CLKPIN_CNTL); 1242 if (tmp & XTALIN_DIVIDE) 1243 return reference_clock / 4; 1244 1245 return reference_clock; 1246 } 1247 1248 /* get temperature in millidegrees */ 1249 int si_get_temp(struct radeon_device *rdev) 1250 { 1251 u32 temp; 1252 int actual_temp = 0; 1253 1254 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 1255 CTF_TEMP_SHIFT; 1256 1257 if (temp & 0x200) 1258 actual_temp = 255; 1259 else 1260 actual_temp = temp & 0x1ff; 1261 1262 actual_temp = (actual_temp * 1000); 1263 1264 return actual_temp; 1265 } 1266 1267 #define TAHITI_IO_MC_REGS_SIZE 36 1268 1269 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1270 {0x0000006f, 0x03044000}, 1271 {0x00000070, 0x0480c018}, 1272 {0x00000071, 0x00000040}, 1273 {0x00000072, 0x01000000}, 1274 {0x00000074, 0x000000ff}, 1275 {0x00000075, 0x00143400}, 1276 {0x00000076, 0x08ec0800}, 1277 {0x00000077, 0x040000cc}, 1278 {0x00000079, 0x00000000}, 1279 {0x0000007a, 0x21000409}, 1280 {0x0000007c, 0x00000000}, 1281 {0x0000007d, 0xe8000000}, 1282 {0x0000007e, 0x044408a8}, 1283 {0x0000007f, 0x00000003}, 1284 {0x00000080, 0x00000000}, 1285 {0x00000081, 0x01000000}, 1286 {0x00000082, 0x02000000}, 1287 {0x00000083, 0x00000000}, 1288 {0x00000084, 0xe3f3e4f4}, 1289 {0x00000085, 0x00052024}, 1290 {0x00000087, 0x00000000}, 1291 {0x00000088, 0x66036603}, 1292 {0x00000089, 0x01000000}, 1293 {0x0000008b, 0x1c0a0000}, 1294 {0x0000008c, 0xff010000}, 1295 {0x0000008e, 0xffffefff}, 1296 {0x0000008f, 0xfff3efff}, 1297 {0x00000090, 0xfff3efbf}, 1298 {0x00000094, 0x00101101}, 1299 {0x00000095, 0x00000fff}, 1300 {0x00000096, 0x00116fff}, 1301 {0x00000097, 0x60010000}, 1302 {0x00000098, 0x10010000}, 1303 {0x00000099, 0x00006000}, 1304 {0x0000009a, 0x00001000}, 1305 {0x0000009f, 0x00a77400} 1306 }; 1307 1308 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1309 {0x0000006f, 0x03044000}, 1310 {0x00000070, 0x0480c018}, 1311 {0x00000071, 0x00000040}, 1312 {0x00000072, 0x01000000}, 1313 {0x00000074, 0x000000ff}, 1314 {0x00000075, 0x00143400}, 1315 {0x00000076, 0x08ec0800}, 1316 {0x00000077, 0x040000cc}, 1317 {0x00000079, 0x00000000}, 1318 {0x0000007a, 0x21000409}, 1319 {0x0000007c, 0x00000000}, 1320 {0x0000007d, 0xe8000000}, 1321 {0x0000007e, 0x044408a8}, 1322 {0x0000007f, 0x00000003}, 1323 {0x00000080, 0x00000000}, 1324 {0x00000081, 0x01000000}, 1325 {0x00000082, 0x02000000}, 1326 {0x00000083, 0x00000000}, 1327 {0x00000084, 0xe3f3e4f4}, 1328 {0x00000085, 0x00052024}, 1329 {0x00000087, 0x00000000}, 1330 {0x00000088, 0x66036603}, 1331 {0x00000089, 0x01000000}, 1332 {0x0000008b, 0x1c0a0000}, 1333 {0x0000008c, 0xff010000}, 1334 {0x0000008e, 0xffffefff}, 1335 {0x0000008f, 0xfff3efff}, 1336 {0x00000090, 0xfff3efbf}, 1337 {0x00000094, 0x00101101}, 1338 {0x00000095, 0x00000fff}, 1339 {0x00000096, 0x00116fff}, 1340 {0x00000097, 0x60010000}, 1341 {0x00000098, 0x10010000}, 1342 {0x00000099, 0x00006000}, 1343 {0x0000009a, 0x00001000}, 1344 {0x0000009f, 0x00a47400} 1345 }; 1346 1347 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1348 {0x0000006f, 0x03044000}, 1349 {0x00000070, 0x0480c018}, 1350 {0x00000071, 0x00000040}, 1351 {0x00000072, 0x01000000}, 1352 {0x00000074, 0x000000ff}, 1353 {0x00000075, 0x00143400}, 1354 {0x00000076, 0x08ec0800}, 1355 {0x00000077, 0x040000cc}, 1356 {0x00000079, 0x00000000}, 1357 {0x0000007a, 0x21000409}, 1358 {0x0000007c, 0x00000000}, 1359 {0x0000007d, 0xe8000000}, 1360 {0x0000007e, 0x044408a8}, 1361 {0x0000007f, 0x00000003}, 1362 {0x00000080, 0x00000000}, 1363 {0x00000081, 0x01000000}, 1364 {0x00000082, 0x02000000}, 1365 {0x00000083, 0x00000000}, 1366 {0x00000084, 0xe3f3e4f4}, 1367 {0x00000085, 0x00052024}, 1368 {0x00000087, 0x00000000}, 1369 {0x00000088, 0x66036603}, 1370 {0x00000089, 0x01000000}, 1371 {0x0000008b, 0x1c0a0000}, 1372 {0x0000008c, 0xff010000}, 1373 {0x0000008e, 0xffffefff}, 1374 {0x0000008f, 0xfff3efff}, 1375 {0x00000090, 0xfff3efbf}, 1376 {0x00000094, 0x00101101}, 1377 {0x00000095, 0x00000fff}, 1378 {0x00000096, 0x00116fff}, 1379 {0x00000097, 0x60010000}, 1380 {0x00000098, 0x10010000}, 1381 {0x00000099, 0x00006000}, 1382 {0x0000009a, 0x00001000}, 1383 {0x0000009f, 0x00a37400} 1384 }; 1385 1386 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1387 {0x0000006f, 0x03044000}, 1388 {0x00000070, 0x0480c018}, 1389 {0x00000071, 0x00000040}, 1390 {0x00000072, 0x01000000}, 1391 {0x00000074, 0x000000ff}, 1392 {0x00000075, 0x00143400}, 1393 {0x00000076, 0x08ec0800}, 1394 {0x00000077, 0x040000cc}, 1395 {0x00000079, 0x00000000}, 1396 {0x0000007a, 0x21000409}, 1397 {0x0000007c, 0x00000000}, 1398 {0x0000007d, 0xe8000000}, 1399 {0x0000007e, 0x044408a8}, 1400 {0x0000007f, 0x00000003}, 1401 {0x00000080, 0x00000000}, 1402 {0x00000081, 0x01000000}, 1403 {0x00000082, 0x02000000}, 1404 {0x00000083, 0x00000000}, 1405 {0x00000084, 0xe3f3e4f4}, 1406 {0x00000085, 0x00052024}, 1407 {0x00000087, 0x00000000}, 1408 {0x00000088, 0x66036603}, 1409 {0x00000089, 0x01000000}, 1410 {0x0000008b, 0x1c0a0000}, 1411 {0x0000008c, 0xff010000}, 1412 {0x0000008e, 0xffffefff}, 1413 {0x0000008f, 0xfff3efff}, 1414 {0x00000090, 0xfff3efbf}, 1415 {0x00000094, 0x00101101}, 1416 {0x00000095, 0x00000fff}, 1417 {0x00000096, 0x00116fff}, 1418 {0x00000097, 0x60010000}, 1419 {0x00000098, 0x10010000}, 1420 {0x00000099, 0x00006000}, 1421 {0x0000009a, 0x00001000}, 1422 {0x0000009f, 0x00a17730} 1423 }; 1424 1425 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1426 {0x0000006f, 0x03044000}, 1427 {0x00000070, 0x0480c018}, 1428 {0x00000071, 0x00000040}, 1429 {0x00000072, 0x01000000}, 1430 {0x00000074, 0x000000ff}, 1431 {0x00000075, 0x00143400}, 1432 {0x00000076, 0x08ec0800}, 1433 {0x00000077, 0x040000cc}, 1434 {0x00000079, 0x00000000}, 1435 {0x0000007a, 0x21000409}, 1436 {0x0000007c, 0x00000000}, 1437 {0x0000007d, 0xe8000000}, 1438 {0x0000007e, 0x044408a8}, 1439 {0x0000007f, 0x00000003}, 1440 {0x00000080, 0x00000000}, 1441 {0x00000081, 0x01000000}, 1442 {0x00000082, 0x02000000}, 1443 {0x00000083, 0x00000000}, 1444 {0x00000084, 0xe3f3e4f4}, 1445 {0x00000085, 0x00052024}, 1446 {0x00000087, 0x00000000}, 1447 {0x00000088, 0x66036603}, 1448 {0x00000089, 0x01000000}, 1449 {0x0000008b, 0x1c0a0000}, 1450 {0x0000008c, 0xff010000}, 1451 {0x0000008e, 0xffffefff}, 1452 {0x0000008f, 0xfff3efff}, 1453 {0x00000090, 0xfff3efbf}, 1454 {0x00000094, 0x00101101}, 1455 {0x00000095, 0x00000fff}, 1456 {0x00000096, 0x00116fff}, 1457 {0x00000097, 0x60010000}, 1458 {0x00000098, 0x10010000}, 1459 {0x00000099, 0x00006000}, 1460 {0x0000009a, 0x00001000}, 1461 {0x0000009f, 0x00a07730} 1462 }; 1463 1464 /* ucode loading */ 1465 int si_mc_load_microcode(struct radeon_device *rdev) 1466 { 1467 const __be32 *fw_data; 1468 u32 running, blackout = 0; 1469 u32 *io_mc_regs; 1470 int i, ucode_size, regs_size; 1471 1472 if (!rdev->mc_fw) 1473 return -EINVAL; 1474 1475 switch (rdev->family) { 1476 case CHIP_TAHITI: 1477 io_mc_regs = (u32 *)&tahiti_io_mc_regs; 1478 ucode_size = SI_MC_UCODE_SIZE; 1479 regs_size = TAHITI_IO_MC_REGS_SIZE; 1480 break; 1481 case CHIP_PITCAIRN: 1482 io_mc_regs = (u32 *)&pitcairn_io_mc_regs; 1483 ucode_size = SI_MC_UCODE_SIZE; 1484 regs_size = TAHITI_IO_MC_REGS_SIZE; 1485 break; 1486 case CHIP_VERDE: 1487 default: 1488 io_mc_regs = (u32 *)&verde_io_mc_regs; 1489 ucode_size = SI_MC_UCODE_SIZE; 1490 regs_size = TAHITI_IO_MC_REGS_SIZE; 1491 break; 1492 case CHIP_OLAND: 1493 io_mc_regs = (u32 *)&oland_io_mc_regs; 1494 ucode_size = OLAND_MC_UCODE_SIZE; 1495 regs_size = TAHITI_IO_MC_REGS_SIZE; 1496 break; 1497 case CHIP_HAINAN: 1498 io_mc_regs = (u32 *)&hainan_io_mc_regs; 1499 ucode_size = OLAND_MC_UCODE_SIZE; 1500 regs_size = TAHITI_IO_MC_REGS_SIZE; 1501 break; 1502 } 1503 1504 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1505 1506 if (running == 0) { 1507 if (running) { 1508 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1509 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1510 } 1511 1512 /* reset the engine and set to writable */ 1513 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1514 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1515 1516 /* load mc io regs */ 1517 for (i = 0; i < regs_size; i++) { 1518 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1519 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1520 } 1521 /* load the MC ucode */ 1522 fw_data = (const __be32 *)rdev->mc_fw->data; 1523 for (i = 0; i < ucode_size; i++) 1524 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1525 1526 /* put the engine back into the active state */ 1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1528 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1530 1531 /* wait for training to complete */ 1532 for (i = 0; i < rdev->usec_timeout; i++) { 1533 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1534 break; 1535 udelay(1); 1536 } 1537 for (i = 0; i < rdev->usec_timeout; i++) { 1538 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1539 break; 1540 udelay(1); 1541 } 1542 1543 if (running) 1544 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1545 } 1546 1547 return 0; 1548 } 1549 1550 static int si_init_microcode(struct radeon_device *rdev) 1551 { 1552 const char *chip_name; 1553 const char *rlc_chip_name; 1554 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size; 1555 size_t smc_req_size; 1556 char fw_name[30]; 1557 int err; 1558 1559 DRM_DEBUG("\n"); 1560 1561 switch (rdev->family) { 1562 case CHIP_TAHITI: 1563 chip_name = "TAHITI"; 1564 rlc_chip_name = "TAHITI"; 1565 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1566 me_req_size = SI_PM4_UCODE_SIZE * 4; 1567 ce_req_size = SI_CE_UCODE_SIZE * 4; 1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1569 mc_req_size = SI_MC_UCODE_SIZE * 4; 1570 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4); 1571 break; 1572 case CHIP_PITCAIRN: 1573 chip_name = "PITCAIRN"; 1574 rlc_chip_name = "PITCAIRN"; 1575 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1576 me_req_size = SI_PM4_UCODE_SIZE * 4; 1577 ce_req_size = SI_CE_UCODE_SIZE * 4; 1578 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1579 mc_req_size = SI_MC_UCODE_SIZE * 4; 1580 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4); 1581 break; 1582 case CHIP_VERDE: 1583 chip_name = "VERDE"; 1584 rlc_chip_name = "VERDE"; 1585 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1586 me_req_size = SI_PM4_UCODE_SIZE * 4; 1587 ce_req_size = SI_CE_UCODE_SIZE * 4; 1588 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1589 mc_req_size = SI_MC_UCODE_SIZE * 4; 1590 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4); 1591 break; 1592 case CHIP_OLAND: 1593 chip_name = "OLAND"; 1594 rlc_chip_name = "OLAND"; 1595 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1596 me_req_size = SI_PM4_UCODE_SIZE * 4; 1597 ce_req_size = SI_CE_UCODE_SIZE * 4; 1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1600 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4); 1601 break; 1602 case CHIP_HAINAN: 1603 chip_name = "HAINAN"; 1604 rlc_chip_name = "HAINAN"; 1605 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1606 me_req_size = SI_PM4_UCODE_SIZE * 4; 1607 ce_req_size = SI_CE_UCODE_SIZE * 4; 1608 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1609 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1610 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4); 1611 break; 1612 default: BUG(); 1613 } 1614 1615 DRM_INFO("Loading %s Microcode\n", chip_name); 1616 1617 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 1618 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1619 if (err) 1620 goto out; 1621 if (rdev->pfp_fw->size != pfp_req_size) { 1622 printk(KERN_ERR 1623 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1624 rdev->pfp_fw->size, fw_name); 1625 err = -EINVAL; 1626 goto out; 1627 } 1628 1629 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 1630 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 1631 if (err) 1632 goto out; 1633 if (rdev->me_fw->size != me_req_size) { 1634 printk(KERN_ERR 1635 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1636 rdev->me_fw->size, fw_name); 1637 err = -EINVAL; 1638 } 1639 1640 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 1641 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 1642 if (err) 1643 goto out; 1644 if (rdev->ce_fw->size != ce_req_size) { 1645 printk(KERN_ERR 1646 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1647 rdev->ce_fw->size, fw_name); 1648 err = -EINVAL; 1649 } 1650 1651 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name); 1652 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 1653 if (err) 1654 goto out; 1655 if (rdev->rlc_fw->size != rlc_req_size) { 1656 printk(KERN_ERR 1657 "si_rlc: Bogus length %zu in firmware \"%s\"\n", 1658 rdev->rlc_fw->size, fw_name); 1659 err = -EINVAL; 1660 } 1661 1662 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 1663 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 1664 if (err) 1665 goto out; 1666 if (rdev->mc_fw->size != mc_req_size) { 1667 printk(KERN_ERR 1668 "si_mc: Bogus length %zu in firmware \"%s\"\n", 1669 rdev->mc_fw->size, fw_name); 1670 err = -EINVAL; 1671 } 1672 1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 1674 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 1675 if (err) { 1676 printk(KERN_ERR 1677 "smc: error loading firmware \"%s\"\n", 1678 fw_name); 1679 release_firmware(rdev->smc_fw); 1680 rdev->smc_fw = NULL; 1681 err = 0; 1682 } else if (rdev->smc_fw->size != smc_req_size) { 1683 printk(KERN_ERR 1684 "si_smc: Bogus length %zu in firmware \"%s\"\n", 1685 rdev->smc_fw->size, fw_name); 1686 err = -EINVAL; 1687 } 1688 1689 out: 1690 if (err) { 1691 if (err != -EINVAL) 1692 printk(KERN_ERR 1693 "si_cp: Failed to load firmware \"%s\"\n", 1694 fw_name); 1695 release_firmware(rdev->pfp_fw); 1696 rdev->pfp_fw = NULL; 1697 release_firmware(rdev->me_fw); 1698 rdev->me_fw = NULL; 1699 release_firmware(rdev->ce_fw); 1700 rdev->ce_fw = NULL; 1701 release_firmware(rdev->rlc_fw); 1702 rdev->rlc_fw = NULL; 1703 release_firmware(rdev->mc_fw); 1704 rdev->mc_fw = NULL; 1705 release_firmware(rdev->smc_fw); 1706 rdev->smc_fw = NULL; 1707 } 1708 return err; 1709 } 1710 1711 /* watermark setup */ 1712 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev, 1713 struct radeon_crtc *radeon_crtc, 1714 struct drm_display_mode *mode, 1715 struct drm_display_mode *other_mode) 1716 { 1717 u32 tmp, buffer_alloc, i; 1718 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1719 /* 1720 * Line Buffer Setup 1721 * There are 3 line buffers, each one shared by 2 display controllers. 1722 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1723 * the display controllers. The paritioning is done via one of four 1724 * preset allocations specified in bits 21:20: 1725 * 0 - half lb 1726 * 2 - whole lb, other crtc must be disabled 1727 */ 1728 /* this can get tricky if we have two large displays on a paired group 1729 * of crtcs. Ideally for multiple large displays we'd assign them to 1730 * non-linked crtcs for maximum line buffer allocation. 1731 */ 1732 if (radeon_crtc->base.enabled && mode) { 1733 if (other_mode) { 1734 tmp = 0; /* 1/2 */ 1735 buffer_alloc = 1; 1736 } else { 1737 tmp = 2; /* whole */ 1738 buffer_alloc = 2; 1739 } 1740 } else { 1741 tmp = 0; 1742 buffer_alloc = 0; 1743 } 1744 1745 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, 1746 DC_LB_MEMORY_CONFIG(tmp)); 1747 1748 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1749 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1750 for (i = 0; i < rdev->usec_timeout; i++) { 1751 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1752 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1753 break; 1754 udelay(1); 1755 } 1756 1757 if (radeon_crtc->base.enabled && mode) { 1758 switch (tmp) { 1759 case 0: 1760 default: 1761 return 4096 * 2; 1762 case 2: 1763 return 8192 * 2; 1764 } 1765 } 1766 1767 /* controller not enabled, so no lb used */ 1768 return 0; 1769 } 1770 1771 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev) 1772 { 1773 u32 tmp = RREG32(MC_SHARED_CHMAP); 1774 1775 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1776 case 0: 1777 default: 1778 return 1; 1779 case 1: 1780 return 2; 1781 case 2: 1782 return 4; 1783 case 3: 1784 return 8; 1785 case 4: 1786 return 3; 1787 case 5: 1788 return 6; 1789 case 6: 1790 return 10; 1791 case 7: 1792 return 12; 1793 case 8: 1794 return 16; 1795 } 1796 } 1797 1798 struct dce6_wm_params { 1799 u32 dram_channels; /* number of dram channels */ 1800 u32 yclk; /* bandwidth per dram data pin in kHz */ 1801 u32 sclk; /* engine clock in kHz */ 1802 u32 disp_clk; /* display clock in kHz */ 1803 u32 src_width; /* viewport width */ 1804 u32 active_time; /* active display time in ns */ 1805 u32 blank_time; /* blank time in ns */ 1806 bool interlaced; /* mode is interlaced */ 1807 fixed20_12 vsc; /* vertical scale ratio */ 1808 u32 num_heads; /* number of active crtcs */ 1809 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1810 u32 lb_size; /* line buffer allocated to pipe */ 1811 u32 vtaps; /* vertical scaler taps */ 1812 }; 1813 1814 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm) 1815 { 1816 /* Calculate raw DRAM Bandwidth */ 1817 fixed20_12 dram_efficiency; /* 0.7 */ 1818 fixed20_12 yclk, dram_channels, bandwidth; 1819 fixed20_12 a; 1820 1821 a.full = dfixed_const(1000); 1822 yclk.full = dfixed_const(wm->yclk); 1823 yclk.full = dfixed_div(yclk, a); 1824 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1825 a.full = dfixed_const(10); 1826 dram_efficiency.full = dfixed_const(7); 1827 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1828 bandwidth.full = dfixed_mul(dram_channels, yclk); 1829 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1830 1831 return dfixed_trunc(bandwidth); 1832 } 1833 1834 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm) 1835 { 1836 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1837 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 1838 fixed20_12 yclk, dram_channels, bandwidth; 1839 fixed20_12 a; 1840 1841 a.full = dfixed_const(1000); 1842 yclk.full = dfixed_const(wm->yclk); 1843 yclk.full = dfixed_div(yclk, a); 1844 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1845 a.full = dfixed_const(10); 1846 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 1847 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 1848 bandwidth.full = dfixed_mul(dram_channels, yclk); 1849 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 1850 1851 return dfixed_trunc(bandwidth); 1852 } 1853 1854 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm) 1855 { 1856 /* Calculate the display Data return Bandwidth */ 1857 fixed20_12 return_efficiency; /* 0.8 */ 1858 fixed20_12 sclk, bandwidth; 1859 fixed20_12 a; 1860 1861 a.full = dfixed_const(1000); 1862 sclk.full = dfixed_const(wm->sclk); 1863 sclk.full = dfixed_div(sclk, a); 1864 a.full = dfixed_const(10); 1865 return_efficiency.full = dfixed_const(8); 1866 return_efficiency.full = dfixed_div(return_efficiency, a); 1867 a.full = dfixed_const(32); 1868 bandwidth.full = dfixed_mul(a, sclk); 1869 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 1870 1871 return dfixed_trunc(bandwidth); 1872 } 1873 1874 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm) 1875 { 1876 return 32; 1877 } 1878 1879 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm) 1880 { 1881 /* Calculate the DMIF Request Bandwidth */ 1882 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 1883 fixed20_12 disp_clk, sclk, bandwidth; 1884 fixed20_12 a, b1, b2; 1885 u32 min_bandwidth; 1886 1887 a.full = dfixed_const(1000); 1888 disp_clk.full = dfixed_const(wm->disp_clk); 1889 disp_clk.full = dfixed_div(disp_clk, a); 1890 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2); 1891 b1.full = dfixed_mul(a, disp_clk); 1892 1893 a.full = dfixed_const(1000); 1894 sclk.full = dfixed_const(wm->sclk); 1895 sclk.full = dfixed_div(sclk, a); 1896 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm)); 1897 b2.full = dfixed_mul(a, sclk); 1898 1899 a.full = dfixed_const(10); 1900 disp_clk_request_efficiency.full = dfixed_const(8); 1901 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 1902 1903 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2)); 1904 1905 a.full = dfixed_const(min_bandwidth); 1906 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency); 1907 1908 return dfixed_trunc(bandwidth); 1909 } 1910 1911 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm) 1912 { 1913 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 1914 u32 dram_bandwidth = dce6_dram_bandwidth(wm); 1915 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm); 1916 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm); 1917 1918 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 1919 } 1920 1921 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm) 1922 { 1923 /* Calculate the display mode Average Bandwidth 1924 * DisplayMode should contain the source and destination dimensions, 1925 * timing, etc. 1926 */ 1927 fixed20_12 bpp; 1928 fixed20_12 line_time; 1929 fixed20_12 src_width; 1930 fixed20_12 bandwidth; 1931 fixed20_12 a; 1932 1933 a.full = dfixed_const(1000); 1934 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 1935 line_time.full = dfixed_div(line_time, a); 1936 bpp.full = dfixed_const(wm->bytes_per_pixel); 1937 src_width.full = dfixed_const(wm->src_width); 1938 bandwidth.full = dfixed_mul(src_width, bpp); 1939 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 1940 bandwidth.full = dfixed_div(bandwidth, line_time); 1941 1942 return dfixed_trunc(bandwidth); 1943 } 1944 1945 static u32 dce6_latency_watermark(struct dce6_wm_params *wm) 1946 { 1947 /* First calcualte the latency in ns */ 1948 u32 mc_latency = 2000; /* 2000 ns. */ 1949 u32 available_bandwidth = dce6_available_bandwidth(wm); 1950 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 1951 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 1952 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 1953 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 1954 (wm->num_heads * cursor_line_pair_return_time); 1955 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 1956 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 1957 u32 tmp, dmif_size = 12288; 1958 fixed20_12 a, b, c; 1959 1960 if (wm->num_heads == 0) 1961 return 0; 1962 1963 a.full = dfixed_const(2); 1964 b.full = dfixed_const(1); 1965 if ((wm->vsc.full > a.full) || 1966 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 1967 (wm->vtaps >= 5) || 1968 ((wm->vsc.full >= a.full) && wm->interlaced)) 1969 max_src_lines_per_dst_line = 4; 1970 else 1971 max_src_lines_per_dst_line = 2; 1972 1973 a.full = dfixed_const(available_bandwidth); 1974 b.full = dfixed_const(wm->num_heads); 1975 a.full = dfixed_div(a, b); 1976 1977 b.full = dfixed_const(mc_latency + 512); 1978 c.full = dfixed_const(wm->disp_clk); 1979 b.full = dfixed_div(b, c); 1980 1981 c.full = dfixed_const(dmif_size); 1982 b.full = dfixed_div(c, b); 1983 1984 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 1985 1986 b.full = dfixed_const(1000); 1987 c.full = dfixed_const(wm->disp_clk); 1988 b.full = dfixed_div(c, b); 1989 c.full = dfixed_const(wm->bytes_per_pixel); 1990 b.full = dfixed_mul(b, c); 1991 1992 lb_fill_bw = min(tmp, dfixed_trunc(b)); 1993 1994 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 1995 b.full = dfixed_const(1000); 1996 c.full = dfixed_const(lb_fill_bw); 1997 b.full = dfixed_div(c, b); 1998 a.full = dfixed_div(a, b); 1999 line_fill_time = dfixed_trunc(a); 2000 2001 if (line_fill_time < wm->active_time) 2002 return latency; 2003 else 2004 return latency + (line_fill_time - wm->active_time); 2005 2006 } 2007 2008 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm) 2009 { 2010 if (dce6_average_bandwidth(wm) <= 2011 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads)) 2012 return true; 2013 else 2014 return false; 2015 }; 2016 2017 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm) 2018 { 2019 if (dce6_average_bandwidth(wm) <= 2020 (dce6_available_bandwidth(wm) / wm->num_heads)) 2021 return true; 2022 else 2023 return false; 2024 }; 2025 2026 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm) 2027 { 2028 u32 lb_partitions = wm->lb_size / wm->src_width; 2029 u32 line_time = wm->active_time + wm->blank_time; 2030 u32 latency_tolerant_lines; 2031 u32 latency_hiding; 2032 fixed20_12 a; 2033 2034 a.full = dfixed_const(1); 2035 if (wm->vsc.full > a.full) 2036 latency_tolerant_lines = 1; 2037 else { 2038 if (lb_partitions <= (wm->vtaps + 1)) 2039 latency_tolerant_lines = 1; 2040 else 2041 latency_tolerant_lines = 2; 2042 } 2043 2044 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2045 2046 if (dce6_latency_watermark(wm) <= latency_hiding) 2047 return true; 2048 else 2049 return false; 2050 } 2051 2052 static void dce6_program_watermarks(struct radeon_device *rdev, 2053 struct radeon_crtc *radeon_crtc, 2054 u32 lb_size, u32 num_heads) 2055 { 2056 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2057 struct dce6_wm_params wm_low, wm_high; 2058 u32 dram_channels; 2059 u32 pixel_period; 2060 u32 line_time = 0; 2061 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2062 u32 priority_a_mark = 0, priority_b_mark = 0; 2063 u32 priority_a_cnt = PRIORITY_OFF; 2064 u32 priority_b_cnt = PRIORITY_OFF; 2065 u32 tmp, arb_control3; 2066 fixed20_12 a, b, c; 2067 2068 if (radeon_crtc->base.enabled && num_heads && mode) { 2069 pixel_period = 1000000 / (u32)mode->clock; 2070 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 2071 priority_a_cnt = 0; 2072 priority_b_cnt = 0; 2073 2074 if (rdev->family == CHIP_ARUBA) 2075 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2076 else 2077 dram_channels = si_get_number_of_dram_channels(rdev); 2078 2079 /* watermark for high clocks */ 2080 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2081 wm_high.yclk = 2082 radeon_dpm_get_mclk(rdev, false) * 10; 2083 wm_high.sclk = 2084 radeon_dpm_get_sclk(rdev, false) * 10; 2085 } else { 2086 wm_high.yclk = rdev->pm.current_mclk * 10; 2087 wm_high.sclk = rdev->pm.current_sclk * 10; 2088 } 2089 2090 wm_high.disp_clk = mode->clock; 2091 wm_high.src_width = mode->crtc_hdisplay; 2092 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 2093 wm_high.blank_time = line_time - wm_high.active_time; 2094 wm_high.interlaced = false; 2095 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2096 wm_high.interlaced = true; 2097 wm_high.vsc = radeon_crtc->vsc; 2098 wm_high.vtaps = 1; 2099 if (radeon_crtc->rmx_type != RMX_OFF) 2100 wm_high.vtaps = 2; 2101 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2102 wm_high.lb_size = lb_size; 2103 wm_high.dram_channels = dram_channels; 2104 wm_high.num_heads = num_heads; 2105 2106 /* watermark for low clocks */ 2107 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2108 wm_low.yclk = 2109 radeon_dpm_get_mclk(rdev, true) * 10; 2110 wm_low.sclk = 2111 radeon_dpm_get_sclk(rdev, true) * 10; 2112 } else { 2113 wm_low.yclk = rdev->pm.current_mclk * 10; 2114 wm_low.sclk = rdev->pm.current_sclk * 10; 2115 } 2116 2117 wm_low.disp_clk = mode->clock; 2118 wm_low.src_width = mode->crtc_hdisplay; 2119 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 2120 wm_low.blank_time = line_time - wm_low.active_time; 2121 wm_low.interlaced = false; 2122 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2123 wm_low.interlaced = true; 2124 wm_low.vsc = radeon_crtc->vsc; 2125 wm_low.vtaps = 1; 2126 if (radeon_crtc->rmx_type != RMX_OFF) 2127 wm_low.vtaps = 2; 2128 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2129 wm_low.lb_size = lb_size; 2130 wm_low.dram_channels = dram_channels; 2131 wm_low.num_heads = num_heads; 2132 2133 /* set for high clocks */ 2134 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535); 2135 /* set for low clocks */ 2136 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535); 2137 2138 /* possibly force display priority to high */ 2139 /* should really do this at mode validation time... */ 2140 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2141 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) || 2142 !dce6_check_latency_hiding(&wm_high) || 2143 (rdev->disp_priority == 2)) { 2144 DRM_DEBUG_KMS("force priority to high\n"); 2145 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2146 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2147 } 2148 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2149 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) || 2150 !dce6_check_latency_hiding(&wm_low) || 2151 (rdev->disp_priority == 2)) { 2152 DRM_DEBUG_KMS("force priority to high\n"); 2153 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2154 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2155 } 2156 2157 a.full = dfixed_const(1000); 2158 b.full = dfixed_const(mode->clock); 2159 b.full = dfixed_div(b, a); 2160 c.full = dfixed_const(latency_watermark_a); 2161 c.full = dfixed_mul(c, b); 2162 c.full = dfixed_mul(c, radeon_crtc->hsc); 2163 c.full = dfixed_div(c, a); 2164 a.full = dfixed_const(16); 2165 c.full = dfixed_div(c, a); 2166 priority_a_mark = dfixed_trunc(c); 2167 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2168 2169 a.full = dfixed_const(1000); 2170 b.full = dfixed_const(mode->clock); 2171 b.full = dfixed_div(b, a); 2172 c.full = dfixed_const(latency_watermark_b); 2173 c.full = dfixed_mul(c, b); 2174 c.full = dfixed_mul(c, radeon_crtc->hsc); 2175 c.full = dfixed_div(c, a); 2176 a.full = dfixed_const(16); 2177 c.full = dfixed_div(c, a); 2178 priority_b_mark = dfixed_trunc(c); 2179 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2180 } 2181 2182 /* select wm A */ 2183 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2184 tmp = arb_control3; 2185 tmp &= ~LATENCY_WATERMARK_MASK(3); 2186 tmp |= LATENCY_WATERMARK_MASK(1); 2187 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2188 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2189 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2190 LATENCY_HIGH_WATERMARK(line_time))); 2191 /* select wm B */ 2192 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2193 tmp &= ~LATENCY_WATERMARK_MASK(3); 2194 tmp |= LATENCY_WATERMARK_MASK(2); 2195 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2196 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2197 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2198 LATENCY_HIGH_WATERMARK(line_time))); 2199 /* restore original selection */ 2200 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3); 2201 2202 /* write the priority marks */ 2203 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2204 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2205 2206 /* save values for DPM */ 2207 radeon_crtc->line_time = line_time; 2208 radeon_crtc->wm_high = latency_watermark_a; 2209 radeon_crtc->wm_low = latency_watermark_b; 2210 } 2211 2212 void dce6_bandwidth_update(struct radeon_device *rdev) 2213 { 2214 struct drm_display_mode *mode0 = NULL; 2215 struct drm_display_mode *mode1 = NULL; 2216 u32 num_heads = 0, lb_size; 2217 int i; 2218 2219 radeon_update_display_priority(rdev); 2220 2221 for (i = 0; i < rdev->num_crtc; i++) { 2222 if (rdev->mode_info.crtcs[i]->base.enabled) 2223 num_heads++; 2224 } 2225 for (i = 0; i < rdev->num_crtc; i += 2) { 2226 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2227 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2228 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2229 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2230 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2231 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2232 } 2233 } 2234 2235 /* 2236 * Core functions 2237 */ 2238 static void si_tiling_mode_table_init(struct radeon_device *rdev) 2239 { 2240 const u32 num_tile_mode_states = 32; 2241 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2242 2243 switch (rdev->config.si.mem_row_size_in_kb) { 2244 case 1: 2245 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2246 break; 2247 case 2: 2248 default: 2249 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2250 break; 2251 case 4: 2252 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2253 break; 2254 } 2255 2256 if ((rdev->family == CHIP_TAHITI) || 2257 (rdev->family == CHIP_PITCAIRN)) { 2258 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2259 switch (reg_offset) { 2260 case 0: /* non-AA compressed depth or any compressed stencil */ 2261 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2262 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2263 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2264 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2265 NUM_BANKS(ADDR_SURF_16_BANK) | 2266 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2267 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2268 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2269 break; 2270 case 1: /* 2xAA/4xAA compressed depth only */ 2271 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2272 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2273 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2274 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2275 NUM_BANKS(ADDR_SURF_16_BANK) | 2276 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2279 break; 2280 case 2: /* 8xAA compressed depth only */ 2281 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2282 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2283 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2285 NUM_BANKS(ADDR_SURF_16_BANK) | 2286 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2289 break; 2290 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2291 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2292 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2293 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2294 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2295 NUM_BANKS(ADDR_SURF_16_BANK) | 2296 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2297 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2298 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2299 break; 2300 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2301 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2302 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2304 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2305 NUM_BANKS(ADDR_SURF_16_BANK) | 2306 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2309 break; 2310 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2311 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2312 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2313 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2314 TILE_SPLIT(split_equal_to_row_size) | 2315 NUM_BANKS(ADDR_SURF_16_BANK) | 2316 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2319 break; 2320 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2321 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2322 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2323 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2324 TILE_SPLIT(split_equal_to_row_size) | 2325 NUM_BANKS(ADDR_SURF_16_BANK) | 2326 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2327 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2328 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2329 break; 2330 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2331 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2332 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2333 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2334 TILE_SPLIT(split_equal_to_row_size) | 2335 NUM_BANKS(ADDR_SURF_16_BANK) | 2336 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2337 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2338 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2339 break; 2340 case 8: /* 1D and 1D Array Surfaces */ 2341 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2342 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2343 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2344 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2345 NUM_BANKS(ADDR_SURF_16_BANK) | 2346 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2349 break; 2350 case 9: /* Displayable maps. */ 2351 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2352 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2353 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2355 NUM_BANKS(ADDR_SURF_16_BANK) | 2356 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2359 break; 2360 case 10: /* Display 8bpp. */ 2361 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2362 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2363 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2364 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2365 NUM_BANKS(ADDR_SURF_16_BANK) | 2366 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2369 break; 2370 case 11: /* Display 16bpp. */ 2371 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2372 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2373 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2374 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2375 NUM_BANKS(ADDR_SURF_16_BANK) | 2376 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2379 break; 2380 case 12: /* Display 32bpp. */ 2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2382 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2383 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2385 NUM_BANKS(ADDR_SURF_16_BANK) | 2386 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2389 break; 2390 case 13: /* Thin. */ 2391 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2392 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2393 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2394 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2395 NUM_BANKS(ADDR_SURF_16_BANK) | 2396 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2399 break; 2400 case 14: /* Thin 8 bpp. */ 2401 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2402 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2403 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2405 NUM_BANKS(ADDR_SURF_16_BANK) | 2406 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2409 break; 2410 case 15: /* Thin 16 bpp. */ 2411 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2412 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2413 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2414 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2415 NUM_BANKS(ADDR_SURF_16_BANK) | 2416 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2419 break; 2420 case 16: /* Thin 32 bpp. */ 2421 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2422 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2423 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2424 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2425 NUM_BANKS(ADDR_SURF_16_BANK) | 2426 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2427 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2428 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2429 break; 2430 case 17: /* Thin 64 bpp. */ 2431 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2432 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2433 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2434 TILE_SPLIT(split_equal_to_row_size) | 2435 NUM_BANKS(ADDR_SURF_16_BANK) | 2436 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2437 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2438 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2439 break; 2440 case 21: /* 8 bpp PRT. */ 2441 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2442 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2443 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2444 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2445 NUM_BANKS(ADDR_SURF_16_BANK) | 2446 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2447 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2448 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2449 break; 2450 case 22: /* 16 bpp PRT */ 2451 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2452 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2453 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2454 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2455 NUM_BANKS(ADDR_SURF_16_BANK) | 2456 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2459 break; 2460 case 23: /* 32 bpp PRT */ 2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2462 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2463 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2465 NUM_BANKS(ADDR_SURF_16_BANK) | 2466 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2467 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2468 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2469 break; 2470 case 24: /* 64 bpp PRT */ 2471 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2472 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2473 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2475 NUM_BANKS(ADDR_SURF_16_BANK) | 2476 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2479 break; 2480 case 25: /* 128 bpp PRT */ 2481 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2482 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2483 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2484 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2485 NUM_BANKS(ADDR_SURF_8_BANK) | 2486 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2489 break; 2490 default: 2491 gb_tile_moden = 0; 2492 break; 2493 } 2494 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2495 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2496 } 2497 } else if ((rdev->family == CHIP_VERDE) || 2498 (rdev->family == CHIP_OLAND) || 2499 (rdev->family == CHIP_HAINAN)) { 2500 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2501 switch (reg_offset) { 2502 case 0: /* non-AA compressed depth or any compressed stencil */ 2503 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2504 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2505 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2506 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2507 NUM_BANKS(ADDR_SURF_16_BANK) | 2508 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2511 break; 2512 case 1: /* 2xAA/4xAA compressed depth only */ 2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2514 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2515 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2517 NUM_BANKS(ADDR_SURF_16_BANK) | 2518 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2519 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2520 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2521 break; 2522 case 2: /* 8xAA compressed depth only */ 2523 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2524 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2525 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2527 NUM_BANKS(ADDR_SURF_16_BANK) | 2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2531 break; 2532 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2533 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2534 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2535 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2536 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2537 NUM_BANKS(ADDR_SURF_16_BANK) | 2538 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2539 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2540 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2541 break; 2542 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2543 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2544 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2545 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2547 NUM_BANKS(ADDR_SURF_16_BANK) | 2548 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2549 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2550 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2551 break; 2552 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2553 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2554 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2555 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2556 TILE_SPLIT(split_equal_to_row_size) | 2557 NUM_BANKS(ADDR_SURF_16_BANK) | 2558 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2559 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2560 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2561 break; 2562 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2564 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2566 TILE_SPLIT(split_equal_to_row_size) | 2567 NUM_BANKS(ADDR_SURF_16_BANK) | 2568 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2569 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2570 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2571 break; 2572 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2573 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2574 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2575 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2576 TILE_SPLIT(split_equal_to_row_size) | 2577 NUM_BANKS(ADDR_SURF_16_BANK) | 2578 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2579 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2580 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2581 break; 2582 case 8: /* 1D and 1D Array Surfaces */ 2583 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2584 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2585 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2587 NUM_BANKS(ADDR_SURF_16_BANK) | 2588 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2589 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2590 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2591 break; 2592 case 9: /* Displayable maps. */ 2593 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2594 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2595 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2596 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2597 NUM_BANKS(ADDR_SURF_16_BANK) | 2598 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2599 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2600 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2601 break; 2602 case 10: /* Display 8bpp. */ 2603 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2604 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2606 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2607 NUM_BANKS(ADDR_SURF_16_BANK) | 2608 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2609 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2610 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2611 break; 2612 case 11: /* Display 16bpp. */ 2613 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2614 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2615 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2617 NUM_BANKS(ADDR_SURF_16_BANK) | 2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2621 break; 2622 case 12: /* Display 32bpp. */ 2623 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2624 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2625 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2626 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2627 NUM_BANKS(ADDR_SURF_16_BANK) | 2628 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2631 break; 2632 case 13: /* Thin. */ 2633 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2634 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2635 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2636 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2637 NUM_BANKS(ADDR_SURF_16_BANK) | 2638 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2639 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2640 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2641 break; 2642 case 14: /* Thin 8 bpp. */ 2643 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2644 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2645 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2646 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2647 NUM_BANKS(ADDR_SURF_16_BANK) | 2648 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2649 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2650 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2651 break; 2652 case 15: /* Thin 16 bpp. */ 2653 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2654 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2655 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2657 NUM_BANKS(ADDR_SURF_16_BANK) | 2658 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2659 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2660 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2661 break; 2662 case 16: /* Thin 32 bpp. */ 2663 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2664 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2665 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2666 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2667 NUM_BANKS(ADDR_SURF_16_BANK) | 2668 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2671 break; 2672 case 17: /* Thin 64 bpp. */ 2673 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2674 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2676 TILE_SPLIT(split_equal_to_row_size) | 2677 NUM_BANKS(ADDR_SURF_16_BANK) | 2678 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2681 break; 2682 case 21: /* 8 bpp PRT. */ 2683 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2684 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2685 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2686 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2687 NUM_BANKS(ADDR_SURF_16_BANK) | 2688 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2689 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2690 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2691 break; 2692 case 22: /* 16 bpp PRT */ 2693 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2694 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2695 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2696 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2697 NUM_BANKS(ADDR_SURF_16_BANK) | 2698 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2701 break; 2702 case 23: /* 32 bpp PRT */ 2703 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2704 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2705 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2707 NUM_BANKS(ADDR_SURF_16_BANK) | 2708 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2711 break; 2712 case 24: /* 64 bpp PRT */ 2713 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2714 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2715 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2716 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2717 NUM_BANKS(ADDR_SURF_16_BANK) | 2718 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2719 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2720 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2721 break; 2722 case 25: /* 128 bpp PRT */ 2723 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2724 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2725 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2726 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2727 NUM_BANKS(ADDR_SURF_8_BANK) | 2728 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2731 break; 2732 default: 2733 gb_tile_moden = 0; 2734 break; 2735 } 2736 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2737 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2738 } 2739 } else 2740 DRM_ERROR("unknown asic: 0x%x\n", rdev->family); 2741 } 2742 2743 static void si_select_se_sh(struct radeon_device *rdev, 2744 u32 se_num, u32 sh_num) 2745 { 2746 u32 data = INSTANCE_BROADCAST_WRITES; 2747 2748 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 2749 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 2750 else if (se_num == 0xffffffff) 2751 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 2752 else if (sh_num == 0xffffffff) 2753 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 2754 else 2755 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 2756 WREG32(GRBM_GFX_INDEX, data); 2757 } 2758 2759 static u32 si_create_bitmask(u32 bit_width) 2760 { 2761 u32 i, mask = 0; 2762 2763 for (i = 0; i < bit_width; i++) { 2764 mask <<= 1; 2765 mask |= 1; 2766 } 2767 return mask; 2768 } 2769 2770 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh) 2771 { 2772 u32 data, mask; 2773 2774 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 2775 if (data & 1) 2776 data &= INACTIVE_CUS_MASK; 2777 else 2778 data = 0; 2779 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG); 2780 2781 data >>= INACTIVE_CUS_SHIFT; 2782 2783 mask = si_create_bitmask(cu_per_sh); 2784 2785 return ~data & mask; 2786 } 2787 2788 static void si_setup_spi(struct radeon_device *rdev, 2789 u32 se_num, u32 sh_per_se, 2790 u32 cu_per_sh) 2791 { 2792 int i, j, k; 2793 u32 data, mask, active_cu; 2794 2795 for (i = 0; i < se_num; i++) { 2796 for (j = 0; j < sh_per_se; j++) { 2797 si_select_se_sh(rdev, i, j); 2798 data = RREG32(SPI_STATIC_THREAD_MGMT_3); 2799 active_cu = si_get_cu_enabled(rdev, cu_per_sh); 2800 2801 mask = 1; 2802 for (k = 0; k < 16; k++) { 2803 mask <<= k; 2804 if (active_cu & mask) { 2805 data &= ~mask; 2806 WREG32(SPI_STATIC_THREAD_MGMT_3, data); 2807 break; 2808 } 2809 } 2810 } 2811 } 2812 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2813 } 2814 2815 static u32 si_get_rb_disabled(struct radeon_device *rdev, 2816 u32 max_rb_num_per_se, 2817 u32 sh_per_se) 2818 { 2819 u32 data, mask; 2820 2821 data = RREG32(CC_RB_BACKEND_DISABLE); 2822 if (data & 1) 2823 data &= BACKEND_DISABLE_MASK; 2824 else 2825 data = 0; 2826 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 2827 2828 data >>= BACKEND_DISABLE_SHIFT; 2829 2830 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se); 2831 2832 return data & mask; 2833 } 2834 2835 static void si_setup_rb(struct radeon_device *rdev, 2836 u32 se_num, u32 sh_per_se, 2837 u32 max_rb_num_per_se) 2838 { 2839 int i, j; 2840 u32 data, mask; 2841 u32 disabled_rbs = 0; 2842 u32 enabled_rbs = 0; 2843 2844 for (i = 0; i < se_num; i++) { 2845 for (j = 0; j < sh_per_se; j++) { 2846 si_select_se_sh(rdev, i, j); 2847 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); 2848 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH); 2849 } 2850 } 2851 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2852 2853 mask = 1; 2854 for (i = 0; i < max_rb_num_per_se * se_num; i++) { 2855 if (!(disabled_rbs & mask)) 2856 enabled_rbs |= mask; 2857 mask <<= 1; 2858 } 2859 2860 rdev->config.si.backend_enable_mask = enabled_rbs; 2861 2862 for (i = 0; i < se_num; i++) { 2863 si_select_se_sh(rdev, i, 0xffffffff); 2864 data = 0; 2865 for (j = 0; j < sh_per_se; j++) { 2866 switch (enabled_rbs & 3) { 2867 case 1: 2868 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 2869 break; 2870 case 2: 2871 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 2872 break; 2873 case 3: 2874 default: 2875 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 2876 break; 2877 } 2878 enabled_rbs >>= 2; 2879 } 2880 WREG32(PA_SC_RASTER_CONFIG, data); 2881 } 2882 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2883 } 2884 2885 static void si_gpu_init(struct radeon_device *rdev) 2886 { 2887 u32 gb_addr_config = 0; 2888 u32 mc_shared_chmap, mc_arb_ramcfg; 2889 u32 sx_debug_1; 2890 u32 hdp_host_path_cntl; 2891 u32 tmp; 2892 int i, j; 2893 2894 switch (rdev->family) { 2895 case CHIP_TAHITI: 2896 rdev->config.si.max_shader_engines = 2; 2897 rdev->config.si.max_tile_pipes = 12; 2898 rdev->config.si.max_cu_per_sh = 8; 2899 rdev->config.si.max_sh_per_se = 2; 2900 rdev->config.si.max_backends_per_se = 4; 2901 rdev->config.si.max_texture_channel_caches = 12; 2902 rdev->config.si.max_gprs = 256; 2903 rdev->config.si.max_gs_threads = 32; 2904 rdev->config.si.max_hw_contexts = 8; 2905 2906 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2907 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 2908 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2909 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2910 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 2911 break; 2912 case CHIP_PITCAIRN: 2913 rdev->config.si.max_shader_engines = 2; 2914 rdev->config.si.max_tile_pipes = 8; 2915 rdev->config.si.max_cu_per_sh = 5; 2916 rdev->config.si.max_sh_per_se = 2; 2917 rdev->config.si.max_backends_per_se = 4; 2918 rdev->config.si.max_texture_channel_caches = 8; 2919 rdev->config.si.max_gprs = 256; 2920 rdev->config.si.max_gs_threads = 32; 2921 rdev->config.si.max_hw_contexts = 8; 2922 2923 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2924 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 2925 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2926 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2927 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 2928 break; 2929 case CHIP_VERDE: 2930 default: 2931 rdev->config.si.max_shader_engines = 1; 2932 rdev->config.si.max_tile_pipes = 4; 2933 rdev->config.si.max_cu_per_sh = 5; 2934 rdev->config.si.max_sh_per_se = 2; 2935 rdev->config.si.max_backends_per_se = 4; 2936 rdev->config.si.max_texture_channel_caches = 4; 2937 rdev->config.si.max_gprs = 256; 2938 rdev->config.si.max_gs_threads = 32; 2939 rdev->config.si.max_hw_contexts = 8; 2940 2941 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2942 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2943 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2944 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2945 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 2946 break; 2947 case CHIP_OLAND: 2948 rdev->config.si.max_shader_engines = 1; 2949 rdev->config.si.max_tile_pipes = 4; 2950 rdev->config.si.max_cu_per_sh = 6; 2951 rdev->config.si.max_sh_per_se = 1; 2952 rdev->config.si.max_backends_per_se = 2; 2953 rdev->config.si.max_texture_channel_caches = 4; 2954 rdev->config.si.max_gprs = 256; 2955 rdev->config.si.max_gs_threads = 16; 2956 rdev->config.si.max_hw_contexts = 8; 2957 2958 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2959 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2960 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2961 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2962 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 2963 break; 2964 case CHIP_HAINAN: 2965 rdev->config.si.max_shader_engines = 1; 2966 rdev->config.si.max_tile_pipes = 4; 2967 rdev->config.si.max_cu_per_sh = 5; 2968 rdev->config.si.max_sh_per_se = 1; 2969 rdev->config.si.max_backends_per_se = 1; 2970 rdev->config.si.max_texture_channel_caches = 2; 2971 rdev->config.si.max_gprs = 256; 2972 rdev->config.si.max_gs_threads = 16; 2973 rdev->config.si.max_hw_contexts = 8; 2974 2975 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2976 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2977 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2978 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2979 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN; 2980 break; 2981 } 2982 2983 /* Initialize HDP */ 2984 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2985 WREG32((0x2c14 + j), 0x00000000); 2986 WREG32((0x2c18 + j), 0x00000000); 2987 WREG32((0x2c1c + j), 0x00000000); 2988 WREG32((0x2c20 + j), 0x00000000); 2989 WREG32((0x2c24 + j), 0x00000000); 2990 } 2991 2992 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 2993 2994 evergreen_fix_pci_max_read_req_size(rdev); 2995 2996 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2997 2998 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 2999 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3000 3001 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes; 3002 rdev->config.si.mem_max_burst_length_bytes = 256; 3003 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 3004 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3005 if (rdev->config.si.mem_row_size_in_kb > 4) 3006 rdev->config.si.mem_row_size_in_kb = 4; 3007 /* XXX use MC settings? */ 3008 rdev->config.si.shader_engine_tile_size = 32; 3009 rdev->config.si.num_gpus = 1; 3010 rdev->config.si.multi_gpu_tile_size = 64; 3011 3012 /* fix up row size */ 3013 gb_addr_config &= ~ROW_SIZE_MASK; 3014 switch (rdev->config.si.mem_row_size_in_kb) { 3015 case 1: 3016 default: 3017 gb_addr_config |= ROW_SIZE(0); 3018 break; 3019 case 2: 3020 gb_addr_config |= ROW_SIZE(1); 3021 break; 3022 case 4: 3023 gb_addr_config |= ROW_SIZE(2); 3024 break; 3025 } 3026 3027 /* setup tiling info dword. gb_addr_config is not adequate since it does 3028 * not have bank info, so create a custom tiling dword. 3029 * bits 3:0 num_pipes 3030 * bits 7:4 num_banks 3031 * bits 11:8 group_size 3032 * bits 15:12 row_size 3033 */ 3034 rdev->config.si.tile_config = 0; 3035 switch (rdev->config.si.num_tile_pipes) { 3036 case 1: 3037 rdev->config.si.tile_config |= (0 << 0); 3038 break; 3039 case 2: 3040 rdev->config.si.tile_config |= (1 << 0); 3041 break; 3042 case 4: 3043 rdev->config.si.tile_config |= (2 << 0); 3044 break; 3045 case 8: 3046 default: 3047 /* XXX what about 12? */ 3048 rdev->config.si.tile_config |= (3 << 0); 3049 break; 3050 } 3051 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3052 case 0: /* four banks */ 3053 rdev->config.si.tile_config |= 0 << 4; 3054 break; 3055 case 1: /* eight banks */ 3056 rdev->config.si.tile_config |= 1 << 4; 3057 break; 3058 case 2: /* sixteen banks */ 3059 default: 3060 rdev->config.si.tile_config |= 2 << 4; 3061 break; 3062 } 3063 rdev->config.si.tile_config |= 3064 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3065 rdev->config.si.tile_config |= 3066 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3067 3068 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3069 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3070 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3071 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3072 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); 3073 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); 3074 if (rdev->has_uvd) { 3075 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3076 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3077 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3078 } 3079 3080 si_tiling_mode_table_init(rdev); 3081 3082 si_setup_rb(rdev, rdev->config.si.max_shader_engines, 3083 rdev->config.si.max_sh_per_se, 3084 rdev->config.si.max_backends_per_se); 3085 3086 si_setup_spi(rdev, rdev->config.si.max_shader_engines, 3087 rdev->config.si.max_sh_per_se, 3088 rdev->config.si.max_cu_per_sh); 3089 3090 3091 /* set HW defaults for 3D engine */ 3092 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3093 ROQ_IB2_START(0x2b))); 3094 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3095 3096 sx_debug_1 = RREG32(SX_DEBUG_1); 3097 WREG32(SX_DEBUG_1, sx_debug_1); 3098 3099 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3100 3101 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) | 3102 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) | 3103 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) | 3104 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size))); 3105 3106 WREG32(VGT_NUM_INSTANCES, 1); 3107 3108 WREG32(CP_PERFMON_CNTL, 0); 3109 3110 WREG32(SQ_CONFIG, 0); 3111 3112 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3113 FORCE_EOV_MAX_REZ_CNT(255))); 3114 3115 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3116 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3117 3118 WREG32(VGT_GS_VERTEX_REUSE, 16); 3119 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3120 3121 WREG32(CB_PERFCOUNTER0_SELECT0, 0); 3122 WREG32(CB_PERFCOUNTER0_SELECT1, 0); 3123 WREG32(CB_PERFCOUNTER1_SELECT0, 0); 3124 WREG32(CB_PERFCOUNTER1_SELECT1, 0); 3125 WREG32(CB_PERFCOUNTER2_SELECT0, 0); 3126 WREG32(CB_PERFCOUNTER2_SELECT1, 0); 3127 WREG32(CB_PERFCOUNTER3_SELECT0, 0); 3128 WREG32(CB_PERFCOUNTER3_SELECT1, 0); 3129 3130 tmp = RREG32(HDP_MISC_CNTL); 3131 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3132 WREG32(HDP_MISC_CNTL, tmp); 3133 3134 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3135 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3136 3137 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3138 3139 udelay(50); 3140 } 3141 3142 /* 3143 * GPU scratch registers helpers function. 3144 */ 3145 static void si_scratch_init(struct radeon_device *rdev) 3146 { 3147 int i; 3148 3149 rdev->scratch.num_reg = 7; 3150 rdev->scratch.reg_base = SCRATCH_REG0; 3151 for (i = 0; i < rdev->scratch.num_reg; i++) { 3152 rdev->scratch.free[i] = true; 3153 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3154 } 3155 } 3156 3157 void si_fence_ring_emit(struct radeon_device *rdev, 3158 struct radeon_fence *fence) 3159 { 3160 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3161 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3162 3163 /* flush read cache over gart */ 3164 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3165 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3166 radeon_ring_write(ring, 0); 3167 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3168 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3169 PACKET3_TC_ACTION_ENA | 3170 PACKET3_SH_KCACHE_ACTION_ENA | 3171 PACKET3_SH_ICACHE_ACTION_ENA); 3172 radeon_ring_write(ring, 0xFFFFFFFF); 3173 radeon_ring_write(ring, 0); 3174 radeon_ring_write(ring, 10); /* poll interval */ 3175 /* EVENT_WRITE_EOP - flush caches, send int */ 3176 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3177 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5)); 3178 radeon_ring_write(ring, addr & 0xffffffff); 3179 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); 3180 radeon_ring_write(ring, fence->seq); 3181 radeon_ring_write(ring, 0); 3182 } 3183 3184 /* 3185 * IB stuff 3186 */ 3187 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3188 { 3189 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3190 u32 header; 3191 3192 if (ib->is_const_ib) { 3193 /* set switch buffer packet before const IB */ 3194 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 3195 radeon_ring_write(ring, 0); 3196 3197 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 3198 } else { 3199 u32 next_rptr; 3200 if (ring->rptr_save_reg) { 3201 next_rptr = ring->wptr + 3 + 4 + 8; 3202 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3203 radeon_ring_write(ring, ((ring->rptr_save_reg - 3204 PACKET3_SET_CONFIG_REG_START) >> 2)); 3205 radeon_ring_write(ring, next_rptr); 3206 } else if (rdev->wb.enabled) { 3207 next_rptr = ring->wptr + 5 + 4 + 8; 3208 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3209 radeon_ring_write(ring, (1 << 8)); 3210 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3211 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); 3212 radeon_ring_write(ring, next_rptr); 3213 } 3214 3215 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 3216 } 3217 3218 radeon_ring_write(ring, header); 3219 radeon_ring_write(ring, 3220 #ifdef __BIG_ENDIAN 3221 (2 << 0) | 3222 #endif 3223 (ib->gpu_addr & 0xFFFFFFFC)); 3224 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 3225 radeon_ring_write(ring, ib->length_dw | 3226 (ib->vm ? (ib->vm->id << 24) : 0)); 3227 3228 if (!ib->is_const_ib) { 3229 /* flush read cache over gart for this vmid */ 3230 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3231 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3232 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); 3233 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3234 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3235 PACKET3_TC_ACTION_ENA | 3236 PACKET3_SH_KCACHE_ACTION_ENA | 3237 PACKET3_SH_ICACHE_ACTION_ENA); 3238 radeon_ring_write(ring, 0xFFFFFFFF); 3239 radeon_ring_write(ring, 0); 3240 radeon_ring_write(ring, 10); /* poll interval */ 3241 } 3242 } 3243 3244 /* 3245 * CP. 3246 */ 3247 static void si_cp_enable(struct radeon_device *rdev, bool enable) 3248 { 3249 if (enable) 3250 WREG32(CP_ME_CNTL, 0); 3251 else { 3252 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 3253 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 3254 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 3255 WREG32(SCRATCH_UMSK, 0); 3256 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3257 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3258 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3259 } 3260 udelay(50); 3261 } 3262 3263 static int si_cp_load_microcode(struct radeon_device *rdev) 3264 { 3265 const __be32 *fw_data; 3266 int i; 3267 3268 if (!rdev->me_fw || !rdev->pfp_fw) 3269 return -EINVAL; 3270 3271 si_cp_enable(rdev, false); 3272 3273 /* PFP */ 3274 fw_data = (const __be32 *)rdev->pfp_fw->data; 3275 WREG32(CP_PFP_UCODE_ADDR, 0); 3276 for (i = 0; i < SI_PFP_UCODE_SIZE; i++) 3277 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3278 WREG32(CP_PFP_UCODE_ADDR, 0); 3279 3280 /* CE */ 3281 fw_data = (const __be32 *)rdev->ce_fw->data; 3282 WREG32(CP_CE_UCODE_ADDR, 0); 3283 for (i = 0; i < SI_CE_UCODE_SIZE; i++) 3284 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 3285 WREG32(CP_CE_UCODE_ADDR, 0); 3286 3287 /* ME */ 3288 fw_data = (const __be32 *)rdev->me_fw->data; 3289 WREG32(CP_ME_RAM_WADDR, 0); 3290 for (i = 0; i < SI_PM4_UCODE_SIZE; i++) 3291 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3292 WREG32(CP_ME_RAM_WADDR, 0); 3293 3294 WREG32(CP_PFP_UCODE_ADDR, 0); 3295 WREG32(CP_CE_UCODE_ADDR, 0); 3296 WREG32(CP_ME_RAM_WADDR, 0); 3297 WREG32(CP_ME_RAM_RADDR, 0); 3298 return 0; 3299 } 3300 3301 static int si_cp_start(struct radeon_device *rdev) 3302 { 3303 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3304 int r, i; 3305 3306 r = radeon_ring_lock(rdev, ring, 7 + 4); 3307 if (r) { 3308 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3309 return r; 3310 } 3311 /* init the CP */ 3312 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3313 radeon_ring_write(ring, 0x1); 3314 radeon_ring_write(ring, 0x0); 3315 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1); 3316 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3317 radeon_ring_write(ring, 0); 3318 radeon_ring_write(ring, 0); 3319 3320 /* init the CE partitions */ 3321 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3322 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3323 radeon_ring_write(ring, 0xc000); 3324 radeon_ring_write(ring, 0xe000); 3325 radeon_ring_unlock_commit(rdev, ring); 3326 3327 si_cp_enable(rdev, true); 3328 3329 r = radeon_ring_lock(rdev, ring, si_default_size + 10); 3330 if (r) { 3331 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3332 return r; 3333 } 3334 3335 /* setup clear context state */ 3336 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3337 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3338 3339 for (i = 0; i < si_default_size; i++) 3340 radeon_ring_write(ring, si_default_state[i]); 3341 3342 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3343 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3344 3345 /* set clear context state */ 3346 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3347 radeon_ring_write(ring, 0); 3348 3349 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 3350 radeon_ring_write(ring, 0x00000316); 3351 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3352 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 3353 3354 radeon_ring_unlock_commit(rdev, ring); 3355 3356 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) { 3357 ring = &rdev->ring[i]; 3358 r = radeon_ring_lock(rdev, ring, 2); 3359 3360 /* clear the compute context state */ 3361 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0)); 3362 radeon_ring_write(ring, 0); 3363 3364 radeon_ring_unlock_commit(rdev, ring); 3365 } 3366 3367 return 0; 3368 } 3369 3370 static void si_cp_fini(struct radeon_device *rdev) 3371 { 3372 struct radeon_ring *ring; 3373 si_cp_enable(rdev, false); 3374 3375 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3376 radeon_ring_fini(rdev, ring); 3377 radeon_scratch_free(rdev, ring->rptr_save_reg); 3378 3379 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3380 radeon_ring_fini(rdev, ring); 3381 radeon_scratch_free(rdev, ring->rptr_save_reg); 3382 3383 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3384 radeon_ring_fini(rdev, ring); 3385 radeon_scratch_free(rdev, ring->rptr_save_reg); 3386 } 3387 3388 static int si_cp_resume(struct radeon_device *rdev) 3389 { 3390 struct radeon_ring *ring; 3391 u32 tmp; 3392 u32 rb_bufsz; 3393 int r; 3394 3395 si_enable_gui_idle_interrupt(rdev, false); 3396 3397 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3398 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3399 3400 /* Set the write pointer delay */ 3401 WREG32(CP_RB_WPTR_DELAY, 0); 3402 3403 WREG32(CP_DEBUG, 0); 3404 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3405 3406 /* ring 0 - compute and gfx */ 3407 /* Set ring buffer size */ 3408 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3409 rb_bufsz = order_base_2(ring->ring_size / 8); 3410 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3411 #ifdef __BIG_ENDIAN 3412 tmp |= BUF_SWAP_32BIT; 3413 #endif 3414 WREG32(CP_RB0_CNTL, tmp); 3415 3416 /* Initialize the ring buffer's read and write pointers */ 3417 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 3418 ring->wptr = 0; 3419 WREG32(CP_RB0_WPTR, ring->wptr); 3420 3421 /* set the wb address whether it's enabled or not */ 3422 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 3423 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3424 3425 if (rdev->wb.enabled) 3426 WREG32(SCRATCH_UMSK, 0xff); 3427 else { 3428 tmp |= RB_NO_UPDATE; 3429 WREG32(SCRATCH_UMSK, 0); 3430 } 3431 3432 mdelay(1); 3433 WREG32(CP_RB0_CNTL, tmp); 3434 3435 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8); 3436 3437 /* ring1 - compute only */ 3438 /* Set ring buffer size */ 3439 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3440 rb_bufsz = order_base_2(ring->ring_size / 8); 3441 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3442 #ifdef __BIG_ENDIAN 3443 tmp |= BUF_SWAP_32BIT; 3444 #endif 3445 WREG32(CP_RB1_CNTL, tmp); 3446 3447 /* Initialize the ring buffer's read and write pointers */ 3448 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA); 3449 ring->wptr = 0; 3450 WREG32(CP_RB1_WPTR, ring->wptr); 3451 3452 /* set the wb address whether it's enabled or not */ 3453 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC); 3454 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF); 3455 3456 mdelay(1); 3457 WREG32(CP_RB1_CNTL, tmp); 3458 3459 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8); 3460 3461 /* ring2 - compute only */ 3462 /* Set ring buffer size */ 3463 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3464 rb_bufsz = order_base_2(ring->ring_size / 8); 3465 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3466 #ifdef __BIG_ENDIAN 3467 tmp |= BUF_SWAP_32BIT; 3468 #endif 3469 WREG32(CP_RB2_CNTL, tmp); 3470 3471 /* Initialize the ring buffer's read and write pointers */ 3472 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA); 3473 ring->wptr = 0; 3474 WREG32(CP_RB2_WPTR, ring->wptr); 3475 3476 /* set the wb address whether it's enabled or not */ 3477 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC); 3478 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF); 3479 3480 mdelay(1); 3481 WREG32(CP_RB2_CNTL, tmp); 3482 3483 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8); 3484 3485 /* start the rings */ 3486 si_cp_start(rdev); 3487 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 3488 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true; 3489 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true; 3490 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 3491 if (r) { 3492 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3493 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3494 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3495 return r; 3496 } 3497 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); 3498 if (r) { 3499 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3500 } 3501 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); 3502 if (r) { 3503 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3504 } 3505 3506 si_enable_gui_idle_interrupt(rdev, true); 3507 3508 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) 3509 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 3510 3511 return 0; 3512 } 3513 3514 u32 si_gpu_check_soft_reset(struct radeon_device *rdev) 3515 { 3516 u32 reset_mask = 0; 3517 u32 tmp; 3518 3519 /* GRBM_STATUS */ 3520 tmp = RREG32(GRBM_STATUS); 3521 if (tmp & (PA_BUSY | SC_BUSY | 3522 BCI_BUSY | SX_BUSY | 3523 TA_BUSY | VGT_BUSY | 3524 DB_BUSY | CB_BUSY | 3525 GDS_BUSY | SPI_BUSY | 3526 IA_BUSY | IA_BUSY_NO_DMA)) 3527 reset_mask |= RADEON_RESET_GFX; 3528 3529 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3530 CP_BUSY | CP_COHERENCY_BUSY)) 3531 reset_mask |= RADEON_RESET_CP; 3532 3533 if (tmp & GRBM_EE_BUSY) 3534 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3535 3536 /* GRBM_STATUS2 */ 3537 tmp = RREG32(GRBM_STATUS2); 3538 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3539 reset_mask |= RADEON_RESET_RLC; 3540 3541 /* DMA_STATUS_REG 0 */ 3542 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); 3543 if (!(tmp & DMA_IDLE)) 3544 reset_mask |= RADEON_RESET_DMA; 3545 3546 /* DMA_STATUS_REG 1 */ 3547 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); 3548 if (!(tmp & DMA_IDLE)) 3549 reset_mask |= RADEON_RESET_DMA1; 3550 3551 /* SRBM_STATUS2 */ 3552 tmp = RREG32(SRBM_STATUS2); 3553 if (tmp & DMA_BUSY) 3554 reset_mask |= RADEON_RESET_DMA; 3555 3556 if (tmp & DMA1_BUSY) 3557 reset_mask |= RADEON_RESET_DMA1; 3558 3559 /* SRBM_STATUS */ 3560 tmp = RREG32(SRBM_STATUS); 3561 3562 if (tmp & IH_BUSY) 3563 reset_mask |= RADEON_RESET_IH; 3564 3565 if (tmp & SEM_BUSY) 3566 reset_mask |= RADEON_RESET_SEM; 3567 3568 if (tmp & GRBM_RQ_PENDING) 3569 reset_mask |= RADEON_RESET_GRBM; 3570 3571 if (tmp & VMC_BUSY) 3572 reset_mask |= RADEON_RESET_VMC; 3573 3574 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3575 MCC_BUSY | MCD_BUSY)) 3576 reset_mask |= RADEON_RESET_MC; 3577 3578 if (evergreen_is_display_hung(rdev)) 3579 reset_mask |= RADEON_RESET_DISPLAY; 3580 3581 /* VM_L2_STATUS */ 3582 tmp = RREG32(VM_L2_STATUS); 3583 if (tmp & L2_BUSY) 3584 reset_mask |= RADEON_RESET_VMC; 3585 3586 /* Skip MC reset as it's mostly likely not hung, just busy */ 3587 if (reset_mask & RADEON_RESET_MC) { 3588 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3589 reset_mask &= ~RADEON_RESET_MC; 3590 } 3591 3592 return reset_mask; 3593 } 3594 3595 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3596 { 3597 struct evergreen_mc_save save; 3598 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3599 u32 tmp; 3600 3601 if (reset_mask == 0) 3602 return; 3603 3604 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3605 3606 evergreen_print_gpu_status_regs(rdev); 3607 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 3608 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 3609 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 3610 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 3611 3612 /* disable PG/CG */ 3613 si_fini_pg(rdev); 3614 si_fini_cg(rdev); 3615 3616 /* stop the rlc */ 3617 si_rlc_stop(rdev); 3618 3619 /* Disable CP parsing/prefetching */ 3620 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 3621 3622 if (reset_mask & RADEON_RESET_DMA) { 3623 /* dma0 */ 3624 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 3625 tmp &= ~DMA_RB_ENABLE; 3626 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 3627 } 3628 if (reset_mask & RADEON_RESET_DMA1) { 3629 /* dma1 */ 3630 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 3631 tmp &= ~DMA_RB_ENABLE; 3632 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 3633 } 3634 3635 udelay(50); 3636 3637 evergreen_mc_stop(rdev, &save); 3638 if (evergreen_mc_wait_for_idle(rdev)) { 3639 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3640 } 3641 3642 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) { 3643 grbm_soft_reset = SOFT_RESET_CB | 3644 SOFT_RESET_DB | 3645 SOFT_RESET_GDS | 3646 SOFT_RESET_PA | 3647 SOFT_RESET_SC | 3648 SOFT_RESET_BCI | 3649 SOFT_RESET_SPI | 3650 SOFT_RESET_SX | 3651 SOFT_RESET_TC | 3652 SOFT_RESET_TA | 3653 SOFT_RESET_VGT | 3654 SOFT_RESET_IA; 3655 } 3656 3657 if (reset_mask & RADEON_RESET_CP) { 3658 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT; 3659 3660 srbm_soft_reset |= SOFT_RESET_GRBM; 3661 } 3662 3663 if (reset_mask & RADEON_RESET_DMA) 3664 srbm_soft_reset |= SOFT_RESET_DMA; 3665 3666 if (reset_mask & RADEON_RESET_DMA1) 3667 srbm_soft_reset |= SOFT_RESET_DMA1; 3668 3669 if (reset_mask & RADEON_RESET_DISPLAY) 3670 srbm_soft_reset |= SOFT_RESET_DC; 3671 3672 if (reset_mask & RADEON_RESET_RLC) 3673 grbm_soft_reset |= SOFT_RESET_RLC; 3674 3675 if (reset_mask & RADEON_RESET_SEM) 3676 srbm_soft_reset |= SOFT_RESET_SEM; 3677 3678 if (reset_mask & RADEON_RESET_IH) 3679 srbm_soft_reset |= SOFT_RESET_IH; 3680 3681 if (reset_mask & RADEON_RESET_GRBM) 3682 srbm_soft_reset |= SOFT_RESET_GRBM; 3683 3684 if (reset_mask & RADEON_RESET_VMC) 3685 srbm_soft_reset |= SOFT_RESET_VMC; 3686 3687 if (reset_mask & RADEON_RESET_MC) 3688 srbm_soft_reset |= SOFT_RESET_MC; 3689 3690 if (grbm_soft_reset) { 3691 tmp = RREG32(GRBM_SOFT_RESET); 3692 tmp |= grbm_soft_reset; 3693 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3694 WREG32(GRBM_SOFT_RESET, tmp); 3695 tmp = RREG32(GRBM_SOFT_RESET); 3696 3697 udelay(50); 3698 3699 tmp &= ~grbm_soft_reset; 3700 WREG32(GRBM_SOFT_RESET, tmp); 3701 tmp = RREG32(GRBM_SOFT_RESET); 3702 } 3703 3704 if (srbm_soft_reset) { 3705 tmp = RREG32(SRBM_SOFT_RESET); 3706 tmp |= srbm_soft_reset; 3707 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3708 WREG32(SRBM_SOFT_RESET, tmp); 3709 tmp = RREG32(SRBM_SOFT_RESET); 3710 3711 udelay(50); 3712 3713 tmp &= ~srbm_soft_reset; 3714 WREG32(SRBM_SOFT_RESET, tmp); 3715 tmp = RREG32(SRBM_SOFT_RESET); 3716 } 3717 3718 /* Wait a little for things to settle down */ 3719 udelay(50); 3720 3721 evergreen_mc_resume(rdev, &save); 3722 udelay(50); 3723 3724 evergreen_print_gpu_status_regs(rdev); 3725 } 3726 3727 static void si_set_clk_bypass_mode(struct radeon_device *rdev) 3728 { 3729 u32 tmp, i; 3730 3731 tmp = RREG32(CG_SPLL_FUNC_CNTL); 3732 tmp |= SPLL_BYPASS_EN; 3733 WREG32(CG_SPLL_FUNC_CNTL, tmp); 3734 3735 tmp = RREG32(CG_SPLL_FUNC_CNTL_2); 3736 tmp |= SPLL_CTLREQ_CHG; 3737 WREG32(CG_SPLL_FUNC_CNTL_2, tmp); 3738 3739 for (i = 0; i < rdev->usec_timeout; i++) { 3740 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS) 3741 break; 3742 udelay(1); 3743 } 3744 3745 tmp = RREG32(CG_SPLL_FUNC_CNTL_2); 3746 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE); 3747 WREG32(CG_SPLL_FUNC_CNTL_2, tmp); 3748 3749 tmp = RREG32(MPLL_CNTL_MODE); 3750 tmp &= ~MPLL_MCLK_SEL; 3751 WREG32(MPLL_CNTL_MODE, tmp); 3752 } 3753 3754 static void si_spll_powerdown(struct radeon_device *rdev) 3755 { 3756 u32 tmp; 3757 3758 tmp = RREG32(SPLL_CNTL_MODE); 3759 tmp |= SPLL_SW_DIR_CONTROL; 3760 WREG32(SPLL_CNTL_MODE, tmp); 3761 3762 tmp = RREG32(CG_SPLL_FUNC_CNTL); 3763 tmp |= SPLL_RESET; 3764 WREG32(CG_SPLL_FUNC_CNTL, tmp); 3765 3766 tmp = RREG32(CG_SPLL_FUNC_CNTL); 3767 tmp |= SPLL_SLEEP; 3768 WREG32(CG_SPLL_FUNC_CNTL, tmp); 3769 3770 tmp = RREG32(SPLL_CNTL_MODE); 3771 tmp &= ~SPLL_SW_DIR_CONTROL; 3772 WREG32(SPLL_CNTL_MODE, tmp); 3773 } 3774 3775 static void si_gpu_pci_config_reset(struct radeon_device *rdev) 3776 { 3777 struct evergreen_mc_save save; 3778 u32 tmp, i; 3779 3780 dev_info(rdev->dev, "GPU pci config reset\n"); 3781 3782 /* disable dpm? */ 3783 3784 /* disable cg/pg */ 3785 si_fini_pg(rdev); 3786 si_fini_cg(rdev); 3787 3788 /* Disable CP parsing/prefetching */ 3789 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 3790 /* dma0 */ 3791 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 3792 tmp &= ~DMA_RB_ENABLE; 3793 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 3794 /* dma1 */ 3795 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 3796 tmp &= ~DMA_RB_ENABLE; 3797 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 3798 /* XXX other engines? */ 3799 3800 /* halt the rlc, disable cp internal ints */ 3801 si_rlc_stop(rdev); 3802 3803 udelay(50); 3804 3805 /* disable mem access */ 3806 evergreen_mc_stop(rdev, &save); 3807 if (evergreen_mc_wait_for_idle(rdev)) { 3808 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 3809 } 3810 3811 /* set mclk/sclk to bypass */ 3812 si_set_clk_bypass_mode(rdev); 3813 /* powerdown spll */ 3814 si_spll_powerdown(rdev); 3815 /* disable BM */ 3816 pci_clear_master(rdev->pdev); 3817 /* reset */ 3818 radeon_pci_config_reset(rdev); 3819 /* wait for asic to come out of reset */ 3820 for (i = 0; i < rdev->usec_timeout; i++) { 3821 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 3822 break; 3823 udelay(1); 3824 } 3825 } 3826 3827 int si_asic_reset(struct radeon_device *rdev) 3828 { 3829 u32 reset_mask; 3830 3831 reset_mask = si_gpu_check_soft_reset(rdev); 3832 3833 if (reset_mask) 3834 r600_set_bios_scratch_engine_hung(rdev, true); 3835 3836 /* try soft reset */ 3837 si_gpu_soft_reset(rdev, reset_mask); 3838 3839 reset_mask = si_gpu_check_soft_reset(rdev); 3840 3841 /* try pci config reset */ 3842 if (reset_mask && radeon_hard_reset) 3843 si_gpu_pci_config_reset(rdev); 3844 3845 reset_mask = si_gpu_check_soft_reset(rdev); 3846 3847 if (!reset_mask) 3848 r600_set_bios_scratch_engine_hung(rdev, false); 3849 3850 return 0; 3851 } 3852 3853 /** 3854 * si_gfx_is_lockup - Check if the GFX engine is locked up 3855 * 3856 * @rdev: radeon_device pointer 3857 * @ring: radeon_ring structure holding ring information 3858 * 3859 * Check if the GFX engine is locked up. 3860 * Returns true if the engine appears to be locked up, false if not. 3861 */ 3862 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 3863 { 3864 u32 reset_mask = si_gpu_check_soft_reset(rdev); 3865 3866 if (!(reset_mask & (RADEON_RESET_GFX | 3867 RADEON_RESET_COMPUTE | 3868 RADEON_RESET_CP))) { 3869 radeon_ring_lockup_update(rdev, ring); 3870 return false; 3871 } 3872 return radeon_ring_test_lockup(rdev, ring); 3873 } 3874 3875 /* MC */ 3876 static void si_mc_program(struct radeon_device *rdev) 3877 { 3878 struct evergreen_mc_save save; 3879 u32 tmp; 3880 int i, j; 3881 3882 /* Initialize HDP */ 3883 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3884 WREG32((0x2c14 + j), 0x00000000); 3885 WREG32((0x2c18 + j), 0x00000000); 3886 WREG32((0x2c1c + j), 0x00000000); 3887 WREG32((0x2c20 + j), 0x00000000); 3888 WREG32((0x2c24 + j), 0x00000000); 3889 } 3890 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 3891 3892 evergreen_mc_stop(rdev, &save); 3893 if (radeon_mc_wait_for_idle(rdev)) { 3894 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3895 } 3896 if (!ASIC_IS_NODCE(rdev)) 3897 /* Lockout access through VGA aperture*/ 3898 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 3899 /* Update configuration */ 3900 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 3901 rdev->mc.vram_start >> 12); 3902 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 3903 rdev->mc.vram_end >> 12); 3904 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 3905 rdev->vram_scratch.gpu_addr >> 12); 3906 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 3907 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 3908 WREG32(MC_VM_FB_LOCATION, tmp); 3909 /* XXX double check these! */ 3910 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 3911 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 3912 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 3913 WREG32(MC_VM_AGP_BASE, 0); 3914 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 3915 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 3916 if (radeon_mc_wait_for_idle(rdev)) { 3917 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3918 } 3919 evergreen_mc_resume(rdev, &save); 3920 if (!ASIC_IS_NODCE(rdev)) { 3921 /* we need to own VRAM, so turn off the VGA renderer here 3922 * to stop it overwriting our objects */ 3923 rv515_vga_render_disable(rdev); 3924 } 3925 } 3926 3927 void si_vram_gtt_location(struct radeon_device *rdev, 3928 struct radeon_mc *mc) 3929 { 3930 if (mc->mc_vram_size > 0xFFC0000000ULL) { 3931 /* leave room for at least 1024M GTT */ 3932 dev_warn(rdev->dev, "limiting VRAM\n"); 3933 mc->real_vram_size = 0xFFC0000000ULL; 3934 mc->mc_vram_size = 0xFFC0000000ULL; 3935 } 3936 radeon_vram_location(rdev, &rdev->mc, 0); 3937 rdev->mc.gtt_base_align = 0; 3938 radeon_gtt_location(rdev, mc); 3939 } 3940 3941 static int si_mc_init(struct radeon_device *rdev) 3942 { 3943 u32 tmp; 3944 int chansize, numchan; 3945 3946 /* Get VRAM informations */ 3947 rdev->mc.vram_is_ddr = true; 3948 tmp = RREG32(MC_ARB_RAMCFG); 3949 if (tmp & CHANSIZE_OVERRIDE) { 3950 chansize = 16; 3951 } else if (tmp & CHANSIZE_MASK) { 3952 chansize = 64; 3953 } else { 3954 chansize = 32; 3955 } 3956 tmp = RREG32(MC_SHARED_CHMAP); 3957 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3958 case 0: 3959 default: 3960 numchan = 1; 3961 break; 3962 case 1: 3963 numchan = 2; 3964 break; 3965 case 2: 3966 numchan = 4; 3967 break; 3968 case 3: 3969 numchan = 8; 3970 break; 3971 case 4: 3972 numchan = 3; 3973 break; 3974 case 5: 3975 numchan = 6; 3976 break; 3977 case 6: 3978 numchan = 10; 3979 break; 3980 case 7: 3981 numchan = 12; 3982 break; 3983 case 8: 3984 numchan = 16; 3985 break; 3986 } 3987 rdev->mc.vram_width = numchan * chansize; 3988 /* Could aper size report 0 ? */ 3989 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 3990 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 3991 /* size in MB on si */ 3992 tmp = RREG32(CONFIG_MEMSIZE); 3993 /* some boards may have garbage in the upper 16 bits */ 3994 if (tmp & 0xffff0000) { 3995 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp); 3996 if (tmp & 0xffff) 3997 tmp &= 0xffff; 3998 } 3999 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL; 4000 rdev->mc.real_vram_size = rdev->mc.mc_vram_size; 4001 rdev->mc.visible_vram_size = rdev->mc.aper_size; 4002 si_vram_gtt_location(rdev, &rdev->mc); 4003 radeon_update_bandwidth_info(rdev); 4004 4005 return 0; 4006 } 4007 4008 /* 4009 * GART 4010 */ 4011 void si_pcie_gart_tlb_flush(struct radeon_device *rdev) 4012 { 4013 /* flush hdp cache */ 4014 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 4015 4016 /* bits 0-15 are the VM contexts0-15 */ 4017 WREG32(VM_INVALIDATE_REQUEST, 1); 4018 } 4019 4020 static int si_pcie_gart_enable(struct radeon_device *rdev) 4021 { 4022 int r, i; 4023 4024 if (rdev->gart.robj == NULL) { 4025 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 4026 return -EINVAL; 4027 } 4028 r = radeon_gart_table_vram_pin(rdev); 4029 if (r) 4030 return r; 4031 radeon_gart_restore(rdev); 4032 /* Setup TLB control */ 4033 WREG32(MC_VM_MX_L1_TLB_CNTL, 4034 (0xA << 7) | 4035 ENABLE_L1_TLB | 4036 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 4037 ENABLE_ADVANCED_DRIVER_MODEL | 4038 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 4039 /* Setup L2 cache */ 4040 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 4041 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 4042 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 4043 EFFECTIVE_L2_QUEUE_SIZE(7) | 4044 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 4045 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 4046 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 4047 L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 4048 /* setup context0 */ 4049 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 4050 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 4051 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 4052 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 4053 (u32)(rdev->dummy_page.addr >> 12)); 4054 WREG32(VM_CONTEXT0_CNTL2, 0); 4055 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 4056 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 4057 4058 WREG32(0x15D4, 0); 4059 WREG32(0x15D8, 0); 4060 WREG32(0x15DC, 0); 4061 4062 /* empty context1-15 */ 4063 /* set vm size, must be a multiple of 4 */ 4064 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 4065 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 4066 /* Assign the pt base to something valid for now; the pts used for 4067 * the VMs are determined by the application and setup and assigned 4068 * on the fly in the vm part of radeon_gart.c 4069 */ 4070 for (i = 1; i < 16; i++) { 4071 if (i < 8) 4072 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 4073 rdev->gart.table_addr >> 12); 4074 else 4075 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 4076 rdev->gart.table_addr >> 12); 4077 } 4078 4079 /* enable context1-15 */ 4080 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 4081 (u32)(rdev->dummy_page.addr >> 12)); 4082 WREG32(VM_CONTEXT1_CNTL2, 4); 4083 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 4084 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4085 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 4086 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4087 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 4088 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 4089 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 4090 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 4091 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 4092 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 4093 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 4094 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 4095 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 4096 4097 si_pcie_gart_tlb_flush(rdev); 4098 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 4099 (unsigned)(rdev->mc.gtt_size >> 20), 4100 (unsigned long long)rdev->gart.table_addr); 4101 rdev->gart.ready = true; 4102 return 0; 4103 } 4104 4105 static void si_pcie_gart_disable(struct radeon_device *rdev) 4106 { 4107 /* Disable all tables */ 4108 WREG32(VM_CONTEXT0_CNTL, 0); 4109 WREG32(VM_CONTEXT1_CNTL, 0); 4110 /* Setup TLB control */ 4111 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 4112 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 4113 /* Setup L2 cache */ 4114 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 4115 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 4116 EFFECTIVE_L2_QUEUE_SIZE(7) | 4117 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 4118 WREG32(VM_L2_CNTL2, 0); 4119 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 4120 L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 4121 radeon_gart_table_vram_unpin(rdev); 4122 } 4123 4124 static void si_pcie_gart_fini(struct radeon_device *rdev) 4125 { 4126 si_pcie_gart_disable(rdev); 4127 radeon_gart_table_vram_free(rdev); 4128 radeon_gart_fini(rdev); 4129 } 4130 4131 /* vm parser */ 4132 static bool si_vm_reg_valid(u32 reg) 4133 { 4134 /* context regs are fine */ 4135 if (reg >= 0x28000) 4136 return true; 4137 4138 /* check config regs */ 4139 switch (reg) { 4140 case GRBM_GFX_INDEX: 4141 case CP_STRMOUT_CNTL: 4142 case VGT_VTX_VECT_EJECT_REG: 4143 case VGT_CACHE_INVALIDATION: 4144 case VGT_ESGS_RING_SIZE: 4145 case VGT_GSVS_RING_SIZE: 4146 case VGT_GS_VERTEX_REUSE: 4147 case VGT_PRIMITIVE_TYPE: 4148 case VGT_INDEX_TYPE: 4149 case VGT_NUM_INDICES: 4150 case VGT_NUM_INSTANCES: 4151 case VGT_TF_RING_SIZE: 4152 case VGT_HS_OFFCHIP_PARAM: 4153 case VGT_TF_MEMORY_BASE: 4154 case PA_CL_ENHANCE: 4155 case PA_SU_LINE_STIPPLE_VALUE: 4156 case PA_SC_LINE_STIPPLE_STATE: 4157 case PA_SC_ENHANCE: 4158 case SQC_CACHES: 4159 case SPI_STATIC_THREAD_MGMT_1: 4160 case SPI_STATIC_THREAD_MGMT_2: 4161 case SPI_STATIC_THREAD_MGMT_3: 4162 case SPI_PS_MAX_WAVE_ID: 4163 case SPI_CONFIG_CNTL: 4164 case SPI_CONFIG_CNTL_1: 4165 case TA_CNTL_AUX: 4166 return true; 4167 default: 4168 DRM_ERROR("Invalid register 0x%x in CS\n", reg); 4169 return false; 4170 } 4171 } 4172 4173 static int si_vm_packet3_ce_check(struct radeon_device *rdev, 4174 u32 *ib, struct radeon_cs_packet *pkt) 4175 { 4176 switch (pkt->opcode) { 4177 case PACKET3_NOP: 4178 case PACKET3_SET_BASE: 4179 case PACKET3_SET_CE_DE_COUNTERS: 4180 case PACKET3_LOAD_CONST_RAM: 4181 case PACKET3_WRITE_CONST_RAM: 4182 case PACKET3_WRITE_CONST_RAM_OFFSET: 4183 case PACKET3_DUMP_CONST_RAM: 4184 case PACKET3_INCREMENT_CE_COUNTER: 4185 case PACKET3_WAIT_ON_DE_COUNTER: 4186 case PACKET3_CE_WRITE: 4187 break; 4188 default: 4189 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode); 4190 return -EINVAL; 4191 } 4192 return 0; 4193 } 4194 4195 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx) 4196 { 4197 u32 start_reg, reg, i; 4198 u32 command = ib[idx + 4]; 4199 u32 info = ib[idx + 1]; 4200 u32 idx_value = ib[idx]; 4201 if (command & PACKET3_CP_DMA_CMD_SAS) { 4202 /* src address space is register */ 4203 if (((info & 0x60000000) >> 29) == 0) { 4204 start_reg = idx_value << 2; 4205 if (command & PACKET3_CP_DMA_CMD_SAIC) { 4206 reg = start_reg; 4207 if (!si_vm_reg_valid(reg)) { 4208 DRM_ERROR("CP DMA Bad SRC register\n"); 4209 return -EINVAL; 4210 } 4211 } else { 4212 for (i = 0; i < (command & 0x1fffff); i++) { 4213 reg = start_reg + (4 * i); 4214 if (!si_vm_reg_valid(reg)) { 4215 DRM_ERROR("CP DMA Bad SRC register\n"); 4216 return -EINVAL; 4217 } 4218 } 4219 } 4220 } 4221 } 4222 if (command & PACKET3_CP_DMA_CMD_DAS) { 4223 /* dst address space is register */ 4224 if (((info & 0x00300000) >> 20) == 0) { 4225 start_reg = ib[idx + 2]; 4226 if (command & PACKET3_CP_DMA_CMD_DAIC) { 4227 reg = start_reg; 4228 if (!si_vm_reg_valid(reg)) { 4229 DRM_ERROR("CP DMA Bad DST register\n"); 4230 return -EINVAL; 4231 } 4232 } else { 4233 for (i = 0; i < (command & 0x1fffff); i++) { 4234 reg = start_reg + (4 * i); 4235 if (!si_vm_reg_valid(reg)) { 4236 DRM_ERROR("CP DMA Bad DST register\n"); 4237 return -EINVAL; 4238 } 4239 } 4240 } 4241 } 4242 } 4243 return 0; 4244 } 4245 4246 static int si_vm_packet3_gfx_check(struct radeon_device *rdev, 4247 u32 *ib, struct radeon_cs_packet *pkt) 4248 { 4249 int r; 4250 u32 idx = pkt->idx + 1; 4251 u32 idx_value = ib[idx]; 4252 u32 start_reg, end_reg, reg, i; 4253 4254 switch (pkt->opcode) { 4255 case PACKET3_NOP: 4256 case PACKET3_SET_BASE: 4257 case PACKET3_CLEAR_STATE: 4258 case PACKET3_INDEX_BUFFER_SIZE: 4259 case PACKET3_DISPATCH_DIRECT: 4260 case PACKET3_DISPATCH_INDIRECT: 4261 case PACKET3_ALLOC_GDS: 4262 case PACKET3_WRITE_GDS_RAM: 4263 case PACKET3_ATOMIC_GDS: 4264 case PACKET3_ATOMIC: 4265 case PACKET3_OCCLUSION_QUERY: 4266 case PACKET3_SET_PREDICATION: 4267 case PACKET3_COND_EXEC: 4268 case PACKET3_PRED_EXEC: 4269 case PACKET3_DRAW_INDIRECT: 4270 case PACKET3_DRAW_INDEX_INDIRECT: 4271 case PACKET3_INDEX_BASE: 4272 case PACKET3_DRAW_INDEX_2: 4273 case PACKET3_CONTEXT_CONTROL: 4274 case PACKET3_INDEX_TYPE: 4275 case PACKET3_DRAW_INDIRECT_MULTI: 4276 case PACKET3_DRAW_INDEX_AUTO: 4277 case PACKET3_DRAW_INDEX_IMMD: 4278 case PACKET3_NUM_INSTANCES: 4279 case PACKET3_DRAW_INDEX_MULTI_AUTO: 4280 case PACKET3_STRMOUT_BUFFER_UPDATE: 4281 case PACKET3_DRAW_INDEX_OFFSET_2: 4282 case PACKET3_DRAW_INDEX_MULTI_ELEMENT: 4283 case PACKET3_DRAW_INDEX_INDIRECT_MULTI: 4284 case PACKET3_MPEG_INDEX: 4285 case PACKET3_WAIT_REG_MEM: 4286 case PACKET3_MEM_WRITE: 4287 case PACKET3_PFP_SYNC_ME: 4288 case PACKET3_SURFACE_SYNC: 4289 case PACKET3_EVENT_WRITE: 4290 case PACKET3_EVENT_WRITE_EOP: 4291 case PACKET3_EVENT_WRITE_EOS: 4292 case PACKET3_SET_CONTEXT_REG: 4293 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4294 case PACKET3_SET_SH_REG: 4295 case PACKET3_SET_SH_REG_OFFSET: 4296 case PACKET3_INCREMENT_DE_COUNTER: 4297 case PACKET3_WAIT_ON_CE_COUNTER: 4298 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4299 case PACKET3_ME_WRITE: 4300 break; 4301 case PACKET3_COPY_DATA: 4302 if ((idx_value & 0xf00) == 0) { 4303 reg = ib[idx + 3] * 4; 4304 if (!si_vm_reg_valid(reg)) 4305 return -EINVAL; 4306 } 4307 break; 4308 case PACKET3_WRITE_DATA: 4309 if ((idx_value & 0xf00) == 0) { 4310 start_reg = ib[idx + 1] * 4; 4311 if (idx_value & 0x10000) { 4312 if (!si_vm_reg_valid(start_reg)) 4313 return -EINVAL; 4314 } else { 4315 for (i = 0; i < (pkt->count - 2); i++) { 4316 reg = start_reg + (4 * i); 4317 if (!si_vm_reg_valid(reg)) 4318 return -EINVAL; 4319 } 4320 } 4321 } 4322 break; 4323 case PACKET3_COND_WRITE: 4324 if (idx_value & 0x100) { 4325 reg = ib[idx + 5] * 4; 4326 if (!si_vm_reg_valid(reg)) 4327 return -EINVAL; 4328 } 4329 break; 4330 case PACKET3_COPY_DW: 4331 if (idx_value & 0x2) { 4332 reg = ib[idx + 3] * 4; 4333 if (!si_vm_reg_valid(reg)) 4334 return -EINVAL; 4335 } 4336 break; 4337 case PACKET3_SET_CONFIG_REG: 4338 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START; 4339 end_reg = 4 * pkt->count + start_reg - 4; 4340 if ((start_reg < PACKET3_SET_CONFIG_REG_START) || 4341 (start_reg >= PACKET3_SET_CONFIG_REG_END) || 4342 (end_reg >= PACKET3_SET_CONFIG_REG_END)) { 4343 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); 4344 return -EINVAL; 4345 } 4346 for (i = 0; i < pkt->count; i++) { 4347 reg = start_reg + (4 * i); 4348 if (!si_vm_reg_valid(reg)) 4349 return -EINVAL; 4350 } 4351 break; 4352 case PACKET3_CP_DMA: 4353 r = si_vm_packet3_cp_dma_check(ib, idx); 4354 if (r) 4355 return r; 4356 break; 4357 default: 4358 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode); 4359 return -EINVAL; 4360 } 4361 return 0; 4362 } 4363 4364 static int si_vm_packet3_compute_check(struct radeon_device *rdev, 4365 u32 *ib, struct radeon_cs_packet *pkt) 4366 { 4367 int r; 4368 u32 idx = pkt->idx + 1; 4369 u32 idx_value = ib[idx]; 4370 u32 start_reg, reg, i; 4371 4372 switch (pkt->opcode) { 4373 case PACKET3_NOP: 4374 case PACKET3_SET_BASE: 4375 case PACKET3_CLEAR_STATE: 4376 case PACKET3_DISPATCH_DIRECT: 4377 case PACKET3_DISPATCH_INDIRECT: 4378 case PACKET3_ALLOC_GDS: 4379 case PACKET3_WRITE_GDS_RAM: 4380 case PACKET3_ATOMIC_GDS: 4381 case PACKET3_ATOMIC: 4382 case PACKET3_OCCLUSION_QUERY: 4383 case PACKET3_SET_PREDICATION: 4384 case PACKET3_COND_EXEC: 4385 case PACKET3_PRED_EXEC: 4386 case PACKET3_CONTEXT_CONTROL: 4387 case PACKET3_STRMOUT_BUFFER_UPDATE: 4388 case PACKET3_WAIT_REG_MEM: 4389 case PACKET3_MEM_WRITE: 4390 case PACKET3_PFP_SYNC_ME: 4391 case PACKET3_SURFACE_SYNC: 4392 case PACKET3_EVENT_WRITE: 4393 case PACKET3_EVENT_WRITE_EOP: 4394 case PACKET3_EVENT_WRITE_EOS: 4395 case PACKET3_SET_CONTEXT_REG: 4396 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4397 case PACKET3_SET_SH_REG: 4398 case PACKET3_SET_SH_REG_OFFSET: 4399 case PACKET3_INCREMENT_DE_COUNTER: 4400 case PACKET3_WAIT_ON_CE_COUNTER: 4401 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4402 case PACKET3_ME_WRITE: 4403 break; 4404 case PACKET3_COPY_DATA: 4405 if ((idx_value & 0xf00) == 0) { 4406 reg = ib[idx + 3] * 4; 4407 if (!si_vm_reg_valid(reg)) 4408 return -EINVAL; 4409 } 4410 break; 4411 case PACKET3_WRITE_DATA: 4412 if ((idx_value & 0xf00) == 0) { 4413 start_reg = ib[idx + 1] * 4; 4414 if (idx_value & 0x10000) { 4415 if (!si_vm_reg_valid(start_reg)) 4416 return -EINVAL; 4417 } else { 4418 for (i = 0; i < (pkt->count - 2); i++) { 4419 reg = start_reg + (4 * i); 4420 if (!si_vm_reg_valid(reg)) 4421 return -EINVAL; 4422 } 4423 } 4424 } 4425 break; 4426 case PACKET3_COND_WRITE: 4427 if (idx_value & 0x100) { 4428 reg = ib[idx + 5] * 4; 4429 if (!si_vm_reg_valid(reg)) 4430 return -EINVAL; 4431 } 4432 break; 4433 case PACKET3_COPY_DW: 4434 if (idx_value & 0x2) { 4435 reg = ib[idx + 3] * 4; 4436 if (!si_vm_reg_valid(reg)) 4437 return -EINVAL; 4438 } 4439 break; 4440 case PACKET3_CP_DMA: 4441 r = si_vm_packet3_cp_dma_check(ib, idx); 4442 if (r) 4443 return r; 4444 break; 4445 default: 4446 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode); 4447 return -EINVAL; 4448 } 4449 return 0; 4450 } 4451 4452 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 4453 { 4454 int ret = 0; 4455 u32 idx = 0; 4456 struct radeon_cs_packet pkt; 4457 4458 do { 4459 pkt.idx = idx; 4460 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]); 4461 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]); 4462 pkt.one_reg_wr = 0; 4463 switch (pkt.type) { 4464 case RADEON_PACKET_TYPE0: 4465 dev_err(rdev->dev, "Packet0 not allowed!\n"); 4466 ret = -EINVAL; 4467 break; 4468 case RADEON_PACKET_TYPE2: 4469 idx += 1; 4470 break; 4471 case RADEON_PACKET_TYPE3: 4472 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]); 4473 if (ib->is_const_ib) 4474 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt); 4475 else { 4476 switch (ib->ring) { 4477 case RADEON_RING_TYPE_GFX_INDEX: 4478 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt); 4479 break; 4480 case CAYMAN_RING_TYPE_CP1_INDEX: 4481 case CAYMAN_RING_TYPE_CP2_INDEX: 4482 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt); 4483 break; 4484 default: 4485 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring); 4486 ret = -EINVAL; 4487 break; 4488 } 4489 } 4490 idx += pkt.count + 2; 4491 break; 4492 default: 4493 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type); 4494 ret = -EINVAL; 4495 break; 4496 } 4497 if (ret) 4498 break; 4499 } while (idx < ib->length_dw); 4500 4501 return ret; 4502 } 4503 4504 /* 4505 * vm 4506 */ 4507 int si_vm_init(struct radeon_device *rdev) 4508 { 4509 /* number of VMs */ 4510 rdev->vm_manager.nvm = 16; 4511 /* base offset of vram pages */ 4512 rdev->vm_manager.vram_base_offset = 0; 4513 4514 return 0; 4515 } 4516 4517 void si_vm_fini(struct radeon_device *rdev) 4518 { 4519 } 4520 4521 /** 4522 * si_vm_decode_fault - print human readable fault info 4523 * 4524 * @rdev: radeon_device pointer 4525 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 4526 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 4527 * 4528 * Print human readable fault information (SI). 4529 */ 4530 static void si_vm_decode_fault(struct radeon_device *rdev, 4531 u32 status, u32 addr) 4532 { 4533 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 4534 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 4535 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 4536 char *block; 4537 4538 if (rdev->family == CHIP_TAHITI) { 4539 switch (mc_id) { 4540 case 160: 4541 case 144: 4542 case 96: 4543 case 80: 4544 case 224: 4545 case 208: 4546 case 32: 4547 case 16: 4548 block = "CB"; 4549 break; 4550 case 161: 4551 case 145: 4552 case 97: 4553 case 81: 4554 case 225: 4555 case 209: 4556 case 33: 4557 case 17: 4558 block = "CB_FMASK"; 4559 break; 4560 case 162: 4561 case 146: 4562 case 98: 4563 case 82: 4564 case 226: 4565 case 210: 4566 case 34: 4567 case 18: 4568 block = "CB_CMASK"; 4569 break; 4570 case 163: 4571 case 147: 4572 case 99: 4573 case 83: 4574 case 227: 4575 case 211: 4576 case 35: 4577 case 19: 4578 block = "CB_IMMED"; 4579 break; 4580 case 164: 4581 case 148: 4582 case 100: 4583 case 84: 4584 case 228: 4585 case 212: 4586 case 36: 4587 case 20: 4588 block = "DB"; 4589 break; 4590 case 165: 4591 case 149: 4592 case 101: 4593 case 85: 4594 case 229: 4595 case 213: 4596 case 37: 4597 case 21: 4598 block = "DB_HTILE"; 4599 break; 4600 case 167: 4601 case 151: 4602 case 103: 4603 case 87: 4604 case 231: 4605 case 215: 4606 case 39: 4607 case 23: 4608 block = "DB_STEN"; 4609 break; 4610 case 72: 4611 case 68: 4612 case 64: 4613 case 8: 4614 case 4: 4615 case 0: 4616 case 136: 4617 case 132: 4618 case 128: 4619 case 200: 4620 case 196: 4621 case 192: 4622 block = "TC"; 4623 break; 4624 case 112: 4625 case 48: 4626 block = "CP"; 4627 break; 4628 case 49: 4629 case 177: 4630 case 50: 4631 case 178: 4632 block = "SH"; 4633 break; 4634 case 53: 4635 case 190: 4636 block = "VGT"; 4637 break; 4638 case 117: 4639 block = "IH"; 4640 break; 4641 case 51: 4642 case 115: 4643 block = "RLC"; 4644 break; 4645 case 119: 4646 case 183: 4647 block = "DMA0"; 4648 break; 4649 case 61: 4650 block = "DMA1"; 4651 break; 4652 case 248: 4653 case 120: 4654 block = "HDP"; 4655 break; 4656 default: 4657 block = "unknown"; 4658 break; 4659 } 4660 } else { 4661 switch (mc_id) { 4662 case 32: 4663 case 16: 4664 case 96: 4665 case 80: 4666 case 160: 4667 case 144: 4668 case 224: 4669 case 208: 4670 block = "CB"; 4671 break; 4672 case 33: 4673 case 17: 4674 case 97: 4675 case 81: 4676 case 161: 4677 case 145: 4678 case 225: 4679 case 209: 4680 block = "CB_FMASK"; 4681 break; 4682 case 34: 4683 case 18: 4684 case 98: 4685 case 82: 4686 case 162: 4687 case 146: 4688 case 226: 4689 case 210: 4690 block = "CB_CMASK"; 4691 break; 4692 case 35: 4693 case 19: 4694 case 99: 4695 case 83: 4696 case 163: 4697 case 147: 4698 case 227: 4699 case 211: 4700 block = "CB_IMMED"; 4701 break; 4702 case 36: 4703 case 20: 4704 case 100: 4705 case 84: 4706 case 164: 4707 case 148: 4708 case 228: 4709 case 212: 4710 block = "DB"; 4711 break; 4712 case 37: 4713 case 21: 4714 case 101: 4715 case 85: 4716 case 165: 4717 case 149: 4718 case 229: 4719 case 213: 4720 block = "DB_HTILE"; 4721 break; 4722 case 39: 4723 case 23: 4724 case 103: 4725 case 87: 4726 case 167: 4727 case 151: 4728 case 231: 4729 case 215: 4730 block = "DB_STEN"; 4731 break; 4732 case 72: 4733 case 68: 4734 case 8: 4735 case 4: 4736 case 136: 4737 case 132: 4738 case 200: 4739 case 196: 4740 block = "TC"; 4741 break; 4742 case 112: 4743 case 48: 4744 block = "CP"; 4745 break; 4746 case 49: 4747 case 177: 4748 case 50: 4749 case 178: 4750 block = "SH"; 4751 break; 4752 case 53: 4753 block = "VGT"; 4754 break; 4755 case 117: 4756 block = "IH"; 4757 break; 4758 case 51: 4759 case 115: 4760 block = "RLC"; 4761 break; 4762 case 119: 4763 case 183: 4764 block = "DMA0"; 4765 break; 4766 case 61: 4767 block = "DMA1"; 4768 break; 4769 case 248: 4770 case 120: 4771 block = "HDP"; 4772 break; 4773 default: 4774 block = "unknown"; 4775 break; 4776 } 4777 } 4778 4779 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n", 4780 protections, vmid, addr, 4781 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 4782 block, mc_id); 4783 } 4784 4785 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 4786 { 4787 struct radeon_ring *ring = &rdev->ring[ridx]; 4788 4789 if (vm == NULL) 4790 return; 4791 4792 /* write new base address */ 4793 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4794 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4795 WRITE_DATA_DST_SEL(0))); 4796 4797 if (vm->id < 8) { 4798 radeon_ring_write(ring, 4799 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); 4800 } else { 4801 radeon_ring_write(ring, 4802 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); 4803 } 4804 radeon_ring_write(ring, 0); 4805 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 4806 4807 /* flush hdp cache */ 4808 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4809 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4810 WRITE_DATA_DST_SEL(0))); 4811 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2); 4812 radeon_ring_write(ring, 0); 4813 radeon_ring_write(ring, 0x1); 4814 4815 /* bits 0-15 are the VM contexts0-15 */ 4816 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4817 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4818 WRITE_DATA_DST_SEL(0))); 4819 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 4820 radeon_ring_write(ring, 0); 4821 radeon_ring_write(ring, 1 << vm->id); 4822 4823 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 4824 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 4825 radeon_ring_write(ring, 0x0); 4826 } 4827 4828 /* 4829 * Power and clock gating 4830 */ 4831 static void si_wait_for_rlc_serdes(struct radeon_device *rdev) 4832 { 4833 int i; 4834 4835 for (i = 0; i < rdev->usec_timeout; i++) { 4836 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0) 4837 break; 4838 udelay(1); 4839 } 4840 4841 for (i = 0; i < rdev->usec_timeout; i++) { 4842 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0) 4843 break; 4844 udelay(1); 4845 } 4846 } 4847 4848 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, 4849 bool enable) 4850 { 4851 u32 tmp = RREG32(CP_INT_CNTL_RING0); 4852 u32 mask; 4853 int i; 4854 4855 if (enable) 4856 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4857 else 4858 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4859 WREG32(CP_INT_CNTL_RING0, tmp); 4860 4861 if (!enable) { 4862 /* read a gfx register */ 4863 tmp = RREG32(DB_DEPTH_INFO); 4864 4865 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS; 4866 for (i = 0; i < rdev->usec_timeout; i++) { 4867 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS)) 4868 break; 4869 udelay(1); 4870 } 4871 } 4872 } 4873 4874 static void si_set_uvd_dcm(struct radeon_device *rdev, 4875 bool sw_mode) 4876 { 4877 u32 tmp, tmp2; 4878 4879 tmp = RREG32(UVD_CGC_CTRL); 4880 tmp &= ~(CLK_OD_MASK | CG_DT_MASK); 4881 tmp |= DCM | CG_DT(1) | CLK_OD(4); 4882 4883 if (sw_mode) { 4884 tmp &= ~0x7ffff800; 4885 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7); 4886 } else { 4887 tmp |= 0x7ffff800; 4888 tmp2 = 0; 4889 } 4890 4891 WREG32(UVD_CGC_CTRL, tmp); 4892 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2); 4893 } 4894 4895 void si_init_uvd_internal_cg(struct radeon_device *rdev) 4896 { 4897 bool hw_mode = true; 4898 4899 if (hw_mode) { 4900 si_set_uvd_dcm(rdev, false); 4901 } else { 4902 u32 tmp = RREG32(UVD_CGC_CTRL); 4903 tmp &= ~DCM; 4904 WREG32(UVD_CGC_CTRL, tmp); 4905 } 4906 } 4907 4908 static u32 si_halt_rlc(struct radeon_device *rdev) 4909 { 4910 u32 data, orig; 4911 4912 orig = data = RREG32(RLC_CNTL); 4913 4914 if (data & RLC_ENABLE) { 4915 data &= ~RLC_ENABLE; 4916 WREG32(RLC_CNTL, data); 4917 4918 si_wait_for_rlc_serdes(rdev); 4919 } 4920 4921 return orig; 4922 } 4923 4924 static void si_update_rlc(struct radeon_device *rdev, u32 rlc) 4925 { 4926 u32 tmp; 4927 4928 tmp = RREG32(RLC_CNTL); 4929 if (tmp != rlc) 4930 WREG32(RLC_CNTL, rlc); 4931 } 4932 4933 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable) 4934 { 4935 u32 data, orig; 4936 4937 orig = data = RREG32(DMA_PG); 4938 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA)) 4939 data |= PG_CNTL_ENABLE; 4940 else 4941 data &= ~PG_CNTL_ENABLE; 4942 if (orig != data) 4943 WREG32(DMA_PG, data); 4944 } 4945 4946 static void si_init_dma_pg(struct radeon_device *rdev) 4947 { 4948 u32 tmp; 4949 4950 WREG32(DMA_PGFSM_WRITE, 0x00002000); 4951 WREG32(DMA_PGFSM_CONFIG, 0x100010ff); 4952 4953 for (tmp = 0; tmp < 5; tmp++) 4954 WREG32(DMA_PGFSM_WRITE, 0); 4955 } 4956 4957 static void si_enable_gfx_cgpg(struct radeon_device *rdev, 4958 bool enable) 4959 { 4960 u32 tmp; 4961 4962 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { 4963 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10); 4964 WREG32(RLC_TTOP_D, tmp); 4965 4966 tmp = RREG32(RLC_PG_CNTL); 4967 tmp |= GFX_PG_ENABLE; 4968 WREG32(RLC_PG_CNTL, tmp); 4969 4970 tmp = RREG32(RLC_AUTO_PG_CTRL); 4971 tmp |= AUTO_PG_EN; 4972 WREG32(RLC_AUTO_PG_CTRL, tmp); 4973 } else { 4974 tmp = RREG32(RLC_AUTO_PG_CTRL); 4975 tmp &= ~AUTO_PG_EN; 4976 WREG32(RLC_AUTO_PG_CTRL, tmp); 4977 4978 tmp = RREG32(DB_RENDER_CONTROL); 4979 } 4980 } 4981 4982 static void si_init_gfx_cgpg(struct radeon_device *rdev) 4983 { 4984 u32 tmp; 4985 4986 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4987 4988 tmp = RREG32(RLC_PG_CNTL); 4989 tmp |= GFX_PG_SRC; 4990 WREG32(RLC_PG_CNTL, tmp); 4991 4992 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4993 4994 tmp = RREG32(RLC_AUTO_PG_CTRL); 4995 4996 tmp &= ~GRBM_REG_SGIT_MASK; 4997 tmp |= GRBM_REG_SGIT(0x700); 4998 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK; 4999 WREG32(RLC_AUTO_PG_CTRL, tmp); 5000 } 5001 5002 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 5003 { 5004 u32 mask = 0, tmp, tmp1; 5005 int i; 5006 5007 si_select_se_sh(rdev, se, sh); 5008 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 5009 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 5010 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5011 5012 tmp &= 0xffff0000; 5013 5014 tmp |= tmp1; 5015 tmp >>= 16; 5016 5017 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) { 5018 mask <<= 1; 5019 mask |= 1; 5020 } 5021 5022 return (~tmp) & mask; 5023 } 5024 5025 static void si_init_ao_cu_mask(struct radeon_device *rdev) 5026 { 5027 u32 i, j, k, active_cu_number = 0; 5028 u32 mask, counter, cu_bitmap; 5029 u32 tmp = 0; 5030 5031 for (i = 0; i < rdev->config.si.max_shader_engines; i++) { 5032 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { 5033 mask = 1; 5034 cu_bitmap = 0; 5035 counter = 0; 5036 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) { 5037 if (si_get_cu_active_bitmap(rdev, i, j) & mask) { 5038 if (counter < 2) 5039 cu_bitmap |= mask; 5040 counter++; 5041 } 5042 mask <<= 1; 5043 } 5044 5045 active_cu_number += counter; 5046 tmp |= (cu_bitmap << (i * 16 + j * 8)); 5047 } 5048 } 5049 5050 WREG32(RLC_PG_AO_CU_MASK, tmp); 5051 5052 tmp = RREG32(RLC_MAX_PG_CU); 5053 tmp &= ~MAX_PU_CU_MASK; 5054 tmp |= MAX_PU_CU(active_cu_number); 5055 WREG32(RLC_MAX_PG_CU, tmp); 5056 } 5057 5058 static void si_enable_cgcg(struct radeon_device *rdev, 5059 bool enable) 5060 { 5061 u32 data, orig, tmp; 5062 5063 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 5064 5065 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 5066 si_enable_gui_idle_interrupt(rdev, true); 5067 5068 WREG32(RLC_GCPM_GENERAL_3, 0x00000080); 5069 5070 tmp = si_halt_rlc(rdev); 5071 5072 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5073 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5074 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff); 5075 5076 si_wait_for_rlc_serdes(rdev); 5077 5078 si_update_rlc(rdev, tmp); 5079 5080 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff); 5081 5082 data |= CGCG_EN | CGLS_EN; 5083 } else { 5084 si_enable_gui_idle_interrupt(rdev, false); 5085 5086 RREG32(CB_CGTT_SCLK_CTRL); 5087 RREG32(CB_CGTT_SCLK_CTRL); 5088 RREG32(CB_CGTT_SCLK_CTRL); 5089 RREG32(CB_CGTT_SCLK_CTRL); 5090 5091 data &= ~(CGCG_EN | CGLS_EN); 5092 } 5093 5094 if (orig != data) 5095 WREG32(RLC_CGCG_CGLS_CTRL, data); 5096 } 5097 5098 static void si_enable_mgcg(struct radeon_device *rdev, 5099 bool enable) 5100 { 5101 u32 data, orig, tmp = 0; 5102 5103 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 5104 orig = data = RREG32(CGTS_SM_CTRL_REG); 5105 data = 0x96940200; 5106 if (orig != data) 5107 WREG32(CGTS_SM_CTRL_REG, data); 5108 5109 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 5110 orig = data = RREG32(CP_MEM_SLP_CNTL); 5111 data |= CP_MEM_LS_EN; 5112 if (orig != data) 5113 WREG32(CP_MEM_SLP_CNTL, data); 5114 } 5115 5116 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5117 data &= 0xffffffc0; 5118 if (orig != data) 5119 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5120 5121 tmp = si_halt_rlc(rdev); 5122 5123 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5124 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5125 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff); 5126 5127 si_update_rlc(rdev, tmp); 5128 } else { 5129 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5130 data |= 0x00000003; 5131 if (orig != data) 5132 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5133 5134 data = RREG32(CP_MEM_SLP_CNTL); 5135 if (data & CP_MEM_LS_EN) { 5136 data &= ~CP_MEM_LS_EN; 5137 WREG32(CP_MEM_SLP_CNTL, data); 5138 } 5139 orig = data = RREG32(CGTS_SM_CTRL_REG); 5140 data |= LS_OVERRIDE | OVERRIDE; 5141 if (orig != data) 5142 WREG32(CGTS_SM_CTRL_REG, data); 5143 5144 tmp = si_halt_rlc(rdev); 5145 5146 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5147 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5148 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff); 5149 5150 si_update_rlc(rdev, tmp); 5151 } 5152 } 5153 5154 static void si_enable_uvd_mgcg(struct radeon_device *rdev, 5155 bool enable) 5156 { 5157 u32 orig, data, tmp; 5158 5159 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 5160 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5161 tmp |= 0x3fff; 5162 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5163 5164 orig = data = RREG32(UVD_CGC_CTRL); 5165 data |= DCM; 5166 if (orig != data) 5167 WREG32(UVD_CGC_CTRL, data); 5168 5169 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0); 5170 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0); 5171 } else { 5172 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5173 tmp &= ~0x3fff; 5174 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5175 5176 orig = data = RREG32(UVD_CGC_CTRL); 5177 data &= ~DCM; 5178 if (orig != data) 5179 WREG32(UVD_CGC_CTRL, data); 5180 5181 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff); 5182 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff); 5183 } 5184 } 5185 5186 static const u32 mc_cg_registers[] = 5187 { 5188 MC_HUB_MISC_HUB_CG, 5189 MC_HUB_MISC_SIP_CG, 5190 MC_HUB_MISC_VM_CG, 5191 MC_XPB_CLK_GAT, 5192 ATC_MISC_CG, 5193 MC_CITF_MISC_WR_CG, 5194 MC_CITF_MISC_RD_CG, 5195 MC_CITF_MISC_VM_CG, 5196 VM_L2_CG, 5197 }; 5198 5199 static void si_enable_mc_ls(struct radeon_device *rdev, 5200 bool enable) 5201 { 5202 int i; 5203 u32 orig, data; 5204 5205 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5206 orig = data = RREG32(mc_cg_registers[i]); 5207 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 5208 data |= MC_LS_ENABLE; 5209 else 5210 data &= ~MC_LS_ENABLE; 5211 if (data != orig) 5212 WREG32(mc_cg_registers[i], data); 5213 } 5214 } 5215 5216 static void si_enable_mc_mgcg(struct radeon_device *rdev, 5217 bool enable) 5218 { 5219 int i; 5220 u32 orig, data; 5221 5222 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5223 orig = data = RREG32(mc_cg_registers[i]); 5224 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 5225 data |= MC_CG_ENABLE; 5226 else 5227 data &= ~MC_CG_ENABLE; 5228 if (data != orig) 5229 WREG32(mc_cg_registers[i], data); 5230 } 5231 } 5232 5233 static void si_enable_dma_mgcg(struct radeon_device *rdev, 5234 bool enable) 5235 { 5236 u32 orig, data, offset; 5237 int i; 5238 5239 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 5240 for (i = 0; i < 2; i++) { 5241 if (i == 0) 5242 offset = DMA0_REGISTER_OFFSET; 5243 else 5244 offset = DMA1_REGISTER_OFFSET; 5245 orig = data = RREG32(DMA_POWER_CNTL + offset); 5246 data &= ~MEM_POWER_OVERRIDE; 5247 if (data != orig) 5248 WREG32(DMA_POWER_CNTL + offset, data); 5249 WREG32(DMA_CLK_CTRL + offset, 0x00000100); 5250 } 5251 } else { 5252 for (i = 0; i < 2; i++) { 5253 if (i == 0) 5254 offset = DMA0_REGISTER_OFFSET; 5255 else 5256 offset = DMA1_REGISTER_OFFSET; 5257 orig = data = RREG32(DMA_POWER_CNTL + offset); 5258 data |= MEM_POWER_OVERRIDE; 5259 if (data != orig) 5260 WREG32(DMA_POWER_CNTL + offset, data); 5261 5262 orig = data = RREG32(DMA_CLK_CTRL + offset); 5263 data = 0xff000000; 5264 if (data != orig) 5265 WREG32(DMA_CLK_CTRL + offset, data); 5266 } 5267 } 5268 } 5269 5270 static void si_enable_bif_mgls(struct radeon_device *rdev, 5271 bool enable) 5272 { 5273 u32 orig, data; 5274 5275 orig = data = RREG32_PCIE(PCIE_CNTL2); 5276 5277 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 5278 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 5279 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 5280 else 5281 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 5282 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 5283 5284 if (orig != data) 5285 WREG32_PCIE(PCIE_CNTL2, data); 5286 } 5287 5288 static void si_enable_hdp_mgcg(struct radeon_device *rdev, 5289 bool enable) 5290 { 5291 u32 orig, data; 5292 5293 orig = data = RREG32(HDP_HOST_PATH_CNTL); 5294 5295 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 5296 data &= ~CLOCK_GATING_DIS; 5297 else 5298 data |= CLOCK_GATING_DIS; 5299 5300 if (orig != data) 5301 WREG32(HDP_HOST_PATH_CNTL, data); 5302 } 5303 5304 static void si_enable_hdp_ls(struct radeon_device *rdev, 5305 bool enable) 5306 { 5307 u32 orig, data; 5308 5309 orig = data = RREG32(HDP_MEM_POWER_LS); 5310 5311 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 5312 data |= HDP_LS_ENABLE; 5313 else 5314 data &= ~HDP_LS_ENABLE; 5315 5316 if (orig != data) 5317 WREG32(HDP_MEM_POWER_LS, data); 5318 } 5319 5320 static void si_update_cg(struct radeon_device *rdev, 5321 u32 block, bool enable) 5322 { 5323 if (block & RADEON_CG_BLOCK_GFX) { 5324 si_enable_gui_idle_interrupt(rdev, false); 5325 /* order matters! */ 5326 if (enable) { 5327 si_enable_mgcg(rdev, true); 5328 si_enable_cgcg(rdev, true); 5329 } else { 5330 si_enable_cgcg(rdev, false); 5331 si_enable_mgcg(rdev, false); 5332 } 5333 si_enable_gui_idle_interrupt(rdev, true); 5334 } 5335 5336 if (block & RADEON_CG_BLOCK_MC) { 5337 si_enable_mc_mgcg(rdev, enable); 5338 si_enable_mc_ls(rdev, enable); 5339 } 5340 5341 if (block & RADEON_CG_BLOCK_SDMA) { 5342 si_enable_dma_mgcg(rdev, enable); 5343 } 5344 5345 if (block & RADEON_CG_BLOCK_BIF) { 5346 si_enable_bif_mgls(rdev, enable); 5347 } 5348 5349 if (block & RADEON_CG_BLOCK_UVD) { 5350 if (rdev->has_uvd) { 5351 si_enable_uvd_mgcg(rdev, enable); 5352 } 5353 } 5354 5355 if (block & RADEON_CG_BLOCK_HDP) { 5356 si_enable_hdp_mgcg(rdev, enable); 5357 si_enable_hdp_ls(rdev, enable); 5358 } 5359 } 5360 5361 static void si_init_cg(struct radeon_device *rdev) 5362 { 5363 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | 5364 RADEON_CG_BLOCK_MC | 5365 RADEON_CG_BLOCK_SDMA | 5366 RADEON_CG_BLOCK_BIF | 5367 RADEON_CG_BLOCK_HDP), true); 5368 if (rdev->has_uvd) { 5369 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true); 5370 si_init_uvd_internal_cg(rdev); 5371 } 5372 } 5373 5374 static void si_fini_cg(struct radeon_device *rdev) 5375 { 5376 if (rdev->has_uvd) { 5377 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false); 5378 } 5379 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | 5380 RADEON_CG_BLOCK_MC | 5381 RADEON_CG_BLOCK_SDMA | 5382 RADEON_CG_BLOCK_BIF | 5383 RADEON_CG_BLOCK_HDP), false); 5384 } 5385 5386 u32 si_get_csb_size(struct radeon_device *rdev) 5387 { 5388 u32 count = 0; 5389 const struct cs_section_def *sect = NULL; 5390 const struct cs_extent_def *ext = NULL; 5391 5392 if (rdev->rlc.cs_data == NULL) 5393 return 0; 5394 5395 /* begin clear state */ 5396 count += 2; 5397 /* context control state */ 5398 count += 3; 5399 5400 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 5401 for (ext = sect->section; ext->extent != NULL; ++ext) { 5402 if (sect->id == SECT_CONTEXT) 5403 count += 2 + ext->reg_count; 5404 else 5405 return 0; 5406 } 5407 } 5408 /* pa_sc_raster_config */ 5409 count += 3; 5410 /* end clear state */ 5411 count += 2; 5412 /* clear state */ 5413 count += 2; 5414 5415 return count; 5416 } 5417 5418 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 5419 { 5420 u32 count = 0, i; 5421 const struct cs_section_def *sect = NULL; 5422 const struct cs_extent_def *ext = NULL; 5423 5424 if (rdev->rlc.cs_data == NULL) 5425 return; 5426 if (buffer == NULL) 5427 return; 5428 5429 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 5430 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 5431 5432 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5433 buffer[count++] = cpu_to_le32(0x80000000); 5434 buffer[count++] = cpu_to_le32(0x80000000); 5435 5436 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 5437 for (ext = sect->section; ext->extent != NULL; ++ext) { 5438 if (sect->id == SECT_CONTEXT) { 5439 buffer[count++] = 5440 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 5441 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000); 5442 for (i = 0; i < ext->reg_count; i++) 5443 buffer[count++] = cpu_to_le32(ext->extent[i]); 5444 } else { 5445 return; 5446 } 5447 } 5448 } 5449 5450 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 5451 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 5452 switch (rdev->family) { 5453 case CHIP_TAHITI: 5454 case CHIP_PITCAIRN: 5455 buffer[count++] = cpu_to_le32(0x2a00126a); 5456 break; 5457 case CHIP_VERDE: 5458 buffer[count++] = cpu_to_le32(0x0000124a); 5459 break; 5460 case CHIP_OLAND: 5461 buffer[count++] = cpu_to_le32(0x00000082); 5462 break; 5463 case CHIP_HAINAN: 5464 buffer[count++] = cpu_to_le32(0x00000000); 5465 break; 5466 default: 5467 buffer[count++] = cpu_to_le32(0x00000000); 5468 break; 5469 } 5470 5471 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 5472 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 5473 5474 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 5475 buffer[count++] = cpu_to_le32(0); 5476 } 5477 5478 static void si_init_pg(struct radeon_device *rdev) 5479 { 5480 if (rdev->pg_flags) { 5481 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) { 5482 si_init_dma_pg(rdev); 5483 } 5484 si_init_ao_cu_mask(rdev); 5485 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { 5486 si_init_gfx_cgpg(rdev); 5487 } else { 5488 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5489 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5490 } 5491 si_enable_dma_pg(rdev, true); 5492 si_enable_gfx_cgpg(rdev, true); 5493 } else { 5494 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5495 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5496 } 5497 } 5498 5499 static void si_fini_pg(struct radeon_device *rdev) 5500 { 5501 if (rdev->pg_flags) { 5502 si_enable_dma_pg(rdev, false); 5503 si_enable_gfx_cgpg(rdev, false); 5504 } 5505 } 5506 5507 /* 5508 * RLC 5509 */ 5510 void si_rlc_reset(struct radeon_device *rdev) 5511 { 5512 u32 tmp = RREG32(GRBM_SOFT_RESET); 5513 5514 tmp |= SOFT_RESET_RLC; 5515 WREG32(GRBM_SOFT_RESET, tmp); 5516 udelay(50); 5517 tmp &= ~SOFT_RESET_RLC; 5518 WREG32(GRBM_SOFT_RESET, tmp); 5519 udelay(50); 5520 } 5521 5522 static void si_rlc_stop(struct radeon_device *rdev) 5523 { 5524 WREG32(RLC_CNTL, 0); 5525 5526 si_enable_gui_idle_interrupt(rdev, false); 5527 5528 si_wait_for_rlc_serdes(rdev); 5529 } 5530 5531 static void si_rlc_start(struct radeon_device *rdev) 5532 { 5533 WREG32(RLC_CNTL, RLC_ENABLE); 5534 5535 si_enable_gui_idle_interrupt(rdev, true); 5536 5537 udelay(50); 5538 } 5539 5540 static bool si_lbpw_supported(struct radeon_device *rdev) 5541 { 5542 u32 tmp; 5543 5544 /* Enable LBPW only for DDR3 */ 5545 tmp = RREG32(MC_SEQ_MISC0); 5546 if ((tmp & 0xF0000000) == 0xB0000000) 5547 return true; 5548 return false; 5549 } 5550 5551 static void si_enable_lbpw(struct radeon_device *rdev, bool enable) 5552 { 5553 u32 tmp; 5554 5555 tmp = RREG32(RLC_LB_CNTL); 5556 if (enable) 5557 tmp |= LOAD_BALANCE_ENABLE; 5558 else 5559 tmp &= ~LOAD_BALANCE_ENABLE; 5560 WREG32(RLC_LB_CNTL, tmp); 5561 5562 if (!enable) { 5563 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5564 WREG32(SPI_LB_CU_MASK, 0x00ff); 5565 } 5566 } 5567 5568 static int si_rlc_resume(struct radeon_device *rdev) 5569 { 5570 u32 i; 5571 const __be32 *fw_data; 5572 5573 if (!rdev->rlc_fw) 5574 return -EINVAL; 5575 5576 si_rlc_stop(rdev); 5577 5578 si_rlc_reset(rdev); 5579 5580 si_init_pg(rdev); 5581 5582 si_init_cg(rdev); 5583 5584 WREG32(RLC_RL_BASE, 0); 5585 WREG32(RLC_RL_SIZE, 0); 5586 WREG32(RLC_LB_CNTL, 0); 5587 WREG32(RLC_LB_CNTR_MAX, 0xffffffff); 5588 WREG32(RLC_LB_CNTR_INIT, 0); 5589 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 5590 5591 WREG32(RLC_MC_CNTL, 0); 5592 WREG32(RLC_UCODE_CNTL, 0); 5593 5594 fw_data = (const __be32 *)rdev->rlc_fw->data; 5595 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) { 5596 WREG32(RLC_UCODE_ADDR, i); 5597 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 5598 } 5599 WREG32(RLC_UCODE_ADDR, 0); 5600 5601 si_enable_lbpw(rdev, si_lbpw_supported(rdev)); 5602 5603 si_rlc_start(rdev); 5604 5605 return 0; 5606 } 5607 5608 static void si_enable_interrupts(struct radeon_device *rdev) 5609 { 5610 u32 ih_cntl = RREG32(IH_CNTL); 5611 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5612 5613 ih_cntl |= ENABLE_INTR; 5614 ih_rb_cntl |= IH_RB_ENABLE; 5615 WREG32(IH_CNTL, ih_cntl); 5616 WREG32(IH_RB_CNTL, ih_rb_cntl); 5617 rdev->ih.enabled = true; 5618 } 5619 5620 static void si_disable_interrupts(struct radeon_device *rdev) 5621 { 5622 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5623 u32 ih_cntl = RREG32(IH_CNTL); 5624 5625 ih_rb_cntl &= ~IH_RB_ENABLE; 5626 ih_cntl &= ~ENABLE_INTR; 5627 WREG32(IH_RB_CNTL, ih_rb_cntl); 5628 WREG32(IH_CNTL, ih_cntl); 5629 /* set rptr, wptr to 0 */ 5630 WREG32(IH_RB_RPTR, 0); 5631 WREG32(IH_RB_WPTR, 0); 5632 rdev->ih.enabled = false; 5633 rdev->ih.rptr = 0; 5634 } 5635 5636 static void si_disable_interrupt_state(struct radeon_device *rdev) 5637 { 5638 u32 tmp; 5639 5640 tmp = RREG32(CP_INT_CNTL_RING0) & 5641 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5642 WREG32(CP_INT_CNTL_RING0, tmp); 5643 WREG32(CP_INT_CNTL_RING1, 0); 5644 WREG32(CP_INT_CNTL_RING2, 0); 5645 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5646 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp); 5647 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5648 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp); 5649 WREG32(GRBM_INT_CNTL, 0); 5650 if (rdev->num_crtc >= 2) { 5651 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5652 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5653 } 5654 if (rdev->num_crtc >= 4) { 5655 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5656 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5657 } 5658 if (rdev->num_crtc >= 6) { 5659 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5660 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5661 } 5662 5663 if (rdev->num_crtc >= 2) { 5664 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5665 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5666 } 5667 if (rdev->num_crtc >= 4) { 5668 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5669 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5670 } 5671 if (rdev->num_crtc >= 6) { 5672 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5673 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5674 } 5675 5676 if (!ASIC_IS_NODCE(rdev)) { 5677 WREG32(DAC_AUTODETECT_INT_CONTROL, 0); 5678 5679 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5680 WREG32(DC_HPD1_INT_CONTROL, tmp); 5681 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5682 WREG32(DC_HPD2_INT_CONTROL, tmp); 5683 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5684 WREG32(DC_HPD3_INT_CONTROL, tmp); 5685 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5686 WREG32(DC_HPD4_INT_CONTROL, tmp); 5687 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5688 WREG32(DC_HPD5_INT_CONTROL, tmp); 5689 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5690 WREG32(DC_HPD6_INT_CONTROL, tmp); 5691 } 5692 } 5693 5694 static int si_irq_init(struct radeon_device *rdev) 5695 { 5696 int ret = 0; 5697 int rb_bufsz; 5698 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 5699 5700 /* allocate ring */ 5701 ret = r600_ih_ring_alloc(rdev); 5702 if (ret) 5703 return ret; 5704 5705 /* disable irqs */ 5706 si_disable_interrupts(rdev); 5707 5708 /* init rlc */ 5709 ret = si_rlc_resume(rdev); 5710 if (ret) { 5711 r600_ih_ring_fini(rdev); 5712 return ret; 5713 } 5714 5715 /* setup interrupt control */ 5716 /* set dummy read address to ring address */ 5717 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 5718 interrupt_cntl = RREG32(INTERRUPT_CNTL); 5719 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 5720 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 5721 */ 5722 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 5723 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 5724 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 5725 WREG32(INTERRUPT_CNTL, interrupt_cntl); 5726 5727 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 5728 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 5729 5730 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 5731 IH_WPTR_OVERFLOW_CLEAR | 5732 (rb_bufsz << 1)); 5733 5734 if (rdev->wb.enabled) 5735 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 5736 5737 /* set the writeback address whether it's enabled or not */ 5738 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 5739 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 5740 5741 WREG32(IH_RB_CNTL, ih_rb_cntl); 5742 5743 /* set rptr, wptr to 0 */ 5744 WREG32(IH_RB_RPTR, 0); 5745 WREG32(IH_RB_WPTR, 0); 5746 5747 /* Default settings for IH_CNTL (disabled at first) */ 5748 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 5749 /* RPTR_REARM only works if msi's are enabled */ 5750 if (rdev->msi_enabled) 5751 ih_cntl |= RPTR_REARM; 5752 WREG32(IH_CNTL, ih_cntl); 5753 5754 /* force the active interrupt state to all disabled */ 5755 si_disable_interrupt_state(rdev); 5756 5757 pci_set_master(rdev->pdev); 5758 5759 /* enable irqs */ 5760 si_enable_interrupts(rdev); 5761 5762 return ret; 5763 } 5764 5765 int si_irq_set(struct radeon_device *rdev) 5766 { 5767 u32 cp_int_cntl; 5768 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 5769 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 5770 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0; 5771 u32 grbm_int_cntl = 0; 5772 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; 5773 u32 dma_cntl, dma_cntl1; 5774 u32 thermal_int = 0; 5775 5776 if (!rdev->irq.installed) { 5777 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 5778 return -EINVAL; 5779 } 5780 /* don't enable anything if the ih is disabled */ 5781 if (!rdev->ih.enabled) { 5782 si_disable_interrupts(rdev); 5783 /* force the active interrupt state to all disabled */ 5784 si_disable_interrupt_state(rdev); 5785 return 0; 5786 } 5787 5788 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) & 5789 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5790 5791 if (!ASIC_IS_NODCE(rdev)) { 5792 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 5793 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 5794 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 5795 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 5796 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 5797 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 5798 } 5799 5800 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5801 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5802 5803 thermal_int = RREG32(CG_THERMAL_INT) & 5804 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 5805 5806 /* enable CP interrupts on all rings */ 5807 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 5808 DRM_DEBUG("si_irq_set: sw int gfx\n"); 5809 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 5810 } 5811 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 5812 DRM_DEBUG("si_irq_set: sw int cp1\n"); 5813 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 5814 } 5815 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 5816 DRM_DEBUG("si_irq_set: sw int cp2\n"); 5817 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 5818 } 5819 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 5820 DRM_DEBUG("si_irq_set: sw int dma\n"); 5821 dma_cntl |= TRAP_ENABLE; 5822 } 5823 5824 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 5825 DRM_DEBUG("si_irq_set: sw int dma1\n"); 5826 dma_cntl1 |= TRAP_ENABLE; 5827 } 5828 if (rdev->irq.crtc_vblank_int[0] || 5829 atomic_read(&rdev->irq.pflip[0])) { 5830 DRM_DEBUG("si_irq_set: vblank 0\n"); 5831 crtc1 |= VBLANK_INT_MASK; 5832 } 5833 if (rdev->irq.crtc_vblank_int[1] || 5834 atomic_read(&rdev->irq.pflip[1])) { 5835 DRM_DEBUG("si_irq_set: vblank 1\n"); 5836 crtc2 |= VBLANK_INT_MASK; 5837 } 5838 if (rdev->irq.crtc_vblank_int[2] || 5839 atomic_read(&rdev->irq.pflip[2])) { 5840 DRM_DEBUG("si_irq_set: vblank 2\n"); 5841 crtc3 |= VBLANK_INT_MASK; 5842 } 5843 if (rdev->irq.crtc_vblank_int[3] || 5844 atomic_read(&rdev->irq.pflip[3])) { 5845 DRM_DEBUG("si_irq_set: vblank 3\n"); 5846 crtc4 |= VBLANK_INT_MASK; 5847 } 5848 if (rdev->irq.crtc_vblank_int[4] || 5849 atomic_read(&rdev->irq.pflip[4])) { 5850 DRM_DEBUG("si_irq_set: vblank 4\n"); 5851 crtc5 |= VBLANK_INT_MASK; 5852 } 5853 if (rdev->irq.crtc_vblank_int[5] || 5854 atomic_read(&rdev->irq.pflip[5])) { 5855 DRM_DEBUG("si_irq_set: vblank 5\n"); 5856 crtc6 |= VBLANK_INT_MASK; 5857 } 5858 if (rdev->irq.hpd[0]) { 5859 DRM_DEBUG("si_irq_set: hpd 1\n"); 5860 hpd1 |= DC_HPDx_INT_EN; 5861 } 5862 if (rdev->irq.hpd[1]) { 5863 DRM_DEBUG("si_irq_set: hpd 2\n"); 5864 hpd2 |= DC_HPDx_INT_EN; 5865 } 5866 if (rdev->irq.hpd[2]) { 5867 DRM_DEBUG("si_irq_set: hpd 3\n"); 5868 hpd3 |= DC_HPDx_INT_EN; 5869 } 5870 if (rdev->irq.hpd[3]) { 5871 DRM_DEBUG("si_irq_set: hpd 4\n"); 5872 hpd4 |= DC_HPDx_INT_EN; 5873 } 5874 if (rdev->irq.hpd[4]) { 5875 DRM_DEBUG("si_irq_set: hpd 5\n"); 5876 hpd5 |= DC_HPDx_INT_EN; 5877 } 5878 if (rdev->irq.hpd[5]) { 5879 DRM_DEBUG("si_irq_set: hpd 6\n"); 5880 hpd6 |= DC_HPDx_INT_EN; 5881 } 5882 5883 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 5884 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1); 5885 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2); 5886 5887 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl); 5888 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1); 5889 5890 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 5891 5892 if (rdev->irq.dpm_thermal) { 5893 DRM_DEBUG("dpm thermal\n"); 5894 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 5895 } 5896 5897 if (rdev->num_crtc >= 2) { 5898 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 5899 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 5900 } 5901 if (rdev->num_crtc >= 4) { 5902 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 5903 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 5904 } 5905 if (rdev->num_crtc >= 6) { 5906 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 5907 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 5908 } 5909 5910 if (rdev->num_crtc >= 2) { 5911 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); 5912 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); 5913 } 5914 if (rdev->num_crtc >= 4) { 5915 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); 5916 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); 5917 } 5918 if (rdev->num_crtc >= 6) { 5919 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); 5920 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); 5921 } 5922 5923 if (!ASIC_IS_NODCE(rdev)) { 5924 WREG32(DC_HPD1_INT_CONTROL, hpd1); 5925 WREG32(DC_HPD2_INT_CONTROL, hpd2); 5926 WREG32(DC_HPD3_INT_CONTROL, hpd3); 5927 WREG32(DC_HPD4_INT_CONTROL, hpd4); 5928 WREG32(DC_HPD5_INT_CONTROL, hpd5); 5929 WREG32(DC_HPD6_INT_CONTROL, hpd6); 5930 } 5931 5932 WREG32(CG_THERMAL_INT, thermal_int); 5933 5934 return 0; 5935 } 5936 5937 static inline void si_irq_ack(struct radeon_device *rdev) 5938 { 5939 u32 tmp; 5940 5941 if (ASIC_IS_NODCE(rdev)) 5942 return; 5943 5944 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 5945 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 5946 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 5947 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 5948 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 5949 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 5950 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 5951 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 5952 if (rdev->num_crtc >= 4) { 5953 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 5954 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 5955 } 5956 if (rdev->num_crtc >= 6) { 5957 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 5958 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 5959 } 5960 5961 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 5962 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5963 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 5964 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5965 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 5966 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 5967 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 5968 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 5969 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 5970 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 5971 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 5972 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 5973 5974 if (rdev->num_crtc >= 4) { 5975 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 5976 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5977 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 5978 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5979 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 5980 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 5981 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 5982 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 5983 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 5984 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 5985 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 5986 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 5987 } 5988 5989 if (rdev->num_crtc >= 6) { 5990 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 5991 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5992 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 5993 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5994 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 5995 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 5996 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 5997 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 5998 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 5999 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 6000 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 6001 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 6002 } 6003 6004 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 6005 tmp = RREG32(DC_HPD1_INT_CONTROL); 6006 tmp |= DC_HPDx_INT_ACK; 6007 WREG32(DC_HPD1_INT_CONTROL, tmp); 6008 } 6009 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 6010 tmp = RREG32(DC_HPD2_INT_CONTROL); 6011 tmp |= DC_HPDx_INT_ACK; 6012 WREG32(DC_HPD2_INT_CONTROL, tmp); 6013 } 6014 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 6015 tmp = RREG32(DC_HPD3_INT_CONTROL); 6016 tmp |= DC_HPDx_INT_ACK; 6017 WREG32(DC_HPD3_INT_CONTROL, tmp); 6018 } 6019 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 6020 tmp = RREG32(DC_HPD4_INT_CONTROL); 6021 tmp |= DC_HPDx_INT_ACK; 6022 WREG32(DC_HPD4_INT_CONTROL, tmp); 6023 } 6024 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 6025 tmp = RREG32(DC_HPD5_INT_CONTROL); 6026 tmp |= DC_HPDx_INT_ACK; 6027 WREG32(DC_HPD5_INT_CONTROL, tmp); 6028 } 6029 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 6030 tmp = RREG32(DC_HPD5_INT_CONTROL); 6031 tmp |= DC_HPDx_INT_ACK; 6032 WREG32(DC_HPD6_INT_CONTROL, tmp); 6033 } 6034 } 6035 6036 static void si_irq_disable(struct radeon_device *rdev) 6037 { 6038 si_disable_interrupts(rdev); 6039 /* Wait and acknowledge irq */ 6040 mdelay(1); 6041 si_irq_ack(rdev); 6042 si_disable_interrupt_state(rdev); 6043 } 6044 6045 static void si_irq_suspend(struct radeon_device *rdev) 6046 { 6047 si_irq_disable(rdev); 6048 si_rlc_stop(rdev); 6049 } 6050 6051 static void si_irq_fini(struct radeon_device *rdev) 6052 { 6053 si_irq_suspend(rdev); 6054 r600_ih_ring_fini(rdev); 6055 } 6056 6057 static inline u32 si_get_ih_wptr(struct radeon_device *rdev) 6058 { 6059 u32 wptr, tmp; 6060 6061 if (rdev->wb.enabled) 6062 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 6063 else 6064 wptr = RREG32(IH_RB_WPTR); 6065 6066 if (wptr & RB_OVERFLOW) { 6067 /* When a ring buffer overflow happen start parsing interrupt 6068 * from the last not overwritten vector (wptr + 16). Hopefully 6069 * this should allow us to catchup. 6070 */ 6071 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 6072 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 6073 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 6074 tmp = RREG32(IH_RB_CNTL); 6075 tmp |= IH_WPTR_OVERFLOW_CLEAR; 6076 WREG32(IH_RB_CNTL, tmp); 6077 } 6078 return (wptr & rdev->ih.ptr_mask); 6079 } 6080 6081 /* SI IV Ring 6082 * Each IV ring entry is 128 bits: 6083 * [7:0] - interrupt source id 6084 * [31:8] - reserved 6085 * [59:32] - interrupt source data 6086 * [63:60] - reserved 6087 * [71:64] - RINGID 6088 * [79:72] - VMID 6089 * [127:80] - reserved 6090 */ 6091 int si_irq_process(struct radeon_device *rdev) 6092 { 6093 u32 wptr; 6094 u32 rptr; 6095 u32 src_id, src_data, ring_id; 6096 u32 ring_index; 6097 bool queue_hotplug = false; 6098 bool queue_thermal = false; 6099 u32 status, addr; 6100 6101 if (!rdev->ih.enabled || rdev->shutdown) 6102 return IRQ_NONE; 6103 6104 wptr = si_get_ih_wptr(rdev); 6105 6106 restart_ih: 6107 /* is somebody else already processing irqs? */ 6108 if (atomic_xchg(&rdev->ih.lock, 1)) 6109 return IRQ_NONE; 6110 6111 rptr = rdev->ih.rptr; 6112 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 6113 6114 /* Order reading of wptr vs. reading of IH ring data */ 6115 rmb(); 6116 6117 /* display interrupts */ 6118 si_irq_ack(rdev); 6119 6120 while (rptr != wptr) { 6121 /* wptr/rptr are in bytes! */ 6122 ring_index = rptr / 4; 6123 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 6124 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 6125 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 6126 6127 switch (src_id) { 6128 case 1: /* D1 vblank/vline */ 6129 switch (src_data) { 6130 case 0: /* D1 vblank */ 6131 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 6132 if (rdev->irq.crtc_vblank_int[0]) { 6133 drm_handle_vblank(rdev->ddev, 0); 6134 rdev->pm.vblank_sync = true; 6135 wake_up(&rdev->irq.vblank_queue); 6136 } 6137 if (atomic_read(&rdev->irq.pflip[0])) 6138 radeon_crtc_handle_flip(rdev, 0); 6139 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 6140 DRM_DEBUG("IH: D1 vblank\n"); 6141 } 6142 break; 6143 case 1: /* D1 vline */ 6144 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 6145 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 6146 DRM_DEBUG("IH: D1 vline\n"); 6147 } 6148 break; 6149 default: 6150 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6151 break; 6152 } 6153 break; 6154 case 2: /* D2 vblank/vline */ 6155 switch (src_data) { 6156 case 0: /* D2 vblank */ 6157 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 6158 if (rdev->irq.crtc_vblank_int[1]) { 6159 drm_handle_vblank(rdev->ddev, 1); 6160 rdev->pm.vblank_sync = true; 6161 wake_up(&rdev->irq.vblank_queue); 6162 } 6163 if (atomic_read(&rdev->irq.pflip[1])) 6164 radeon_crtc_handle_flip(rdev, 1); 6165 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 6166 DRM_DEBUG("IH: D2 vblank\n"); 6167 } 6168 break; 6169 case 1: /* D2 vline */ 6170 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 6171 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 6172 DRM_DEBUG("IH: D2 vline\n"); 6173 } 6174 break; 6175 default: 6176 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6177 break; 6178 } 6179 break; 6180 case 3: /* D3 vblank/vline */ 6181 switch (src_data) { 6182 case 0: /* D3 vblank */ 6183 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 6184 if (rdev->irq.crtc_vblank_int[2]) { 6185 drm_handle_vblank(rdev->ddev, 2); 6186 rdev->pm.vblank_sync = true; 6187 wake_up(&rdev->irq.vblank_queue); 6188 } 6189 if (atomic_read(&rdev->irq.pflip[2])) 6190 radeon_crtc_handle_flip(rdev, 2); 6191 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 6192 DRM_DEBUG("IH: D3 vblank\n"); 6193 } 6194 break; 6195 case 1: /* D3 vline */ 6196 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 6197 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 6198 DRM_DEBUG("IH: D3 vline\n"); 6199 } 6200 break; 6201 default: 6202 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6203 break; 6204 } 6205 break; 6206 case 4: /* D4 vblank/vline */ 6207 switch (src_data) { 6208 case 0: /* D4 vblank */ 6209 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 6210 if (rdev->irq.crtc_vblank_int[3]) { 6211 drm_handle_vblank(rdev->ddev, 3); 6212 rdev->pm.vblank_sync = true; 6213 wake_up(&rdev->irq.vblank_queue); 6214 } 6215 if (atomic_read(&rdev->irq.pflip[3])) 6216 radeon_crtc_handle_flip(rdev, 3); 6217 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 6218 DRM_DEBUG("IH: D4 vblank\n"); 6219 } 6220 break; 6221 case 1: /* D4 vline */ 6222 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 6223 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 6224 DRM_DEBUG("IH: D4 vline\n"); 6225 } 6226 break; 6227 default: 6228 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6229 break; 6230 } 6231 break; 6232 case 5: /* D5 vblank/vline */ 6233 switch (src_data) { 6234 case 0: /* D5 vblank */ 6235 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 6236 if (rdev->irq.crtc_vblank_int[4]) { 6237 drm_handle_vblank(rdev->ddev, 4); 6238 rdev->pm.vblank_sync = true; 6239 wake_up(&rdev->irq.vblank_queue); 6240 } 6241 if (atomic_read(&rdev->irq.pflip[4])) 6242 radeon_crtc_handle_flip(rdev, 4); 6243 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 6244 DRM_DEBUG("IH: D5 vblank\n"); 6245 } 6246 break; 6247 case 1: /* D5 vline */ 6248 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 6249 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 6250 DRM_DEBUG("IH: D5 vline\n"); 6251 } 6252 break; 6253 default: 6254 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6255 break; 6256 } 6257 break; 6258 case 6: /* D6 vblank/vline */ 6259 switch (src_data) { 6260 case 0: /* D6 vblank */ 6261 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 6262 if (rdev->irq.crtc_vblank_int[5]) { 6263 drm_handle_vblank(rdev->ddev, 5); 6264 rdev->pm.vblank_sync = true; 6265 wake_up(&rdev->irq.vblank_queue); 6266 } 6267 if (atomic_read(&rdev->irq.pflip[5])) 6268 radeon_crtc_handle_flip(rdev, 5); 6269 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 6270 DRM_DEBUG("IH: D6 vblank\n"); 6271 } 6272 break; 6273 case 1: /* D6 vline */ 6274 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 6275 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 6276 DRM_DEBUG("IH: D6 vline\n"); 6277 } 6278 break; 6279 default: 6280 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6281 break; 6282 } 6283 break; 6284 case 42: /* HPD hotplug */ 6285 switch (src_data) { 6286 case 0: 6287 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 6288 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 6289 queue_hotplug = true; 6290 DRM_DEBUG("IH: HPD1\n"); 6291 } 6292 break; 6293 case 1: 6294 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 6295 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 6296 queue_hotplug = true; 6297 DRM_DEBUG("IH: HPD2\n"); 6298 } 6299 break; 6300 case 2: 6301 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 6302 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 6303 queue_hotplug = true; 6304 DRM_DEBUG("IH: HPD3\n"); 6305 } 6306 break; 6307 case 3: 6308 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 6309 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 6310 queue_hotplug = true; 6311 DRM_DEBUG("IH: HPD4\n"); 6312 } 6313 break; 6314 case 4: 6315 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 6316 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 6317 queue_hotplug = true; 6318 DRM_DEBUG("IH: HPD5\n"); 6319 } 6320 break; 6321 case 5: 6322 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 6323 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 6324 queue_hotplug = true; 6325 DRM_DEBUG("IH: HPD6\n"); 6326 } 6327 break; 6328 default: 6329 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6330 break; 6331 } 6332 break; 6333 case 124: /* UVD */ 6334 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 6335 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 6336 break; 6337 case 146: 6338 case 147: 6339 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 6340 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 6341 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 6342 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 6343 addr); 6344 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 6345 status); 6346 si_vm_decode_fault(rdev, status, addr); 6347 /* reset addr and status */ 6348 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 6349 break; 6350 case 176: /* RINGID0 CP_INT */ 6351 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6352 break; 6353 case 177: /* RINGID1 CP_INT */ 6354 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6355 break; 6356 case 178: /* RINGID2 CP_INT */ 6357 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6358 break; 6359 case 181: /* CP EOP event */ 6360 DRM_DEBUG("IH: CP EOP\n"); 6361 switch (ring_id) { 6362 case 0: 6363 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6364 break; 6365 case 1: 6366 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6367 break; 6368 case 2: 6369 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6370 break; 6371 } 6372 break; 6373 case 224: /* DMA trap event */ 6374 DRM_DEBUG("IH: DMA trap\n"); 6375 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 6376 break; 6377 case 230: /* thermal low to high */ 6378 DRM_DEBUG("IH: thermal low to high\n"); 6379 rdev->pm.dpm.thermal.high_to_low = false; 6380 queue_thermal = true; 6381 break; 6382 case 231: /* thermal high to low */ 6383 DRM_DEBUG("IH: thermal high to low\n"); 6384 rdev->pm.dpm.thermal.high_to_low = true; 6385 queue_thermal = true; 6386 break; 6387 case 233: /* GUI IDLE */ 6388 DRM_DEBUG("IH: GUI idle\n"); 6389 break; 6390 case 244: /* DMA trap event */ 6391 DRM_DEBUG("IH: DMA1 trap\n"); 6392 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6393 break; 6394 default: 6395 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6396 break; 6397 } 6398 6399 /* wptr/rptr are in bytes! */ 6400 rptr += 16; 6401 rptr &= rdev->ih.ptr_mask; 6402 } 6403 if (queue_hotplug) 6404 schedule_work(&rdev->hotplug_work); 6405 if (queue_thermal && rdev->pm.dpm_enabled) 6406 schedule_work(&rdev->pm.dpm.thermal.work); 6407 rdev->ih.rptr = rptr; 6408 WREG32(IH_RB_RPTR, rdev->ih.rptr); 6409 atomic_set(&rdev->ih.lock, 0); 6410 6411 /* make sure wptr hasn't changed while processing */ 6412 wptr = si_get_ih_wptr(rdev); 6413 if (wptr != rptr) 6414 goto restart_ih; 6415 6416 return IRQ_HANDLED; 6417 } 6418 6419 /* 6420 * startup/shutdown callbacks 6421 */ 6422 static int si_startup(struct radeon_device *rdev) 6423 { 6424 struct radeon_ring *ring; 6425 int r; 6426 6427 /* enable pcie gen2/3 link */ 6428 si_pcie_gen3_enable(rdev); 6429 /* enable aspm */ 6430 si_program_aspm(rdev); 6431 6432 /* scratch needs to be initialized before MC */ 6433 r = r600_vram_scratch_init(rdev); 6434 if (r) 6435 return r; 6436 6437 si_mc_program(rdev); 6438 6439 if (!rdev->pm.dpm_enabled) { 6440 r = si_mc_load_microcode(rdev); 6441 if (r) { 6442 DRM_ERROR("Failed to load MC firmware!\n"); 6443 return r; 6444 } 6445 } 6446 6447 r = si_pcie_gart_enable(rdev); 6448 if (r) 6449 return r; 6450 si_gpu_init(rdev); 6451 6452 /* allocate rlc buffers */ 6453 if (rdev->family == CHIP_VERDE) { 6454 rdev->rlc.reg_list = verde_rlc_save_restore_register_list; 6455 rdev->rlc.reg_list_size = 6456 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list); 6457 } 6458 rdev->rlc.cs_data = si_cs_data; 6459 r = sumo_rlc_init(rdev); 6460 if (r) { 6461 DRM_ERROR("Failed to init rlc BOs!\n"); 6462 return r; 6463 } 6464 6465 /* allocate wb buffer */ 6466 r = radeon_wb_init(rdev); 6467 if (r) 6468 return r; 6469 6470 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 6471 if (r) { 6472 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6473 return r; 6474 } 6475 6476 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6477 if (r) { 6478 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6479 return r; 6480 } 6481 6482 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6483 if (r) { 6484 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6485 return r; 6486 } 6487 6488 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 6489 if (r) { 6490 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6491 return r; 6492 } 6493 6494 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6495 if (r) { 6496 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6497 return r; 6498 } 6499 6500 if (rdev->has_uvd) { 6501 r = uvd_v2_2_resume(rdev); 6502 if (!r) { 6503 r = radeon_fence_driver_start_ring(rdev, 6504 R600_RING_TYPE_UVD_INDEX); 6505 if (r) 6506 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 6507 } 6508 if (r) 6509 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 6510 } 6511 6512 /* Enable IRQ */ 6513 if (!rdev->irq.installed) { 6514 r = radeon_irq_kms_init(rdev); 6515 if (r) 6516 return r; 6517 } 6518 6519 r = si_irq_init(rdev); 6520 if (r) { 6521 DRM_ERROR("radeon: IH init failed (%d).\n", r); 6522 radeon_irq_kms_fini(rdev); 6523 return r; 6524 } 6525 si_irq_set(rdev); 6526 6527 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6528 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 6529 RADEON_CP_PACKET2); 6530 if (r) 6531 return r; 6532 6533 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6534 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 6535 RADEON_CP_PACKET2); 6536 if (r) 6537 return r; 6538 6539 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6540 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 6541 RADEON_CP_PACKET2); 6542 if (r) 6543 return r; 6544 6545 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6546 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 6547 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6548 if (r) 6549 return r; 6550 6551 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6552 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 6553 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6554 if (r) 6555 return r; 6556 6557 r = si_cp_load_microcode(rdev); 6558 if (r) 6559 return r; 6560 r = si_cp_resume(rdev); 6561 if (r) 6562 return r; 6563 6564 r = cayman_dma_resume(rdev); 6565 if (r) 6566 return r; 6567 6568 if (rdev->has_uvd) { 6569 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6570 if (ring->ring_size) { 6571 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 6572 RADEON_CP_PACKET2); 6573 if (!r) 6574 r = uvd_v1_0_init(rdev); 6575 if (r) 6576 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 6577 } 6578 } 6579 6580 r = radeon_ib_pool_init(rdev); 6581 if (r) { 6582 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 6583 return r; 6584 } 6585 6586 r = radeon_vm_manager_init(rdev); 6587 if (r) { 6588 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 6589 return r; 6590 } 6591 6592 r = dce6_audio_init(rdev); 6593 if (r) 6594 return r; 6595 6596 return 0; 6597 } 6598 6599 int si_resume(struct radeon_device *rdev) 6600 { 6601 int r; 6602 6603 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 6604 * posting will perform necessary task to bring back GPU into good 6605 * shape. 6606 */ 6607 /* post card */ 6608 atom_asic_init(rdev->mode_info.atom_context); 6609 6610 /* init golden registers */ 6611 si_init_golden_registers(rdev); 6612 6613 if (rdev->pm.pm_method == PM_METHOD_DPM) 6614 radeon_pm_resume(rdev); 6615 6616 rdev->accel_working = true; 6617 r = si_startup(rdev); 6618 if (r) { 6619 DRM_ERROR("si startup failed on resume\n"); 6620 rdev->accel_working = false; 6621 return r; 6622 } 6623 6624 return r; 6625 6626 } 6627 6628 int si_suspend(struct radeon_device *rdev) 6629 { 6630 radeon_pm_suspend(rdev); 6631 dce6_audio_fini(rdev); 6632 radeon_vm_manager_fini(rdev); 6633 si_cp_enable(rdev, false); 6634 cayman_dma_stop(rdev); 6635 if (rdev->has_uvd) { 6636 uvd_v1_0_fini(rdev); 6637 radeon_uvd_suspend(rdev); 6638 } 6639 si_fini_pg(rdev); 6640 si_fini_cg(rdev); 6641 si_irq_suspend(rdev); 6642 radeon_wb_disable(rdev); 6643 si_pcie_gart_disable(rdev); 6644 return 0; 6645 } 6646 6647 /* Plan is to move initialization in that function and use 6648 * helper function so that radeon_device_init pretty much 6649 * do nothing more than calling asic specific function. This 6650 * should also allow to remove a bunch of callback function 6651 * like vram_info. 6652 */ 6653 int si_init(struct radeon_device *rdev) 6654 { 6655 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6656 int r; 6657 6658 /* Read BIOS */ 6659 if (!radeon_get_bios(rdev)) { 6660 if (ASIC_IS_AVIVO(rdev)) 6661 return -EINVAL; 6662 } 6663 /* Must be an ATOMBIOS */ 6664 if (!rdev->is_atom_bios) { 6665 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 6666 return -EINVAL; 6667 } 6668 r = radeon_atombios_init(rdev); 6669 if (r) 6670 return r; 6671 6672 /* Post card if necessary */ 6673 if (!radeon_card_posted(rdev)) { 6674 if (!rdev->bios) { 6675 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 6676 return -EINVAL; 6677 } 6678 DRM_INFO("GPU not posted. posting now...\n"); 6679 atom_asic_init(rdev->mode_info.atom_context); 6680 } 6681 /* init golden registers */ 6682 si_init_golden_registers(rdev); 6683 /* Initialize scratch registers */ 6684 si_scratch_init(rdev); 6685 /* Initialize surface registers */ 6686 radeon_surface_init(rdev); 6687 /* Initialize clocks */ 6688 radeon_get_clock_info(rdev->ddev); 6689 6690 /* Fence driver */ 6691 r = radeon_fence_driver_init(rdev); 6692 if (r) 6693 return r; 6694 6695 /* initialize memory controller */ 6696 r = si_mc_init(rdev); 6697 if (r) 6698 return r; 6699 /* Memory manager */ 6700 r = radeon_bo_init(rdev); 6701 if (r) 6702 return r; 6703 6704 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 6705 !rdev->rlc_fw || !rdev->mc_fw) { 6706 r = si_init_microcode(rdev); 6707 if (r) { 6708 DRM_ERROR("Failed to load firmware!\n"); 6709 return r; 6710 } 6711 } 6712 6713 /* Initialize power management */ 6714 radeon_pm_init(rdev); 6715 6716 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6717 ring->ring_obj = NULL; 6718 r600_ring_init(rdev, ring, 1024 * 1024); 6719 6720 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6721 ring->ring_obj = NULL; 6722 r600_ring_init(rdev, ring, 1024 * 1024); 6723 6724 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6725 ring->ring_obj = NULL; 6726 r600_ring_init(rdev, ring, 1024 * 1024); 6727 6728 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6729 ring->ring_obj = NULL; 6730 r600_ring_init(rdev, ring, 64 * 1024); 6731 6732 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6733 ring->ring_obj = NULL; 6734 r600_ring_init(rdev, ring, 64 * 1024); 6735 6736 if (rdev->has_uvd) { 6737 r = radeon_uvd_init(rdev); 6738 if (!r) { 6739 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6740 ring->ring_obj = NULL; 6741 r600_ring_init(rdev, ring, 4096); 6742 } 6743 } 6744 6745 rdev->ih.ring_obj = NULL; 6746 r600_ih_ring_init(rdev, 64 * 1024); 6747 6748 r = r600_pcie_gart_init(rdev); 6749 if (r) 6750 return r; 6751 6752 rdev->accel_working = true; 6753 r = si_startup(rdev); 6754 if (r) { 6755 dev_err(rdev->dev, "disabling GPU acceleration\n"); 6756 si_cp_fini(rdev); 6757 cayman_dma_fini(rdev); 6758 si_irq_fini(rdev); 6759 sumo_rlc_fini(rdev); 6760 radeon_wb_fini(rdev); 6761 radeon_ib_pool_fini(rdev); 6762 radeon_vm_manager_fini(rdev); 6763 radeon_irq_kms_fini(rdev); 6764 si_pcie_gart_fini(rdev); 6765 rdev->accel_working = false; 6766 } 6767 6768 /* Don't start up if the MC ucode is missing. 6769 * The default clocks and voltages before the MC ucode 6770 * is loaded are not suffient for advanced operations. 6771 */ 6772 if (!rdev->mc_fw) { 6773 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 6774 return -EINVAL; 6775 } 6776 6777 return 0; 6778 } 6779 6780 void si_fini(struct radeon_device *rdev) 6781 { 6782 radeon_pm_fini(rdev); 6783 si_cp_fini(rdev); 6784 cayman_dma_fini(rdev); 6785 si_fini_pg(rdev); 6786 si_fini_cg(rdev); 6787 si_irq_fini(rdev); 6788 sumo_rlc_fini(rdev); 6789 radeon_wb_fini(rdev); 6790 radeon_vm_manager_fini(rdev); 6791 radeon_ib_pool_fini(rdev); 6792 radeon_irq_kms_fini(rdev); 6793 if (rdev->has_uvd) { 6794 uvd_v1_0_fini(rdev); 6795 radeon_uvd_fini(rdev); 6796 } 6797 si_pcie_gart_fini(rdev); 6798 r600_vram_scratch_fini(rdev); 6799 radeon_gem_fini(rdev); 6800 radeon_fence_driver_fini(rdev); 6801 radeon_bo_fini(rdev); 6802 radeon_atombios_fini(rdev); 6803 kfree(rdev->bios); 6804 rdev->bios = NULL; 6805 } 6806 6807 /** 6808 * si_get_gpu_clock_counter - return GPU clock counter snapshot 6809 * 6810 * @rdev: radeon_device pointer 6811 * 6812 * Fetches a GPU clock counter snapshot (SI). 6813 * Returns the 64 bit clock counter snapshot. 6814 */ 6815 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev) 6816 { 6817 uint64_t clock; 6818 6819 mutex_lock(&rdev->gpu_clock_mutex); 6820 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 6821 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 6822 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 6823 mutex_unlock(&rdev->gpu_clock_mutex); 6824 return clock; 6825 } 6826 6827 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 6828 { 6829 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 6830 int r; 6831 6832 /* bypass vclk and dclk with bclk */ 6833 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6834 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 6835 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 6836 6837 /* put PLL in bypass mode */ 6838 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 6839 6840 if (!vclk || !dclk) { 6841 /* keep the Bypass mode, put PLL to sleep */ 6842 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 6843 return 0; 6844 } 6845 6846 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 6847 16384, 0x03FFFFFF, 0, 128, 5, 6848 &fb_div, &vclk_div, &dclk_div); 6849 if (r) 6850 return r; 6851 6852 /* set RESET_ANTI_MUX to 0 */ 6853 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK); 6854 6855 /* set VCO_MODE to 1 */ 6856 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 6857 6858 /* toggle UPLL_SLEEP to 1 then back to 0 */ 6859 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 6860 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 6861 6862 /* deassert UPLL_RESET */ 6863 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 6864 6865 mdelay(1); 6866 6867 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 6868 if (r) 6869 return r; 6870 6871 /* assert UPLL_RESET again */ 6872 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 6873 6874 /* disable spread spectrum. */ 6875 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 6876 6877 /* set feedback divider */ 6878 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 6879 6880 /* set ref divider to 0 */ 6881 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 6882 6883 if (fb_div < 307200) 6884 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 6885 else 6886 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 6887 6888 /* set PDIV_A and PDIV_B */ 6889 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6890 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 6891 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 6892 6893 /* give the PLL some time to settle */ 6894 mdelay(15); 6895 6896 /* deassert PLL_RESET */ 6897 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 6898 6899 mdelay(15); 6900 6901 /* switch from bypass mode to normal mode */ 6902 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 6903 6904 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 6905 if (r) 6906 return r; 6907 6908 /* switch VCLK and DCLK selection */ 6909 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6910 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 6911 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 6912 6913 mdelay(100); 6914 6915 return 0; 6916 } 6917 6918 static void si_pcie_gen3_enable(struct radeon_device *rdev) 6919 { 6920 struct pci_dev *root = rdev->pdev->bus->self; 6921 int bridge_pos, gpu_pos; 6922 u32 speed_cntl, mask, current_data_rate; 6923 int ret, i; 6924 u16 tmp16; 6925 6926 if (radeon_pcie_gen2 == 0) 6927 return; 6928 6929 if (rdev->flags & RADEON_IS_IGP) 6930 return; 6931 6932 if (!(rdev->flags & RADEON_IS_PCIE)) 6933 return; 6934 6935 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 6936 if (ret != 0) 6937 return; 6938 6939 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 6940 return; 6941 6942 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 6943 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 6944 LC_CURRENT_DATA_RATE_SHIFT; 6945 if (mask & DRM_PCIE_SPEED_80) { 6946 if (current_data_rate == 2) { 6947 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 6948 return; 6949 } 6950 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 6951 } else if (mask & DRM_PCIE_SPEED_50) { 6952 if (current_data_rate == 1) { 6953 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 6954 return; 6955 } 6956 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 6957 } 6958 6959 bridge_pos = pci_pcie_cap(root); 6960 if (!bridge_pos) 6961 return; 6962 6963 gpu_pos = pci_pcie_cap(rdev->pdev); 6964 if (!gpu_pos) 6965 return; 6966 6967 if (mask & DRM_PCIE_SPEED_80) { 6968 /* re-try equalization if gen3 is not already enabled */ 6969 if (current_data_rate != 2) { 6970 u16 bridge_cfg, gpu_cfg; 6971 u16 bridge_cfg2, gpu_cfg2; 6972 u32 max_lw, current_lw, tmp; 6973 6974 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 6975 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 6976 6977 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 6978 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 6979 6980 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 6981 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 6982 6983 tmp = RREG32_PCIE(PCIE_LC_STATUS1); 6984 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 6985 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 6986 6987 if (current_lw < max_lw) { 6988 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 6989 if (tmp & LC_RENEGOTIATION_SUPPORT) { 6990 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 6991 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 6992 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 6993 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 6994 } 6995 } 6996 6997 for (i = 0; i < 10; i++) { 6998 /* check status */ 6999 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 7000 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 7001 break; 7002 7003 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 7004 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 7005 7006 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 7007 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 7008 7009 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7010 tmp |= LC_SET_QUIESCE; 7011 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7012 7013 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7014 tmp |= LC_REDO_EQ; 7015 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7016 7017 mdelay(100); 7018 7019 /* linkctl */ 7020 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 7021 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 7022 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 7023 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 7024 7025 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 7026 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 7027 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 7028 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 7029 7030 /* linkctl2 */ 7031 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 7032 tmp16 &= ~((1 << 4) | (7 << 9)); 7033 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 7034 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 7035 7036 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 7037 tmp16 &= ~((1 << 4) | (7 << 9)); 7038 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 7039 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 7040 7041 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 7042 tmp &= ~LC_SET_QUIESCE; 7043 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 7044 } 7045 } 7046 } 7047 7048 /* set the link speed */ 7049 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 7050 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 7051 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 7052 7053 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 7054 tmp16 &= ~0xf; 7055 if (mask & DRM_PCIE_SPEED_80) 7056 tmp16 |= 3; /* gen3 */ 7057 else if (mask & DRM_PCIE_SPEED_50) 7058 tmp16 |= 2; /* gen2 */ 7059 else 7060 tmp16 |= 1; /* gen1 */ 7061 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 7062 7063 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 7064 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 7065 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 7066 7067 for (i = 0; i < rdev->usec_timeout; i++) { 7068 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 7069 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 7070 break; 7071 udelay(1); 7072 } 7073 } 7074 7075 static void si_program_aspm(struct radeon_device *rdev) 7076 { 7077 u32 data, orig; 7078 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 7079 bool disable_clkreq = false; 7080 7081 if (radeon_aspm == 0) 7082 return; 7083 7084 if (!(rdev->flags & RADEON_IS_PCIE)) 7085 return; 7086 7087 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7088 data &= ~LC_XMIT_N_FTS_MASK; 7089 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 7090 if (orig != data) 7091 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 7092 7093 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 7094 data |= LC_GO_TO_RECOVERY; 7095 if (orig != data) 7096 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 7097 7098 orig = data = RREG32_PCIE(PCIE_P_CNTL); 7099 data |= P_IGNORE_EDB_ERR; 7100 if (orig != data) 7101 WREG32_PCIE(PCIE_P_CNTL, data); 7102 7103 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7104 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 7105 data |= LC_PMI_TO_L1_DIS; 7106 if (!disable_l0s) 7107 data |= LC_L0S_INACTIVITY(7); 7108 7109 if (!disable_l1) { 7110 data |= LC_L1_INACTIVITY(7); 7111 data &= ~LC_PMI_TO_L1_DIS; 7112 if (orig != data) 7113 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7114 7115 if (!disable_plloff_in_l1) { 7116 bool clk_req_support; 7117 7118 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7119 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7120 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7121 if (orig != data) 7122 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7123 7124 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7125 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7126 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7127 if (orig != data) 7128 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7129 7130 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7131 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7132 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7133 if (orig != data) 7134 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7135 7136 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7137 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7138 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7139 if (orig != data) 7140 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7141 7142 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) { 7143 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7144 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7145 if (orig != data) 7146 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7147 7148 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7149 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7150 if (orig != data) 7151 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7152 7153 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2); 7154 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7155 if (orig != data) 7156 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data); 7157 7158 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3); 7159 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7160 if (orig != data) 7161 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data); 7162 7163 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7164 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7165 if (orig != data) 7166 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7167 7168 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7169 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7170 if (orig != data) 7171 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7172 7173 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2); 7174 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7175 if (orig != data) 7176 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data); 7177 7178 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3); 7179 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7180 if (orig != data) 7181 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data); 7182 } 7183 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 7184 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 7185 data |= LC_DYN_LANES_PWR_STATE(3); 7186 if (orig != data) 7187 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 7188 7189 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL); 7190 data &= ~LS2_EXIT_TIME_MASK; 7191 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7192 data |= LS2_EXIT_TIME(5); 7193 if (orig != data) 7194 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 7195 7196 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL); 7197 data &= ~LS2_EXIT_TIME_MASK; 7198 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7199 data |= LS2_EXIT_TIME(5); 7200 if (orig != data) 7201 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 7202 7203 if (!disable_clkreq) { 7204 struct pci_dev *root = rdev->pdev->bus->self; 7205 u32 lnkcap; 7206 7207 clk_req_support = false; 7208 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 7209 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 7210 clk_req_support = true; 7211 } else { 7212 clk_req_support = false; 7213 } 7214 7215 if (clk_req_support) { 7216 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 7217 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 7218 if (orig != data) 7219 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 7220 7221 orig = data = RREG32(THM_CLK_CNTL); 7222 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 7223 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 7224 if (orig != data) 7225 WREG32(THM_CLK_CNTL, data); 7226 7227 orig = data = RREG32(MISC_CLK_CNTL); 7228 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 7229 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 7230 if (orig != data) 7231 WREG32(MISC_CLK_CNTL, data); 7232 7233 orig = data = RREG32(CG_CLKPIN_CNTL); 7234 data &= ~BCLK_AS_XCLK; 7235 if (orig != data) 7236 WREG32(CG_CLKPIN_CNTL, data); 7237 7238 orig = data = RREG32(CG_CLKPIN_CNTL_2); 7239 data &= ~FORCE_BIF_REFCLK_EN; 7240 if (orig != data) 7241 WREG32(CG_CLKPIN_CNTL_2, data); 7242 7243 orig = data = RREG32(MPLL_BYPASSCLK_SEL); 7244 data &= ~MPLL_CLKOUT_SEL_MASK; 7245 data |= MPLL_CLKOUT_SEL(4); 7246 if (orig != data) 7247 WREG32(MPLL_BYPASSCLK_SEL, data); 7248 7249 orig = data = RREG32(SPLL_CNTL_MODE); 7250 data &= ~SPLL_REFCLK_SEL_MASK; 7251 if (orig != data) 7252 WREG32(SPLL_CNTL_MODE, data); 7253 } 7254 } 7255 } else { 7256 if (orig != data) 7257 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7258 } 7259 7260 orig = data = RREG32_PCIE(PCIE_CNTL2); 7261 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 7262 if (orig != data) 7263 WREG32_PCIE(PCIE_CNTL2, data); 7264 7265 if (!disable_l0s) { 7266 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7267 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 7268 data = RREG32_PCIE(PCIE_LC_STATUS1); 7269 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 7270 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7271 data &= ~LC_L0S_INACTIVITY_MASK; 7272 if (orig != data) 7273 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7274 } 7275 } 7276 } 7277 } 7278