1 /* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include <drm/drmP.h> 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include <drm/radeon_drm.h> 31 #include "nid.h" 32 #include "atom.h" 33 #include "ni_reg.h" 34 #include "cayman_blit_shaders.h" 35 #include "radeon_ucode.h" 36 #include "clearstate_cayman.h" 37 38 static u32 tn_rlc_save_restore_register_list[] = 39 { 40 0x98fc, 41 0x98f0, 42 0x9834, 43 0x9838, 44 0x9870, 45 0x9874, 46 0x8a14, 47 0x8b24, 48 0x8bcc, 49 0x8b10, 50 0x8c30, 51 0x8d00, 52 0x8d04, 53 0x8c00, 54 0x8c04, 55 0x8c10, 56 0x8c14, 57 0x8d8c, 58 0x8cf0, 59 0x8e38, 60 0x9508, 61 0x9688, 62 0x9608, 63 0x960c, 64 0x9610, 65 0x9614, 66 0x88c4, 67 0x8978, 68 0x88d4, 69 0x900c, 70 0x9100, 71 0x913c, 72 0x90e8, 73 0x9354, 74 0xa008, 75 0x98f8, 76 0x9148, 77 0x914c, 78 0x3f94, 79 0x98f4, 80 0x9b7c, 81 0x3f8c, 82 0x8950, 83 0x8954, 84 0x8a18, 85 0x8b28, 86 0x9144, 87 0x3f90, 88 0x915c, 89 0x9160, 90 0x9178, 91 0x917c, 92 0x9180, 93 0x918c, 94 0x9190, 95 0x9194, 96 0x9198, 97 0x919c, 98 0x91a8, 99 0x91ac, 100 0x91b0, 101 0x91b4, 102 0x91b8, 103 0x91c4, 104 0x91c8, 105 0x91cc, 106 0x91d0, 107 0x91d4, 108 0x91e0, 109 0x91e4, 110 0x91ec, 111 0x91f0, 112 0x91f4, 113 0x9200, 114 0x9204, 115 0x929c, 116 0x8030, 117 0x9150, 118 0x9a60, 119 0x920c, 120 0x9210, 121 0x9228, 122 0x922c, 123 0x9244, 124 0x9248, 125 0x91e8, 126 0x9294, 127 0x9208, 128 0x9224, 129 0x9240, 130 0x9220, 131 0x923c, 132 0x9258, 133 0x9744, 134 0xa200, 135 0xa204, 136 0xa208, 137 0xa20c, 138 0x8d58, 139 0x9030, 140 0x9034, 141 0x9038, 142 0x903c, 143 0x9040, 144 0x9654, 145 0x897c, 146 0xa210, 147 0xa214, 148 0x9868, 149 0xa02c, 150 0x9664, 151 0x9698, 152 0x949c, 153 0x8e10, 154 0x8e18, 155 0x8c50, 156 0x8c58, 157 0x8c60, 158 0x8c68, 159 0x89b4, 160 0x9830, 161 0x802c, 162 }; 163 static u32 tn_rlc_save_restore_register_list_size = ARRAY_SIZE(tn_rlc_save_restore_register_list); 164 165 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 166 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev); 167 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 168 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 169 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev); 170 extern void evergreen_mc_program(struct radeon_device *rdev); 171 extern void evergreen_irq_suspend(struct radeon_device *rdev); 172 extern int evergreen_mc_init(struct radeon_device *rdev); 173 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev); 174 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 175 extern void evergreen_program_aspm(struct radeon_device *rdev); 176 extern void sumo_rlc_fini(struct radeon_device *rdev); 177 extern int sumo_rlc_init(struct radeon_device *rdev); 178 179 /* Firmware Names */ 180 MODULE_FIRMWARE("radeon/BARTS_pfp.bin"); 181 MODULE_FIRMWARE("radeon/BARTS_me.bin"); 182 MODULE_FIRMWARE("radeon/BARTS_mc.bin"); 183 MODULE_FIRMWARE("radeon/BARTS_smc.bin"); 184 MODULE_FIRMWARE("radeon/BTC_rlc.bin"); 185 MODULE_FIRMWARE("radeon/TURKS_pfp.bin"); 186 MODULE_FIRMWARE("radeon/TURKS_me.bin"); 187 MODULE_FIRMWARE("radeon/TURKS_mc.bin"); 188 MODULE_FIRMWARE("radeon/TURKS_smc.bin"); 189 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin"); 190 MODULE_FIRMWARE("radeon/CAICOS_me.bin"); 191 MODULE_FIRMWARE("radeon/CAICOS_mc.bin"); 192 MODULE_FIRMWARE("radeon/CAICOS_smc.bin"); 193 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin"); 194 MODULE_FIRMWARE("radeon/CAYMAN_me.bin"); 195 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin"); 196 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin"); 197 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin"); 198 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin"); 199 MODULE_FIRMWARE("radeon/ARUBA_me.bin"); 200 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin"); 201 202 203 static const u32 cayman_golden_registers2[] = 204 { 205 0x3e5c, 0xffffffff, 0x00000000, 206 0x3e48, 0xffffffff, 0x00000000, 207 0x3e4c, 0xffffffff, 0x00000000, 208 0x3e64, 0xffffffff, 0x00000000, 209 0x3e50, 0xffffffff, 0x00000000, 210 0x3e60, 0xffffffff, 0x00000000 211 }; 212 213 static const u32 cayman_golden_registers[] = 214 { 215 0x5eb4, 0xffffffff, 0x00000002, 216 0x5e78, 0x8f311ff1, 0x001000f0, 217 0x3f90, 0xffff0000, 0xff000000, 218 0x9148, 0xffff0000, 0xff000000, 219 0x3f94, 0xffff0000, 0xff000000, 220 0x914c, 0xffff0000, 0xff000000, 221 0xc78, 0x00000080, 0x00000080, 222 0xbd4, 0x70073777, 0x00011003, 223 0xd02c, 0xbfffff1f, 0x08421000, 224 0xd0b8, 0x73773777, 0x02011003, 225 0x5bc0, 0x00200000, 0x50100000, 226 0x98f8, 0x33773777, 0x02011003, 227 0x98fc, 0xffffffff, 0x76541032, 228 0x7030, 0x31000311, 0x00000011, 229 0x2f48, 0x33773777, 0x42010001, 230 0x6b28, 0x00000010, 0x00000012, 231 0x7728, 0x00000010, 0x00000012, 232 0x10328, 0x00000010, 0x00000012, 233 0x10f28, 0x00000010, 0x00000012, 234 0x11b28, 0x00000010, 0x00000012, 235 0x12728, 0x00000010, 0x00000012, 236 0x240c, 0x000007ff, 0x00000000, 237 0x8a14, 0xf000001f, 0x00000007, 238 0x8b24, 0x3fff3fff, 0x00ff0fff, 239 0x8b10, 0x0000ff0f, 0x00000000, 240 0x28a4c, 0x07ffffff, 0x06000000, 241 0x10c, 0x00000001, 0x00010003, 242 0xa02c, 0xffffffff, 0x0000009b, 243 0x913c, 0x0000010f, 0x01000100, 244 0x8c04, 0xf8ff00ff, 0x40600060, 245 0x28350, 0x00000f01, 0x00000000, 246 0x9508, 0x3700001f, 0x00000002, 247 0x960c, 0xffffffff, 0x54763210, 248 0x88c4, 0x001f3ae3, 0x00000082, 249 0x88d0, 0xffffffff, 0x0f40df40, 250 0x88d4, 0x0000001f, 0x00000010, 251 0x8974, 0xffffffff, 0x00000000 252 }; 253 254 static const u32 dvst_golden_registers2[] = 255 { 256 0x8f8, 0xffffffff, 0, 257 0x8fc, 0x00380000, 0, 258 0x8f8, 0xffffffff, 1, 259 0x8fc, 0x0e000000, 0 260 }; 261 262 static const u32 dvst_golden_registers[] = 263 { 264 0x690, 0x3fff3fff, 0x20c00033, 265 0x918c, 0x0fff0fff, 0x00010006, 266 0x91a8, 0x0fff0fff, 0x00010006, 267 0x9150, 0xffffdfff, 0x6e944040, 268 0x917c, 0x0fff0fff, 0x00030002, 269 0x9198, 0x0fff0fff, 0x00030002, 270 0x915c, 0x0fff0fff, 0x00010000, 271 0x3f90, 0xffff0001, 0xff000000, 272 0x9178, 0x0fff0fff, 0x00070000, 273 0x9194, 0x0fff0fff, 0x00070000, 274 0x9148, 0xffff0001, 0xff000000, 275 0x9190, 0x0fff0fff, 0x00090008, 276 0x91ac, 0x0fff0fff, 0x00090008, 277 0x3f94, 0xffff0000, 0xff000000, 278 0x914c, 0xffff0000, 0xff000000, 279 0x929c, 0x00000fff, 0x00000001, 280 0x55e4, 0xff607fff, 0xfc000100, 281 0x8a18, 0xff000fff, 0x00000100, 282 0x8b28, 0xff000fff, 0x00000100, 283 0x9144, 0xfffc0fff, 0x00000100, 284 0x6ed8, 0x00010101, 0x00010000, 285 0x9830, 0xffffffff, 0x00000000, 286 0x9834, 0xf00fffff, 0x00000400, 287 0x9838, 0xfffffffe, 0x00000000, 288 0xd0c0, 0xff000fff, 0x00000100, 289 0xd02c, 0xbfffff1f, 0x08421000, 290 0xd0b8, 0x73773777, 0x12010001, 291 0x5bb0, 0x000000f0, 0x00000070, 292 0x98f8, 0x73773777, 0x12010001, 293 0x98fc, 0xffffffff, 0x00000010, 294 0x9b7c, 0x00ff0000, 0x00fc0000, 295 0x8030, 0x00001f0f, 0x0000100a, 296 0x2f48, 0x73773777, 0x12010001, 297 0x2408, 0x00030000, 0x000c007f, 298 0x8a14, 0xf000003f, 0x00000007, 299 0x8b24, 0x3fff3fff, 0x00ff0fff, 300 0x8b10, 0x0000ff0f, 0x00000000, 301 0x28a4c, 0x07ffffff, 0x06000000, 302 0x4d8, 0x00000fff, 0x00000100, 303 0xa008, 0xffffffff, 0x00010000, 304 0x913c, 0xffff03ff, 0x01000100, 305 0x8c00, 0x000000ff, 0x00000003, 306 0x8c04, 0xf8ff00ff, 0x40600060, 307 0x8cf0, 0x1fff1fff, 0x08e00410, 308 0x28350, 0x00000f01, 0x00000000, 309 0x9508, 0xf700071f, 0x00000002, 310 0x960c, 0xffffffff, 0x54763210, 311 0x20ef8, 0x01ff01ff, 0x00000002, 312 0x20e98, 0xfffffbff, 0x00200000, 313 0x2015c, 0xffffffff, 0x00000f40, 314 0x88c4, 0x001f3ae3, 0x00000082, 315 0x8978, 0x3fffffff, 0x04050140, 316 0x88d4, 0x0000001f, 0x00000010, 317 0x8974, 0xffffffff, 0x00000000 318 }; 319 320 static const u32 scrapper_golden_registers[] = 321 { 322 0x690, 0x3fff3fff, 0x20c00033, 323 0x918c, 0x0fff0fff, 0x00010006, 324 0x918c, 0x0fff0fff, 0x00010006, 325 0x91a8, 0x0fff0fff, 0x00010006, 326 0x91a8, 0x0fff0fff, 0x00010006, 327 0x9150, 0xffffdfff, 0x6e944040, 328 0x9150, 0xffffdfff, 0x6e944040, 329 0x917c, 0x0fff0fff, 0x00030002, 330 0x917c, 0x0fff0fff, 0x00030002, 331 0x9198, 0x0fff0fff, 0x00030002, 332 0x9198, 0x0fff0fff, 0x00030002, 333 0x915c, 0x0fff0fff, 0x00010000, 334 0x915c, 0x0fff0fff, 0x00010000, 335 0x3f90, 0xffff0001, 0xff000000, 336 0x3f90, 0xffff0001, 0xff000000, 337 0x9178, 0x0fff0fff, 0x00070000, 338 0x9178, 0x0fff0fff, 0x00070000, 339 0x9194, 0x0fff0fff, 0x00070000, 340 0x9194, 0x0fff0fff, 0x00070000, 341 0x9148, 0xffff0001, 0xff000000, 342 0x9148, 0xffff0001, 0xff000000, 343 0x9190, 0x0fff0fff, 0x00090008, 344 0x9190, 0x0fff0fff, 0x00090008, 345 0x91ac, 0x0fff0fff, 0x00090008, 346 0x91ac, 0x0fff0fff, 0x00090008, 347 0x3f94, 0xffff0000, 0xff000000, 348 0x3f94, 0xffff0000, 0xff000000, 349 0x914c, 0xffff0000, 0xff000000, 350 0x914c, 0xffff0000, 0xff000000, 351 0x929c, 0x00000fff, 0x00000001, 352 0x929c, 0x00000fff, 0x00000001, 353 0x55e4, 0xff607fff, 0xfc000100, 354 0x8a18, 0xff000fff, 0x00000100, 355 0x8a18, 0xff000fff, 0x00000100, 356 0x8b28, 0xff000fff, 0x00000100, 357 0x8b28, 0xff000fff, 0x00000100, 358 0x9144, 0xfffc0fff, 0x00000100, 359 0x9144, 0xfffc0fff, 0x00000100, 360 0x6ed8, 0x00010101, 0x00010000, 361 0x9830, 0xffffffff, 0x00000000, 362 0x9830, 0xffffffff, 0x00000000, 363 0x9834, 0xf00fffff, 0x00000400, 364 0x9834, 0xf00fffff, 0x00000400, 365 0x9838, 0xfffffffe, 0x00000000, 366 0x9838, 0xfffffffe, 0x00000000, 367 0xd0c0, 0xff000fff, 0x00000100, 368 0xd02c, 0xbfffff1f, 0x08421000, 369 0xd02c, 0xbfffff1f, 0x08421000, 370 0xd0b8, 0x73773777, 0x12010001, 371 0xd0b8, 0x73773777, 0x12010001, 372 0x5bb0, 0x000000f0, 0x00000070, 373 0x98f8, 0x73773777, 0x12010001, 374 0x98f8, 0x73773777, 0x12010001, 375 0x98fc, 0xffffffff, 0x00000010, 376 0x98fc, 0xffffffff, 0x00000010, 377 0x9b7c, 0x00ff0000, 0x00fc0000, 378 0x9b7c, 0x00ff0000, 0x00fc0000, 379 0x8030, 0x00001f0f, 0x0000100a, 380 0x8030, 0x00001f0f, 0x0000100a, 381 0x2f48, 0x73773777, 0x12010001, 382 0x2f48, 0x73773777, 0x12010001, 383 0x2408, 0x00030000, 0x000c007f, 384 0x8a14, 0xf000003f, 0x00000007, 385 0x8a14, 0xf000003f, 0x00000007, 386 0x8b24, 0x3fff3fff, 0x00ff0fff, 387 0x8b24, 0x3fff3fff, 0x00ff0fff, 388 0x8b10, 0x0000ff0f, 0x00000000, 389 0x8b10, 0x0000ff0f, 0x00000000, 390 0x28a4c, 0x07ffffff, 0x06000000, 391 0x28a4c, 0x07ffffff, 0x06000000, 392 0x4d8, 0x00000fff, 0x00000100, 393 0x4d8, 0x00000fff, 0x00000100, 394 0xa008, 0xffffffff, 0x00010000, 395 0xa008, 0xffffffff, 0x00010000, 396 0x913c, 0xffff03ff, 0x01000100, 397 0x913c, 0xffff03ff, 0x01000100, 398 0x90e8, 0x001fffff, 0x010400c0, 399 0x8c00, 0x000000ff, 0x00000003, 400 0x8c00, 0x000000ff, 0x00000003, 401 0x8c04, 0xf8ff00ff, 0x40600060, 402 0x8c04, 0xf8ff00ff, 0x40600060, 403 0x8c30, 0x0000000f, 0x00040005, 404 0x8cf0, 0x1fff1fff, 0x08e00410, 405 0x8cf0, 0x1fff1fff, 0x08e00410, 406 0x900c, 0x00ffffff, 0x0017071f, 407 0x28350, 0x00000f01, 0x00000000, 408 0x28350, 0x00000f01, 0x00000000, 409 0x9508, 0xf700071f, 0x00000002, 410 0x9508, 0xf700071f, 0x00000002, 411 0x9688, 0x00300000, 0x0017000f, 412 0x960c, 0xffffffff, 0x54763210, 413 0x960c, 0xffffffff, 0x54763210, 414 0x20ef8, 0x01ff01ff, 0x00000002, 415 0x20e98, 0xfffffbff, 0x00200000, 416 0x2015c, 0xffffffff, 0x00000f40, 417 0x88c4, 0x001f3ae3, 0x00000082, 418 0x88c4, 0x001f3ae3, 0x00000082, 419 0x8978, 0x3fffffff, 0x04050140, 420 0x8978, 0x3fffffff, 0x04050140, 421 0x88d4, 0x0000001f, 0x00000010, 422 0x88d4, 0x0000001f, 0x00000010, 423 0x8974, 0xffffffff, 0x00000000, 424 0x8974, 0xffffffff, 0x00000000 425 }; 426 427 static void ni_init_golden_registers(struct radeon_device *rdev) 428 { 429 switch (rdev->family) { 430 case CHIP_CAYMAN: 431 radeon_program_register_sequence(rdev, 432 cayman_golden_registers, 433 (const u32)ARRAY_SIZE(cayman_golden_registers)); 434 radeon_program_register_sequence(rdev, 435 cayman_golden_registers2, 436 (const u32)ARRAY_SIZE(cayman_golden_registers2)); 437 break; 438 case CHIP_ARUBA: 439 if ((rdev->pdev->device == 0x9900) || 440 (rdev->pdev->device == 0x9901) || 441 (rdev->pdev->device == 0x9903) || 442 (rdev->pdev->device == 0x9904) || 443 (rdev->pdev->device == 0x9905) || 444 (rdev->pdev->device == 0x9906) || 445 (rdev->pdev->device == 0x9907) || 446 (rdev->pdev->device == 0x9908) || 447 (rdev->pdev->device == 0x9909) || 448 (rdev->pdev->device == 0x990A) || 449 (rdev->pdev->device == 0x990B) || 450 (rdev->pdev->device == 0x990C) || 451 (rdev->pdev->device == 0x990D) || 452 (rdev->pdev->device == 0x990E) || 453 (rdev->pdev->device == 0x990F) || 454 (rdev->pdev->device == 0x9910) || 455 (rdev->pdev->device == 0x9913) || 456 (rdev->pdev->device == 0x9917) || 457 (rdev->pdev->device == 0x9918)) { 458 radeon_program_register_sequence(rdev, 459 dvst_golden_registers, 460 (const u32)ARRAY_SIZE(dvst_golden_registers)); 461 radeon_program_register_sequence(rdev, 462 dvst_golden_registers2, 463 (const u32)ARRAY_SIZE(dvst_golden_registers2)); 464 } else { 465 radeon_program_register_sequence(rdev, 466 scrapper_golden_registers, 467 (const u32)ARRAY_SIZE(scrapper_golden_registers)); 468 radeon_program_register_sequence(rdev, 469 dvst_golden_registers2, 470 (const u32)ARRAY_SIZE(dvst_golden_registers2)); 471 } 472 break; 473 default: 474 break; 475 } 476 } 477 478 #define BTC_IO_MC_REGS_SIZE 29 479 480 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = { 481 {0x00000077, 0xff010100}, 482 {0x00000078, 0x00000000}, 483 {0x00000079, 0x00001434}, 484 {0x0000007a, 0xcc08ec08}, 485 {0x0000007b, 0x00040000}, 486 {0x0000007c, 0x000080c0}, 487 {0x0000007d, 0x09000000}, 488 {0x0000007e, 0x00210404}, 489 {0x00000081, 0x08a8e800}, 490 {0x00000082, 0x00030444}, 491 {0x00000083, 0x00000000}, 492 {0x00000085, 0x00000001}, 493 {0x00000086, 0x00000002}, 494 {0x00000087, 0x48490000}, 495 {0x00000088, 0x20244647}, 496 {0x00000089, 0x00000005}, 497 {0x0000008b, 0x66030000}, 498 {0x0000008c, 0x00006603}, 499 {0x0000008d, 0x00000100}, 500 {0x0000008f, 0x00001c0a}, 501 {0x00000090, 0xff000001}, 502 {0x00000094, 0x00101101}, 503 {0x00000095, 0x00000fff}, 504 {0x00000096, 0x00116fff}, 505 {0x00000097, 0x60010000}, 506 {0x00000098, 0x10010000}, 507 {0x00000099, 0x00006000}, 508 {0x0000009a, 0x00001000}, 509 {0x0000009f, 0x00946a00} 510 }; 511 512 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = { 513 {0x00000077, 0xff010100}, 514 {0x00000078, 0x00000000}, 515 {0x00000079, 0x00001434}, 516 {0x0000007a, 0xcc08ec08}, 517 {0x0000007b, 0x00040000}, 518 {0x0000007c, 0x000080c0}, 519 {0x0000007d, 0x09000000}, 520 {0x0000007e, 0x00210404}, 521 {0x00000081, 0x08a8e800}, 522 {0x00000082, 0x00030444}, 523 {0x00000083, 0x00000000}, 524 {0x00000085, 0x00000001}, 525 {0x00000086, 0x00000002}, 526 {0x00000087, 0x48490000}, 527 {0x00000088, 0x20244647}, 528 {0x00000089, 0x00000005}, 529 {0x0000008b, 0x66030000}, 530 {0x0000008c, 0x00006603}, 531 {0x0000008d, 0x00000100}, 532 {0x0000008f, 0x00001c0a}, 533 {0x00000090, 0xff000001}, 534 {0x00000094, 0x00101101}, 535 {0x00000095, 0x00000fff}, 536 {0x00000096, 0x00116fff}, 537 {0x00000097, 0x60010000}, 538 {0x00000098, 0x10010000}, 539 {0x00000099, 0x00006000}, 540 {0x0000009a, 0x00001000}, 541 {0x0000009f, 0x00936a00} 542 }; 543 544 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = { 545 {0x00000077, 0xff010100}, 546 {0x00000078, 0x00000000}, 547 {0x00000079, 0x00001434}, 548 {0x0000007a, 0xcc08ec08}, 549 {0x0000007b, 0x00040000}, 550 {0x0000007c, 0x000080c0}, 551 {0x0000007d, 0x09000000}, 552 {0x0000007e, 0x00210404}, 553 {0x00000081, 0x08a8e800}, 554 {0x00000082, 0x00030444}, 555 {0x00000083, 0x00000000}, 556 {0x00000085, 0x00000001}, 557 {0x00000086, 0x00000002}, 558 {0x00000087, 0x48490000}, 559 {0x00000088, 0x20244647}, 560 {0x00000089, 0x00000005}, 561 {0x0000008b, 0x66030000}, 562 {0x0000008c, 0x00006603}, 563 {0x0000008d, 0x00000100}, 564 {0x0000008f, 0x00001c0a}, 565 {0x00000090, 0xff000001}, 566 {0x00000094, 0x00101101}, 567 {0x00000095, 0x00000fff}, 568 {0x00000096, 0x00116fff}, 569 {0x00000097, 0x60010000}, 570 {0x00000098, 0x10010000}, 571 {0x00000099, 0x00006000}, 572 {0x0000009a, 0x00001000}, 573 {0x0000009f, 0x00916a00} 574 }; 575 576 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = { 577 {0x00000077, 0xff010100}, 578 {0x00000078, 0x00000000}, 579 {0x00000079, 0x00001434}, 580 {0x0000007a, 0xcc08ec08}, 581 {0x0000007b, 0x00040000}, 582 {0x0000007c, 0x000080c0}, 583 {0x0000007d, 0x09000000}, 584 {0x0000007e, 0x00210404}, 585 {0x00000081, 0x08a8e800}, 586 {0x00000082, 0x00030444}, 587 {0x00000083, 0x00000000}, 588 {0x00000085, 0x00000001}, 589 {0x00000086, 0x00000002}, 590 {0x00000087, 0x48490000}, 591 {0x00000088, 0x20244647}, 592 {0x00000089, 0x00000005}, 593 {0x0000008b, 0x66030000}, 594 {0x0000008c, 0x00006603}, 595 {0x0000008d, 0x00000100}, 596 {0x0000008f, 0x00001c0a}, 597 {0x00000090, 0xff000001}, 598 {0x00000094, 0x00101101}, 599 {0x00000095, 0x00000fff}, 600 {0x00000096, 0x00116fff}, 601 {0x00000097, 0x60010000}, 602 {0x00000098, 0x10010000}, 603 {0x00000099, 0x00006000}, 604 {0x0000009a, 0x00001000}, 605 {0x0000009f, 0x00976b00} 606 }; 607 608 int ni_mc_load_microcode(struct radeon_device *rdev) 609 { 610 const __be32 *fw_data; 611 u32 mem_type, running, blackout = 0; 612 u32 *io_mc_regs; 613 int i, ucode_size, regs_size; 614 615 if (!rdev->mc_fw) 616 return -EINVAL; 617 618 switch (rdev->family) { 619 case CHIP_BARTS: 620 io_mc_regs = (u32 *)&barts_io_mc_regs; 621 ucode_size = BTC_MC_UCODE_SIZE; 622 regs_size = BTC_IO_MC_REGS_SIZE; 623 break; 624 case CHIP_TURKS: 625 io_mc_regs = (u32 *)&turks_io_mc_regs; 626 ucode_size = BTC_MC_UCODE_SIZE; 627 regs_size = BTC_IO_MC_REGS_SIZE; 628 break; 629 case CHIP_CAICOS: 630 default: 631 io_mc_regs = (u32 *)&caicos_io_mc_regs; 632 ucode_size = BTC_MC_UCODE_SIZE; 633 regs_size = BTC_IO_MC_REGS_SIZE; 634 break; 635 case CHIP_CAYMAN: 636 io_mc_regs = (u32 *)&cayman_io_mc_regs; 637 ucode_size = CAYMAN_MC_UCODE_SIZE; 638 regs_size = BTC_IO_MC_REGS_SIZE; 639 break; 640 } 641 642 mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT; 643 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 644 645 if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) { 646 if (running) { 647 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 648 WREG32(MC_SHARED_BLACKOUT_CNTL, 1); 649 } 650 651 /* reset the engine and set to writable */ 652 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 653 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 654 655 /* load mc io regs */ 656 for (i = 0; i < regs_size; i++) { 657 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 658 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 659 } 660 /* load the MC ucode */ 661 fw_data = (const __be32 *)rdev->mc_fw->data; 662 for (i = 0; i < ucode_size; i++) 663 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 664 665 /* put the engine back into the active state */ 666 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 667 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 668 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 669 670 /* wait for training to complete */ 671 for (i = 0; i < rdev->usec_timeout; i++) { 672 if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD) 673 break; 674 udelay(1); 675 } 676 677 if (running) 678 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 679 } 680 681 return 0; 682 } 683 684 int ni_init_microcode(struct radeon_device *rdev) 685 { 686 const char *chip_name; 687 const char *rlc_chip_name; 688 size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size; 689 size_t smc_req_size = 0; 690 char fw_name[30]; 691 int err; 692 693 DRM_DEBUG("\n"); 694 695 switch (rdev->family) { 696 case CHIP_BARTS: 697 chip_name = "BARTS"; 698 rlc_chip_name = "BTC"; 699 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4; 700 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4; 701 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4; 702 mc_req_size = BTC_MC_UCODE_SIZE * 4; 703 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4); 704 break; 705 case CHIP_TURKS: 706 chip_name = "TURKS"; 707 rlc_chip_name = "BTC"; 708 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4; 709 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4; 710 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4; 711 mc_req_size = BTC_MC_UCODE_SIZE * 4; 712 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4); 713 break; 714 case CHIP_CAICOS: 715 chip_name = "CAICOS"; 716 rlc_chip_name = "BTC"; 717 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4; 718 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4; 719 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4; 720 mc_req_size = BTC_MC_UCODE_SIZE * 4; 721 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4); 722 break; 723 case CHIP_CAYMAN: 724 chip_name = "CAYMAN"; 725 rlc_chip_name = "CAYMAN"; 726 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4; 727 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4; 728 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4; 729 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4; 730 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4); 731 break; 732 case CHIP_ARUBA: 733 chip_name = "ARUBA"; 734 rlc_chip_name = "ARUBA"; 735 /* pfp/me same size as CAYMAN */ 736 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4; 737 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4; 738 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4; 739 mc_req_size = 0; 740 break; 741 default: BUG(); 742 } 743 744 DRM_INFO("Loading %s Microcode\n", chip_name); 745 746 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 747 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 748 if (err) 749 goto out; 750 if (rdev->pfp_fw->size != pfp_req_size) { 751 printk(KERN_ERR 752 "ni_cp: Bogus length %zu in firmware \"%s\"\n", 753 rdev->pfp_fw->size, fw_name); 754 err = -EINVAL; 755 goto out; 756 } 757 758 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 759 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 760 if (err) 761 goto out; 762 if (rdev->me_fw->size != me_req_size) { 763 printk(KERN_ERR 764 "ni_cp: Bogus length %zu in firmware \"%s\"\n", 765 rdev->me_fw->size, fw_name); 766 err = -EINVAL; 767 } 768 769 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name); 770 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 771 if (err) 772 goto out; 773 if (rdev->rlc_fw->size != rlc_req_size) { 774 printk(KERN_ERR 775 "ni_rlc: Bogus length %zu in firmware \"%s\"\n", 776 rdev->rlc_fw->size, fw_name); 777 err = -EINVAL; 778 } 779 780 /* no MC ucode on TN */ 781 if (!(rdev->flags & RADEON_IS_IGP)) { 782 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 783 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 784 if (err) 785 goto out; 786 if (rdev->mc_fw->size != mc_req_size) { 787 printk(KERN_ERR 788 "ni_mc: Bogus length %zu in firmware \"%s\"\n", 789 rdev->mc_fw->size, fw_name); 790 err = -EINVAL; 791 } 792 } 793 794 if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) { 795 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 796 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 797 if (err) 798 goto out; 799 if (rdev->smc_fw->size != smc_req_size) { 800 printk(KERN_ERR 801 "ni_mc: Bogus length %zu in firmware \"%s\"\n", 802 rdev->mc_fw->size, fw_name); 803 err = -EINVAL; 804 } 805 } 806 807 out: 808 if (err) { 809 if (err != -EINVAL) 810 printk(KERN_ERR 811 "ni_cp: Failed to load firmware \"%s\"\n", 812 fw_name); 813 release_firmware(rdev->pfp_fw); 814 rdev->pfp_fw = NULL; 815 release_firmware(rdev->me_fw); 816 rdev->me_fw = NULL; 817 release_firmware(rdev->rlc_fw); 818 rdev->rlc_fw = NULL; 819 release_firmware(rdev->mc_fw); 820 rdev->mc_fw = NULL; 821 } 822 return err; 823 } 824 825 int tn_get_temp(struct radeon_device *rdev) 826 { 827 u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff; 828 int actual_temp = (temp / 8) - 49; 829 830 return actual_temp * 1000; 831 } 832 833 /* 834 * Core functions 835 */ 836 static void cayman_gpu_init(struct radeon_device *rdev) 837 { 838 u32 gb_addr_config = 0; 839 u32 mc_shared_chmap, mc_arb_ramcfg; 840 u32 cgts_tcc_disable; 841 u32 sx_debug_1; 842 u32 smx_dc_ctl0; 843 u32 cgts_sm_ctrl_reg; 844 u32 hdp_host_path_cntl; 845 u32 tmp; 846 u32 disabled_rb_mask; 847 int i, j; 848 849 switch (rdev->family) { 850 case CHIP_CAYMAN: 851 rdev->config.cayman.max_shader_engines = 2; 852 rdev->config.cayman.max_pipes_per_simd = 4; 853 rdev->config.cayman.max_tile_pipes = 8; 854 rdev->config.cayman.max_simds_per_se = 12; 855 rdev->config.cayman.max_backends_per_se = 4; 856 rdev->config.cayman.max_texture_channel_caches = 8; 857 rdev->config.cayman.max_gprs = 256; 858 rdev->config.cayman.max_threads = 256; 859 rdev->config.cayman.max_gs_threads = 32; 860 rdev->config.cayman.max_stack_entries = 512; 861 rdev->config.cayman.sx_num_of_sets = 8; 862 rdev->config.cayman.sx_max_export_size = 256; 863 rdev->config.cayman.sx_max_export_pos_size = 64; 864 rdev->config.cayman.sx_max_export_smx_size = 192; 865 rdev->config.cayman.max_hw_contexts = 8; 866 rdev->config.cayman.sq_num_cf_insts = 2; 867 868 rdev->config.cayman.sc_prim_fifo_size = 0x100; 869 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30; 870 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130; 871 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN; 872 break; 873 case CHIP_ARUBA: 874 default: 875 rdev->config.cayman.max_shader_engines = 1; 876 rdev->config.cayman.max_pipes_per_simd = 4; 877 rdev->config.cayman.max_tile_pipes = 2; 878 if ((rdev->pdev->device == 0x9900) || 879 (rdev->pdev->device == 0x9901) || 880 (rdev->pdev->device == 0x9905) || 881 (rdev->pdev->device == 0x9906) || 882 (rdev->pdev->device == 0x9907) || 883 (rdev->pdev->device == 0x9908) || 884 (rdev->pdev->device == 0x9909) || 885 (rdev->pdev->device == 0x990B) || 886 (rdev->pdev->device == 0x990C) || 887 (rdev->pdev->device == 0x990F) || 888 (rdev->pdev->device == 0x9910) || 889 (rdev->pdev->device == 0x9917) || 890 (rdev->pdev->device == 0x9999) || 891 (rdev->pdev->device == 0x999C)) { 892 rdev->config.cayman.max_simds_per_se = 6; 893 rdev->config.cayman.max_backends_per_se = 2; 894 } else if ((rdev->pdev->device == 0x9903) || 895 (rdev->pdev->device == 0x9904) || 896 (rdev->pdev->device == 0x990A) || 897 (rdev->pdev->device == 0x990D) || 898 (rdev->pdev->device == 0x990E) || 899 (rdev->pdev->device == 0x9913) || 900 (rdev->pdev->device == 0x9918) || 901 (rdev->pdev->device == 0x999D)) { 902 rdev->config.cayman.max_simds_per_se = 4; 903 rdev->config.cayman.max_backends_per_se = 2; 904 } else if ((rdev->pdev->device == 0x9919) || 905 (rdev->pdev->device == 0x9990) || 906 (rdev->pdev->device == 0x9991) || 907 (rdev->pdev->device == 0x9994) || 908 (rdev->pdev->device == 0x9995) || 909 (rdev->pdev->device == 0x9996) || 910 (rdev->pdev->device == 0x999A) || 911 (rdev->pdev->device == 0x99A0)) { 912 rdev->config.cayman.max_simds_per_se = 3; 913 rdev->config.cayman.max_backends_per_se = 1; 914 } else { 915 rdev->config.cayman.max_simds_per_se = 2; 916 rdev->config.cayman.max_backends_per_se = 1; 917 } 918 rdev->config.cayman.max_texture_channel_caches = 2; 919 rdev->config.cayman.max_gprs = 256; 920 rdev->config.cayman.max_threads = 256; 921 rdev->config.cayman.max_gs_threads = 32; 922 rdev->config.cayman.max_stack_entries = 512; 923 rdev->config.cayman.sx_num_of_sets = 8; 924 rdev->config.cayman.sx_max_export_size = 256; 925 rdev->config.cayman.sx_max_export_pos_size = 64; 926 rdev->config.cayman.sx_max_export_smx_size = 192; 927 rdev->config.cayman.max_hw_contexts = 8; 928 rdev->config.cayman.sq_num_cf_insts = 2; 929 930 rdev->config.cayman.sc_prim_fifo_size = 0x40; 931 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30; 932 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130; 933 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN; 934 break; 935 } 936 937 /* Initialize HDP */ 938 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 939 WREG32((0x2c14 + j), 0x00000000); 940 WREG32((0x2c18 + j), 0x00000000); 941 WREG32((0x2c1c + j), 0x00000000); 942 WREG32((0x2c20 + j), 0x00000000); 943 WREG32((0x2c24 + j), 0x00000000); 944 } 945 946 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 947 948 evergreen_fix_pci_max_read_req_size(rdev); 949 950 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 951 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 952 953 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 954 rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 955 if (rdev->config.cayman.mem_row_size_in_kb > 4) 956 rdev->config.cayman.mem_row_size_in_kb = 4; 957 /* XXX use MC settings? */ 958 rdev->config.cayman.shader_engine_tile_size = 32; 959 rdev->config.cayman.num_gpus = 1; 960 rdev->config.cayman.multi_gpu_tile_size = 64; 961 962 tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT; 963 rdev->config.cayman.num_tile_pipes = (1 << tmp); 964 tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT; 965 rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256; 966 tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT; 967 rdev->config.cayman.num_shader_engines = tmp + 1; 968 tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT; 969 rdev->config.cayman.num_gpus = tmp + 1; 970 tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT; 971 rdev->config.cayman.multi_gpu_tile_size = 1 << tmp; 972 tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT; 973 rdev->config.cayman.mem_row_size_in_kb = 1 << tmp; 974 975 976 /* setup tiling info dword. gb_addr_config is not adequate since it does 977 * not have bank info, so create a custom tiling dword. 978 * bits 3:0 num_pipes 979 * bits 7:4 num_banks 980 * bits 11:8 group_size 981 * bits 15:12 row_size 982 */ 983 rdev->config.cayman.tile_config = 0; 984 switch (rdev->config.cayman.num_tile_pipes) { 985 case 1: 986 default: 987 rdev->config.cayman.tile_config |= (0 << 0); 988 break; 989 case 2: 990 rdev->config.cayman.tile_config |= (1 << 0); 991 break; 992 case 4: 993 rdev->config.cayman.tile_config |= (2 << 0); 994 break; 995 case 8: 996 rdev->config.cayman.tile_config |= (3 << 0); 997 break; 998 } 999 1000 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 1001 if (rdev->flags & RADEON_IS_IGP) 1002 rdev->config.cayman.tile_config |= 1 << 4; 1003 else { 1004 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 1005 case 0: /* four banks */ 1006 rdev->config.cayman.tile_config |= 0 << 4; 1007 break; 1008 case 1: /* eight banks */ 1009 rdev->config.cayman.tile_config |= 1 << 4; 1010 break; 1011 case 2: /* sixteen banks */ 1012 default: 1013 rdev->config.cayman.tile_config |= 2 << 4; 1014 break; 1015 } 1016 } 1017 rdev->config.cayman.tile_config |= 1018 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 1019 rdev->config.cayman.tile_config |= 1020 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 1021 1022 tmp = 0; 1023 for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) { 1024 u32 rb_disable_bitmap; 1025 1026 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 1027 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 1028 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 1029 tmp <<= 4; 1030 tmp |= rb_disable_bitmap; 1031 } 1032 /* enabled rb are just the one not disabled :) */ 1033 disabled_rb_mask = tmp; 1034 tmp = 0; 1035 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++) 1036 tmp |= (1 << i); 1037 /* if all the backends are disabled, fix it up here */ 1038 if ((disabled_rb_mask & tmp) == tmp) { 1039 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++) 1040 disabled_rb_mask &= ~(1 << i); 1041 } 1042 1043 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 1044 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 1045 1046 WREG32(GB_ADDR_CONFIG, gb_addr_config); 1047 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 1048 if (ASIC_IS_DCE6(rdev)) 1049 WREG32(DMIF_ADDR_CALC, gb_addr_config); 1050 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 1051 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); 1052 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); 1053 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 1054 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 1055 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 1056 1057 if ((rdev->config.cayman.max_backends_per_se == 1) && 1058 (rdev->flags & RADEON_IS_IGP)) { 1059 if ((disabled_rb_mask & 3) == 1) { 1060 /* RB0 disabled, RB1 enabled */ 1061 tmp = 0x11111111; 1062 } else { 1063 /* RB1 disabled, RB0 enabled */ 1064 tmp = 0x00000000; 1065 } 1066 } else { 1067 tmp = gb_addr_config & NUM_PIPES_MASK; 1068 tmp = r6xx_remap_render_backend(rdev, tmp, 1069 rdev->config.cayman.max_backends_per_se * 1070 rdev->config.cayman.max_shader_engines, 1071 CAYMAN_MAX_BACKENDS, disabled_rb_mask); 1072 } 1073 WREG32(GB_BACKEND_MAP, tmp); 1074 1075 cgts_tcc_disable = 0xffff0000; 1076 for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++) 1077 cgts_tcc_disable &= ~(1 << (16 + i)); 1078 WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable); 1079 WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable); 1080 WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable); 1081 WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable); 1082 1083 /* reprogram the shader complex */ 1084 cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG); 1085 for (i = 0; i < 16; i++) 1086 WREG32(CGTS_SM_CTRL_REG, OVERRIDE); 1087 WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg); 1088 1089 /* set HW defaults for 3D engine */ 1090 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 1091 1092 sx_debug_1 = RREG32(SX_DEBUG_1); 1093 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 1094 WREG32(SX_DEBUG_1, sx_debug_1); 1095 1096 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 1097 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 1098 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets); 1099 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 1100 1101 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE); 1102 1103 /* need to be explicitly zero-ed */ 1104 WREG32(VGT_OFFCHIP_LDS_BASE, 0); 1105 WREG32(SQ_LSTMP_RING_BASE, 0); 1106 WREG32(SQ_HSTMP_RING_BASE, 0); 1107 WREG32(SQ_ESTMP_RING_BASE, 0); 1108 WREG32(SQ_GSTMP_RING_BASE, 0); 1109 WREG32(SQ_VSTMP_RING_BASE, 0); 1110 WREG32(SQ_PSTMP_RING_BASE, 0); 1111 1112 WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO); 1113 1114 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) | 1115 POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) | 1116 SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1))); 1117 1118 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) | 1119 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) | 1120 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size))); 1121 1122 1123 WREG32(VGT_NUM_INSTANCES, 1); 1124 1125 WREG32(CP_PERFMON_CNTL, 0); 1126 1127 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) | 1128 FETCH_FIFO_HIWATER(0x4) | 1129 DONE_FIFO_HIWATER(0xe0) | 1130 ALU_UPDATE_FIFO_HIWATER(0x8))); 1131 1132 WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4)); 1133 WREG32(SQ_CONFIG, (VC_ENABLE | 1134 EXPORT_SRC_C | 1135 GFX_PRIO(0) | 1136 CS1_PRIO(0) | 1137 CS2_PRIO(1))); 1138 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE); 1139 1140 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 1141 FORCE_EOV_MAX_REZ_CNT(255))); 1142 1143 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 1144 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 1145 1146 WREG32(VGT_GS_VERTEX_REUSE, 16); 1147 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 1148 1149 WREG32(CB_PERF_CTR0_SEL_0, 0); 1150 WREG32(CB_PERF_CTR0_SEL_1, 0); 1151 WREG32(CB_PERF_CTR1_SEL_0, 0); 1152 WREG32(CB_PERF_CTR1_SEL_1, 0); 1153 WREG32(CB_PERF_CTR2_SEL_0, 0); 1154 WREG32(CB_PERF_CTR2_SEL_1, 0); 1155 WREG32(CB_PERF_CTR3_SEL_0, 0); 1156 WREG32(CB_PERF_CTR3_SEL_1, 0); 1157 1158 tmp = RREG32(HDP_MISC_CNTL); 1159 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 1160 WREG32(HDP_MISC_CNTL, tmp); 1161 1162 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 1163 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 1164 1165 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 1166 1167 udelay(50); 1168 1169 /* set clockgating golden values on TN */ 1170 if (rdev->family == CHIP_ARUBA) { 1171 tmp = RREG32_CG(CG_CGTT_LOCAL_0); 1172 tmp &= ~0x00380000; 1173 WREG32_CG(CG_CGTT_LOCAL_0, tmp); 1174 tmp = RREG32_CG(CG_CGTT_LOCAL_1); 1175 tmp &= ~0x0e000000; 1176 WREG32_CG(CG_CGTT_LOCAL_1, tmp); 1177 } 1178 } 1179 1180 /* 1181 * GART 1182 */ 1183 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev) 1184 { 1185 /* flush hdp cache */ 1186 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 1187 1188 /* bits 0-7 are the VM contexts0-7 */ 1189 WREG32(VM_INVALIDATE_REQUEST, 1); 1190 } 1191 1192 static int cayman_pcie_gart_enable(struct radeon_device *rdev) 1193 { 1194 int i, r; 1195 1196 if (rdev->gart.robj == NULL) { 1197 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 1198 return -EINVAL; 1199 } 1200 r = radeon_gart_table_vram_pin(rdev); 1201 if (r) 1202 return r; 1203 radeon_gart_restore(rdev); 1204 /* Setup TLB control */ 1205 WREG32(MC_VM_MX_L1_TLB_CNTL, 1206 (0xA << 7) | 1207 ENABLE_L1_TLB | 1208 ENABLE_L1_FRAGMENT_PROCESSING | 1209 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 1210 ENABLE_ADVANCED_DRIVER_MODEL | 1211 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 1212 /* Setup L2 cache */ 1213 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 1214 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 1215 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 1216 EFFECTIVE_L2_QUEUE_SIZE(7) | 1217 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 1218 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 1219 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 1220 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 1221 /* setup context0 */ 1222 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 1223 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 1224 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 1225 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 1226 (u32)(rdev->dummy_page.addr >> 12)); 1227 WREG32(VM_CONTEXT0_CNTL2, 0); 1228 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 1229 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 1230 1231 WREG32(0x15D4, 0); 1232 WREG32(0x15D8, 0); 1233 WREG32(0x15DC, 0); 1234 1235 /* empty context1-7 */ 1236 /* Assign the pt base to something valid for now; the pts used for 1237 * the VMs are determined by the application and setup and assigned 1238 * on the fly in the vm part of radeon_gart.c 1239 */ 1240 for (i = 1; i < 8; i++) { 1241 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0); 1242 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn); 1243 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 1244 rdev->gart.table_addr >> 12); 1245 } 1246 1247 /* enable context1-7 */ 1248 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 1249 (u32)(rdev->dummy_page.addr >> 12)); 1250 WREG32(VM_CONTEXT1_CNTL2, 4); 1251 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 1252 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 1253 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 1254 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 1255 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 1256 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 1257 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 1258 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 1259 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 1260 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 1261 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 1262 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 1263 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 1264 1265 cayman_pcie_gart_tlb_flush(rdev); 1266 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1267 (unsigned)(rdev->mc.gtt_size >> 20), 1268 (unsigned long long)rdev->gart.table_addr); 1269 rdev->gart.ready = true; 1270 return 0; 1271 } 1272 1273 static void cayman_pcie_gart_disable(struct radeon_device *rdev) 1274 { 1275 /* Disable all tables */ 1276 WREG32(VM_CONTEXT0_CNTL, 0); 1277 WREG32(VM_CONTEXT1_CNTL, 0); 1278 /* Setup TLB control */ 1279 WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING | 1280 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 1281 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 1282 /* Setup L2 cache */ 1283 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 1284 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 1285 EFFECTIVE_L2_QUEUE_SIZE(7) | 1286 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 1287 WREG32(VM_L2_CNTL2, 0); 1288 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 1289 L2_CACHE_BIGK_FRAGMENT_SIZE(6)); 1290 radeon_gart_table_vram_unpin(rdev); 1291 } 1292 1293 static void cayman_pcie_gart_fini(struct radeon_device *rdev) 1294 { 1295 cayman_pcie_gart_disable(rdev); 1296 radeon_gart_table_vram_free(rdev); 1297 radeon_gart_fini(rdev); 1298 } 1299 1300 void cayman_cp_int_cntl_setup(struct radeon_device *rdev, 1301 int ring, u32 cp_int_cntl) 1302 { 1303 u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3; 1304 1305 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3)); 1306 WREG32(CP_INT_CNTL, cp_int_cntl); 1307 } 1308 1309 /* 1310 * CP. 1311 */ 1312 void cayman_fence_ring_emit(struct radeon_device *rdev, 1313 struct radeon_fence *fence) 1314 { 1315 struct radeon_ring *ring = &rdev->ring[fence->ring]; 1316 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 1317 1318 /* flush read cache over gart for this vmid */ 1319 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 1320 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 1321 radeon_ring_write(ring, 0); 1322 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 1323 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA); 1324 radeon_ring_write(ring, 0xFFFFFFFF); 1325 radeon_ring_write(ring, 0); 1326 radeon_ring_write(ring, 10); /* poll interval */ 1327 /* EVENT_WRITE_EOP - flush caches, send int */ 1328 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 1329 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5)); 1330 radeon_ring_write(ring, addr & 0xffffffff); 1331 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); 1332 radeon_ring_write(ring, fence->seq); 1333 radeon_ring_write(ring, 0); 1334 } 1335 1336 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 1337 { 1338 struct radeon_ring *ring = &rdev->ring[ib->ring]; 1339 1340 /* set to DX10/11 mode */ 1341 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 1342 radeon_ring_write(ring, 1); 1343 1344 if (ring->rptr_save_reg) { 1345 uint32_t next_rptr = ring->wptr + 3 + 4 + 8; 1346 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 1347 radeon_ring_write(ring, ((ring->rptr_save_reg - 1348 PACKET3_SET_CONFIG_REG_START) >> 2)); 1349 radeon_ring_write(ring, next_rptr); 1350 } 1351 1352 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 1353 radeon_ring_write(ring, 1354 #ifdef __BIG_ENDIAN 1355 (2 << 0) | 1356 #endif 1357 (ib->gpu_addr & 0xFFFFFFFC)); 1358 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 1359 radeon_ring_write(ring, ib->length_dw | 1360 (ib->vm ? (ib->vm->id << 24) : 0)); 1361 1362 /* flush read cache over gart for this vmid */ 1363 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 1364 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 1365 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); 1366 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 1367 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA); 1368 radeon_ring_write(ring, 0xFFFFFFFF); 1369 radeon_ring_write(ring, 0); 1370 radeon_ring_write(ring, 10); /* poll interval */ 1371 } 1372 1373 void cayman_uvd_semaphore_emit(struct radeon_device *rdev, 1374 struct radeon_ring *ring, 1375 struct radeon_semaphore *semaphore, 1376 bool emit_wait) 1377 { 1378 uint64_t addr = semaphore->gpu_addr; 1379 1380 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0)); 1381 radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF); 1382 1383 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0)); 1384 radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF); 1385 1386 radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0)); 1387 radeon_ring_write(ring, 0x80 | (emit_wait ? 1 : 0)); 1388 } 1389 1390 static void cayman_cp_enable(struct radeon_device *rdev, bool enable) 1391 { 1392 if (enable) 1393 WREG32(CP_ME_CNTL, 0); 1394 else { 1395 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 1396 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT)); 1397 WREG32(SCRATCH_UMSK, 0); 1398 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 1399 } 1400 } 1401 1402 static int cayman_cp_load_microcode(struct radeon_device *rdev) 1403 { 1404 const __be32 *fw_data; 1405 int i; 1406 1407 if (!rdev->me_fw || !rdev->pfp_fw) 1408 return -EINVAL; 1409 1410 cayman_cp_enable(rdev, false); 1411 1412 fw_data = (const __be32 *)rdev->pfp_fw->data; 1413 WREG32(CP_PFP_UCODE_ADDR, 0); 1414 for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++) 1415 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 1416 WREG32(CP_PFP_UCODE_ADDR, 0); 1417 1418 fw_data = (const __be32 *)rdev->me_fw->data; 1419 WREG32(CP_ME_RAM_WADDR, 0); 1420 for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++) 1421 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 1422 1423 WREG32(CP_PFP_UCODE_ADDR, 0); 1424 WREG32(CP_ME_RAM_WADDR, 0); 1425 WREG32(CP_ME_RAM_RADDR, 0); 1426 return 0; 1427 } 1428 1429 static int cayman_cp_start(struct radeon_device *rdev) 1430 { 1431 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 1432 int r, i; 1433 1434 r = radeon_ring_lock(rdev, ring, 7); 1435 if (r) { 1436 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1437 return r; 1438 } 1439 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 1440 radeon_ring_write(ring, 0x1); 1441 radeon_ring_write(ring, 0x0); 1442 radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1); 1443 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 1444 radeon_ring_write(ring, 0); 1445 radeon_ring_write(ring, 0); 1446 radeon_ring_unlock_commit(rdev, ring); 1447 1448 cayman_cp_enable(rdev, true); 1449 1450 r = radeon_ring_lock(rdev, ring, cayman_default_size + 19); 1451 if (r) { 1452 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1453 return r; 1454 } 1455 1456 /* setup clear context state */ 1457 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1458 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1459 1460 for (i = 0; i < cayman_default_size; i++) 1461 radeon_ring_write(ring, cayman_default_state[i]); 1462 1463 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1464 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 1465 1466 /* set clear context state */ 1467 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 1468 radeon_ring_write(ring, 0); 1469 1470 /* SQ_VTX_BASE_VTX_LOC */ 1471 radeon_ring_write(ring, 0xc0026f00); 1472 radeon_ring_write(ring, 0x00000000); 1473 radeon_ring_write(ring, 0x00000000); 1474 radeon_ring_write(ring, 0x00000000); 1475 1476 /* Clear consts */ 1477 radeon_ring_write(ring, 0xc0036f00); 1478 radeon_ring_write(ring, 0x00000bc4); 1479 radeon_ring_write(ring, 0xffffffff); 1480 radeon_ring_write(ring, 0xffffffff); 1481 radeon_ring_write(ring, 0xffffffff); 1482 1483 radeon_ring_write(ring, 0xc0026900); 1484 radeon_ring_write(ring, 0x00000316); 1485 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 1486 radeon_ring_write(ring, 0x00000010); /* */ 1487 1488 radeon_ring_unlock_commit(rdev, ring); 1489 1490 /* XXX init other rings */ 1491 1492 return 0; 1493 } 1494 1495 static void cayman_cp_fini(struct radeon_device *rdev) 1496 { 1497 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 1498 cayman_cp_enable(rdev, false); 1499 radeon_ring_fini(rdev, ring); 1500 radeon_scratch_free(rdev, ring->rptr_save_reg); 1501 } 1502 1503 static int cayman_cp_resume(struct radeon_device *rdev) 1504 { 1505 static const int ridx[] = { 1506 RADEON_RING_TYPE_GFX_INDEX, 1507 CAYMAN_RING_TYPE_CP1_INDEX, 1508 CAYMAN_RING_TYPE_CP2_INDEX 1509 }; 1510 static const unsigned cp_rb_cntl[] = { 1511 CP_RB0_CNTL, 1512 CP_RB1_CNTL, 1513 CP_RB2_CNTL, 1514 }; 1515 static const unsigned cp_rb_rptr_addr[] = { 1516 CP_RB0_RPTR_ADDR, 1517 CP_RB1_RPTR_ADDR, 1518 CP_RB2_RPTR_ADDR 1519 }; 1520 static const unsigned cp_rb_rptr_addr_hi[] = { 1521 CP_RB0_RPTR_ADDR_HI, 1522 CP_RB1_RPTR_ADDR_HI, 1523 CP_RB2_RPTR_ADDR_HI 1524 }; 1525 static const unsigned cp_rb_base[] = { 1526 CP_RB0_BASE, 1527 CP_RB1_BASE, 1528 CP_RB2_BASE 1529 }; 1530 struct radeon_ring *ring; 1531 int i, r; 1532 1533 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 1534 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 1535 SOFT_RESET_PA | 1536 SOFT_RESET_SH | 1537 SOFT_RESET_VGT | 1538 SOFT_RESET_SPI | 1539 SOFT_RESET_SX)); 1540 RREG32(GRBM_SOFT_RESET); 1541 mdelay(15); 1542 WREG32(GRBM_SOFT_RESET, 0); 1543 RREG32(GRBM_SOFT_RESET); 1544 1545 WREG32(CP_SEM_WAIT_TIMER, 0x0); 1546 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 1547 1548 /* Set the write pointer delay */ 1549 WREG32(CP_RB_WPTR_DELAY, 0); 1550 1551 WREG32(CP_DEBUG, (1 << 27)); 1552 1553 /* set the wb address whether it's enabled or not */ 1554 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 1555 WREG32(SCRATCH_UMSK, 0xff); 1556 1557 for (i = 0; i < 3; ++i) { 1558 uint32_t rb_cntl; 1559 uint64_t addr; 1560 1561 /* Set ring buffer size */ 1562 ring = &rdev->ring[ridx[i]]; 1563 rb_cntl = drm_order(ring->ring_size / 8); 1564 rb_cntl |= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8; 1565 #ifdef __BIG_ENDIAN 1566 rb_cntl |= BUF_SWAP_32BIT; 1567 #endif 1568 WREG32(cp_rb_cntl[i], rb_cntl); 1569 1570 /* set the wb address whether it's enabled or not */ 1571 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET; 1572 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC); 1573 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF); 1574 } 1575 1576 /* set the rb base addr, this causes an internal reset of ALL rings */ 1577 for (i = 0; i < 3; ++i) { 1578 ring = &rdev->ring[ridx[i]]; 1579 WREG32(cp_rb_base[i], ring->gpu_addr >> 8); 1580 } 1581 1582 for (i = 0; i < 3; ++i) { 1583 /* Initialize the ring buffer's read and write pointers */ 1584 ring = &rdev->ring[ridx[i]]; 1585 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA); 1586 1587 ring->rptr = ring->wptr = 0; 1588 WREG32(ring->rptr_reg, ring->rptr); 1589 WREG32(ring->wptr_reg, ring->wptr); 1590 1591 mdelay(1); 1592 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA); 1593 } 1594 1595 /* start the rings */ 1596 cayman_cp_start(rdev); 1597 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 1598 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 1599 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 1600 /* this only test cp0 */ 1601 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 1602 if (r) { 1603 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 1604 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 1605 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 1606 return r; 1607 } 1608 1609 return 0; 1610 } 1611 1612 /* 1613 * DMA 1614 * Starting with R600, the GPU has an asynchronous 1615 * DMA engine. The programming model is very similar 1616 * to the 3D engine (ring buffer, IBs, etc.), but the 1617 * DMA controller has it's own packet format that is 1618 * different form the PM4 format used by the 3D engine. 1619 * It supports copying data, writing embedded data, 1620 * solid fills, and a number of other things. It also 1621 * has support for tiling/detiling of buffers. 1622 * Cayman and newer support two asynchronous DMA engines. 1623 */ 1624 /** 1625 * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine 1626 * 1627 * @rdev: radeon_device pointer 1628 * @ib: IB object to schedule 1629 * 1630 * Schedule an IB in the DMA ring (cayman-SI). 1631 */ 1632 void cayman_dma_ring_ib_execute(struct radeon_device *rdev, 1633 struct radeon_ib *ib) 1634 { 1635 struct radeon_ring *ring = &rdev->ring[ib->ring]; 1636 1637 if (rdev->wb.enabled) { 1638 u32 next_rptr = ring->wptr + 4; 1639 while ((next_rptr & 7) != 5) 1640 next_rptr++; 1641 next_rptr += 3; 1642 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1)); 1643 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 1644 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff); 1645 radeon_ring_write(ring, next_rptr); 1646 } 1647 1648 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. 1649 * Pad as necessary with NOPs. 1650 */ 1651 while ((ring->wptr & 7) != 5) 1652 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); 1653 radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0)); 1654 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); 1655 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); 1656 1657 } 1658 1659 /** 1660 * cayman_dma_stop - stop the async dma engines 1661 * 1662 * @rdev: radeon_device pointer 1663 * 1664 * Stop the async dma engines (cayman-SI). 1665 */ 1666 void cayman_dma_stop(struct radeon_device *rdev) 1667 { 1668 u32 rb_cntl; 1669 1670 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 1671 1672 /* dma0 */ 1673 rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 1674 rb_cntl &= ~DMA_RB_ENABLE; 1675 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl); 1676 1677 /* dma1 */ 1678 rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 1679 rb_cntl &= ~DMA_RB_ENABLE; 1680 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl); 1681 1682 rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false; 1683 rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false; 1684 } 1685 1686 /** 1687 * cayman_dma_resume - setup and start the async dma engines 1688 * 1689 * @rdev: radeon_device pointer 1690 * 1691 * Set up the DMA ring buffers and enable them. (cayman-SI). 1692 * Returns 0 for success, error for failure. 1693 */ 1694 int cayman_dma_resume(struct radeon_device *rdev) 1695 { 1696 struct radeon_ring *ring; 1697 u32 rb_cntl, dma_cntl, ib_cntl; 1698 u32 rb_bufsz; 1699 u32 reg_offset, wb_offset; 1700 int i, r; 1701 1702 /* Reset dma */ 1703 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1); 1704 RREG32(SRBM_SOFT_RESET); 1705 udelay(50); 1706 WREG32(SRBM_SOFT_RESET, 0); 1707 1708 for (i = 0; i < 2; i++) { 1709 if (i == 0) { 1710 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 1711 reg_offset = DMA0_REGISTER_OFFSET; 1712 wb_offset = R600_WB_DMA_RPTR_OFFSET; 1713 } else { 1714 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 1715 reg_offset = DMA1_REGISTER_OFFSET; 1716 wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET; 1717 } 1718 1719 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0); 1720 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0); 1721 1722 /* Set ring buffer size in dwords */ 1723 rb_bufsz = drm_order(ring->ring_size / 4); 1724 rb_cntl = rb_bufsz << 1; 1725 #ifdef __BIG_ENDIAN 1726 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE; 1727 #endif 1728 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl); 1729 1730 /* Initialize the ring buffer's read and write pointers */ 1731 WREG32(DMA_RB_RPTR + reg_offset, 0); 1732 WREG32(DMA_RB_WPTR + reg_offset, 0); 1733 1734 /* set the wb address whether it's enabled or not */ 1735 WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset, 1736 upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF); 1737 WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset, 1738 ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC)); 1739 1740 if (rdev->wb.enabled) 1741 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE; 1742 1743 WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8); 1744 1745 /* enable DMA IBs */ 1746 ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE; 1747 #ifdef __BIG_ENDIAN 1748 ib_cntl |= DMA_IB_SWAP_ENABLE; 1749 #endif 1750 WREG32(DMA_IB_CNTL + reg_offset, ib_cntl); 1751 1752 dma_cntl = RREG32(DMA_CNTL + reg_offset); 1753 dma_cntl &= ~CTXEMPTY_INT_ENABLE; 1754 WREG32(DMA_CNTL + reg_offset, dma_cntl); 1755 1756 ring->wptr = 0; 1757 WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2); 1758 1759 ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2; 1760 1761 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE); 1762 1763 ring->ready = true; 1764 1765 r = radeon_ring_test(rdev, ring->idx, ring); 1766 if (r) { 1767 ring->ready = false; 1768 return r; 1769 } 1770 } 1771 1772 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 1773 1774 return 0; 1775 } 1776 1777 /** 1778 * cayman_dma_fini - tear down the async dma engines 1779 * 1780 * @rdev: radeon_device pointer 1781 * 1782 * Stop the async dma engines and free the rings (cayman-SI). 1783 */ 1784 void cayman_dma_fini(struct radeon_device *rdev) 1785 { 1786 cayman_dma_stop(rdev); 1787 radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]); 1788 radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]); 1789 } 1790 1791 static u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev) 1792 { 1793 u32 reset_mask = 0; 1794 u32 tmp; 1795 1796 /* GRBM_STATUS */ 1797 tmp = RREG32(GRBM_STATUS); 1798 if (tmp & (PA_BUSY | SC_BUSY | 1799 SH_BUSY | SX_BUSY | 1800 TA_BUSY | VGT_BUSY | 1801 DB_BUSY | CB_BUSY | 1802 GDS_BUSY | SPI_BUSY | 1803 IA_BUSY | IA_BUSY_NO_DMA)) 1804 reset_mask |= RADEON_RESET_GFX; 1805 1806 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 1807 CP_BUSY | CP_COHERENCY_BUSY)) 1808 reset_mask |= RADEON_RESET_CP; 1809 1810 if (tmp & GRBM_EE_BUSY) 1811 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 1812 1813 /* DMA_STATUS_REG 0 */ 1814 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); 1815 if (!(tmp & DMA_IDLE)) 1816 reset_mask |= RADEON_RESET_DMA; 1817 1818 /* DMA_STATUS_REG 1 */ 1819 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); 1820 if (!(tmp & DMA_IDLE)) 1821 reset_mask |= RADEON_RESET_DMA1; 1822 1823 /* SRBM_STATUS2 */ 1824 tmp = RREG32(SRBM_STATUS2); 1825 if (tmp & DMA_BUSY) 1826 reset_mask |= RADEON_RESET_DMA; 1827 1828 if (tmp & DMA1_BUSY) 1829 reset_mask |= RADEON_RESET_DMA1; 1830 1831 /* SRBM_STATUS */ 1832 tmp = RREG32(SRBM_STATUS); 1833 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 1834 reset_mask |= RADEON_RESET_RLC; 1835 1836 if (tmp & IH_BUSY) 1837 reset_mask |= RADEON_RESET_IH; 1838 1839 if (tmp & SEM_BUSY) 1840 reset_mask |= RADEON_RESET_SEM; 1841 1842 if (tmp & GRBM_RQ_PENDING) 1843 reset_mask |= RADEON_RESET_GRBM; 1844 1845 if (tmp & VMC_BUSY) 1846 reset_mask |= RADEON_RESET_VMC; 1847 1848 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 1849 MCC_BUSY | MCD_BUSY)) 1850 reset_mask |= RADEON_RESET_MC; 1851 1852 if (evergreen_is_display_hung(rdev)) 1853 reset_mask |= RADEON_RESET_DISPLAY; 1854 1855 /* VM_L2_STATUS */ 1856 tmp = RREG32(VM_L2_STATUS); 1857 if (tmp & L2_BUSY) 1858 reset_mask |= RADEON_RESET_VMC; 1859 1860 /* Skip MC reset as it's mostly likely not hung, just busy */ 1861 if (reset_mask & RADEON_RESET_MC) { 1862 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 1863 reset_mask &= ~RADEON_RESET_MC; 1864 } 1865 1866 return reset_mask; 1867 } 1868 1869 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 1870 { 1871 struct evergreen_mc_save save; 1872 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 1873 u32 tmp; 1874 1875 if (reset_mask == 0) 1876 return; 1877 1878 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 1879 1880 evergreen_print_gpu_status_regs(rdev); 1881 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n", 1882 RREG32(0x14F8)); 1883 dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n", 1884 RREG32(0x14D8)); 1885 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 1886 RREG32(0x14FC)); 1887 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 1888 RREG32(0x14DC)); 1889 1890 /* Disable CP parsing/prefetching */ 1891 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 1892 1893 if (reset_mask & RADEON_RESET_DMA) { 1894 /* dma0 */ 1895 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 1896 tmp &= ~DMA_RB_ENABLE; 1897 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 1898 } 1899 1900 if (reset_mask & RADEON_RESET_DMA1) { 1901 /* dma1 */ 1902 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 1903 tmp &= ~DMA_RB_ENABLE; 1904 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 1905 } 1906 1907 udelay(50); 1908 1909 evergreen_mc_stop(rdev, &save); 1910 if (evergreen_mc_wait_for_idle(rdev)) { 1911 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 1912 } 1913 1914 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 1915 grbm_soft_reset = SOFT_RESET_CB | 1916 SOFT_RESET_DB | 1917 SOFT_RESET_GDS | 1918 SOFT_RESET_PA | 1919 SOFT_RESET_SC | 1920 SOFT_RESET_SPI | 1921 SOFT_RESET_SH | 1922 SOFT_RESET_SX | 1923 SOFT_RESET_TC | 1924 SOFT_RESET_TA | 1925 SOFT_RESET_VGT | 1926 SOFT_RESET_IA; 1927 } 1928 1929 if (reset_mask & RADEON_RESET_CP) { 1930 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT; 1931 1932 srbm_soft_reset |= SOFT_RESET_GRBM; 1933 } 1934 1935 if (reset_mask & RADEON_RESET_DMA) 1936 srbm_soft_reset |= SOFT_RESET_DMA; 1937 1938 if (reset_mask & RADEON_RESET_DMA1) 1939 srbm_soft_reset |= SOFT_RESET_DMA1; 1940 1941 if (reset_mask & RADEON_RESET_DISPLAY) 1942 srbm_soft_reset |= SOFT_RESET_DC; 1943 1944 if (reset_mask & RADEON_RESET_RLC) 1945 srbm_soft_reset |= SOFT_RESET_RLC; 1946 1947 if (reset_mask & RADEON_RESET_SEM) 1948 srbm_soft_reset |= SOFT_RESET_SEM; 1949 1950 if (reset_mask & RADEON_RESET_IH) 1951 srbm_soft_reset |= SOFT_RESET_IH; 1952 1953 if (reset_mask & RADEON_RESET_GRBM) 1954 srbm_soft_reset |= SOFT_RESET_GRBM; 1955 1956 if (reset_mask & RADEON_RESET_VMC) 1957 srbm_soft_reset |= SOFT_RESET_VMC; 1958 1959 if (!(rdev->flags & RADEON_IS_IGP)) { 1960 if (reset_mask & RADEON_RESET_MC) 1961 srbm_soft_reset |= SOFT_RESET_MC; 1962 } 1963 1964 if (grbm_soft_reset) { 1965 tmp = RREG32(GRBM_SOFT_RESET); 1966 tmp |= grbm_soft_reset; 1967 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 1968 WREG32(GRBM_SOFT_RESET, tmp); 1969 tmp = RREG32(GRBM_SOFT_RESET); 1970 1971 udelay(50); 1972 1973 tmp &= ~grbm_soft_reset; 1974 WREG32(GRBM_SOFT_RESET, tmp); 1975 tmp = RREG32(GRBM_SOFT_RESET); 1976 } 1977 1978 if (srbm_soft_reset) { 1979 tmp = RREG32(SRBM_SOFT_RESET); 1980 tmp |= srbm_soft_reset; 1981 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 1982 WREG32(SRBM_SOFT_RESET, tmp); 1983 tmp = RREG32(SRBM_SOFT_RESET); 1984 1985 udelay(50); 1986 1987 tmp &= ~srbm_soft_reset; 1988 WREG32(SRBM_SOFT_RESET, tmp); 1989 tmp = RREG32(SRBM_SOFT_RESET); 1990 } 1991 1992 /* Wait a little for things to settle down */ 1993 udelay(50); 1994 1995 evergreen_mc_resume(rdev, &save); 1996 udelay(50); 1997 1998 evergreen_print_gpu_status_regs(rdev); 1999 } 2000 2001 int cayman_asic_reset(struct radeon_device *rdev) 2002 { 2003 u32 reset_mask; 2004 2005 reset_mask = cayman_gpu_check_soft_reset(rdev); 2006 2007 if (reset_mask) 2008 r600_set_bios_scratch_engine_hung(rdev, true); 2009 2010 cayman_gpu_soft_reset(rdev, reset_mask); 2011 2012 reset_mask = cayman_gpu_check_soft_reset(rdev); 2013 2014 if (!reset_mask) 2015 r600_set_bios_scratch_engine_hung(rdev, false); 2016 2017 return 0; 2018 } 2019 2020 /** 2021 * cayman_gfx_is_lockup - Check if the GFX engine is locked up 2022 * 2023 * @rdev: radeon_device pointer 2024 * @ring: radeon_ring structure holding ring information 2025 * 2026 * Check if the GFX engine is locked up. 2027 * Returns true if the engine appears to be locked up, false if not. 2028 */ 2029 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 2030 { 2031 u32 reset_mask = cayman_gpu_check_soft_reset(rdev); 2032 2033 if (!(reset_mask & (RADEON_RESET_GFX | 2034 RADEON_RESET_COMPUTE | 2035 RADEON_RESET_CP))) { 2036 radeon_ring_lockup_update(ring); 2037 return false; 2038 } 2039 /* force CP activities */ 2040 radeon_ring_force_activity(rdev, ring); 2041 return radeon_ring_test_lockup(rdev, ring); 2042 } 2043 2044 /** 2045 * cayman_dma_is_lockup - Check if the DMA engine is locked up 2046 * 2047 * @rdev: radeon_device pointer 2048 * @ring: radeon_ring structure holding ring information 2049 * 2050 * Check if the async DMA engine is locked up. 2051 * Returns true if the engine appears to be locked up, false if not. 2052 */ 2053 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 2054 { 2055 u32 reset_mask = cayman_gpu_check_soft_reset(rdev); 2056 u32 mask; 2057 2058 if (ring->idx == R600_RING_TYPE_DMA_INDEX) 2059 mask = RADEON_RESET_DMA; 2060 else 2061 mask = RADEON_RESET_DMA1; 2062 2063 if (!(reset_mask & mask)) { 2064 radeon_ring_lockup_update(ring); 2065 return false; 2066 } 2067 /* force ring activities */ 2068 radeon_ring_force_activity(rdev, ring); 2069 return radeon_ring_test_lockup(rdev, ring); 2070 } 2071 2072 static int cayman_startup(struct radeon_device *rdev) 2073 { 2074 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 2075 int r; 2076 2077 /* enable pcie gen2 link */ 2078 evergreen_pcie_gen2_enable(rdev); 2079 /* enable aspm */ 2080 evergreen_program_aspm(rdev); 2081 2082 if (rdev->flags & RADEON_IS_IGP) { 2083 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 2084 r = ni_init_microcode(rdev); 2085 if (r) { 2086 DRM_ERROR("Failed to load firmware!\n"); 2087 return r; 2088 } 2089 } 2090 } else { 2091 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 2092 r = ni_init_microcode(rdev); 2093 if (r) { 2094 DRM_ERROR("Failed to load firmware!\n"); 2095 return r; 2096 } 2097 } 2098 2099 r = ni_mc_load_microcode(rdev); 2100 if (r) { 2101 DRM_ERROR("Failed to load MC firmware!\n"); 2102 return r; 2103 } 2104 } 2105 2106 r = r600_vram_scratch_init(rdev); 2107 if (r) 2108 return r; 2109 2110 evergreen_mc_program(rdev); 2111 r = cayman_pcie_gart_enable(rdev); 2112 if (r) 2113 return r; 2114 cayman_gpu_init(rdev); 2115 2116 r = evergreen_blit_init(rdev); 2117 if (r) { 2118 r600_blit_fini(rdev); 2119 rdev->asic->copy.copy = NULL; 2120 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); 2121 } 2122 2123 /* allocate rlc buffers */ 2124 if (rdev->flags & RADEON_IS_IGP) { 2125 rdev->rlc.reg_list = tn_rlc_save_restore_register_list; 2126 rdev->rlc.reg_list_size = tn_rlc_save_restore_register_list_size; 2127 rdev->rlc.cs_data = cayman_cs_data; 2128 r = sumo_rlc_init(rdev); 2129 if (r) { 2130 DRM_ERROR("Failed to init rlc BOs!\n"); 2131 return r; 2132 } 2133 } 2134 2135 /* allocate wb buffer */ 2136 r = radeon_wb_init(rdev); 2137 if (r) 2138 return r; 2139 2140 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 2141 if (r) { 2142 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 2143 return r; 2144 } 2145 2146 r = rv770_uvd_resume(rdev); 2147 if (!r) { 2148 r = radeon_fence_driver_start_ring(rdev, 2149 R600_RING_TYPE_UVD_INDEX); 2150 if (r) 2151 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 2152 } 2153 if (r) 2154 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 2155 2156 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 2157 if (r) { 2158 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 2159 return r; 2160 } 2161 2162 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 2163 if (r) { 2164 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 2165 return r; 2166 } 2167 2168 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 2169 if (r) { 2170 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 2171 return r; 2172 } 2173 2174 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 2175 if (r) { 2176 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 2177 return r; 2178 } 2179 2180 /* Enable IRQ */ 2181 if (!rdev->irq.installed) { 2182 r = radeon_irq_kms_init(rdev); 2183 if (r) 2184 return r; 2185 } 2186 2187 r = r600_irq_init(rdev); 2188 if (r) { 2189 DRM_ERROR("radeon: IH init failed (%d).\n", r); 2190 radeon_irq_kms_fini(rdev); 2191 return r; 2192 } 2193 evergreen_irq_set(rdev); 2194 2195 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 2196 CP_RB0_RPTR, CP_RB0_WPTR, 2197 0, 0xfffff, RADEON_CP_PACKET2); 2198 if (r) 2199 return r; 2200 2201 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 2202 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 2203 DMA_RB_RPTR + DMA0_REGISTER_OFFSET, 2204 DMA_RB_WPTR + DMA0_REGISTER_OFFSET, 2205 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); 2206 if (r) 2207 return r; 2208 2209 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 2210 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 2211 DMA_RB_RPTR + DMA1_REGISTER_OFFSET, 2212 DMA_RB_WPTR + DMA1_REGISTER_OFFSET, 2213 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); 2214 if (r) 2215 return r; 2216 2217 r = cayman_cp_load_microcode(rdev); 2218 if (r) 2219 return r; 2220 r = cayman_cp_resume(rdev); 2221 if (r) 2222 return r; 2223 2224 r = cayman_dma_resume(rdev); 2225 if (r) 2226 return r; 2227 2228 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 2229 if (ring->ring_size) { 2230 r = radeon_ring_init(rdev, ring, ring->ring_size, 2231 R600_WB_UVD_RPTR_OFFSET, 2232 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR, 2233 0, 0xfffff, RADEON_CP_PACKET2); 2234 if (!r) 2235 r = r600_uvd_init(rdev); 2236 if (r) 2237 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 2238 } 2239 2240 r = radeon_ib_pool_init(rdev); 2241 if (r) { 2242 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 2243 return r; 2244 } 2245 2246 r = radeon_vm_manager_init(rdev); 2247 if (r) { 2248 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 2249 return r; 2250 } 2251 2252 r = r600_audio_init(rdev); 2253 if (r) 2254 return r; 2255 2256 return 0; 2257 } 2258 2259 int cayman_resume(struct radeon_device *rdev) 2260 { 2261 int r; 2262 2263 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 2264 * posting will perform necessary task to bring back GPU into good 2265 * shape. 2266 */ 2267 /* post card */ 2268 atom_asic_init(rdev->mode_info.atom_context); 2269 2270 /* init golden registers */ 2271 ni_init_golden_registers(rdev); 2272 2273 rdev->accel_working = true; 2274 r = cayman_startup(rdev); 2275 if (r) { 2276 DRM_ERROR("cayman startup failed on resume\n"); 2277 rdev->accel_working = false; 2278 return r; 2279 } 2280 return r; 2281 } 2282 2283 int cayman_suspend(struct radeon_device *rdev) 2284 { 2285 r600_audio_fini(rdev); 2286 radeon_vm_manager_fini(rdev); 2287 cayman_cp_enable(rdev, false); 2288 cayman_dma_stop(rdev); 2289 r600_uvd_rbc_stop(rdev); 2290 radeon_uvd_suspend(rdev); 2291 evergreen_irq_suspend(rdev); 2292 radeon_wb_disable(rdev); 2293 cayman_pcie_gart_disable(rdev); 2294 return 0; 2295 } 2296 2297 /* Plan is to move initialization in that function and use 2298 * helper function so that radeon_device_init pretty much 2299 * do nothing more than calling asic specific function. This 2300 * should also allow to remove a bunch of callback function 2301 * like vram_info. 2302 */ 2303 int cayman_init(struct radeon_device *rdev) 2304 { 2305 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 2306 int r; 2307 2308 /* Read BIOS */ 2309 if (!radeon_get_bios(rdev)) { 2310 if (ASIC_IS_AVIVO(rdev)) 2311 return -EINVAL; 2312 } 2313 /* Must be an ATOMBIOS */ 2314 if (!rdev->is_atom_bios) { 2315 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 2316 return -EINVAL; 2317 } 2318 r = radeon_atombios_init(rdev); 2319 if (r) 2320 return r; 2321 2322 /* Post card if necessary */ 2323 if (!radeon_card_posted(rdev)) { 2324 if (!rdev->bios) { 2325 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 2326 return -EINVAL; 2327 } 2328 DRM_INFO("GPU not posted. posting now...\n"); 2329 atom_asic_init(rdev->mode_info.atom_context); 2330 } 2331 /* init golden registers */ 2332 ni_init_golden_registers(rdev); 2333 /* Initialize scratch registers */ 2334 r600_scratch_init(rdev); 2335 /* Initialize surface registers */ 2336 radeon_surface_init(rdev); 2337 /* Initialize clocks */ 2338 radeon_get_clock_info(rdev->ddev); 2339 /* Fence driver */ 2340 r = radeon_fence_driver_init(rdev); 2341 if (r) 2342 return r; 2343 /* initialize memory controller */ 2344 r = evergreen_mc_init(rdev); 2345 if (r) 2346 return r; 2347 /* Memory manager */ 2348 r = radeon_bo_init(rdev); 2349 if (r) 2350 return r; 2351 2352 ring->ring_obj = NULL; 2353 r600_ring_init(rdev, ring, 1024 * 1024); 2354 2355 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 2356 ring->ring_obj = NULL; 2357 r600_ring_init(rdev, ring, 64 * 1024); 2358 2359 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 2360 ring->ring_obj = NULL; 2361 r600_ring_init(rdev, ring, 64 * 1024); 2362 2363 r = radeon_uvd_init(rdev); 2364 if (!r) { 2365 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 2366 ring->ring_obj = NULL; 2367 r600_ring_init(rdev, ring, 4096); 2368 } 2369 2370 rdev->ih.ring_obj = NULL; 2371 r600_ih_ring_init(rdev, 64 * 1024); 2372 2373 r = r600_pcie_gart_init(rdev); 2374 if (r) 2375 return r; 2376 2377 rdev->accel_working = true; 2378 r = cayman_startup(rdev); 2379 if (r) { 2380 dev_err(rdev->dev, "disabling GPU acceleration\n"); 2381 cayman_cp_fini(rdev); 2382 cayman_dma_fini(rdev); 2383 r600_irq_fini(rdev); 2384 if (rdev->flags & RADEON_IS_IGP) 2385 sumo_rlc_fini(rdev); 2386 radeon_wb_fini(rdev); 2387 radeon_ib_pool_fini(rdev); 2388 radeon_vm_manager_fini(rdev); 2389 radeon_irq_kms_fini(rdev); 2390 cayman_pcie_gart_fini(rdev); 2391 rdev->accel_working = false; 2392 } 2393 2394 /* Don't start up if the MC ucode is missing. 2395 * The default clocks and voltages before the MC ucode 2396 * is loaded are not suffient for advanced operations. 2397 * 2398 * We can skip this check for TN, because there is no MC 2399 * ucode. 2400 */ 2401 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 2402 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 2403 return -EINVAL; 2404 } 2405 2406 return 0; 2407 } 2408 2409 void cayman_fini(struct radeon_device *rdev) 2410 { 2411 r600_blit_fini(rdev); 2412 cayman_cp_fini(rdev); 2413 cayman_dma_fini(rdev); 2414 r600_irq_fini(rdev); 2415 if (rdev->flags & RADEON_IS_IGP) 2416 sumo_rlc_fini(rdev); 2417 radeon_wb_fini(rdev); 2418 radeon_vm_manager_fini(rdev); 2419 radeon_ib_pool_fini(rdev); 2420 radeon_irq_kms_fini(rdev); 2421 radeon_uvd_fini(rdev); 2422 cayman_pcie_gart_fini(rdev); 2423 r600_vram_scratch_fini(rdev); 2424 radeon_gem_fini(rdev); 2425 radeon_fence_driver_fini(rdev); 2426 radeon_bo_fini(rdev); 2427 radeon_atombios_fini(rdev); 2428 kfree(rdev->bios); 2429 rdev->bios = NULL; 2430 } 2431 2432 /* 2433 * vm 2434 */ 2435 int cayman_vm_init(struct radeon_device *rdev) 2436 { 2437 /* number of VMs */ 2438 rdev->vm_manager.nvm = 8; 2439 /* base offset of vram pages */ 2440 if (rdev->flags & RADEON_IS_IGP) { 2441 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET); 2442 tmp <<= 22; 2443 rdev->vm_manager.vram_base_offset = tmp; 2444 } else 2445 rdev->vm_manager.vram_base_offset = 0; 2446 return 0; 2447 } 2448 2449 void cayman_vm_fini(struct radeon_device *rdev) 2450 { 2451 } 2452 2453 /** 2454 * cayman_vm_decode_fault - print human readable fault info 2455 * 2456 * @rdev: radeon_device pointer 2457 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 2458 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 2459 * 2460 * Print human readable fault information (cayman/TN). 2461 */ 2462 void cayman_vm_decode_fault(struct radeon_device *rdev, 2463 u32 status, u32 addr) 2464 { 2465 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 2466 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 2467 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 2468 char *block; 2469 2470 switch (mc_id) { 2471 case 32: 2472 case 16: 2473 case 96: 2474 case 80: 2475 case 160: 2476 case 144: 2477 case 224: 2478 case 208: 2479 block = "CB"; 2480 break; 2481 case 33: 2482 case 17: 2483 case 97: 2484 case 81: 2485 case 161: 2486 case 145: 2487 case 225: 2488 case 209: 2489 block = "CB_FMASK"; 2490 break; 2491 case 34: 2492 case 18: 2493 case 98: 2494 case 82: 2495 case 162: 2496 case 146: 2497 case 226: 2498 case 210: 2499 block = "CB_CMASK"; 2500 break; 2501 case 35: 2502 case 19: 2503 case 99: 2504 case 83: 2505 case 163: 2506 case 147: 2507 case 227: 2508 case 211: 2509 block = "CB_IMMED"; 2510 break; 2511 case 36: 2512 case 20: 2513 case 100: 2514 case 84: 2515 case 164: 2516 case 148: 2517 case 228: 2518 case 212: 2519 block = "DB"; 2520 break; 2521 case 37: 2522 case 21: 2523 case 101: 2524 case 85: 2525 case 165: 2526 case 149: 2527 case 229: 2528 case 213: 2529 block = "DB_HTILE"; 2530 break; 2531 case 38: 2532 case 22: 2533 case 102: 2534 case 86: 2535 case 166: 2536 case 150: 2537 case 230: 2538 case 214: 2539 block = "SX"; 2540 break; 2541 case 39: 2542 case 23: 2543 case 103: 2544 case 87: 2545 case 167: 2546 case 151: 2547 case 231: 2548 case 215: 2549 block = "DB_STEN"; 2550 break; 2551 case 40: 2552 case 24: 2553 case 104: 2554 case 88: 2555 case 232: 2556 case 216: 2557 case 168: 2558 case 152: 2559 block = "TC_TFETCH"; 2560 break; 2561 case 41: 2562 case 25: 2563 case 105: 2564 case 89: 2565 case 233: 2566 case 217: 2567 case 169: 2568 case 153: 2569 block = "TC_VFETCH"; 2570 break; 2571 case 42: 2572 case 26: 2573 case 106: 2574 case 90: 2575 case 234: 2576 case 218: 2577 case 170: 2578 case 154: 2579 block = "VC"; 2580 break; 2581 case 112: 2582 block = "CP"; 2583 break; 2584 case 113: 2585 case 114: 2586 block = "SH"; 2587 break; 2588 case 115: 2589 block = "VGT"; 2590 break; 2591 case 178: 2592 block = "IH"; 2593 break; 2594 case 51: 2595 block = "RLC"; 2596 break; 2597 case 55: 2598 block = "DMA"; 2599 break; 2600 case 56: 2601 block = "HDP"; 2602 break; 2603 default: 2604 block = "unknown"; 2605 break; 2606 } 2607 2608 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n", 2609 protections, vmid, addr, 2610 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 2611 block, mc_id); 2612 } 2613 2614 #define R600_ENTRY_VALID (1 << 0) 2615 #define R600_PTE_SYSTEM (1 << 1) 2616 #define R600_PTE_SNOOPED (1 << 2) 2617 #define R600_PTE_READABLE (1 << 5) 2618 #define R600_PTE_WRITEABLE (1 << 6) 2619 2620 uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags) 2621 { 2622 uint32_t r600_flags = 0; 2623 r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0; 2624 r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0; 2625 r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0; 2626 if (flags & RADEON_VM_PAGE_SYSTEM) { 2627 r600_flags |= R600_PTE_SYSTEM; 2628 r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0; 2629 } 2630 return r600_flags; 2631 } 2632 2633 /** 2634 * cayman_vm_set_page - update the page tables using the CP 2635 * 2636 * @rdev: radeon_device pointer 2637 * @ib: indirect buffer to fill with commands 2638 * @pe: addr of the page entry 2639 * @addr: dst addr to write into pe 2640 * @count: number of page entries to update 2641 * @incr: increase next addr by incr bytes 2642 * @flags: access flags 2643 * 2644 * Update the page tables using the CP (cayman/TN). 2645 */ 2646 void cayman_vm_set_page(struct radeon_device *rdev, 2647 struct radeon_ib *ib, 2648 uint64_t pe, 2649 uint64_t addr, unsigned count, 2650 uint32_t incr, uint32_t flags) 2651 { 2652 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); 2653 uint64_t value; 2654 unsigned ndw; 2655 2656 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) { 2657 while (count) { 2658 ndw = 1 + count * 2; 2659 if (ndw > 0x3FFF) 2660 ndw = 0x3FFF; 2661 2662 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_ME_WRITE, ndw); 2663 ib->ptr[ib->length_dw++] = pe; 2664 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 2665 for (; ndw > 1; ndw -= 2, --count, pe += 8) { 2666 if (flags & RADEON_VM_PAGE_SYSTEM) { 2667 value = radeon_vm_map_gart(rdev, addr); 2668 value &= 0xFFFFFFFFFFFFF000ULL; 2669 } else if (flags & RADEON_VM_PAGE_VALID) { 2670 value = addr; 2671 } else { 2672 value = 0; 2673 } 2674 addr += incr; 2675 value |= r600_flags; 2676 ib->ptr[ib->length_dw++] = value; 2677 ib->ptr[ib->length_dw++] = upper_32_bits(value); 2678 } 2679 } 2680 } else { 2681 if ((flags & RADEON_VM_PAGE_SYSTEM) || 2682 (count == 1)) { 2683 while (count) { 2684 ndw = count * 2; 2685 if (ndw > 0xFFFFE) 2686 ndw = 0xFFFFE; 2687 2688 /* for non-physically contiguous pages (system) */ 2689 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw); 2690 ib->ptr[ib->length_dw++] = pe; 2691 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 2692 for (; ndw > 0; ndw -= 2, --count, pe += 8) { 2693 if (flags & RADEON_VM_PAGE_SYSTEM) { 2694 value = radeon_vm_map_gart(rdev, addr); 2695 value &= 0xFFFFFFFFFFFFF000ULL; 2696 } else if (flags & RADEON_VM_PAGE_VALID) { 2697 value = addr; 2698 } else { 2699 value = 0; 2700 } 2701 addr += incr; 2702 value |= r600_flags; 2703 ib->ptr[ib->length_dw++] = value; 2704 ib->ptr[ib->length_dw++] = upper_32_bits(value); 2705 } 2706 } 2707 while (ib->length_dw & 0x7) 2708 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0); 2709 } else { 2710 while (count) { 2711 ndw = count * 2; 2712 if (ndw > 0xFFFFE) 2713 ndw = 0xFFFFE; 2714 2715 if (flags & RADEON_VM_PAGE_VALID) 2716 value = addr; 2717 else 2718 value = 0; 2719 /* for physically contiguous pages (vram) */ 2720 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw); 2721 ib->ptr[ib->length_dw++] = pe; /* dst addr */ 2722 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 2723 ib->ptr[ib->length_dw++] = r600_flags; /* mask */ 2724 ib->ptr[ib->length_dw++] = 0; 2725 ib->ptr[ib->length_dw++] = value; /* value */ 2726 ib->ptr[ib->length_dw++] = upper_32_bits(value); 2727 ib->ptr[ib->length_dw++] = incr; /* increment size */ 2728 ib->ptr[ib->length_dw++] = 0; 2729 pe += ndw * 4; 2730 addr += (ndw / 2) * incr; 2731 count -= ndw / 2; 2732 } 2733 } 2734 while (ib->length_dw & 0x7) 2735 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0); 2736 } 2737 } 2738 2739 /** 2740 * cayman_vm_flush - vm flush using the CP 2741 * 2742 * @rdev: radeon_device pointer 2743 * 2744 * Update the page table base and flush the VM TLB 2745 * using the CP (cayman-si). 2746 */ 2747 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 2748 { 2749 struct radeon_ring *ring = &rdev->ring[ridx]; 2750 2751 if (vm == NULL) 2752 return; 2753 2754 radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0)); 2755 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 2756 2757 /* flush hdp cache */ 2758 radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0)); 2759 radeon_ring_write(ring, 0x1); 2760 2761 /* bits 0-7 are the VM contexts0-7 */ 2762 radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0)); 2763 radeon_ring_write(ring, 1 << vm->id); 2764 2765 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 2766 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 2767 radeon_ring_write(ring, 0x0); 2768 } 2769 2770 void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 2771 { 2772 struct radeon_ring *ring = &rdev->ring[ridx]; 2773 2774 if (vm == NULL) 2775 return; 2776 2777 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); 2778 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2)); 2779 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 2780 2781 /* flush hdp cache */ 2782 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); 2783 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); 2784 radeon_ring_write(ring, 1); 2785 2786 /* bits 0-7 are the VM contexts0-7 */ 2787 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); 2788 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); 2789 radeon_ring_write(ring, 1 << vm->id); 2790 } 2791 2792