xref: /openbmc/linux/drivers/gpu/drm/radeon/r600.c (revision 31b90347)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/slab.h>
29 #include <linux/seq_file.h>
30 #include <linux/firmware.h>
31 #include <linux/module.h>
32 #include <drm/drmP.h>
33 #include <drm/radeon_drm.h>
34 #include "radeon.h"
35 #include "radeon_asic.h"
36 #include "radeon_mode.h"
37 #include "r600d.h"
38 #include "atom.h"
39 #include "avivod.h"
40 #include "radeon_ucode.h"
41 
42 /* Firmware Names */
43 MODULE_FIRMWARE("radeon/R600_pfp.bin");
44 MODULE_FIRMWARE("radeon/R600_me.bin");
45 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
46 MODULE_FIRMWARE("radeon/RV610_me.bin");
47 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
48 MODULE_FIRMWARE("radeon/RV630_me.bin");
49 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
50 MODULE_FIRMWARE("radeon/RV620_me.bin");
51 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
52 MODULE_FIRMWARE("radeon/RV635_me.bin");
53 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
54 MODULE_FIRMWARE("radeon/RV670_me.bin");
55 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
56 MODULE_FIRMWARE("radeon/RS780_me.bin");
57 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
58 MODULE_FIRMWARE("radeon/RV770_me.bin");
59 MODULE_FIRMWARE("radeon/RV770_smc.bin");
60 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
61 MODULE_FIRMWARE("radeon/RV730_me.bin");
62 MODULE_FIRMWARE("radeon/RV730_smc.bin");
63 MODULE_FIRMWARE("radeon/RV740_smc.bin");
64 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
65 MODULE_FIRMWARE("radeon/RV710_me.bin");
66 MODULE_FIRMWARE("radeon/RV710_smc.bin");
67 MODULE_FIRMWARE("radeon/R600_rlc.bin");
68 MODULE_FIRMWARE("radeon/R700_rlc.bin");
69 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
70 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
71 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
72 MODULE_FIRMWARE("radeon/CEDAR_smc.bin");
73 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
74 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
75 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
76 MODULE_FIRMWARE("radeon/REDWOOD_smc.bin");
77 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
78 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
79 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
80 MODULE_FIRMWARE("radeon/JUNIPER_smc.bin");
81 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
82 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
83 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
84 MODULE_FIRMWARE("radeon/CYPRESS_smc.bin");
85 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
86 MODULE_FIRMWARE("radeon/PALM_me.bin");
87 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
88 MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
89 MODULE_FIRMWARE("radeon/SUMO_me.bin");
90 MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
91 MODULE_FIRMWARE("radeon/SUMO2_me.bin");
92 
93 static const u32 crtc_offsets[2] =
94 {
95 	0,
96 	AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
97 };
98 
99 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
100 
101 /* r600,rv610,rv630,rv620,rv635,rv670 */
102 int r600_mc_wait_for_idle(struct radeon_device *rdev);
103 static void r600_gpu_init(struct radeon_device *rdev);
104 void r600_fini(struct radeon_device *rdev);
105 void r600_irq_disable(struct radeon_device *rdev);
106 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
107 extern int evergreen_rlc_resume(struct radeon_device *rdev);
108 
109 /**
110  * r600_get_xclk - get the xclk
111  *
112  * @rdev: radeon_device pointer
113  *
114  * Returns the reference clock used by the gfx engine
115  * (r6xx, IGPs, APUs).
116  */
117 u32 r600_get_xclk(struct radeon_device *rdev)
118 {
119 	return rdev->clock.spll.reference_freq;
120 }
121 
122 int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
123 {
124 	return 0;
125 }
126 
127 void dce3_program_fmt(struct drm_encoder *encoder)
128 {
129 	struct drm_device *dev = encoder->dev;
130 	struct radeon_device *rdev = dev->dev_private;
131 	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
132 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
133 	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
134 	int bpc = 0;
135 	u32 tmp = 0;
136 	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
137 
138 	if (connector) {
139 		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
140 		bpc = radeon_get_monitor_bpc(connector);
141 		dither = radeon_connector->dither;
142 	}
143 
144 	/* LVDS FMT is set up by atom */
145 	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
146 		return;
147 
148 	/* not needed for analog */
149 	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
150 	    (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
151 		return;
152 
153 	if (bpc == 0)
154 		return;
155 
156 	switch (bpc) {
157 	case 6:
158 		if (dither == RADEON_FMT_DITHER_ENABLE)
159 			/* XXX sort out optimal dither settings */
160 			tmp |= FMT_SPATIAL_DITHER_EN;
161 		else
162 			tmp |= FMT_TRUNCATE_EN;
163 		break;
164 	case 8:
165 		if (dither == RADEON_FMT_DITHER_ENABLE)
166 			/* XXX sort out optimal dither settings */
167 			tmp |= (FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
168 		else
169 			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
170 		break;
171 	case 10:
172 	default:
173 		/* not needed */
174 		break;
175 	}
176 
177 	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
178 }
179 
180 /* get temperature in millidegrees */
181 int rv6xx_get_temp(struct radeon_device *rdev)
182 {
183 	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
184 		ASIC_T_SHIFT;
185 	int actual_temp = temp & 0xff;
186 
187 	if (temp & 0x100)
188 		actual_temp -= 256;
189 
190 	return actual_temp * 1000;
191 }
192 
193 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
194 {
195 	int i;
196 
197 	rdev->pm.dynpm_can_upclock = true;
198 	rdev->pm.dynpm_can_downclock = true;
199 
200 	/* power state array is low to high, default is first */
201 	if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
202 		int min_power_state_index = 0;
203 
204 		if (rdev->pm.num_power_states > 2)
205 			min_power_state_index = 1;
206 
207 		switch (rdev->pm.dynpm_planned_action) {
208 		case DYNPM_ACTION_MINIMUM:
209 			rdev->pm.requested_power_state_index = min_power_state_index;
210 			rdev->pm.requested_clock_mode_index = 0;
211 			rdev->pm.dynpm_can_downclock = false;
212 			break;
213 		case DYNPM_ACTION_DOWNCLOCK:
214 			if (rdev->pm.current_power_state_index == min_power_state_index) {
215 				rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
216 				rdev->pm.dynpm_can_downclock = false;
217 			} else {
218 				if (rdev->pm.active_crtc_count > 1) {
219 					for (i = 0; i < rdev->pm.num_power_states; i++) {
220 						if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
221 							continue;
222 						else if (i >= rdev->pm.current_power_state_index) {
223 							rdev->pm.requested_power_state_index =
224 								rdev->pm.current_power_state_index;
225 							break;
226 						} else {
227 							rdev->pm.requested_power_state_index = i;
228 							break;
229 						}
230 					}
231 				} else {
232 					if (rdev->pm.current_power_state_index == 0)
233 						rdev->pm.requested_power_state_index =
234 							rdev->pm.num_power_states - 1;
235 					else
236 						rdev->pm.requested_power_state_index =
237 							rdev->pm.current_power_state_index - 1;
238 				}
239 			}
240 			rdev->pm.requested_clock_mode_index = 0;
241 			/* don't use the power state if crtcs are active and no display flag is set */
242 			if ((rdev->pm.active_crtc_count > 0) &&
243 			    (rdev->pm.power_state[rdev->pm.requested_power_state_index].
244 			     clock_info[rdev->pm.requested_clock_mode_index].flags &
245 			     RADEON_PM_MODE_NO_DISPLAY)) {
246 				rdev->pm.requested_power_state_index++;
247 			}
248 			break;
249 		case DYNPM_ACTION_UPCLOCK:
250 			if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
251 				rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
252 				rdev->pm.dynpm_can_upclock = false;
253 			} else {
254 				if (rdev->pm.active_crtc_count > 1) {
255 					for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
256 						if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
257 							continue;
258 						else if (i <= rdev->pm.current_power_state_index) {
259 							rdev->pm.requested_power_state_index =
260 								rdev->pm.current_power_state_index;
261 							break;
262 						} else {
263 							rdev->pm.requested_power_state_index = i;
264 							break;
265 						}
266 					}
267 				} else
268 					rdev->pm.requested_power_state_index =
269 						rdev->pm.current_power_state_index + 1;
270 			}
271 			rdev->pm.requested_clock_mode_index = 0;
272 			break;
273 		case DYNPM_ACTION_DEFAULT:
274 			rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
275 			rdev->pm.requested_clock_mode_index = 0;
276 			rdev->pm.dynpm_can_upclock = false;
277 			break;
278 		case DYNPM_ACTION_NONE:
279 		default:
280 			DRM_ERROR("Requested mode for not defined action\n");
281 			return;
282 		}
283 	} else {
284 		/* XXX select a power state based on AC/DC, single/dualhead, etc. */
285 		/* for now just select the first power state and switch between clock modes */
286 		/* power state array is low to high, default is first (0) */
287 		if (rdev->pm.active_crtc_count > 1) {
288 			rdev->pm.requested_power_state_index = -1;
289 			/* start at 1 as we don't want the default mode */
290 			for (i = 1; i < rdev->pm.num_power_states; i++) {
291 				if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
292 					continue;
293 				else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
294 					 (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
295 					rdev->pm.requested_power_state_index = i;
296 					break;
297 				}
298 			}
299 			/* if nothing selected, grab the default state. */
300 			if (rdev->pm.requested_power_state_index == -1)
301 				rdev->pm.requested_power_state_index = 0;
302 		} else
303 			rdev->pm.requested_power_state_index = 1;
304 
305 		switch (rdev->pm.dynpm_planned_action) {
306 		case DYNPM_ACTION_MINIMUM:
307 			rdev->pm.requested_clock_mode_index = 0;
308 			rdev->pm.dynpm_can_downclock = false;
309 			break;
310 		case DYNPM_ACTION_DOWNCLOCK:
311 			if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
312 				if (rdev->pm.current_clock_mode_index == 0) {
313 					rdev->pm.requested_clock_mode_index = 0;
314 					rdev->pm.dynpm_can_downclock = false;
315 				} else
316 					rdev->pm.requested_clock_mode_index =
317 						rdev->pm.current_clock_mode_index - 1;
318 			} else {
319 				rdev->pm.requested_clock_mode_index = 0;
320 				rdev->pm.dynpm_can_downclock = false;
321 			}
322 			/* don't use the power state if crtcs are active and no display flag is set */
323 			if ((rdev->pm.active_crtc_count > 0) &&
324 			    (rdev->pm.power_state[rdev->pm.requested_power_state_index].
325 			     clock_info[rdev->pm.requested_clock_mode_index].flags &
326 			     RADEON_PM_MODE_NO_DISPLAY)) {
327 				rdev->pm.requested_clock_mode_index++;
328 			}
329 			break;
330 		case DYNPM_ACTION_UPCLOCK:
331 			if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
332 				if (rdev->pm.current_clock_mode_index ==
333 				    (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
334 					rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
335 					rdev->pm.dynpm_can_upclock = false;
336 				} else
337 					rdev->pm.requested_clock_mode_index =
338 						rdev->pm.current_clock_mode_index + 1;
339 			} else {
340 				rdev->pm.requested_clock_mode_index =
341 					rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
342 				rdev->pm.dynpm_can_upclock = false;
343 			}
344 			break;
345 		case DYNPM_ACTION_DEFAULT:
346 			rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
347 			rdev->pm.requested_clock_mode_index = 0;
348 			rdev->pm.dynpm_can_upclock = false;
349 			break;
350 		case DYNPM_ACTION_NONE:
351 		default:
352 			DRM_ERROR("Requested mode for not defined action\n");
353 			return;
354 		}
355 	}
356 
357 	DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
358 		  rdev->pm.power_state[rdev->pm.requested_power_state_index].
359 		  clock_info[rdev->pm.requested_clock_mode_index].sclk,
360 		  rdev->pm.power_state[rdev->pm.requested_power_state_index].
361 		  clock_info[rdev->pm.requested_clock_mode_index].mclk,
362 		  rdev->pm.power_state[rdev->pm.requested_power_state_index].
363 		  pcie_lanes);
364 }
365 
366 void rs780_pm_init_profile(struct radeon_device *rdev)
367 {
368 	if (rdev->pm.num_power_states == 2) {
369 		/* default */
370 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
371 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
372 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
373 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
374 		/* low sh */
375 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
376 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
377 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
378 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
379 		/* mid sh */
380 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
381 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
382 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
383 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
384 		/* high sh */
385 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
386 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
387 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
388 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
389 		/* low mh */
390 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
391 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
392 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
393 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
394 		/* mid mh */
395 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
396 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
397 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
398 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
399 		/* high mh */
400 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
401 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
402 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
403 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
404 	} else if (rdev->pm.num_power_states == 3) {
405 		/* default */
406 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
407 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
408 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
409 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
410 		/* low sh */
411 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
412 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
413 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
414 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
415 		/* mid sh */
416 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
417 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
418 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
419 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
420 		/* high sh */
421 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
422 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
423 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
424 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
425 		/* low mh */
426 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
427 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
428 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
429 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
430 		/* mid mh */
431 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
432 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
433 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
434 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
435 		/* high mh */
436 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
437 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
438 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
439 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
440 	} else {
441 		/* default */
442 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
443 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
444 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
445 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
446 		/* low sh */
447 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
448 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
449 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
450 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
451 		/* mid sh */
452 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
453 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
454 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
455 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
456 		/* high sh */
457 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
458 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
459 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
460 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
461 		/* low mh */
462 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
463 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
464 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
465 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
466 		/* mid mh */
467 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
468 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
469 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
470 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
471 		/* high mh */
472 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
473 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
474 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
475 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
476 	}
477 }
478 
479 void r600_pm_init_profile(struct radeon_device *rdev)
480 {
481 	int idx;
482 
483 	if (rdev->family == CHIP_R600) {
484 		/* XXX */
485 		/* default */
486 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
487 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
488 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
489 		rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
490 		/* low sh */
491 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
492 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
493 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
494 		rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
495 		/* mid sh */
496 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
497 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
498 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
499 		rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
500 		/* high sh */
501 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
502 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
503 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
504 		rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
505 		/* low mh */
506 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
507 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
508 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
509 		rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
510 		/* mid mh */
511 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
512 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
513 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
514 		rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
515 		/* high mh */
516 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
517 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
518 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
519 		rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
520 	} else {
521 		if (rdev->pm.num_power_states < 4) {
522 			/* default */
523 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
524 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
525 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
526 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
527 			/* low sh */
528 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
529 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
530 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
531 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
532 			/* mid sh */
533 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
534 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
535 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
536 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
537 			/* high sh */
538 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
539 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
540 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
541 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
542 			/* low mh */
543 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
544 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
545 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
546 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
547 			/* low mh */
548 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
549 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
550 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
551 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
552 			/* high mh */
553 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
554 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
555 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
556 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
557 		} else {
558 			/* default */
559 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
560 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
561 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
562 			rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
563 			/* low sh */
564 			if (rdev->flags & RADEON_IS_MOBILITY)
565 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
566 			else
567 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
568 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
569 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
570 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
571 			rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
572 			/* mid sh */
573 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
574 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
575 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
576 			rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
577 			/* high sh */
578 			idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
579 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
580 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
581 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
582 			rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
583 			/* low mh */
584 			if (rdev->flags & RADEON_IS_MOBILITY)
585 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
586 			else
587 				idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
588 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
589 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
590 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
591 			rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
592 			/* mid mh */
593 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
594 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
595 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
596 			rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
597 			/* high mh */
598 			idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
599 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
600 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
601 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
602 			rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
603 		}
604 	}
605 }
606 
607 void r600_pm_misc(struct radeon_device *rdev)
608 {
609 	int req_ps_idx = rdev->pm.requested_power_state_index;
610 	int req_cm_idx = rdev->pm.requested_clock_mode_index;
611 	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
612 	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
613 
614 	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
615 		/* 0xff01 is a flag rather then an actual voltage */
616 		if (voltage->voltage == 0xff01)
617 			return;
618 		if (voltage->voltage != rdev->pm.current_vddc) {
619 			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
620 			rdev->pm.current_vddc = voltage->voltage;
621 			DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
622 		}
623 	}
624 }
625 
626 bool r600_gui_idle(struct radeon_device *rdev)
627 {
628 	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
629 		return false;
630 	else
631 		return true;
632 }
633 
634 /* hpd for digital panel detect/disconnect */
635 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
636 {
637 	bool connected = false;
638 
639 	if (ASIC_IS_DCE3(rdev)) {
640 		switch (hpd) {
641 		case RADEON_HPD_1:
642 			if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
643 				connected = true;
644 			break;
645 		case RADEON_HPD_2:
646 			if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
647 				connected = true;
648 			break;
649 		case RADEON_HPD_3:
650 			if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
651 				connected = true;
652 			break;
653 		case RADEON_HPD_4:
654 			if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
655 				connected = true;
656 			break;
657 			/* DCE 3.2 */
658 		case RADEON_HPD_5:
659 			if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
660 				connected = true;
661 			break;
662 		case RADEON_HPD_6:
663 			if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
664 				connected = true;
665 			break;
666 		default:
667 			break;
668 		}
669 	} else {
670 		switch (hpd) {
671 		case RADEON_HPD_1:
672 			if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
673 				connected = true;
674 			break;
675 		case RADEON_HPD_2:
676 			if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
677 				connected = true;
678 			break;
679 		case RADEON_HPD_3:
680 			if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
681 				connected = true;
682 			break;
683 		default:
684 			break;
685 		}
686 	}
687 	return connected;
688 }
689 
690 void r600_hpd_set_polarity(struct radeon_device *rdev,
691 			   enum radeon_hpd_id hpd)
692 {
693 	u32 tmp;
694 	bool connected = r600_hpd_sense(rdev, hpd);
695 
696 	if (ASIC_IS_DCE3(rdev)) {
697 		switch (hpd) {
698 		case RADEON_HPD_1:
699 			tmp = RREG32(DC_HPD1_INT_CONTROL);
700 			if (connected)
701 				tmp &= ~DC_HPDx_INT_POLARITY;
702 			else
703 				tmp |= DC_HPDx_INT_POLARITY;
704 			WREG32(DC_HPD1_INT_CONTROL, tmp);
705 			break;
706 		case RADEON_HPD_2:
707 			tmp = RREG32(DC_HPD2_INT_CONTROL);
708 			if (connected)
709 				tmp &= ~DC_HPDx_INT_POLARITY;
710 			else
711 				tmp |= DC_HPDx_INT_POLARITY;
712 			WREG32(DC_HPD2_INT_CONTROL, tmp);
713 			break;
714 		case RADEON_HPD_3:
715 			tmp = RREG32(DC_HPD3_INT_CONTROL);
716 			if (connected)
717 				tmp &= ~DC_HPDx_INT_POLARITY;
718 			else
719 				tmp |= DC_HPDx_INT_POLARITY;
720 			WREG32(DC_HPD3_INT_CONTROL, tmp);
721 			break;
722 		case RADEON_HPD_4:
723 			tmp = RREG32(DC_HPD4_INT_CONTROL);
724 			if (connected)
725 				tmp &= ~DC_HPDx_INT_POLARITY;
726 			else
727 				tmp |= DC_HPDx_INT_POLARITY;
728 			WREG32(DC_HPD4_INT_CONTROL, tmp);
729 			break;
730 		case RADEON_HPD_5:
731 			tmp = RREG32(DC_HPD5_INT_CONTROL);
732 			if (connected)
733 				tmp &= ~DC_HPDx_INT_POLARITY;
734 			else
735 				tmp |= DC_HPDx_INT_POLARITY;
736 			WREG32(DC_HPD5_INT_CONTROL, tmp);
737 			break;
738 			/* DCE 3.2 */
739 		case RADEON_HPD_6:
740 			tmp = RREG32(DC_HPD6_INT_CONTROL);
741 			if (connected)
742 				tmp &= ~DC_HPDx_INT_POLARITY;
743 			else
744 				tmp |= DC_HPDx_INT_POLARITY;
745 			WREG32(DC_HPD6_INT_CONTROL, tmp);
746 			break;
747 		default:
748 			break;
749 		}
750 	} else {
751 		switch (hpd) {
752 		case RADEON_HPD_1:
753 			tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
754 			if (connected)
755 				tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
756 			else
757 				tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
758 			WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
759 			break;
760 		case RADEON_HPD_2:
761 			tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
762 			if (connected)
763 				tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
764 			else
765 				tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
766 			WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
767 			break;
768 		case RADEON_HPD_3:
769 			tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
770 			if (connected)
771 				tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
772 			else
773 				tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
774 			WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
775 			break;
776 		default:
777 			break;
778 		}
779 	}
780 }
781 
782 void r600_hpd_init(struct radeon_device *rdev)
783 {
784 	struct drm_device *dev = rdev->ddev;
785 	struct drm_connector *connector;
786 	unsigned enable = 0;
787 
788 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
789 		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
790 
791 		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
792 		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
793 			/* don't try to enable hpd on eDP or LVDS avoid breaking the
794 			 * aux dp channel on imac and help (but not completely fix)
795 			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
796 			 */
797 			continue;
798 		}
799 		if (ASIC_IS_DCE3(rdev)) {
800 			u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
801 			if (ASIC_IS_DCE32(rdev))
802 				tmp |= DC_HPDx_EN;
803 
804 			switch (radeon_connector->hpd.hpd) {
805 			case RADEON_HPD_1:
806 				WREG32(DC_HPD1_CONTROL, tmp);
807 				break;
808 			case RADEON_HPD_2:
809 				WREG32(DC_HPD2_CONTROL, tmp);
810 				break;
811 			case RADEON_HPD_3:
812 				WREG32(DC_HPD3_CONTROL, tmp);
813 				break;
814 			case RADEON_HPD_4:
815 				WREG32(DC_HPD4_CONTROL, tmp);
816 				break;
817 				/* DCE 3.2 */
818 			case RADEON_HPD_5:
819 				WREG32(DC_HPD5_CONTROL, tmp);
820 				break;
821 			case RADEON_HPD_6:
822 				WREG32(DC_HPD6_CONTROL, tmp);
823 				break;
824 			default:
825 				break;
826 			}
827 		} else {
828 			switch (radeon_connector->hpd.hpd) {
829 			case RADEON_HPD_1:
830 				WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
831 				break;
832 			case RADEON_HPD_2:
833 				WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
834 				break;
835 			case RADEON_HPD_3:
836 				WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
837 				break;
838 			default:
839 				break;
840 			}
841 		}
842 		enable |= 1 << radeon_connector->hpd.hpd;
843 		radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
844 	}
845 	radeon_irq_kms_enable_hpd(rdev, enable);
846 }
847 
848 void r600_hpd_fini(struct radeon_device *rdev)
849 {
850 	struct drm_device *dev = rdev->ddev;
851 	struct drm_connector *connector;
852 	unsigned disable = 0;
853 
854 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
855 		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
856 		if (ASIC_IS_DCE3(rdev)) {
857 			switch (radeon_connector->hpd.hpd) {
858 			case RADEON_HPD_1:
859 				WREG32(DC_HPD1_CONTROL, 0);
860 				break;
861 			case RADEON_HPD_2:
862 				WREG32(DC_HPD2_CONTROL, 0);
863 				break;
864 			case RADEON_HPD_3:
865 				WREG32(DC_HPD3_CONTROL, 0);
866 				break;
867 			case RADEON_HPD_4:
868 				WREG32(DC_HPD4_CONTROL, 0);
869 				break;
870 				/* DCE 3.2 */
871 			case RADEON_HPD_5:
872 				WREG32(DC_HPD5_CONTROL, 0);
873 				break;
874 			case RADEON_HPD_6:
875 				WREG32(DC_HPD6_CONTROL, 0);
876 				break;
877 			default:
878 				break;
879 			}
880 		} else {
881 			switch (radeon_connector->hpd.hpd) {
882 			case RADEON_HPD_1:
883 				WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
884 				break;
885 			case RADEON_HPD_2:
886 				WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
887 				break;
888 			case RADEON_HPD_3:
889 				WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
890 				break;
891 			default:
892 				break;
893 			}
894 		}
895 		disable |= 1 << radeon_connector->hpd.hpd;
896 	}
897 	radeon_irq_kms_disable_hpd(rdev, disable);
898 }
899 
900 /*
901  * R600 PCIE GART
902  */
903 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
904 {
905 	unsigned i;
906 	u32 tmp;
907 
908 	/* flush hdp cache so updates hit vram */
909 	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
910 	    !(rdev->flags & RADEON_IS_AGP)) {
911 		void __iomem *ptr = (void *)rdev->gart.ptr;
912 		u32 tmp;
913 
914 		/* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
915 		 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
916 		 * This seems to cause problems on some AGP cards. Just use the old
917 		 * method for them.
918 		 */
919 		WREG32(HDP_DEBUG1, 0);
920 		tmp = readl((void __iomem *)ptr);
921 	} else
922 		WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
923 
924 	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
925 	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
926 	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
927 	for (i = 0; i < rdev->usec_timeout; i++) {
928 		/* read MC_STATUS */
929 		tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
930 		tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
931 		if (tmp == 2) {
932 			printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
933 			return;
934 		}
935 		if (tmp) {
936 			return;
937 		}
938 		udelay(1);
939 	}
940 }
941 
942 int r600_pcie_gart_init(struct radeon_device *rdev)
943 {
944 	int r;
945 
946 	if (rdev->gart.robj) {
947 		WARN(1, "R600 PCIE GART already initialized\n");
948 		return 0;
949 	}
950 	/* Initialize common gart structure */
951 	r = radeon_gart_init(rdev);
952 	if (r)
953 		return r;
954 	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
955 	return radeon_gart_table_vram_alloc(rdev);
956 }
957 
958 static int r600_pcie_gart_enable(struct radeon_device *rdev)
959 {
960 	u32 tmp;
961 	int r, i;
962 
963 	if (rdev->gart.robj == NULL) {
964 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
965 		return -EINVAL;
966 	}
967 	r = radeon_gart_table_vram_pin(rdev);
968 	if (r)
969 		return r;
970 	radeon_gart_restore(rdev);
971 
972 	/* Setup L2 cache */
973 	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
974 				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
975 				EFFECTIVE_L2_QUEUE_SIZE(7));
976 	WREG32(VM_L2_CNTL2, 0);
977 	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
978 	/* Setup TLB control */
979 	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
980 		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
981 		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
982 		ENABLE_WAIT_L2_QUERY;
983 	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
984 	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
985 	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
986 	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
987 	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
988 	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
989 	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
990 	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
991 	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
992 	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
993 	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
994 	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
995 	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
996 	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
997 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
998 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
999 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1000 	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1001 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1002 	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1003 			(u32)(rdev->dummy_page.addr >> 12));
1004 	for (i = 1; i < 7; i++)
1005 		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1006 
1007 	r600_pcie_gart_tlb_flush(rdev);
1008 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1009 		 (unsigned)(rdev->mc.gtt_size >> 20),
1010 		 (unsigned long long)rdev->gart.table_addr);
1011 	rdev->gart.ready = true;
1012 	return 0;
1013 }
1014 
1015 static void r600_pcie_gart_disable(struct radeon_device *rdev)
1016 {
1017 	u32 tmp;
1018 	int i;
1019 
1020 	/* Disable all tables */
1021 	for (i = 0; i < 7; i++)
1022 		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1023 
1024 	/* Disable L2 cache */
1025 	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
1026 				EFFECTIVE_L2_QUEUE_SIZE(7));
1027 	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1028 	/* Setup L1 TLB control */
1029 	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1030 		ENABLE_WAIT_L2_QUERY;
1031 	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1032 	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1033 	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1034 	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1035 	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1036 	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1037 	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1038 	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1039 	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
1040 	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
1041 	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1042 	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1043 	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
1044 	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1045 	radeon_gart_table_vram_unpin(rdev);
1046 }
1047 
1048 static void r600_pcie_gart_fini(struct radeon_device *rdev)
1049 {
1050 	radeon_gart_fini(rdev);
1051 	r600_pcie_gart_disable(rdev);
1052 	radeon_gart_table_vram_free(rdev);
1053 }
1054 
1055 static void r600_agp_enable(struct radeon_device *rdev)
1056 {
1057 	u32 tmp;
1058 	int i;
1059 
1060 	/* Setup L2 cache */
1061 	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1062 				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1063 				EFFECTIVE_L2_QUEUE_SIZE(7));
1064 	WREG32(VM_L2_CNTL2, 0);
1065 	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1066 	/* Setup TLB control */
1067 	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1068 		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1069 		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1070 		ENABLE_WAIT_L2_QUERY;
1071 	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1072 	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1073 	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1074 	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1075 	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1076 	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1077 	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1078 	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1079 	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1080 	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1081 	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1082 	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1083 	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1084 	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1085 	for (i = 0; i < 7; i++)
1086 		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1087 }
1088 
1089 int r600_mc_wait_for_idle(struct radeon_device *rdev)
1090 {
1091 	unsigned i;
1092 	u32 tmp;
1093 
1094 	for (i = 0; i < rdev->usec_timeout; i++) {
1095 		/* read MC_STATUS */
1096 		tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1097 		if (!tmp)
1098 			return 0;
1099 		udelay(1);
1100 	}
1101 	return -1;
1102 }
1103 
1104 uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
1105 {
1106 	unsigned long flags;
1107 	uint32_t r;
1108 
1109 	spin_lock_irqsave(&rdev->mc_idx_lock, flags);
1110 	WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
1111 	r = RREG32(R_0028FC_MC_DATA);
1112 	WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
1113 	spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);
1114 	return r;
1115 }
1116 
1117 void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
1118 {
1119 	unsigned long flags;
1120 
1121 	spin_lock_irqsave(&rdev->mc_idx_lock, flags);
1122 	WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) |
1123 		S_0028F8_MC_IND_WR_EN(1));
1124 	WREG32(R_0028FC_MC_DATA, v);
1125 	WREG32(R_0028F8_MC_INDEX, 0x7F);
1126 	spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);
1127 }
1128 
1129 static void r600_mc_program(struct radeon_device *rdev)
1130 {
1131 	struct rv515_mc_save save;
1132 	u32 tmp;
1133 	int i, j;
1134 
1135 	/* Initialize HDP */
1136 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1137 		WREG32((0x2c14 + j), 0x00000000);
1138 		WREG32((0x2c18 + j), 0x00000000);
1139 		WREG32((0x2c1c + j), 0x00000000);
1140 		WREG32((0x2c20 + j), 0x00000000);
1141 		WREG32((0x2c24 + j), 0x00000000);
1142 	}
1143 	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1144 
1145 	rv515_mc_stop(rdev, &save);
1146 	if (r600_mc_wait_for_idle(rdev)) {
1147 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1148 	}
1149 	/* Lockout access through VGA aperture (doesn't exist before R600) */
1150 	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1151 	/* Update configuration */
1152 	if (rdev->flags & RADEON_IS_AGP) {
1153 		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1154 			/* VRAM before AGP */
1155 			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1156 				rdev->mc.vram_start >> 12);
1157 			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1158 				rdev->mc.gtt_end >> 12);
1159 		} else {
1160 			/* VRAM after AGP */
1161 			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1162 				rdev->mc.gtt_start >> 12);
1163 			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1164 				rdev->mc.vram_end >> 12);
1165 		}
1166 	} else {
1167 		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1168 		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1169 	}
1170 	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1171 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1172 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1173 	WREG32(MC_VM_FB_LOCATION, tmp);
1174 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1175 	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1176 	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1177 	if (rdev->flags & RADEON_IS_AGP) {
1178 		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1179 		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
1180 		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1181 	} else {
1182 		WREG32(MC_VM_AGP_BASE, 0);
1183 		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1184 		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1185 	}
1186 	if (r600_mc_wait_for_idle(rdev)) {
1187 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1188 	}
1189 	rv515_mc_resume(rdev, &save);
1190 	/* we need to own VRAM, so turn off the VGA renderer here
1191 	 * to stop it overwriting our objects */
1192 	rv515_vga_render_disable(rdev);
1193 }
1194 
1195 /**
1196  * r600_vram_gtt_location - try to find VRAM & GTT location
1197  * @rdev: radeon device structure holding all necessary informations
1198  * @mc: memory controller structure holding memory informations
1199  *
1200  * Function will place try to place VRAM at same place as in CPU (PCI)
1201  * address space as some GPU seems to have issue when we reprogram at
1202  * different address space.
1203  *
1204  * If there is not enough space to fit the unvisible VRAM after the
1205  * aperture then we limit the VRAM size to the aperture.
1206  *
1207  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1208  * them to be in one from GPU point of view so that we can program GPU to
1209  * catch access outside them (weird GPU policy see ??).
1210  *
1211  * This function will never fails, worst case are limiting VRAM or GTT.
1212  *
1213  * Note: GTT start, end, size should be initialized before calling this
1214  * function on AGP platform.
1215  */
1216 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1217 {
1218 	u64 size_bf, size_af;
1219 
1220 	if (mc->mc_vram_size > 0xE0000000) {
1221 		/* leave room for at least 512M GTT */
1222 		dev_warn(rdev->dev, "limiting VRAM\n");
1223 		mc->real_vram_size = 0xE0000000;
1224 		mc->mc_vram_size = 0xE0000000;
1225 	}
1226 	if (rdev->flags & RADEON_IS_AGP) {
1227 		size_bf = mc->gtt_start;
1228 		size_af = mc->mc_mask - mc->gtt_end;
1229 		if (size_bf > size_af) {
1230 			if (mc->mc_vram_size > size_bf) {
1231 				dev_warn(rdev->dev, "limiting VRAM\n");
1232 				mc->real_vram_size = size_bf;
1233 				mc->mc_vram_size = size_bf;
1234 			}
1235 			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1236 		} else {
1237 			if (mc->mc_vram_size > size_af) {
1238 				dev_warn(rdev->dev, "limiting VRAM\n");
1239 				mc->real_vram_size = size_af;
1240 				mc->mc_vram_size = size_af;
1241 			}
1242 			mc->vram_start = mc->gtt_end + 1;
1243 		}
1244 		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1245 		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1246 				mc->mc_vram_size >> 20, mc->vram_start,
1247 				mc->vram_end, mc->real_vram_size >> 20);
1248 	} else {
1249 		u64 base = 0;
1250 		if (rdev->flags & RADEON_IS_IGP) {
1251 			base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1252 			base <<= 24;
1253 		}
1254 		radeon_vram_location(rdev, &rdev->mc, base);
1255 		rdev->mc.gtt_base_align = 0;
1256 		radeon_gtt_location(rdev, mc);
1257 	}
1258 }
1259 
1260 static int r600_mc_init(struct radeon_device *rdev)
1261 {
1262 	u32 tmp;
1263 	int chansize, numchan;
1264 	uint32_t h_addr, l_addr;
1265 	unsigned long long k8_addr;
1266 
1267 	/* Get VRAM informations */
1268 	rdev->mc.vram_is_ddr = true;
1269 	tmp = RREG32(RAMCFG);
1270 	if (tmp & CHANSIZE_OVERRIDE) {
1271 		chansize = 16;
1272 	} else if (tmp & CHANSIZE_MASK) {
1273 		chansize = 64;
1274 	} else {
1275 		chansize = 32;
1276 	}
1277 	tmp = RREG32(CHMAP);
1278 	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1279 	case 0:
1280 	default:
1281 		numchan = 1;
1282 		break;
1283 	case 1:
1284 		numchan = 2;
1285 		break;
1286 	case 2:
1287 		numchan = 4;
1288 		break;
1289 	case 3:
1290 		numchan = 8;
1291 		break;
1292 	}
1293 	rdev->mc.vram_width = numchan * chansize;
1294 	/* Could aper size report 0 ? */
1295 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1296 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1297 	/* Setup GPU memory space */
1298 	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1299 	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1300 	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1301 	r600_vram_gtt_location(rdev, &rdev->mc);
1302 
1303 	if (rdev->flags & RADEON_IS_IGP) {
1304 		rs690_pm_info(rdev);
1305 		rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1306 
1307 		if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) {
1308 			/* Use K8 direct mapping for fast fb access. */
1309 			rdev->fastfb_working = false;
1310 			h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
1311 			l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
1312 			k8_addr = ((unsigned long long)h_addr) << 32 | l_addr;
1313 #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
1314 			if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
1315 #endif
1316 			{
1317 				/* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
1318 		 		* memory is present.
1319 		 		*/
1320 				if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
1321 					DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
1322 						(unsigned long long)rdev->mc.aper_base, k8_addr);
1323 					rdev->mc.aper_base = (resource_size_t)k8_addr;
1324 					rdev->fastfb_working = true;
1325 				}
1326 			}
1327   		}
1328 	}
1329 
1330 	radeon_update_bandwidth_info(rdev);
1331 	return 0;
1332 }
1333 
1334 int r600_vram_scratch_init(struct radeon_device *rdev)
1335 {
1336 	int r;
1337 
1338 	if (rdev->vram_scratch.robj == NULL) {
1339 		r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1340 				     PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
1341 				     NULL, &rdev->vram_scratch.robj);
1342 		if (r) {
1343 			return r;
1344 		}
1345 	}
1346 
1347 	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1348 	if (unlikely(r != 0))
1349 		return r;
1350 	r = radeon_bo_pin(rdev->vram_scratch.robj,
1351 			  RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1352 	if (r) {
1353 		radeon_bo_unreserve(rdev->vram_scratch.robj);
1354 		return r;
1355 	}
1356 	r = radeon_bo_kmap(rdev->vram_scratch.robj,
1357 				(void **)&rdev->vram_scratch.ptr);
1358 	if (r)
1359 		radeon_bo_unpin(rdev->vram_scratch.robj);
1360 	radeon_bo_unreserve(rdev->vram_scratch.robj);
1361 
1362 	return r;
1363 }
1364 
1365 void r600_vram_scratch_fini(struct radeon_device *rdev)
1366 {
1367 	int r;
1368 
1369 	if (rdev->vram_scratch.robj == NULL) {
1370 		return;
1371 	}
1372 	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1373 	if (likely(r == 0)) {
1374 		radeon_bo_kunmap(rdev->vram_scratch.robj);
1375 		radeon_bo_unpin(rdev->vram_scratch.robj);
1376 		radeon_bo_unreserve(rdev->vram_scratch.robj);
1377 	}
1378 	radeon_bo_unref(&rdev->vram_scratch.robj);
1379 }
1380 
1381 void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1382 {
1383 	u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1384 
1385 	if (hung)
1386 		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1387 	else
1388 		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1389 
1390 	WREG32(R600_BIOS_3_SCRATCH, tmp);
1391 }
1392 
1393 static void r600_print_gpu_status_regs(struct radeon_device *rdev)
1394 {
1395 	dev_info(rdev->dev, "  R_008010_GRBM_STATUS      = 0x%08X\n",
1396 		 RREG32(R_008010_GRBM_STATUS));
1397 	dev_info(rdev->dev, "  R_008014_GRBM_STATUS2     = 0x%08X\n",
1398 		 RREG32(R_008014_GRBM_STATUS2));
1399 	dev_info(rdev->dev, "  R_000E50_SRBM_STATUS      = 0x%08X\n",
1400 		 RREG32(R_000E50_SRBM_STATUS));
1401 	dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1402 		 RREG32(CP_STALLED_STAT1));
1403 	dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1404 		 RREG32(CP_STALLED_STAT2));
1405 	dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1406 		 RREG32(CP_BUSY_STAT));
1407 	dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1408 		 RREG32(CP_STAT));
1409 	dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1410 		RREG32(DMA_STATUS_REG));
1411 }
1412 
1413 static bool r600_is_display_hung(struct radeon_device *rdev)
1414 {
1415 	u32 crtc_hung = 0;
1416 	u32 crtc_status[2];
1417 	u32 i, j, tmp;
1418 
1419 	for (i = 0; i < rdev->num_crtc; i++) {
1420 		if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1421 			crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1422 			crtc_hung |= (1 << i);
1423 		}
1424 	}
1425 
1426 	for (j = 0; j < 10; j++) {
1427 		for (i = 0; i < rdev->num_crtc; i++) {
1428 			if (crtc_hung & (1 << i)) {
1429 				tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1430 				if (tmp != crtc_status[i])
1431 					crtc_hung &= ~(1 << i);
1432 			}
1433 		}
1434 		if (crtc_hung == 0)
1435 			return false;
1436 		udelay(100);
1437 	}
1438 
1439 	return true;
1440 }
1441 
1442 u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1443 {
1444 	u32 reset_mask = 0;
1445 	u32 tmp;
1446 
1447 	/* GRBM_STATUS */
1448 	tmp = RREG32(R_008010_GRBM_STATUS);
1449 	if (rdev->family >= CHIP_RV770) {
1450 		if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1451 		    G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1452 		    G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1453 		    G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1454 		    G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1455 			reset_mask |= RADEON_RESET_GFX;
1456 	} else {
1457 		if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1458 		    G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1459 		    G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1460 		    G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1461 		    G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1462 			reset_mask |= RADEON_RESET_GFX;
1463 	}
1464 
1465 	if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1466 	    G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1467 		reset_mask |= RADEON_RESET_CP;
1468 
1469 	if (G_008010_GRBM_EE_BUSY(tmp))
1470 		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1471 
1472 	/* DMA_STATUS_REG */
1473 	tmp = RREG32(DMA_STATUS_REG);
1474 	if (!(tmp & DMA_IDLE))
1475 		reset_mask |= RADEON_RESET_DMA;
1476 
1477 	/* SRBM_STATUS */
1478 	tmp = RREG32(R_000E50_SRBM_STATUS);
1479 	if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1480 		reset_mask |= RADEON_RESET_RLC;
1481 
1482 	if (G_000E50_IH_BUSY(tmp))
1483 		reset_mask |= RADEON_RESET_IH;
1484 
1485 	if (G_000E50_SEM_BUSY(tmp))
1486 		reset_mask |= RADEON_RESET_SEM;
1487 
1488 	if (G_000E50_GRBM_RQ_PENDING(tmp))
1489 		reset_mask |= RADEON_RESET_GRBM;
1490 
1491 	if (G_000E50_VMC_BUSY(tmp))
1492 		reset_mask |= RADEON_RESET_VMC;
1493 
1494 	if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1495 	    G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1496 	    G_000E50_MCDW_BUSY(tmp))
1497 		reset_mask |= RADEON_RESET_MC;
1498 
1499 	if (r600_is_display_hung(rdev))
1500 		reset_mask |= RADEON_RESET_DISPLAY;
1501 
1502 	/* Skip MC reset as it's mostly likely not hung, just busy */
1503 	if (reset_mask & RADEON_RESET_MC) {
1504 		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1505 		reset_mask &= ~RADEON_RESET_MC;
1506 	}
1507 
1508 	return reset_mask;
1509 }
1510 
1511 static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1512 {
1513 	struct rv515_mc_save save;
1514 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1515 	u32 tmp;
1516 
1517 	if (reset_mask == 0)
1518 		return;
1519 
1520 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1521 
1522 	r600_print_gpu_status_regs(rdev);
1523 
1524 	/* Disable CP parsing/prefetching */
1525 	if (rdev->family >= CHIP_RV770)
1526 		WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1527 	else
1528 		WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1529 
1530 	/* disable the RLC */
1531 	WREG32(RLC_CNTL, 0);
1532 
1533 	if (reset_mask & RADEON_RESET_DMA) {
1534 		/* Disable DMA */
1535 		tmp = RREG32(DMA_RB_CNTL);
1536 		tmp &= ~DMA_RB_ENABLE;
1537 		WREG32(DMA_RB_CNTL, tmp);
1538 	}
1539 
1540 	mdelay(50);
1541 
1542 	rv515_mc_stop(rdev, &save);
1543 	if (r600_mc_wait_for_idle(rdev)) {
1544 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1545 	}
1546 
1547 	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1548 		if (rdev->family >= CHIP_RV770)
1549 			grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1550 				S_008020_SOFT_RESET_CB(1) |
1551 				S_008020_SOFT_RESET_PA(1) |
1552 				S_008020_SOFT_RESET_SC(1) |
1553 				S_008020_SOFT_RESET_SPI(1) |
1554 				S_008020_SOFT_RESET_SX(1) |
1555 				S_008020_SOFT_RESET_SH(1) |
1556 				S_008020_SOFT_RESET_TC(1) |
1557 				S_008020_SOFT_RESET_TA(1) |
1558 				S_008020_SOFT_RESET_VC(1) |
1559 				S_008020_SOFT_RESET_VGT(1);
1560 		else
1561 			grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1562 				S_008020_SOFT_RESET_DB(1) |
1563 				S_008020_SOFT_RESET_CB(1) |
1564 				S_008020_SOFT_RESET_PA(1) |
1565 				S_008020_SOFT_RESET_SC(1) |
1566 				S_008020_SOFT_RESET_SMX(1) |
1567 				S_008020_SOFT_RESET_SPI(1) |
1568 				S_008020_SOFT_RESET_SX(1) |
1569 				S_008020_SOFT_RESET_SH(1) |
1570 				S_008020_SOFT_RESET_TC(1) |
1571 				S_008020_SOFT_RESET_TA(1) |
1572 				S_008020_SOFT_RESET_VC(1) |
1573 				S_008020_SOFT_RESET_VGT(1);
1574 	}
1575 
1576 	if (reset_mask & RADEON_RESET_CP) {
1577 		grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1578 			S_008020_SOFT_RESET_VGT(1);
1579 
1580 		srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1581 	}
1582 
1583 	if (reset_mask & RADEON_RESET_DMA) {
1584 		if (rdev->family >= CHIP_RV770)
1585 			srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1586 		else
1587 			srbm_soft_reset |= SOFT_RESET_DMA;
1588 	}
1589 
1590 	if (reset_mask & RADEON_RESET_RLC)
1591 		srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1592 
1593 	if (reset_mask & RADEON_RESET_SEM)
1594 		srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1595 
1596 	if (reset_mask & RADEON_RESET_IH)
1597 		srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1598 
1599 	if (reset_mask & RADEON_RESET_GRBM)
1600 		srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1601 
1602 	if (!(rdev->flags & RADEON_IS_IGP)) {
1603 		if (reset_mask & RADEON_RESET_MC)
1604 			srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1605 	}
1606 
1607 	if (reset_mask & RADEON_RESET_VMC)
1608 		srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1609 
1610 	if (grbm_soft_reset) {
1611 		tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1612 		tmp |= grbm_soft_reset;
1613 		dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1614 		WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1615 		tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1616 
1617 		udelay(50);
1618 
1619 		tmp &= ~grbm_soft_reset;
1620 		WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1621 		tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1622 	}
1623 
1624 	if (srbm_soft_reset) {
1625 		tmp = RREG32(SRBM_SOFT_RESET);
1626 		tmp |= srbm_soft_reset;
1627 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1628 		WREG32(SRBM_SOFT_RESET, tmp);
1629 		tmp = RREG32(SRBM_SOFT_RESET);
1630 
1631 		udelay(50);
1632 
1633 		tmp &= ~srbm_soft_reset;
1634 		WREG32(SRBM_SOFT_RESET, tmp);
1635 		tmp = RREG32(SRBM_SOFT_RESET);
1636 	}
1637 
1638 	/* Wait a little for things to settle down */
1639 	mdelay(1);
1640 
1641 	rv515_mc_resume(rdev, &save);
1642 	udelay(50);
1643 
1644 	r600_print_gpu_status_regs(rdev);
1645 }
1646 
1647 int r600_asic_reset(struct radeon_device *rdev)
1648 {
1649 	u32 reset_mask;
1650 
1651 	reset_mask = r600_gpu_check_soft_reset(rdev);
1652 
1653 	if (reset_mask)
1654 		r600_set_bios_scratch_engine_hung(rdev, true);
1655 
1656 	r600_gpu_soft_reset(rdev, reset_mask);
1657 
1658 	reset_mask = r600_gpu_check_soft_reset(rdev);
1659 
1660 	if (!reset_mask)
1661 		r600_set_bios_scratch_engine_hung(rdev, false);
1662 
1663 	return 0;
1664 }
1665 
1666 /**
1667  * r600_gfx_is_lockup - Check if the GFX engine is locked up
1668  *
1669  * @rdev: radeon_device pointer
1670  * @ring: radeon_ring structure holding ring information
1671  *
1672  * Check if the GFX engine is locked up.
1673  * Returns true if the engine appears to be locked up, false if not.
1674  */
1675 bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1676 {
1677 	u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1678 
1679 	if (!(reset_mask & (RADEON_RESET_GFX |
1680 			    RADEON_RESET_COMPUTE |
1681 			    RADEON_RESET_CP))) {
1682 		radeon_ring_lockup_update(ring);
1683 		return false;
1684 	}
1685 	/* force CP activities */
1686 	radeon_ring_force_activity(rdev, ring);
1687 	return radeon_ring_test_lockup(rdev, ring);
1688 }
1689 
1690 u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1691 			      u32 tiling_pipe_num,
1692 			      u32 max_rb_num,
1693 			      u32 total_max_rb_num,
1694 			      u32 disabled_rb_mask)
1695 {
1696 	u32 rendering_pipe_num, rb_num_width, req_rb_num;
1697 	u32 pipe_rb_ratio, pipe_rb_remain, tmp;
1698 	u32 data = 0, mask = 1 << (max_rb_num - 1);
1699 	unsigned i, j;
1700 
1701 	/* mask out the RBs that don't exist on that asic */
1702 	tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1703 	/* make sure at least one RB is available */
1704 	if ((tmp & 0xff) != 0xff)
1705 		disabled_rb_mask = tmp;
1706 
1707 	rendering_pipe_num = 1 << tiling_pipe_num;
1708 	req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1709 	BUG_ON(rendering_pipe_num < req_rb_num);
1710 
1711 	pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1712 	pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1713 
1714 	if (rdev->family <= CHIP_RV740) {
1715 		/* r6xx/r7xx */
1716 		rb_num_width = 2;
1717 	} else {
1718 		/* eg+ */
1719 		rb_num_width = 4;
1720 	}
1721 
1722 	for (i = 0; i < max_rb_num; i++) {
1723 		if (!(mask & disabled_rb_mask)) {
1724 			for (j = 0; j < pipe_rb_ratio; j++) {
1725 				data <<= rb_num_width;
1726 				data |= max_rb_num - i - 1;
1727 			}
1728 			if (pipe_rb_remain) {
1729 				data <<= rb_num_width;
1730 				data |= max_rb_num - i - 1;
1731 				pipe_rb_remain--;
1732 			}
1733 		}
1734 		mask >>= 1;
1735 	}
1736 
1737 	return data;
1738 }
1739 
1740 int r600_count_pipe_bits(uint32_t val)
1741 {
1742 	return hweight32(val);
1743 }
1744 
1745 static void r600_gpu_init(struct radeon_device *rdev)
1746 {
1747 	u32 tiling_config;
1748 	u32 ramcfg;
1749 	u32 cc_rb_backend_disable;
1750 	u32 cc_gc_shader_pipe_config;
1751 	u32 tmp;
1752 	int i, j;
1753 	u32 sq_config;
1754 	u32 sq_gpr_resource_mgmt_1 = 0;
1755 	u32 sq_gpr_resource_mgmt_2 = 0;
1756 	u32 sq_thread_resource_mgmt = 0;
1757 	u32 sq_stack_resource_mgmt_1 = 0;
1758 	u32 sq_stack_resource_mgmt_2 = 0;
1759 	u32 disabled_rb_mask;
1760 
1761 	rdev->config.r600.tiling_group_size = 256;
1762 	switch (rdev->family) {
1763 	case CHIP_R600:
1764 		rdev->config.r600.max_pipes = 4;
1765 		rdev->config.r600.max_tile_pipes = 8;
1766 		rdev->config.r600.max_simds = 4;
1767 		rdev->config.r600.max_backends = 4;
1768 		rdev->config.r600.max_gprs = 256;
1769 		rdev->config.r600.max_threads = 192;
1770 		rdev->config.r600.max_stack_entries = 256;
1771 		rdev->config.r600.max_hw_contexts = 8;
1772 		rdev->config.r600.max_gs_threads = 16;
1773 		rdev->config.r600.sx_max_export_size = 128;
1774 		rdev->config.r600.sx_max_export_pos_size = 16;
1775 		rdev->config.r600.sx_max_export_smx_size = 128;
1776 		rdev->config.r600.sq_num_cf_insts = 2;
1777 		break;
1778 	case CHIP_RV630:
1779 	case CHIP_RV635:
1780 		rdev->config.r600.max_pipes = 2;
1781 		rdev->config.r600.max_tile_pipes = 2;
1782 		rdev->config.r600.max_simds = 3;
1783 		rdev->config.r600.max_backends = 1;
1784 		rdev->config.r600.max_gprs = 128;
1785 		rdev->config.r600.max_threads = 192;
1786 		rdev->config.r600.max_stack_entries = 128;
1787 		rdev->config.r600.max_hw_contexts = 8;
1788 		rdev->config.r600.max_gs_threads = 4;
1789 		rdev->config.r600.sx_max_export_size = 128;
1790 		rdev->config.r600.sx_max_export_pos_size = 16;
1791 		rdev->config.r600.sx_max_export_smx_size = 128;
1792 		rdev->config.r600.sq_num_cf_insts = 2;
1793 		break;
1794 	case CHIP_RV610:
1795 	case CHIP_RV620:
1796 	case CHIP_RS780:
1797 	case CHIP_RS880:
1798 		rdev->config.r600.max_pipes = 1;
1799 		rdev->config.r600.max_tile_pipes = 1;
1800 		rdev->config.r600.max_simds = 2;
1801 		rdev->config.r600.max_backends = 1;
1802 		rdev->config.r600.max_gprs = 128;
1803 		rdev->config.r600.max_threads = 192;
1804 		rdev->config.r600.max_stack_entries = 128;
1805 		rdev->config.r600.max_hw_contexts = 4;
1806 		rdev->config.r600.max_gs_threads = 4;
1807 		rdev->config.r600.sx_max_export_size = 128;
1808 		rdev->config.r600.sx_max_export_pos_size = 16;
1809 		rdev->config.r600.sx_max_export_smx_size = 128;
1810 		rdev->config.r600.sq_num_cf_insts = 1;
1811 		break;
1812 	case CHIP_RV670:
1813 		rdev->config.r600.max_pipes = 4;
1814 		rdev->config.r600.max_tile_pipes = 4;
1815 		rdev->config.r600.max_simds = 4;
1816 		rdev->config.r600.max_backends = 4;
1817 		rdev->config.r600.max_gprs = 192;
1818 		rdev->config.r600.max_threads = 192;
1819 		rdev->config.r600.max_stack_entries = 256;
1820 		rdev->config.r600.max_hw_contexts = 8;
1821 		rdev->config.r600.max_gs_threads = 16;
1822 		rdev->config.r600.sx_max_export_size = 128;
1823 		rdev->config.r600.sx_max_export_pos_size = 16;
1824 		rdev->config.r600.sx_max_export_smx_size = 128;
1825 		rdev->config.r600.sq_num_cf_insts = 2;
1826 		break;
1827 	default:
1828 		break;
1829 	}
1830 
1831 	/* Initialize HDP */
1832 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1833 		WREG32((0x2c14 + j), 0x00000000);
1834 		WREG32((0x2c18 + j), 0x00000000);
1835 		WREG32((0x2c1c + j), 0x00000000);
1836 		WREG32((0x2c20 + j), 0x00000000);
1837 		WREG32((0x2c24 + j), 0x00000000);
1838 	}
1839 
1840 	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1841 
1842 	/* Setup tiling */
1843 	tiling_config = 0;
1844 	ramcfg = RREG32(RAMCFG);
1845 	switch (rdev->config.r600.max_tile_pipes) {
1846 	case 1:
1847 		tiling_config |= PIPE_TILING(0);
1848 		break;
1849 	case 2:
1850 		tiling_config |= PIPE_TILING(1);
1851 		break;
1852 	case 4:
1853 		tiling_config |= PIPE_TILING(2);
1854 		break;
1855 	case 8:
1856 		tiling_config |= PIPE_TILING(3);
1857 		break;
1858 	default:
1859 		break;
1860 	}
1861 	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1862 	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1863 	tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1864 	tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1865 
1866 	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1867 	if (tmp > 3) {
1868 		tiling_config |= ROW_TILING(3);
1869 		tiling_config |= SAMPLE_SPLIT(3);
1870 	} else {
1871 		tiling_config |= ROW_TILING(tmp);
1872 		tiling_config |= SAMPLE_SPLIT(tmp);
1873 	}
1874 	tiling_config |= BANK_SWAPS(1);
1875 
1876 	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1877 	tmp = R6XX_MAX_BACKENDS -
1878 		r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1879 	if (tmp < rdev->config.r600.max_backends) {
1880 		rdev->config.r600.max_backends = tmp;
1881 	}
1882 
1883 	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1884 	tmp = R6XX_MAX_PIPES -
1885 		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1886 	if (tmp < rdev->config.r600.max_pipes) {
1887 		rdev->config.r600.max_pipes = tmp;
1888 	}
1889 	tmp = R6XX_MAX_SIMDS -
1890 		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1891 	if (tmp < rdev->config.r600.max_simds) {
1892 		rdev->config.r600.max_simds = tmp;
1893 	}
1894 
1895 	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1896 	tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1897 	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1898 					R6XX_MAX_BACKENDS, disabled_rb_mask);
1899 	tiling_config |= tmp << 16;
1900 	rdev->config.r600.backend_map = tmp;
1901 
1902 	rdev->config.r600.tile_config = tiling_config;
1903 	WREG32(GB_TILING_CONFIG, tiling_config);
1904 	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1905 	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1906 	WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1907 
1908 	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1909 	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1910 	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1911 
1912 	/* Setup some CP states */
1913 	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1914 	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1915 
1916 	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1917 			     SYNC_WALKER | SYNC_ALIGNER));
1918 	/* Setup various GPU states */
1919 	if (rdev->family == CHIP_RV670)
1920 		WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1921 
1922 	tmp = RREG32(SX_DEBUG_1);
1923 	tmp |= SMX_EVENT_RELEASE;
1924 	if ((rdev->family > CHIP_R600))
1925 		tmp |= ENABLE_NEW_SMX_ADDRESS;
1926 	WREG32(SX_DEBUG_1, tmp);
1927 
1928 	if (((rdev->family) == CHIP_R600) ||
1929 	    ((rdev->family) == CHIP_RV630) ||
1930 	    ((rdev->family) == CHIP_RV610) ||
1931 	    ((rdev->family) == CHIP_RV620) ||
1932 	    ((rdev->family) == CHIP_RS780) ||
1933 	    ((rdev->family) == CHIP_RS880)) {
1934 		WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1935 	} else {
1936 		WREG32(DB_DEBUG, 0);
1937 	}
1938 	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1939 			       DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1940 
1941 	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1942 	WREG32(VGT_NUM_INSTANCES, 0);
1943 
1944 	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1945 	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1946 
1947 	tmp = RREG32(SQ_MS_FIFO_SIZES);
1948 	if (((rdev->family) == CHIP_RV610) ||
1949 	    ((rdev->family) == CHIP_RV620) ||
1950 	    ((rdev->family) == CHIP_RS780) ||
1951 	    ((rdev->family) == CHIP_RS880)) {
1952 		tmp = (CACHE_FIFO_SIZE(0xa) |
1953 		       FETCH_FIFO_HIWATER(0xa) |
1954 		       DONE_FIFO_HIWATER(0xe0) |
1955 		       ALU_UPDATE_FIFO_HIWATER(0x8));
1956 	} else if (((rdev->family) == CHIP_R600) ||
1957 		   ((rdev->family) == CHIP_RV630)) {
1958 		tmp &= ~DONE_FIFO_HIWATER(0xff);
1959 		tmp |= DONE_FIFO_HIWATER(0x4);
1960 	}
1961 	WREG32(SQ_MS_FIFO_SIZES, tmp);
1962 
1963 	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1964 	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1965 	 */
1966 	sq_config = RREG32(SQ_CONFIG);
1967 	sq_config &= ~(PS_PRIO(3) |
1968 		       VS_PRIO(3) |
1969 		       GS_PRIO(3) |
1970 		       ES_PRIO(3));
1971 	sq_config |= (DX9_CONSTS |
1972 		      VC_ENABLE |
1973 		      PS_PRIO(0) |
1974 		      VS_PRIO(1) |
1975 		      GS_PRIO(2) |
1976 		      ES_PRIO(3));
1977 
1978 	if ((rdev->family) == CHIP_R600) {
1979 		sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1980 					  NUM_VS_GPRS(124) |
1981 					  NUM_CLAUSE_TEMP_GPRS(4));
1982 		sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1983 					  NUM_ES_GPRS(0));
1984 		sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1985 					   NUM_VS_THREADS(48) |
1986 					   NUM_GS_THREADS(4) |
1987 					   NUM_ES_THREADS(4));
1988 		sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1989 					    NUM_VS_STACK_ENTRIES(128));
1990 		sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1991 					    NUM_ES_STACK_ENTRIES(0));
1992 	} else if (((rdev->family) == CHIP_RV610) ||
1993 		   ((rdev->family) == CHIP_RV620) ||
1994 		   ((rdev->family) == CHIP_RS780) ||
1995 		   ((rdev->family) == CHIP_RS880)) {
1996 		/* no vertex cache */
1997 		sq_config &= ~VC_ENABLE;
1998 
1999 		sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
2000 					  NUM_VS_GPRS(44) |
2001 					  NUM_CLAUSE_TEMP_GPRS(2));
2002 		sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
2003 					  NUM_ES_GPRS(17));
2004 		sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
2005 					   NUM_VS_THREADS(78) |
2006 					   NUM_GS_THREADS(4) |
2007 					   NUM_ES_THREADS(31));
2008 		sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
2009 					    NUM_VS_STACK_ENTRIES(40));
2010 		sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
2011 					    NUM_ES_STACK_ENTRIES(16));
2012 	} else if (((rdev->family) == CHIP_RV630) ||
2013 		   ((rdev->family) == CHIP_RV635)) {
2014 		sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
2015 					  NUM_VS_GPRS(44) |
2016 					  NUM_CLAUSE_TEMP_GPRS(2));
2017 		sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
2018 					  NUM_ES_GPRS(18));
2019 		sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
2020 					   NUM_VS_THREADS(78) |
2021 					   NUM_GS_THREADS(4) |
2022 					   NUM_ES_THREADS(31));
2023 		sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
2024 					    NUM_VS_STACK_ENTRIES(40));
2025 		sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
2026 					    NUM_ES_STACK_ENTRIES(16));
2027 	} else if ((rdev->family) == CHIP_RV670) {
2028 		sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
2029 					  NUM_VS_GPRS(44) |
2030 					  NUM_CLAUSE_TEMP_GPRS(2));
2031 		sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
2032 					  NUM_ES_GPRS(17));
2033 		sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
2034 					   NUM_VS_THREADS(78) |
2035 					   NUM_GS_THREADS(4) |
2036 					   NUM_ES_THREADS(31));
2037 		sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
2038 					    NUM_VS_STACK_ENTRIES(64));
2039 		sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
2040 					    NUM_ES_STACK_ENTRIES(64));
2041 	}
2042 
2043 	WREG32(SQ_CONFIG, sq_config);
2044 	WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
2045 	WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
2046 	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2047 	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2048 	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2049 
2050 	if (((rdev->family) == CHIP_RV610) ||
2051 	    ((rdev->family) == CHIP_RV620) ||
2052 	    ((rdev->family) == CHIP_RS780) ||
2053 	    ((rdev->family) == CHIP_RS880)) {
2054 		WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
2055 	} else {
2056 		WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
2057 	}
2058 
2059 	/* More default values. 2D/3D driver should adjust as needed */
2060 	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
2061 					 S1_X(0x4) | S1_Y(0xc)));
2062 	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
2063 					 S1_X(0x2) | S1_Y(0x2) |
2064 					 S2_X(0xa) | S2_Y(0x6) |
2065 					 S3_X(0x6) | S3_Y(0xa)));
2066 	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
2067 					     S1_X(0x4) | S1_Y(0xc) |
2068 					     S2_X(0x1) | S2_Y(0x6) |
2069 					     S3_X(0xa) | S3_Y(0xe)));
2070 	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
2071 					     S5_X(0x0) | S5_Y(0x0) |
2072 					     S6_X(0xb) | S6_Y(0x4) |
2073 					     S7_X(0x7) | S7_Y(0x8)));
2074 
2075 	WREG32(VGT_STRMOUT_EN, 0);
2076 	tmp = rdev->config.r600.max_pipes * 16;
2077 	switch (rdev->family) {
2078 	case CHIP_RV610:
2079 	case CHIP_RV620:
2080 	case CHIP_RS780:
2081 	case CHIP_RS880:
2082 		tmp += 32;
2083 		break;
2084 	case CHIP_RV670:
2085 		tmp += 128;
2086 		break;
2087 	default:
2088 		break;
2089 	}
2090 	if (tmp > 256) {
2091 		tmp = 256;
2092 	}
2093 	WREG32(VGT_ES_PER_GS, 128);
2094 	WREG32(VGT_GS_PER_ES, tmp);
2095 	WREG32(VGT_GS_PER_VS, 2);
2096 	WREG32(VGT_GS_VERTEX_REUSE, 16);
2097 
2098 	/* more default values. 2D/3D driver should adjust as needed */
2099 	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2100 	WREG32(VGT_STRMOUT_EN, 0);
2101 	WREG32(SX_MISC, 0);
2102 	WREG32(PA_SC_MODE_CNTL, 0);
2103 	WREG32(PA_SC_AA_CONFIG, 0);
2104 	WREG32(PA_SC_LINE_STIPPLE, 0);
2105 	WREG32(SPI_INPUT_Z, 0);
2106 	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2107 	WREG32(CB_COLOR7_FRAG, 0);
2108 
2109 	/* Clear render buffer base addresses */
2110 	WREG32(CB_COLOR0_BASE, 0);
2111 	WREG32(CB_COLOR1_BASE, 0);
2112 	WREG32(CB_COLOR2_BASE, 0);
2113 	WREG32(CB_COLOR3_BASE, 0);
2114 	WREG32(CB_COLOR4_BASE, 0);
2115 	WREG32(CB_COLOR5_BASE, 0);
2116 	WREG32(CB_COLOR6_BASE, 0);
2117 	WREG32(CB_COLOR7_BASE, 0);
2118 	WREG32(CB_COLOR7_FRAG, 0);
2119 
2120 	switch (rdev->family) {
2121 	case CHIP_RV610:
2122 	case CHIP_RV620:
2123 	case CHIP_RS780:
2124 	case CHIP_RS880:
2125 		tmp = TC_L2_SIZE(8);
2126 		break;
2127 	case CHIP_RV630:
2128 	case CHIP_RV635:
2129 		tmp = TC_L2_SIZE(4);
2130 		break;
2131 	case CHIP_R600:
2132 		tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2133 		break;
2134 	default:
2135 		tmp = TC_L2_SIZE(0);
2136 		break;
2137 	}
2138 	WREG32(TC_CNTL, tmp);
2139 
2140 	tmp = RREG32(HDP_HOST_PATH_CNTL);
2141 	WREG32(HDP_HOST_PATH_CNTL, tmp);
2142 
2143 	tmp = RREG32(ARB_POP);
2144 	tmp |= ENABLE_TC128;
2145 	WREG32(ARB_POP, tmp);
2146 
2147 	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2148 	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2149 			       NUM_CLIP_SEQ(3)));
2150 	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2151 	WREG32(VC_ENHANCE, 0);
2152 }
2153 
2154 
2155 /*
2156  * Indirect registers accessor
2157  */
2158 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2159 {
2160 	unsigned long flags;
2161 	u32 r;
2162 
2163 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
2164 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2165 	(void)RREG32(PCIE_PORT_INDEX);
2166 	r = RREG32(PCIE_PORT_DATA);
2167 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
2168 	return r;
2169 }
2170 
2171 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2172 {
2173 	unsigned long flags;
2174 
2175 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
2176 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2177 	(void)RREG32(PCIE_PORT_INDEX);
2178 	WREG32(PCIE_PORT_DATA, (v));
2179 	(void)RREG32(PCIE_PORT_DATA);
2180 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
2181 }
2182 
2183 /*
2184  * CP & Ring
2185  */
2186 void r600_cp_stop(struct radeon_device *rdev)
2187 {
2188 	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2189 	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
2190 	WREG32(SCRATCH_UMSK, 0);
2191 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2192 }
2193 
2194 int r600_init_microcode(struct radeon_device *rdev)
2195 {
2196 	const char *chip_name;
2197 	const char *rlc_chip_name;
2198 	const char *smc_chip_name = "RV770";
2199 	size_t pfp_req_size, me_req_size, rlc_req_size, smc_req_size = 0;
2200 	char fw_name[30];
2201 	int err;
2202 
2203 	DRM_DEBUG("\n");
2204 
2205 	switch (rdev->family) {
2206 	case CHIP_R600:
2207 		chip_name = "R600";
2208 		rlc_chip_name = "R600";
2209 		break;
2210 	case CHIP_RV610:
2211 		chip_name = "RV610";
2212 		rlc_chip_name = "R600";
2213 		break;
2214 	case CHIP_RV630:
2215 		chip_name = "RV630";
2216 		rlc_chip_name = "R600";
2217 		break;
2218 	case CHIP_RV620:
2219 		chip_name = "RV620";
2220 		rlc_chip_name = "R600";
2221 		break;
2222 	case CHIP_RV635:
2223 		chip_name = "RV635";
2224 		rlc_chip_name = "R600";
2225 		break;
2226 	case CHIP_RV670:
2227 		chip_name = "RV670";
2228 		rlc_chip_name = "R600";
2229 		break;
2230 	case CHIP_RS780:
2231 	case CHIP_RS880:
2232 		chip_name = "RS780";
2233 		rlc_chip_name = "R600";
2234 		break;
2235 	case CHIP_RV770:
2236 		chip_name = "RV770";
2237 		rlc_chip_name = "R700";
2238 		smc_chip_name = "RV770";
2239 		smc_req_size = ALIGN(RV770_SMC_UCODE_SIZE, 4);
2240 		break;
2241 	case CHIP_RV730:
2242 		chip_name = "RV730";
2243 		rlc_chip_name = "R700";
2244 		smc_chip_name = "RV730";
2245 		smc_req_size = ALIGN(RV730_SMC_UCODE_SIZE, 4);
2246 		break;
2247 	case CHIP_RV710:
2248 		chip_name = "RV710";
2249 		rlc_chip_name = "R700";
2250 		smc_chip_name = "RV710";
2251 		smc_req_size = ALIGN(RV710_SMC_UCODE_SIZE, 4);
2252 		break;
2253 	case CHIP_RV740:
2254 		chip_name = "RV730";
2255 		rlc_chip_name = "R700";
2256 		smc_chip_name = "RV740";
2257 		smc_req_size = ALIGN(RV740_SMC_UCODE_SIZE, 4);
2258 		break;
2259 	case CHIP_CEDAR:
2260 		chip_name = "CEDAR";
2261 		rlc_chip_name = "CEDAR";
2262 		smc_chip_name = "CEDAR";
2263 		smc_req_size = ALIGN(CEDAR_SMC_UCODE_SIZE, 4);
2264 		break;
2265 	case CHIP_REDWOOD:
2266 		chip_name = "REDWOOD";
2267 		rlc_chip_name = "REDWOOD";
2268 		smc_chip_name = "REDWOOD";
2269 		smc_req_size = ALIGN(REDWOOD_SMC_UCODE_SIZE, 4);
2270 		break;
2271 	case CHIP_JUNIPER:
2272 		chip_name = "JUNIPER";
2273 		rlc_chip_name = "JUNIPER";
2274 		smc_chip_name = "JUNIPER";
2275 		smc_req_size = ALIGN(JUNIPER_SMC_UCODE_SIZE, 4);
2276 		break;
2277 	case CHIP_CYPRESS:
2278 	case CHIP_HEMLOCK:
2279 		chip_name = "CYPRESS";
2280 		rlc_chip_name = "CYPRESS";
2281 		smc_chip_name = "CYPRESS";
2282 		smc_req_size = ALIGN(CYPRESS_SMC_UCODE_SIZE, 4);
2283 		break;
2284 	case CHIP_PALM:
2285 		chip_name = "PALM";
2286 		rlc_chip_name = "SUMO";
2287 		break;
2288 	case CHIP_SUMO:
2289 		chip_name = "SUMO";
2290 		rlc_chip_name = "SUMO";
2291 		break;
2292 	case CHIP_SUMO2:
2293 		chip_name = "SUMO2";
2294 		rlc_chip_name = "SUMO";
2295 		break;
2296 	default: BUG();
2297 	}
2298 
2299 	if (rdev->family >= CHIP_CEDAR) {
2300 		pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2301 		me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
2302 		rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
2303 	} else if (rdev->family >= CHIP_RV770) {
2304 		pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2305 		me_req_size = R700_PM4_UCODE_SIZE * 4;
2306 		rlc_req_size = R700_RLC_UCODE_SIZE * 4;
2307 	} else {
2308 		pfp_req_size = R600_PFP_UCODE_SIZE * 4;
2309 		me_req_size = R600_PM4_UCODE_SIZE * 12;
2310 		rlc_req_size = R600_RLC_UCODE_SIZE * 4;
2311 	}
2312 
2313 	DRM_INFO("Loading %s Microcode\n", chip_name);
2314 
2315 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2316 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2317 	if (err)
2318 		goto out;
2319 	if (rdev->pfp_fw->size != pfp_req_size) {
2320 		printk(KERN_ERR
2321 		       "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2322 		       rdev->pfp_fw->size, fw_name);
2323 		err = -EINVAL;
2324 		goto out;
2325 	}
2326 
2327 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2328 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2329 	if (err)
2330 		goto out;
2331 	if (rdev->me_fw->size != me_req_size) {
2332 		printk(KERN_ERR
2333 		       "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2334 		       rdev->me_fw->size, fw_name);
2335 		err = -EINVAL;
2336 	}
2337 
2338 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
2339 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2340 	if (err)
2341 		goto out;
2342 	if (rdev->rlc_fw->size != rlc_req_size) {
2343 		printk(KERN_ERR
2344 		       "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2345 		       rdev->rlc_fw->size, fw_name);
2346 		err = -EINVAL;
2347 	}
2348 
2349 	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_HEMLOCK)) {
2350 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", smc_chip_name);
2351 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2352 		if (err) {
2353 			printk(KERN_ERR
2354 			       "smc: error loading firmware \"%s\"\n",
2355 			       fw_name);
2356 			release_firmware(rdev->smc_fw);
2357 			rdev->smc_fw = NULL;
2358 			err = 0;
2359 		} else if (rdev->smc_fw->size != smc_req_size) {
2360 			printk(KERN_ERR
2361 			       "smc: Bogus length %zu in firmware \"%s\"\n",
2362 			       rdev->smc_fw->size, fw_name);
2363 			err = -EINVAL;
2364 		}
2365 	}
2366 
2367 out:
2368 	if (err) {
2369 		if (err != -EINVAL)
2370 			printk(KERN_ERR
2371 			       "r600_cp: Failed to load firmware \"%s\"\n",
2372 			       fw_name);
2373 		release_firmware(rdev->pfp_fw);
2374 		rdev->pfp_fw = NULL;
2375 		release_firmware(rdev->me_fw);
2376 		rdev->me_fw = NULL;
2377 		release_firmware(rdev->rlc_fw);
2378 		rdev->rlc_fw = NULL;
2379 		release_firmware(rdev->smc_fw);
2380 		rdev->smc_fw = NULL;
2381 	}
2382 	return err;
2383 }
2384 
2385 static int r600_cp_load_microcode(struct radeon_device *rdev)
2386 {
2387 	const __be32 *fw_data;
2388 	int i;
2389 
2390 	if (!rdev->me_fw || !rdev->pfp_fw)
2391 		return -EINVAL;
2392 
2393 	r600_cp_stop(rdev);
2394 
2395 	WREG32(CP_RB_CNTL,
2396 #ifdef __BIG_ENDIAN
2397 	       BUF_SWAP_32BIT |
2398 #endif
2399 	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2400 
2401 	/* Reset cp */
2402 	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2403 	RREG32(GRBM_SOFT_RESET);
2404 	mdelay(15);
2405 	WREG32(GRBM_SOFT_RESET, 0);
2406 
2407 	WREG32(CP_ME_RAM_WADDR, 0);
2408 
2409 	fw_data = (const __be32 *)rdev->me_fw->data;
2410 	WREG32(CP_ME_RAM_WADDR, 0);
2411 	for (i = 0; i < R600_PM4_UCODE_SIZE * 3; i++)
2412 		WREG32(CP_ME_RAM_DATA,
2413 		       be32_to_cpup(fw_data++));
2414 
2415 	fw_data = (const __be32 *)rdev->pfp_fw->data;
2416 	WREG32(CP_PFP_UCODE_ADDR, 0);
2417 	for (i = 0; i < R600_PFP_UCODE_SIZE; i++)
2418 		WREG32(CP_PFP_UCODE_DATA,
2419 		       be32_to_cpup(fw_data++));
2420 
2421 	WREG32(CP_PFP_UCODE_ADDR, 0);
2422 	WREG32(CP_ME_RAM_WADDR, 0);
2423 	WREG32(CP_ME_RAM_RADDR, 0);
2424 	return 0;
2425 }
2426 
2427 int r600_cp_start(struct radeon_device *rdev)
2428 {
2429 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2430 	int r;
2431 	uint32_t cp_me;
2432 
2433 	r = radeon_ring_lock(rdev, ring, 7);
2434 	if (r) {
2435 		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2436 		return r;
2437 	}
2438 	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2439 	radeon_ring_write(ring, 0x1);
2440 	if (rdev->family >= CHIP_RV770) {
2441 		radeon_ring_write(ring, 0x0);
2442 		radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2443 	} else {
2444 		radeon_ring_write(ring, 0x3);
2445 		radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2446 	}
2447 	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2448 	radeon_ring_write(ring, 0);
2449 	radeon_ring_write(ring, 0);
2450 	radeon_ring_unlock_commit(rdev, ring);
2451 
2452 	cp_me = 0xff;
2453 	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2454 	return 0;
2455 }
2456 
2457 int r600_cp_resume(struct radeon_device *rdev)
2458 {
2459 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2460 	u32 tmp;
2461 	u32 rb_bufsz;
2462 	int r;
2463 
2464 	/* Reset cp */
2465 	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2466 	RREG32(GRBM_SOFT_RESET);
2467 	mdelay(15);
2468 	WREG32(GRBM_SOFT_RESET, 0);
2469 
2470 	/* Set ring buffer size */
2471 	rb_bufsz = order_base_2(ring->ring_size / 8);
2472 	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2473 #ifdef __BIG_ENDIAN
2474 	tmp |= BUF_SWAP_32BIT;
2475 #endif
2476 	WREG32(CP_RB_CNTL, tmp);
2477 	WREG32(CP_SEM_WAIT_TIMER, 0x0);
2478 
2479 	/* Set the write pointer delay */
2480 	WREG32(CP_RB_WPTR_DELAY, 0);
2481 
2482 	/* Initialize the ring buffer's read and write pointers */
2483 	WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2484 	WREG32(CP_RB_RPTR_WR, 0);
2485 	ring->wptr = 0;
2486 	WREG32(CP_RB_WPTR, ring->wptr);
2487 
2488 	/* set the wb address whether it's enabled or not */
2489 	WREG32(CP_RB_RPTR_ADDR,
2490 	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2491 	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2492 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2493 
2494 	if (rdev->wb.enabled)
2495 		WREG32(SCRATCH_UMSK, 0xff);
2496 	else {
2497 		tmp |= RB_NO_UPDATE;
2498 		WREG32(SCRATCH_UMSK, 0);
2499 	}
2500 
2501 	mdelay(1);
2502 	WREG32(CP_RB_CNTL, tmp);
2503 
2504 	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2505 	WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2506 
2507 	ring->rptr = RREG32(CP_RB_RPTR);
2508 
2509 	r600_cp_start(rdev);
2510 	ring->ready = true;
2511 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2512 	if (r) {
2513 		ring->ready = false;
2514 		return r;
2515 	}
2516 	return 0;
2517 }
2518 
2519 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2520 {
2521 	u32 rb_bufsz;
2522 	int r;
2523 
2524 	/* Align ring size */
2525 	rb_bufsz = order_base_2(ring_size / 8);
2526 	ring_size = (1 << (rb_bufsz + 1)) * 4;
2527 	ring->ring_size = ring_size;
2528 	ring->align_mask = 16 - 1;
2529 
2530 	if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2531 		r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2532 		if (r) {
2533 			DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2534 			ring->rptr_save_reg = 0;
2535 		}
2536 	}
2537 }
2538 
2539 void r600_cp_fini(struct radeon_device *rdev)
2540 {
2541 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2542 	r600_cp_stop(rdev);
2543 	radeon_ring_fini(rdev, ring);
2544 	radeon_scratch_free(rdev, ring->rptr_save_reg);
2545 }
2546 
2547 /*
2548  * GPU scratch registers helpers function.
2549  */
2550 void r600_scratch_init(struct radeon_device *rdev)
2551 {
2552 	int i;
2553 
2554 	rdev->scratch.num_reg = 7;
2555 	rdev->scratch.reg_base = SCRATCH_REG0;
2556 	for (i = 0; i < rdev->scratch.num_reg; i++) {
2557 		rdev->scratch.free[i] = true;
2558 		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2559 	}
2560 }
2561 
2562 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2563 {
2564 	uint32_t scratch;
2565 	uint32_t tmp = 0;
2566 	unsigned i;
2567 	int r;
2568 
2569 	r = radeon_scratch_get(rdev, &scratch);
2570 	if (r) {
2571 		DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2572 		return r;
2573 	}
2574 	WREG32(scratch, 0xCAFEDEAD);
2575 	r = radeon_ring_lock(rdev, ring, 3);
2576 	if (r) {
2577 		DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2578 		radeon_scratch_free(rdev, scratch);
2579 		return r;
2580 	}
2581 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2582 	radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2583 	radeon_ring_write(ring, 0xDEADBEEF);
2584 	radeon_ring_unlock_commit(rdev, ring);
2585 	for (i = 0; i < rdev->usec_timeout; i++) {
2586 		tmp = RREG32(scratch);
2587 		if (tmp == 0xDEADBEEF)
2588 			break;
2589 		DRM_UDELAY(1);
2590 	}
2591 	if (i < rdev->usec_timeout) {
2592 		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2593 	} else {
2594 		DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2595 			  ring->idx, scratch, tmp);
2596 		r = -EINVAL;
2597 	}
2598 	radeon_scratch_free(rdev, scratch);
2599 	return r;
2600 }
2601 
2602 /*
2603  * CP fences/semaphores
2604  */
2605 
2606 void r600_fence_ring_emit(struct radeon_device *rdev,
2607 			  struct radeon_fence *fence)
2608 {
2609 	struct radeon_ring *ring = &rdev->ring[fence->ring];
2610 
2611 	if (rdev->wb.use_event) {
2612 		u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2613 		/* flush read cache over gart */
2614 		radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2615 		radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2616 					PACKET3_VC_ACTION_ENA |
2617 					PACKET3_SH_ACTION_ENA);
2618 		radeon_ring_write(ring, 0xFFFFFFFF);
2619 		radeon_ring_write(ring, 0);
2620 		radeon_ring_write(ring, 10); /* poll interval */
2621 		/* EVENT_WRITE_EOP - flush caches, send int */
2622 		radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2623 		radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2624 		radeon_ring_write(ring, addr & 0xffffffff);
2625 		radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2626 		radeon_ring_write(ring, fence->seq);
2627 		radeon_ring_write(ring, 0);
2628 	} else {
2629 		/* flush read cache over gart */
2630 		radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2631 		radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2632 					PACKET3_VC_ACTION_ENA |
2633 					PACKET3_SH_ACTION_ENA);
2634 		radeon_ring_write(ring, 0xFFFFFFFF);
2635 		radeon_ring_write(ring, 0);
2636 		radeon_ring_write(ring, 10); /* poll interval */
2637 		radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2638 		radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
2639 		/* wait for 3D idle clean */
2640 		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2641 		radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2642 		radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2643 		/* Emit fence sequence & fire IRQ */
2644 		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2645 		radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2646 		radeon_ring_write(ring, fence->seq);
2647 		/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2648 		radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2649 		radeon_ring_write(ring, RB_INT_STAT);
2650 	}
2651 }
2652 
2653 bool r600_semaphore_ring_emit(struct radeon_device *rdev,
2654 			      struct radeon_ring *ring,
2655 			      struct radeon_semaphore *semaphore,
2656 			      bool emit_wait)
2657 {
2658 	uint64_t addr = semaphore->gpu_addr;
2659 	unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2660 
2661 	if (rdev->family < CHIP_CAYMAN)
2662 		sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
2663 
2664 	radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2665 	radeon_ring_write(ring, addr & 0xffffffff);
2666 	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
2667 
2668 	return true;
2669 }
2670 
2671 /**
2672  * r600_copy_cpdma - copy pages using the CP DMA engine
2673  *
2674  * @rdev: radeon_device pointer
2675  * @src_offset: src GPU address
2676  * @dst_offset: dst GPU address
2677  * @num_gpu_pages: number of GPU pages to xfer
2678  * @fence: radeon fence object
2679  *
2680  * Copy GPU paging using the CP DMA engine (r6xx+).
2681  * Used by the radeon ttm implementation to move pages if
2682  * registered as the asic copy callback.
2683  */
2684 int r600_copy_cpdma(struct radeon_device *rdev,
2685 		    uint64_t src_offset, uint64_t dst_offset,
2686 		    unsigned num_gpu_pages,
2687 		    struct radeon_fence **fence)
2688 {
2689 	struct radeon_semaphore *sem = NULL;
2690 	int ring_index = rdev->asic->copy.blit_ring_index;
2691 	struct radeon_ring *ring = &rdev->ring[ring_index];
2692 	u32 size_in_bytes, cur_size_in_bytes, tmp;
2693 	int i, num_loops;
2694 	int r = 0;
2695 
2696 	r = radeon_semaphore_create(rdev, &sem);
2697 	if (r) {
2698 		DRM_ERROR("radeon: moving bo (%d).\n", r);
2699 		return r;
2700 	}
2701 
2702 	size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
2703 	num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
2704 	r = radeon_ring_lock(rdev, ring, num_loops * 6 + 24);
2705 	if (r) {
2706 		DRM_ERROR("radeon: moving bo (%d).\n", r);
2707 		radeon_semaphore_free(rdev, &sem, NULL);
2708 		return r;
2709 	}
2710 
2711 	radeon_semaphore_sync_to(sem, *fence);
2712 	radeon_semaphore_sync_rings(rdev, sem, ring->idx);
2713 
2714 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2715 	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2716 	radeon_ring_write(ring, WAIT_3D_IDLE_bit);
2717 	for (i = 0; i < num_loops; i++) {
2718 		cur_size_in_bytes = size_in_bytes;
2719 		if (cur_size_in_bytes > 0x1fffff)
2720 			cur_size_in_bytes = 0x1fffff;
2721 		size_in_bytes -= cur_size_in_bytes;
2722 		tmp = upper_32_bits(src_offset) & 0xff;
2723 		if (size_in_bytes == 0)
2724 			tmp |= PACKET3_CP_DMA_CP_SYNC;
2725 		radeon_ring_write(ring, PACKET3(PACKET3_CP_DMA, 4));
2726 		radeon_ring_write(ring, src_offset & 0xffffffff);
2727 		radeon_ring_write(ring, tmp);
2728 		radeon_ring_write(ring, dst_offset & 0xffffffff);
2729 		radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
2730 		radeon_ring_write(ring, cur_size_in_bytes);
2731 		src_offset += cur_size_in_bytes;
2732 		dst_offset += cur_size_in_bytes;
2733 	}
2734 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2735 	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2736 	radeon_ring_write(ring, WAIT_CP_DMA_IDLE_bit);
2737 
2738 	r = radeon_fence_emit(rdev, fence, ring->idx);
2739 	if (r) {
2740 		radeon_ring_unlock_undo(rdev, ring);
2741 		return r;
2742 	}
2743 
2744 	radeon_ring_unlock_commit(rdev, ring);
2745 	radeon_semaphore_free(rdev, &sem, *fence);
2746 
2747 	return r;
2748 }
2749 
2750 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
2751 			 uint32_t tiling_flags, uint32_t pitch,
2752 			 uint32_t offset, uint32_t obj_size)
2753 {
2754 	/* FIXME: implement */
2755 	return 0;
2756 }
2757 
2758 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
2759 {
2760 	/* FIXME: implement */
2761 }
2762 
2763 static int r600_startup(struct radeon_device *rdev)
2764 {
2765 	struct radeon_ring *ring;
2766 	int r;
2767 
2768 	/* enable pcie gen2 link */
2769 	r600_pcie_gen2_enable(rdev);
2770 
2771 	/* scratch needs to be initialized before MC */
2772 	r = r600_vram_scratch_init(rdev);
2773 	if (r)
2774 		return r;
2775 
2776 	r600_mc_program(rdev);
2777 
2778 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2779 		r = r600_init_microcode(rdev);
2780 		if (r) {
2781 			DRM_ERROR("Failed to load firmware!\n");
2782 			return r;
2783 		}
2784 	}
2785 
2786 	if (rdev->flags & RADEON_IS_AGP) {
2787 		r600_agp_enable(rdev);
2788 	} else {
2789 		r = r600_pcie_gart_enable(rdev);
2790 		if (r)
2791 			return r;
2792 	}
2793 	r600_gpu_init(rdev);
2794 
2795 	/* allocate wb buffer */
2796 	r = radeon_wb_init(rdev);
2797 	if (r)
2798 		return r;
2799 
2800 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2801 	if (r) {
2802 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2803 		return r;
2804 	}
2805 
2806 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2807 	if (r) {
2808 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2809 		return r;
2810 	}
2811 
2812 	/* Enable IRQ */
2813 	if (!rdev->irq.installed) {
2814 		r = radeon_irq_kms_init(rdev);
2815 		if (r)
2816 			return r;
2817 	}
2818 
2819 	r = r600_irq_init(rdev);
2820 	if (r) {
2821 		DRM_ERROR("radeon: IH init failed (%d).\n", r);
2822 		radeon_irq_kms_fini(rdev);
2823 		return r;
2824 	}
2825 	r600_irq_set(rdev);
2826 
2827 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2828 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2829 			     R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2830 			     RADEON_CP_PACKET2);
2831 	if (r)
2832 		return r;
2833 
2834 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2835 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2836 			     DMA_RB_RPTR, DMA_RB_WPTR,
2837 			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2838 	if (r)
2839 		return r;
2840 
2841 	r = r600_cp_load_microcode(rdev);
2842 	if (r)
2843 		return r;
2844 	r = r600_cp_resume(rdev);
2845 	if (r)
2846 		return r;
2847 
2848 	r = r600_dma_resume(rdev);
2849 	if (r)
2850 		return r;
2851 
2852 	r = radeon_ib_pool_init(rdev);
2853 	if (r) {
2854 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2855 		return r;
2856 	}
2857 
2858 	r = r600_audio_init(rdev);
2859 	if (r) {
2860 		DRM_ERROR("radeon: audio init failed\n");
2861 		return r;
2862 	}
2863 
2864 	return 0;
2865 }
2866 
2867 void r600_vga_set_state(struct radeon_device *rdev, bool state)
2868 {
2869 	uint32_t temp;
2870 
2871 	temp = RREG32(CONFIG_CNTL);
2872 	if (state == false) {
2873 		temp &= ~(1<<0);
2874 		temp |= (1<<1);
2875 	} else {
2876 		temp &= ~(1<<1);
2877 	}
2878 	WREG32(CONFIG_CNTL, temp);
2879 }
2880 
2881 int r600_resume(struct radeon_device *rdev)
2882 {
2883 	int r;
2884 
2885 	/* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
2886 	 * posting will perform necessary task to bring back GPU into good
2887 	 * shape.
2888 	 */
2889 	/* post card */
2890 	atom_asic_init(rdev->mode_info.atom_context);
2891 
2892 	rdev->accel_working = true;
2893 	r = r600_startup(rdev);
2894 	if (r) {
2895 		DRM_ERROR("r600 startup failed on resume\n");
2896 		rdev->accel_working = false;
2897 		return r;
2898 	}
2899 
2900 	return r;
2901 }
2902 
2903 int r600_suspend(struct radeon_device *rdev)
2904 {
2905 	r600_audio_fini(rdev);
2906 	r600_cp_stop(rdev);
2907 	r600_dma_stop(rdev);
2908 	r600_irq_suspend(rdev);
2909 	radeon_wb_disable(rdev);
2910 	r600_pcie_gart_disable(rdev);
2911 
2912 	return 0;
2913 }
2914 
2915 /* Plan is to move initialization in that function and use
2916  * helper function so that radeon_device_init pretty much
2917  * do nothing more than calling asic specific function. This
2918  * should also allow to remove a bunch of callback function
2919  * like vram_info.
2920  */
2921 int r600_init(struct radeon_device *rdev)
2922 {
2923 	int r;
2924 
2925 	if (r600_debugfs_mc_info_init(rdev)) {
2926 		DRM_ERROR("Failed to register debugfs file for mc !\n");
2927 	}
2928 	/* Read BIOS */
2929 	if (!radeon_get_bios(rdev)) {
2930 		if (ASIC_IS_AVIVO(rdev))
2931 			return -EINVAL;
2932 	}
2933 	/* Must be an ATOMBIOS */
2934 	if (!rdev->is_atom_bios) {
2935 		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
2936 		return -EINVAL;
2937 	}
2938 	r = radeon_atombios_init(rdev);
2939 	if (r)
2940 		return r;
2941 	/* Post card if necessary */
2942 	if (!radeon_card_posted(rdev)) {
2943 		if (!rdev->bios) {
2944 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2945 			return -EINVAL;
2946 		}
2947 		DRM_INFO("GPU not posted. posting now...\n");
2948 		atom_asic_init(rdev->mode_info.atom_context);
2949 	}
2950 	/* Initialize scratch registers */
2951 	r600_scratch_init(rdev);
2952 	/* Initialize surface registers */
2953 	radeon_surface_init(rdev);
2954 	/* Initialize clocks */
2955 	radeon_get_clock_info(rdev->ddev);
2956 	/* Fence driver */
2957 	r = radeon_fence_driver_init(rdev);
2958 	if (r)
2959 		return r;
2960 	if (rdev->flags & RADEON_IS_AGP) {
2961 		r = radeon_agp_init(rdev);
2962 		if (r)
2963 			radeon_agp_disable(rdev);
2964 	}
2965 	r = r600_mc_init(rdev);
2966 	if (r)
2967 		return r;
2968 	/* Memory manager */
2969 	r = radeon_bo_init(rdev);
2970 	if (r)
2971 		return r;
2972 
2973 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
2974 	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
2975 
2976 	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
2977 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
2978 
2979 	rdev->ih.ring_obj = NULL;
2980 	r600_ih_ring_init(rdev, 64 * 1024);
2981 
2982 	r = r600_pcie_gart_init(rdev);
2983 	if (r)
2984 		return r;
2985 
2986 	rdev->accel_working = true;
2987 	r = r600_startup(rdev);
2988 	if (r) {
2989 		dev_err(rdev->dev, "disabling GPU acceleration\n");
2990 		r600_cp_fini(rdev);
2991 		r600_dma_fini(rdev);
2992 		r600_irq_fini(rdev);
2993 		radeon_wb_fini(rdev);
2994 		radeon_ib_pool_fini(rdev);
2995 		radeon_irq_kms_fini(rdev);
2996 		r600_pcie_gart_fini(rdev);
2997 		rdev->accel_working = false;
2998 	}
2999 
3000 	return 0;
3001 }
3002 
3003 void r600_fini(struct radeon_device *rdev)
3004 {
3005 	r600_audio_fini(rdev);
3006 	r600_cp_fini(rdev);
3007 	r600_dma_fini(rdev);
3008 	r600_irq_fini(rdev);
3009 	radeon_wb_fini(rdev);
3010 	radeon_ib_pool_fini(rdev);
3011 	radeon_irq_kms_fini(rdev);
3012 	r600_pcie_gart_fini(rdev);
3013 	r600_vram_scratch_fini(rdev);
3014 	radeon_agp_fini(rdev);
3015 	radeon_gem_fini(rdev);
3016 	radeon_fence_driver_fini(rdev);
3017 	radeon_bo_fini(rdev);
3018 	radeon_atombios_fini(rdev);
3019 	kfree(rdev->bios);
3020 	rdev->bios = NULL;
3021 }
3022 
3023 
3024 /*
3025  * CS stuff
3026  */
3027 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3028 {
3029 	struct radeon_ring *ring = &rdev->ring[ib->ring];
3030 	u32 next_rptr;
3031 
3032 	if (ring->rptr_save_reg) {
3033 		next_rptr = ring->wptr + 3 + 4;
3034 		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3035 		radeon_ring_write(ring, ((ring->rptr_save_reg -
3036 					 PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3037 		radeon_ring_write(ring, next_rptr);
3038 	} else if (rdev->wb.enabled) {
3039 		next_rptr = ring->wptr + 5 + 4;
3040 		radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3041 		radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3042 		radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3043 		radeon_ring_write(ring, next_rptr);
3044 		radeon_ring_write(ring, 0);
3045 	}
3046 
3047 	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3048 	radeon_ring_write(ring,
3049 #ifdef __BIG_ENDIAN
3050 			  (2 << 0) |
3051 #endif
3052 			  (ib->gpu_addr & 0xFFFFFFFC));
3053 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3054 	radeon_ring_write(ring, ib->length_dw);
3055 }
3056 
3057 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3058 {
3059 	struct radeon_ib ib;
3060 	uint32_t scratch;
3061 	uint32_t tmp = 0;
3062 	unsigned i;
3063 	int r;
3064 
3065 	r = radeon_scratch_get(rdev, &scratch);
3066 	if (r) {
3067 		DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3068 		return r;
3069 	}
3070 	WREG32(scratch, 0xCAFEDEAD);
3071 	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3072 	if (r) {
3073 		DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3074 		goto free_scratch;
3075 	}
3076 	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3077 	ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3078 	ib.ptr[2] = 0xDEADBEEF;
3079 	ib.length_dw = 3;
3080 	r = radeon_ib_schedule(rdev, &ib, NULL);
3081 	if (r) {
3082 		DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3083 		goto free_ib;
3084 	}
3085 	r = radeon_fence_wait(ib.fence, false);
3086 	if (r) {
3087 		DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3088 		goto free_ib;
3089 	}
3090 	for (i = 0; i < rdev->usec_timeout; i++) {
3091 		tmp = RREG32(scratch);
3092 		if (tmp == 0xDEADBEEF)
3093 			break;
3094 		DRM_UDELAY(1);
3095 	}
3096 	if (i < rdev->usec_timeout) {
3097 		DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3098 	} else {
3099 		DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3100 			  scratch, tmp);
3101 		r = -EINVAL;
3102 	}
3103 free_ib:
3104 	radeon_ib_free(rdev, &ib);
3105 free_scratch:
3106 	radeon_scratch_free(rdev, scratch);
3107 	return r;
3108 }
3109 
3110 /*
3111  * Interrupts
3112  *
3113  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
3114  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
3115  * writing to the ring and the GPU consuming, the GPU writes to the ring
3116  * and host consumes.  As the host irq handler processes interrupts, it
3117  * increments the rptr.  When the rptr catches up with the wptr, all the
3118  * current interrupts have been processed.
3119  */
3120 
3121 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3122 {
3123 	u32 rb_bufsz;
3124 
3125 	/* Align ring size */
3126 	rb_bufsz = order_base_2(ring_size / 4);
3127 	ring_size = (1 << rb_bufsz) * 4;
3128 	rdev->ih.ring_size = ring_size;
3129 	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3130 	rdev->ih.rptr = 0;
3131 }
3132 
3133 int r600_ih_ring_alloc(struct radeon_device *rdev)
3134 {
3135 	int r;
3136 
3137 	/* Allocate ring buffer */
3138 	if (rdev->ih.ring_obj == NULL) {
3139 		r = radeon_bo_create(rdev, rdev->ih.ring_size,
3140 				     PAGE_SIZE, true,
3141 				     RADEON_GEM_DOMAIN_GTT,
3142 				     NULL, &rdev->ih.ring_obj);
3143 		if (r) {
3144 			DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3145 			return r;
3146 		}
3147 		r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3148 		if (unlikely(r != 0))
3149 			return r;
3150 		r = radeon_bo_pin(rdev->ih.ring_obj,
3151 				  RADEON_GEM_DOMAIN_GTT,
3152 				  &rdev->ih.gpu_addr);
3153 		if (r) {
3154 			radeon_bo_unreserve(rdev->ih.ring_obj);
3155 			DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3156 			return r;
3157 		}
3158 		r = radeon_bo_kmap(rdev->ih.ring_obj,
3159 				   (void **)&rdev->ih.ring);
3160 		radeon_bo_unreserve(rdev->ih.ring_obj);
3161 		if (r) {
3162 			DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3163 			return r;
3164 		}
3165 	}
3166 	return 0;
3167 }
3168 
3169 void r600_ih_ring_fini(struct radeon_device *rdev)
3170 {
3171 	int r;
3172 	if (rdev->ih.ring_obj) {
3173 		r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3174 		if (likely(r == 0)) {
3175 			radeon_bo_kunmap(rdev->ih.ring_obj);
3176 			radeon_bo_unpin(rdev->ih.ring_obj);
3177 			radeon_bo_unreserve(rdev->ih.ring_obj);
3178 		}
3179 		radeon_bo_unref(&rdev->ih.ring_obj);
3180 		rdev->ih.ring = NULL;
3181 		rdev->ih.ring_obj = NULL;
3182 	}
3183 }
3184 
3185 void r600_rlc_stop(struct radeon_device *rdev)
3186 {
3187 
3188 	if ((rdev->family >= CHIP_RV770) &&
3189 	    (rdev->family <= CHIP_RV740)) {
3190 		/* r7xx asics need to soft reset RLC before halting */
3191 		WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3192 		RREG32(SRBM_SOFT_RESET);
3193 		mdelay(15);
3194 		WREG32(SRBM_SOFT_RESET, 0);
3195 		RREG32(SRBM_SOFT_RESET);
3196 	}
3197 
3198 	WREG32(RLC_CNTL, 0);
3199 }
3200 
3201 static void r600_rlc_start(struct radeon_device *rdev)
3202 {
3203 	WREG32(RLC_CNTL, RLC_ENABLE);
3204 }
3205 
3206 static int r600_rlc_resume(struct radeon_device *rdev)
3207 {
3208 	u32 i;
3209 	const __be32 *fw_data;
3210 
3211 	if (!rdev->rlc_fw)
3212 		return -EINVAL;
3213 
3214 	r600_rlc_stop(rdev);
3215 
3216 	WREG32(RLC_HB_CNTL, 0);
3217 
3218 	WREG32(RLC_HB_BASE, 0);
3219 	WREG32(RLC_HB_RPTR, 0);
3220 	WREG32(RLC_HB_WPTR, 0);
3221 	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3222 	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3223 	WREG32(RLC_MC_CNTL, 0);
3224 	WREG32(RLC_UCODE_CNTL, 0);
3225 
3226 	fw_data = (const __be32 *)rdev->rlc_fw->data;
3227 	if (rdev->family >= CHIP_RV770) {
3228 		for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3229 			WREG32(RLC_UCODE_ADDR, i);
3230 			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3231 		}
3232 	} else {
3233 		for (i = 0; i < R600_RLC_UCODE_SIZE; i++) {
3234 			WREG32(RLC_UCODE_ADDR, i);
3235 			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3236 		}
3237 	}
3238 	WREG32(RLC_UCODE_ADDR, 0);
3239 
3240 	r600_rlc_start(rdev);
3241 
3242 	return 0;
3243 }
3244 
3245 static void r600_enable_interrupts(struct radeon_device *rdev)
3246 {
3247 	u32 ih_cntl = RREG32(IH_CNTL);
3248 	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3249 
3250 	ih_cntl |= ENABLE_INTR;
3251 	ih_rb_cntl |= IH_RB_ENABLE;
3252 	WREG32(IH_CNTL, ih_cntl);
3253 	WREG32(IH_RB_CNTL, ih_rb_cntl);
3254 	rdev->ih.enabled = true;
3255 }
3256 
3257 void r600_disable_interrupts(struct radeon_device *rdev)
3258 {
3259 	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3260 	u32 ih_cntl = RREG32(IH_CNTL);
3261 
3262 	ih_rb_cntl &= ~IH_RB_ENABLE;
3263 	ih_cntl &= ~ENABLE_INTR;
3264 	WREG32(IH_RB_CNTL, ih_rb_cntl);
3265 	WREG32(IH_CNTL, ih_cntl);
3266 	/* set rptr, wptr to 0 */
3267 	WREG32(IH_RB_RPTR, 0);
3268 	WREG32(IH_RB_WPTR, 0);
3269 	rdev->ih.enabled = false;
3270 	rdev->ih.rptr = 0;
3271 }
3272 
3273 static void r600_disable_interrupt_state(struct radeon_device *rdev)
3274 {
3275 	u32 tmp;
3276 
3277 	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3278 	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3279 	WREG32(DMA_CNTL, tmp);
3280 	WREG32(GRBM_INT_CNTL, 0);
3281 	WREG32(DxMODE_INT_MASK, 0);
3282 	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3283 	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
3284 	if (ASIC_IS_DCE3(rdev)) {
3285 		WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3286 		WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3287 		tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3288 		WREG32(DC_HPD1_INT_CONTROL, tmp);
3289 		tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3290 		WREG32(DC_HPD2_INT_CONTROL, tmp);
3291 		tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3292 		WREG32(DC_HPD3_INT_CONTROL, tmp);
3293 		tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3294 		WREG32(DC_HPD4_INT_CONTROL, tmp);
3295 		if (ASIC_IS_DCE32(rdev)) {
3296 			tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3297 			WREG32(DC_HPD5_INT_CONTROL, tmp);
3298 			tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3299 			WREG32(DC_HPD6_INT_CONTROL, tmp);
3300 			tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3301 			WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3302 			tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3303 			WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3304 		} else {
3305 			tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3306 			WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3307 			tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3308 			WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3309 		}
3310 	} else {
3311 		WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3312 		WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3313 		tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3314 		WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3315 		tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3316 		WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3317 		tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3318 		WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3319 		tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3320 		WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3321 		tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3322 		WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3323 	}
3324 }
3325 
3326 int r600_irq_init(struct radeon_device *rdev)
3327 {
3328 	int ret = 0;
3329 	int rb_bufsz;
3330 	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3331 
3332 	/* allocate ring */
3333 	ret = r600_ih_ring_alloc(rdev);
3334 	if (ret)
3335 		return ret;
3336 
3337 	/* disable irqs */
3338 	r600_disable_interrupts(rdev);
3339 
3340 	/* init rlc */
3341 	if (rdev->family >= CHIP_CEDAR)
3342 		ret = evergreen_rlc_resume(rdev);
3343 	else
3344 		ret = r600_rlc_resume(rdev);
3345 	if (ret) {
3346 		r600_ih_ring_fini(rdev);
3347 		return ret;
3348 	}
3349 
3350 	/* setup interrupt control */
3351 	/* set dummy read address to ring address */
3352 	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3353 	interrupt_cntl = RREG32(INTERRUPT_CNTL);
3354 	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3355 	 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3356 	 */
3357 	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3358 	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3359 	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3360 	WREG32(INTERRUPT_CNTL, interrupt_cntl);
3361 
3362 	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3363 	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
3364 
3365 	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3366 		      IH_WPTR_OVERFLOW_CLEAR |
3367 		      (rb_bufsz << 1));
3368 
3369 	if (rdev->wb.enabled)
3370 		ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3371 
3372 	/* set the writeback address whether it's enabled or not */
3373 	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3374 	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3375 
3376 	WREG32(IH_RB_CNTL, ih_rb_cntl);
3377 
3378 	/* set rptr, wptr to 0 */
3379 	WREG32(IH_RB_RPTR, 0);
3380 	WREG32(IH_RB_WPTR, 0);
3381 
3382 	/* Default settings for IH_CNTL (disabled at first) */
3383 	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
3384 	/* RPTR_REARM only works if msi's are enabled */
3385 	if (rdev->msi_enabled)
3386 		ih_cntl |= RPTR_REARM;
3387 	WREG32(IH_CNTL, ih_cntl);
3388 
3389 	/* force the active interrupt state to all disabled */
3390 	if (rdev->family >= CHIP_CEDAR)
3391 		evergreen_disable_interrupt_state(rdev);
3392 	else
3393 		r600_disable_interrupt_state(rdev);
3394 
3395 	/* at this point everything should be setup correctly to enable master */
3396 	pci_set_master(rdev->pdev);
3397 
3398 	/* enable irqs */
3399 	r600_enable_interrupts(rdev);
3400 
3401 	return ret;
3402 }
3403 
3404 void r600_irq_suspend(struct radeon_device *rdev)
3405 {
3406 	r600_irq_disable(rdev);
3407 	r600_rlc_stop(rdev);
3408 }
3409 
3410 void r600_irq_fini(struct radeon_device *rdev)
3411 {
3412 	r600_irq_suspend(rdev);
3413 	r600_ih_ring_fini(rdev);
3414 }
3415 
3416 int r600_irq_set(struct radeon_device *rdev)
3417 {
3418 	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3419 	u32 mode_int = 0;
3420 	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
3421 	u32 grbm_int_cntl = 0;
3422 	u32 hdmi0, hdmi1;
3423 	u32 d1grph = 0, d2grph = 0;
3424 	u32 dma_cntl;
3425 	u32 thermal_int = 0;
3426 
3427 	if (!rdev->irq.installed) {
3428 		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
3429 		return -EINVAL;
3430 	}
3431 	/* don't enable anything if the ih is disabled */
3432 	if (!rdev->ih.enabled) {
3433 		r600_disable_interrupts(rdev);
3434 		/* force the active interrupt state to all disabled */
3435 		r600_disable_interrupt_state(rdev);
3436 		return 0;
3437 	}
3438 
3439 	if (ASIC_IS_DCE3(rdev)) {
3440 		hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3441 		hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3442 		hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3443 		hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3444 		if (ASIC_IS_DCE32(rdev)) {
3445 			hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3446 			hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3447 			hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3448 			hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3449 		} else {
3450 			hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3451 			hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3452 		}
3453 	} else {
3454 		hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3455 		hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3456 		hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3457 		hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3458 		hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3459 	}
3460 
3461 	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3462 
3463 	if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
3464 		thermal_int = RREG32(CG_THERMAL_INT) &
3465 			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
3466 	} else if (rdev->family >= CHIP_RV770) {
3467 		thermal_int = RREG32(RV770_CG_THERMAL_INT) &
3468 			~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
3469 	}
3470 	if (rdev->irq.dpm_thermal) {
3471 		DRM_DEBUG("dpm thermal\n");
3472 		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
3473 	}
3474 
3475 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3476 		DRM_DEBUG("r600_irq_set: sw int\n");
3477 		cp_int_cntl |= RB_INT_ENABLE;
3478 		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3479 	}
3480 
3481 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3482 		DRM_DEBUG("r600_irq_set: sw int dma\n");
3483 		dma_cntl |= TRAP_ENABLE;
3484 	}
3485 
3486 	if (rdev->irq.crtc_vblank_int[0] ||
3487 	    atomic_read(&rdev->irq.pflip[0])) {
3488 		DRM_DEBUG("r600_irq_set: vblank 0\n");
3489 		mode_int |= D1MODE_VBLANK_INT_MASK;
3490 	}
3491 	if (rdev->irq.crtc_vblank_int[1] ||
3492 	    atomic_read(&rdev->irq.pflip[1])) {
3493 		DRM_DEBUG("r600_irq_set: vblank 1\n");
3494 		mode_int |= D2MODE_VBLANK_INT_MASK;
3495 	}
3496 	if (rdev->irq.hpd[0]) {
3497 		DRM_DEBUG("r600_irq_set: hpd 1\n");
3498 		hpd1 |= DC_HPDx_INT_EN;
3499 	}
3500 	if (rdev->irq.hpd[1]) {
3501 		DRM_DEBUG("r600_irq_set: hpd 2\n");
3502 		hpd2 |= DC_HPDx_INT_EN;
3503 	}
3504 	if (rdev->irq.hpd[2]) {
3505 		DRM_DEBUG("r600_irq_set: hpd 3\n");
3506 		hpd3 |= DC_HPDx_INT_EN;
3507 	}
3508 	if (rdev->irq.hpd[3]) {
3509 		DRM_DEBUG("r600_irq_set: hpd 4\n");
3510 		hpd4 |= DC_HPDx_INT_EN;
3511 	}
3512 	if (rdev->irq.hpd[4]) {
3513 		DRM_DEBUG("r600_irq_set: hpd 5\n");
3514 		hpd5 |= DC_HPDx_INT_EN;
3515 	}
3516 	if (rdev->irq.hpd[5]) {
3517 		DRM_DEBUG("r600_irq_set: hpd 6\n");
3518 		hpd6 |= DC_HPDx_INT_EN;
3519 	}
3520 	if (rdev->irq.afmt[0]) {
3521 		DRM_DEBUG("r600_irq_set: hdmi 0\n");
3522 		hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3523 	}
3524 	if (rdev->irq.afmt[1]) {
3525 		DRM_DEBUG("r600_irq_set: hdmi 0\n");
3526 		hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3527 	}
3528 
3529 	WREG32(CP_INT_CNTL, cp_int_cntl);
3530 	WREG32(DMA_CNTL, dma_cntl);
3531 	WREG32(DxMODE_INT_MASK, mode_int);
3532 	WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
3533 	WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
3534 	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3535 	if (ASIC_IS_DCE3(rdev)) {
3536 		WREG32(DC_HPD1_INT_CONTROL, hpd1);
3537 		WREG32(DC_HPD2_INT_CONTROL, hpd2);
3538 		WREG32(DC_HPD3_INT_CONTROL, hpd3);
3539 		WREG32(DC_HPD4_INT_CONTROL, hpd4);
3540 		if (ASIC_IS_DCE32(rdev)) {
3541 			WREG32(DC_HPD5_INT_CONTROL, hpd5);
3542 			WREG32(DC_HPD6_INT_CONTROL, hpd6);
3543 			WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
3544 			WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
3545 		} else {
3546 			WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3547 			WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3548 		}
3549 	} else {
3550 		WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
3551 		WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
3552 		WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
3553 		WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3554 		WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3555 	}
3556 	if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
3557 		WREG32(CG_THERMAL_INT, thermal_int);
3558 	} else if (rdev->family >= CHIP_RV770) {
3559 		WREG32(RV770_CG_THERMAL_INT, thermal_int);
3560 	}
3561 
3562 	return 0;
3563 }
3564 
3565 static void r600_irq_ack(struct radeon_device *rdev)
3566 {
3567 	u32 tmp;
3568 
3569 	if (ASIC_IS_DCE3(rdev)) {
3570 		rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
3571 		rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
3572 		rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
3573 		if (ASIC_IS_DCE32(rdev)) {
3574 			rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
3575 			rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
3576 		} else {
3577 			rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3578 			rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
3579 		}
3580 	} else {
3581 		rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3582 		rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3583 		rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
3584 		rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3585 		rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
3586 	}
3587 	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
3588 	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
3589 
3590 	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3591 		WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3592 	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3593 		WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3594 	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
3595 		WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3596 	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
3597 		WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3598 	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
3599 		WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3600 	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
3601 		WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3602 	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3603 		if (ASIC_IS_DCE3(rdev)) {
3604 			tmp = RREG32(DC_HPD1_INT_CONTROL);
3605 			tmp |= DC_HPDx_INT_ACK;
3606 			WREG32(DC_HPD1_INT_CONTROL, tmp);
3607 		} else {
3608 			tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
3609 			tmp |= DC_HPDx_INT_ACK;
3610 			WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3611 		}
3612 	}
3613 	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3614 		if (ASIC_IS_DCE3(rdev)) {
3615 			tmp = RREG32(DC_HPD2_INT_CONTROL);
3616 			tmp |= DC_HPDx_INT_ACK;
3617 			WREG32(DC_HPD2_INT_CONTROL, tmp);
3618 		} else {
3619 			tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
3620 			tmp |= DC_HPDx_INT_ACK;
3621 			WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3622 		}
3623 	}
3624 	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3625 		if (ASIC_IS_DCE3(rdev)) {
3626 			tmp = RREG32(DC_HPD3_INT_CONTROL);
3627 			tmp |= DC_HPDx_INT_ACK;
3628 			WREG32(DC_HPD3_INT_CONTROL, tmp);
3629 		} else {
3630 			tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
3631 			tmp |= DC_HPDx_INT_ACK;
3632 			WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3633 		}
3634 	}
3635 	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3636 		tmp = RREG32(DC_HPD4_INT_CONTROL);
3637 		tmp |= DC_HPDx_INT_ACK;
3638 		WREG32(DC_HPD4_INT_CONTROL, tmp);
3639 	}
3640 	if (ASIC_IS_DCE32(rdev)) {
3641 		if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
3642 			tmp = RREG32(DC_HPD5_INT_CONTROL);
3643 			tmp |= DC_HPDx_INT_ACK;
3644 			WREG32(DC_HPD5_INT_CONTROL, tmp);
3645 		}
3646 		if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
3647 			tmp = RREG32(DC_HPD5_INT_CONTROL);
3648 			tmp |= DC_HPDx_INT_ACK;
3649 			WREG32(DC_HPD6_INT_CONTROL, tmp);
3650 		}
3651 		if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
3652 			tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
3653 			tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3654 			WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3655 		}
3656 		if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
3657 			tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
3658 			tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3659 			WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3660 		}
3661 	} else {
3662 		if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
3663 			tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
3664 			tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3665 			WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3666 		}
3667 		if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
3668 			if (ASIC_IS_DCE3(rdev)) {
3669 				tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
3670 				tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3671 				WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3672 			} else {
3673 				tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
3674 				tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3675 				WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3676 			}
3677 		}
3678 	}
3679 }
3680 
3681 void r600_irq_disable(struct radeon_device *rdev)
3682 {
3683 	r600_disable_interrupts(rdev);
3684 	/* Wait and acknowledge irq */
3685 	mdelay(1);
3686 	r600_irq_ack(rdev);
3687 	r600_disable_interrupt_state(rdev);
3688 }
3689 
3690 static u32 r600_get_ih_wptr(struct radeon_device *rdev)
3691 {
3692 	u32 wptr, tmp;
3693 
3694 	if (rdev->wb.enabled)
3695 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3696 	else
3697 		wptr = RREG32(IH_RB_WPTR);
3698 
3699 	if (wptr & RB_OVERFLOW) {
3700 		/* When a ring buffer overflow happen start parsing interrupt
3701 		 * from the last not overwritten vector (wptr + 16). Hopefully
3702 		 * this should allow us to catchup.
3703 		 */
3704 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3705 			wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3706 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3707 		tmp = RREG32(IH_RB_CNTL);
3708 		tmp |= IH_WPTR_OVERFLOW_CLEAR;
3709 		WREG32(IH_RB_CNTL, tmp);
3710 	}
3711 	return (wptr & rdev->ih.ptr_mask);
3712 }
3713 
3714 /*        r600 IV Ring
3715  * Each IV ring entry is 128 bits:
3716  * [7:0]    - interrupt source id
3717  * [31:8]   - reserved
3718  * [59:32]  - interrupt source data
3719  * [127:60]  - reserved
3720  *
3721  * The basic interrupt vector entries
3722  * are decoded as follows:
3723  * src_id  src_data  description
3724  *      1         0  D1 Vblank
3725  *      1         1  D1 Vline
3726  *      5         0  D2 Vblank
3727  *      5         1  D2 Vline
3728  *     19         0  FP Hot plug detection A
3729  *     19         1  FP Hot plug detection B
3730  *     19         2  DAC A auto-detection
3731  *     19         3  DAC B auto-detection
3732  *     21         4  HDMI block A
3733  *     21         5  HDMI block B
3734  *    176         -  CP_INT RB
3735  *    177         -  CP_INT IB1
3736  *    178         -  CP_INT IB2
3737  *    181         -  EOP Interrupt
3738  *    233         -  GUI Idle
3739  *
3740  * Note, these are based on r600 and may need to be
3741  * adjusted or added to on newer asics
3742  */
3743 
3744 int r600_irq_process(struct radeon_device *rdev)
3745 {
3746 	u32 wptr;
3747 	u32 rptr;
3748 	u32 src_id, src_data;
3749 	u32 ring_index;
3750 	bool queue_hotplug = false;
3751 	bool queue_hdmi = false;
3752 	bool queue_thermal = false;
3753 
3754 	if (!rdev->ih.enabled || rdev->shutdown)
3755 		return IRQ_NONE;
3756 
3757 	/* No MSIs, need a dummy read to flush PCI DMAs */
3758 	if (!rdev->msi_enabled)
3759 		RREG32(IH_RB_WPTR);
3760 
3761 	wptr = r600_get_ih_wptr(rdev);
3762 
3763 restart_ih:
3764 	/* is somebody else already processing irqs? */
3765 	if (atomic_xchg(&rdev->ih.lock, 1))
3766 		return IRQ_NONE;
3767 
3768 	rptr = rdev->ih.rptr;
3769 	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3770 
3771 	/* Order reading of wptr vs. reading of IH ring data */
3772 	rmb();
3773 
3774 	/* display interrupts */
3775 	r600_irq_ack(rdev);
3776 
3777 	while (rptr != wptr) {
3778 		/* wptr/rptr are in bytes! */
3779 		ring_index = rptr / 4;
3780 		src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3781 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
3782 
3783 		switch (src_id) {
3784 		case 1: /* D1 vblank/vline */
3785 			switch (src_data) {
3786 			case 0: /* D1 vblank */
3787 				if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
3788 					if (rdev->irq.crtc_vblank_int[0]) {
3789 						drm_handle_vblank(rdev->ddev, 0);
3790 						rdev->pm.vblank_sync = true;
3791 						wake_up(&rdev->irq.vblank_queue);
3792 					}
3793 					if (atomic_read(&rdev->irq.pflip[0]))
3794 						radeon_crtc_handle_flip(rdev, 0);
3795 					rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3796 					DRM_DEBUG("IH: D1 vblank\n");
3797 				}
3798 				break;
3799 			case 1: /* D1 vline */
3800 				if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
3801 					rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
3802 					DRM_DEBUG("IH: D1 vline\n");
3803 				}
3804 				break;
3805 			default:
3806 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3807 				break;
3808 			}
3809 			break;
3810 		case 5: /* D2 vblank/vline */
3811 			switch (src_data) {
3812 			case 0: /* D2 vblank */
3813 				if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
3814 					if (rdev->irq.crtc_vblank_int[1]) {
3815 						drm_handle_vblank(rdev->ddev, 1);
3816 						rdev->pm.vblank_sync = true;
3817 						wake_up(&rdev->irq.vblank_queue);
3818 					}
3819 					if (atomic_read(&rdev->irq.pflip[1]))
3820 						radeon_crtc_handle_flip(rdev, 1);
3821 					rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
3822 					DRM_DEBUG("IH: D2 vblank\n");
3823 				}
3824 				break;
3825 			case 1: /* D1 vline */
3826 				if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
3827 					rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
3828 					DRM_DEBUG("IH: D2 vline\n");
3829 				}
3830 				break;
3831 			default:
3832 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3833 				break;
3834 			}
3835 			break;
3836 		case 19: /* HPD/DAC hotplug */
3837 			switch (src_data) {
3838 			case 0:
3839 				if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3840 					rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
3841 					queue_hotplug = true;
3842 					DRM_DEBUG("IH: HPD1\n");
3843 				}
3844 				break;
3845 			case 1:
3846 				if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3847 					rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
3848 					queue_hotplug = true;
3849 					DRM_DEBUG("IH: HPD2\n");
3850 				}
3851 				break;
3852 			case 4:
3853 				if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3854 					rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
3855 					queue_hotplug = true;
3856 					DRM_DEBUG("IH: HPD3\n");
3857 				}
3858 				break;
3859 			case 5:
3860 				if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3861 					rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
3862 					queue_hotplug = true;
3863 					DRM_DEBUG("IH: HPD4\n");
3864 				}
3865 				break;
3866 			case 10:
3867 				if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
3868 					rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
3869 					queue_hotplug = true;
3870 					DRM_DEBUG("IH: HPD5\n");
3871 				}
3872 				break;
3873 			case 12:
3874 				if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
3875 					rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
3876 					queue_hotplug = true;
3877 					DRM_DEBUG("IH: HPD6\n");
3878 				}
3879 				break;
3880 			default:
3881 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3882 				break;
3883 			}
3884 			break;
3885 		case 21: /* hdmi */
3886 			switch (src_data) {
3887 			case 4:
3888 				if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
3889 					rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
3890 					queue_hdmi = true;
3891 					DRM_DEBUG("IH: HDMI0\n");
3892 				}
3893 				break;
3894 			case 5:
3895 				if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
3896 					rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
3897 					queue_hdmi = true;
3898 					DRM_DEBUG("IH: HDMI1\n");
3899 				}
3900 				break;
3901 			default:
3902 				DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
3903 				break;
3904 			}
3905 			break;
3906 		case 176: /* CP_INT in ring buffer */
3907 		case 177: /* CP_INT in IB1 */
3908 		case 178: /* CP_INT in IB2 */
3909 			DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
3910 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3911 			break;
3912 		case 181: /* CP EOP event */
3913 			DRM_DEBUG("IH: CP EOP\n");
3914 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3915 			break;
3916 		case 224: /* DMA trap event */
3917 			DRM_DEBUG("IH: DMA trap\n");
3918 			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
3919 			break;
3920 		case 230: /* thermal low to high */
3921 			DRM_DEBUG("IH: thermal low to high\n");
3922 			rdev->pm.dpm.thermal.high_to_low = false;
3923 			queue_thermal = true;
3924 			break;
3925 		case 231: /* thermal high to low */
3926 			DRM_DEBUG("IH: thermal high to low\n");
3927 			rdev->pm.dpm.thermal.high_to_low = true;
3928 			queue_thermal = true;
3929 			break;
3930 		case 233: /* GUI IDLE */
3931 			DRM_DEBUG("IH: GUI idle\n");
3932 			break;
3933 		default:
3934 			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3935 			break;
3936 		}
3937 
3938 		/* wptr/rptr are in bytes! */
3939 		rptr += 16;
3940 		rptr &= rdev->ih.ptr_mask;
3941 	}
3942 	if (queue_hotplug)
3943 		schedule_work(&rdev->hotplug_work);
3944 	if (queue_hdmi)
3945 		schedule_work(&rdev->audio_work);
3946 	if (queue_thermal && rdev->pm.dpm_enabled)
3947 		schedule_work(&rdev->pm.dpm.thermal.work);
3948 	rdev->ih.rptr = rptr;
3949 	WREG32(IH_RB_RPTR, rdev->ih.rptr);
3950 	atomic_set(&rdev->ih.lock, 0);
3951 
3952 	/* make sure wptr hasn't changed while processing */
3953 	wptr = r600_get_ih_wptr(rdev);
3954 	if (wptr != rptr)
3955 		goto restart_ih;
3956 
3957 	return IRQ_HANDLED;
3958 }
3959 
3960 /*
3961  * Debugfs info
3962  */
3963 #if defined(CONFIG_DEBUG_FS)
3964 
3965 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
3966 {
3967 	struct drm_info_node *node = (struct drm_info_node *) m->private;
3968 	struct drm_device *dev = node->minor->dev;
3969 	struct radeon_device *rdev = dev->dev_private;
3970 
3971 	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
3972 	DREG32_SYS(m, rdev, VM_L2_STATUS);
3973 	return 0;
3974 }
3975 
3976 static struct drm_info_list r600_mc_info_list[] = {
3977 	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
3978 };
3979 #endif
3980 
3981 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
3982 {
3983 #if defined(CONFIG_DEBUG_FS)
3984 	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
3985 #else
3986 	return 0;
3987 #endif
3988 }
3989 
3990 /**
3991  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
3992  * rdev: radeon device structure
3993  * bo: buffer object struct which userspace is waiting for idle
3994  *
3995  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
3996  * through ring buffer, this leads to corruption in rendering, see
3997  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
3998  * directly perform HDP flush by writing register through MMIO.
3999  */
4000 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4001 {
4002 	/* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
4003 	 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4004 	 * This seems to cause problems on some AGP cards. Just use the old
4005 	 * method for them.
4006 	 */
4007 	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
4008 	    rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
4009 		void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
4010 		u32 tmp;
4011 
4012 		WREG32(HDP_DEBUG1, 0);
4013 		tmp = readl((void __iomem *)ptr);
4014 	} else
4015 		WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4016 }
4017 
4018 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4019 {
4020 	u32 link_width_cntl, mask;
4021 
4022 	if (rdev->flags & RADEON_IS_IGP)
4023 		return;
4024 
4025 	if (!(rdev->flags & RADEON_IS_PCIE))
4026 		return;
4027 
4028 	/* x2 cards have a special sequence */
4029 	if (ASIC_IS_X2(rdev))
4030 		return;
4031 
4032 	radeon_gui_idle(rdev);
4033 
4034 	switch (lanes) {
4035 	case 0:
4036 		mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4037 		break;
4038 	case 1:
4039 		mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4040 		break;
4041 	case 2:
4042 		mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4043 		break;
4044 	case 4:
4045 		mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4046 		break;
4047 	case 8:
4048 		mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4049 		break;
4050 	case 12:
4051 		/* not actually supported */
4052 		mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4053 		break;
4054 	case 16:
4055 		mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4056 		break;
4057 	default:
4058 		DRM_ERROR("invalid pcie lane request: %d\n", lanes);
4059 		return;
4060 	}
4061 
4062 	link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4063 	link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
4064 	link_width_cntl |= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
4065 	link_width_cntl |= (RADEON_PCIE_LC_RECONFIG_NOW |
4066 			    R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
4067 
4068 	WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4069 }
4070 
4071 int r600_get_pcie_lanes(struct radeon_device *rdev)
4072 {
4073 	u32 link_width_cntl;
4074 
4075 	if (rdev->flags & RADEON_IS_IGP)
4076 		return 0;
4077 
4078 	if (!(rdev->flags & RADEON_IS_PCIE))
4079 		return 0;
4080 
4081 	/* x2 cards have a special sequence */
4082 	if (ASIC_IS_X2(rdev))
4083 		return 0;
4084 
4085 	radeon_gui_idle(rdev);
4086 
4087 	link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4088 
4089 	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
4090 	case RADEON_PCIE_LC_LINK_WIDTH_X1:
4091 		return 1;
4092 	case RADEON_PCIE_LC_LINK_WIDTH_X2:
4093 		return 2;
4094 	case RADEON_PCIE_LC_LINK_WIDTH_X4:
4095 		return 4;
4096 	case RADEON_PCIE_LC_LINK_WIDTH_X8:
4097 		return 8;
4098 	case RADEON_PCIE_LC_LINK_WIDTH_X12:
4099 		/* not actually supported */
4100 		return 12;
4101 	case RADEON_PCIE_LC_LINK_WIDTH_X0:
4102 	case RADEON_PCIE_LC_LINK_WIDTH_X16:
4103 	default:
4104 		return 16;
4105 	}
4106 }
4107 
4108 static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4109 {
4110 	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4111 	u16 link_cntl2;
4112 
4113 	if (radeon_pcie_gen2 == 0)
4114 		return;
4115 
4116 	if (rdev->flags & RADEON_IS_IGP)
4117 		return;
4118 
4119 	if (!(rdev->flags & RADEON_IS_PCIE))
4120 		return;
4121 
4122 	/* x2 cards have a special sequence */
4123 	if (ASIC_IS_X2(rdev))
4124 		return;
4125 
4126 	/* only RV6xx+ chips are supported */
4127 	if (rdev->family <= CHIP_R600)
4128 		return;
4129 
4130 	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
4131 		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
4132 		return;
4133 
4134 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4135 	if (speed_cntl & LC_CURRENT_DATA_RATE) {
4136 		DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4137 		return;
4138 	}
4139 
4140 	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4141 
4142 	/* 55 nm r6xx asics */
4143 	if ((rdev->family == CHIP_RV670) ||
4144 	    (rdev->family == CHIP_RV620) ||
4145 	    (rdev->family == CHIP_RV635)) {
4146 		/* advertise upconfig capability */
4147 		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4148 		link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4149 		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4150 		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4151 		if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4152 			lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4153 			link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4154 					     LC_RECONFIG_ARC_MISSING_ESCAPE);
4155 			link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
4156 			WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4157 		} else {
4158 			link_width_cntl |= LC_UPCONFIGURE_DIS;
4159 			WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4160 		}
4161 	}
4162 
4163 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4164 	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4165 	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4166 
4167 		/* 55 nm r6xx asics */
4168 		if ((rdev->family == CHIP_RV670) ||
4169 		    (rdev->family == CHIP_RV620) ||
4170 		    (rdev->family == CHIP_RV635)) {
4171 			WREG32(MM_CFGREGS_CNTL, 0x8);
4172 			link_cntl2 = RREG32(0x4088);
4173 			WREG32(MM_CFGREGS_CNTL, 0);
4174 			/* not supported yet */
4175 			if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4176 				return;
4177 		}
4178 
4179 		speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4180 		speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4181 		speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4182 		speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4183 		speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
4184 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4185 
4186 		tmp = RREG32(0x541c);
4187 		WREG32(0x541c, tmp | 0x8);
4188 		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4189 		link_cntl2 = RREG16(0x4088);
4190 		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4191 		link_cntl2 |= 0x2;
4192 		WREG16(0x4088, link_cntl2);
4193 		WREG32(MM_CFGREGS_CNTL, 0);
4194 
4195 		if ((rdev->family == CHIP_RV670) ||
4196 		    (rdev->family == CHIP_RV620) ||
4197 		    (rdev->family == CHIP_RV635)) {
4198 			training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
4199 			training_cntl &= ~LC_POINT_7_PLUS_EN;
4200 			WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
4201 		} else {
4202 			speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4203 			speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4204 			WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4205 		}
4206 
4207 		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4208 		speed_cntl |= LC_GEN2_EN_STRAP;
4209 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4210 
4211 	} else {
4212 		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4213 		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4214 		if (1)
4215 			link_width_cntl |= LC_UPCONFIGURE_DIS;
4216 		else
4217 			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4218 		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4219 	}
4220 }
4221 
4222 /**
4223  * r600_get_gpu_clock_counter - return GPU clock counter snapshot
4224  *
4225  * @rdev: radeon_device pointer
4226  *
4227  * Fetches a GPU clock counter snapshot (R6xx-cayman).
4228  * Returns the 64 bit clock counter snapshot.
4229  */
4230 uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
4231 {
4232 	uint64_t clock;
4233 
4234 	mutex_lock(&rdev->gpu_clock_mutex);
4235 	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4236 	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4237 	        ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4238 	mutex_unlock(&rdev->gpu_clock_mutex);
4239 	return clock;
4240 }
4241