xref: /openbmc/linux/drivers/gpu/drm/radeon/rv770.c (revision c819e2cf)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/firmware.h>
29 #include <linux/slab.h>
30 #include <drm/drmP.h>
31 #include "radeon.h"
32 #include "radeon_asic.h"
33 #include <drm/radeon_drm.h>
34 #include "rv770d.h"
35 #include "atom.h"
36 #include "avivod.h"
37 
38 #define R700_PFP_UCODE_SIZE 848
39 #define R700_PM4_UCODE_SIZE 1360
40 
41 static void rv770_gpu_init(struct radeon_device *rdev);
42 void rv770_fini(struct radeon_device *rdev);
43 static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
44 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
45 
46 int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
47 {
48 	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
49 	int r;
50 
51 	/* RV740 uses evergreen uvd clk programming */
52 	if (rdev->family == CHIP_RV740)
53 		return evergreen_set_uvd_clocks(rdev, vclk, dclk);
54 
55 	/* bypass vclk and dclk with bclk */
56 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
57 		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
58 		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
59 
60 	if (!vclk || !dclk) {
61 		/* keep the Bypass mode, put PLL to sleep */
62 		WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
63 		return 0;
64 	}
65 
66 	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
67 					  43663, 0x03FFFFFE, 1, 30, ~0,
68 					  &fb_div, &vclk_div, &dclk_div);
69 	if (r)
70 		return r;
71 
72 	fb_div |= 1;
73 	vclk_div -= 1;
74 	dclk_div -= 1;
75 
76 	/* set UPLL_FB_DIV to 0x50000 */
77 	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
78 
79 	/* deassert UPLL_RESET and UPLL_SLEEP */
80 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
81 
82 	/* assert BYPASS EN and FB_DIV[0] <- ??? why? */
83 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
84 	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
85 
86 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
87 	if (r)
88 		return r;
89 
90 	/* assert PLL_RESET */
91 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
92 
93 	/* set the required FB_DIV, REF_DIV, Post divder values */
94 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
95 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
96 		 UPLL_SW_HILEN(vclk_div >> 1) |
97 		 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
98 		 UPLL_SW_HILEN2(dclk_div >> 1) |
99 		 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
100 		 ~UPLL_SW_MASK);
101 
102 	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
103 		 ~UPLL_FB_DIV_MASK);
104 
105 	/* give the PLL some time to settle */
106 	mdelay(15);
107 
108 	/* deassert PLL_RESET */
109 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
110 
111 	mdelay(15);
112 
113 	/* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
114 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
115 	WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
116 
117 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
118 	if (r)
119 		return r;
120 
121 	/* switch VCLK and DCLK selection */
122 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
123 		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
124 		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
125 
126 	mdelay(100);
127 
128 	return 0;
129 }
130 
131 static const u32 r7xx_golden_registers[] =
132 {
133 	0x8d00, 0xffffffff, 0x0e0e0074,
134 	0x8d04, 0xffffffff, 0x013a2b34,
135 	0x9508, 0xffffffff, 0x00000002,
136 	0x8b20, 0xffffffff, 0,
137 	0x88c4, 0xffffffff, 0x000000c2,
138 	0x28350, 0xffffffff, 0,
139 	0x9058, 0xffffffff, 0x0fffc40f,
140 	0x240c, 0xffffffff, 0x00000380,
141 	0x733c, 0xffffffff, 0x00000002,
142 	0x2650, 0x00040000, 0,
143 	0x20bc, 0x00040000, 0,
144 	0x7300, 0xffffffff, 0x001000f0
145 };
146 
147 static const u32 r7xx_golden_dyn_gpr_registers[] =
148 {
149 	0x8db0, 0xffffffff, 0x98989898,
150 	0x8db4, 0xffffffff, 0x98989898,
151 	0x8db8, 0xffffffff, 0x98989898,
152 	0x8dbc, 0xffffffff, 0x98989898,
153 	0x8dc0, 0xffffffff, 0x98989898,
154 	0x8dc4, 0xffffffff, 0x98989898,
155 	0x8dc8, 0xffffffff, 0x98989898,
156 	0x8dcc, 0xffffffff, 0x98989898,
157 	0x88c4, 0xffffffff, 0x00000082
158 };
159 
160 static const u32 rv770_golden_registers[] =
161 {
162 	0x562c, 0xffffffff, 0,
163 	0x3f90, 0xffffffff, 0,
164 	0x9148, 0xffffffff, 0,
165 	0x3f94, 0xffffffff, 0,
166 	0x914c, 0xffffffff, 0,
167 	0x9698, 0x18000000, 0x18000000
168 };
169 
170 static const u32 rv770ce_golden_registers[] =
171 {
172 	0x562c, 0xffffffff, 0,
173 	0x3f90, 0xffffffff, 0x00cc0000,
174 	0x9148, 0xffffffff, 0x00cc0000,
175 	0x3f94, 0xffffffff, 0x00cc0000,
176 	0x914c, 0xffffffff, 0x00cc0000,
177 	0x9b7c, 0xffffffff, 0x00fa0000,
178 	0x3f8c, 0xffffffff, 0x00fa0000,
179 	0x9698, 0x18000000, 0x18000000
180 };
181 
182 static const u32 rv770_mgcg_init[] =
183 {
184 	0x8bcc, 0xffffffff, 0x130300f9,
185 	0x5448, 0xffffffff, 0x100,
186 	0x55e4, 0xffffffff, 0x100,
187 	0x160c, 0xffffffff, 0x100,
188 	0x5644, 0xffffffff, 0x100,
189 	0xc164, 0xffffffff, 0x100,
190 	0x8a18, 0xffffffff, 0x100,
191 	0x897c, 0xffffffff, 0x8000100,
192 	0x8b28, 0xffffffff, 0x3c000100,
193 	0x9144, 0xffffffff, 0x100,
194 	0x9a1c, 0xffffffff, 0x10000,
195 	0x9a50, 0xffffffff, 0x100,
196 	0x9a1c, 0xffffffff, 0x10001,
197 	0x9a50, 0xffffffff, 0x100,
198 	0x9a1c, 0xffffffff, 0x10002,
199 	0x9a50, 0xffffffff, 0x100,
200 	0x9a1c, 0xffffffff, 0x10003,
201 	0x9a50, 0xffffffff, 0x100,
202 	0x9a1c, 0xffffffff, 0x0,
203 	0x9870, 0xffffffff, 0x100,
204 	0x8d58, 0xffffffff, 0x100,
205 	0x9500, 0xffffffff, 0x0,
206 	0x9510, 0xffffffff, 0x100,
207 	0x9500, 0xffffffff, 0x1,
208 	0x9510, 0xffffffff, 0x100,
209 	0x9500, 0xffffffff, 0x2,
210 	0x9510, 0xffffffff, 0x100,
211 	0x9500, 0xffffffff, 0x3,
212 	0x9510, 0xffffffff, 0x100,
213 	0x9500, 0xffffffff, 0x4,
214 	0x9510, 0xffffffff, 0x100,
215 	0x9500, 0xffffffff, 0x5,
216 	0x9510, 0xffffffff, 0x100,
217 	0x9500, 0xffffffff, 0x6,
218 	0x9510, 0xffffffff, 0x100,
219 	0x9500, 0xffffffff, 0x7,
220 	0x9510, 0xffffffff, 0x100,
221 	0x9500, 0xffffffff, 0x8,
222 	0x9510, 0xffffffff, 0x100,
223 	0x9500, 0xffffffff, 0x9,
224 	0x9510, 0xffffffff, 0x100,
225 	0x9500, 0xffffffff, 0x8000,
226 	0x9490, 0xffffffff, 0x0,
227 	0x949c, 0xffffffff, 0x100,
228 	0x9490, 0xffffffff, 0x1,
229 	0x949c, 0xffffffff, 0x100,
230 	0x9490, 0xffffffff, 0x2,
231 	0x949c, 0xffffffff, 0x100,
232 	0x9490, 0xffffffff, 0x3,
233 	0x949c, 0xffffffff, 0x100,
234 	0x9490, 0xffffffff, 0x4,
235 	0x949c, 0xffffffff, 0x100,
236 	0x9490, 0xffffffff, 0x5,
237 	0x949c, 0xffffffff, 0x100,
238 	0x9490, 0xffffffff, 0x6,
239 	0x949c, 0xffffffff, 0x100,
240 	0x9490, 0xffffffff, 0x7,
241 	0x949c, 0xffffffff, 0x100,
242 	0x9490, 0xffffffff, 0x8,
243 	0x949c, 0xffffffff, 0x100,
244 	0x9490, 0xffffffff, 0x9,
245 	0x949c, 0xffffffff, 0x100,
246 	0x9490, 0xffffffff, 0x8000,
247 	0x9604, 0xffffffff, 0x0,
248 	0x9654, 0xffffffff, 0x100,
249 	0x9604, 0xffffffff, 0x1,
250 	0x9654, 0xffffffff, 0x100,
251 	0x9604, 0xffffffff, 0x2,
252 	0x9654, 0xffffffff, 0x100,
253 	0x9604, 0xffffffff, 0x3,
254 	0x9654, 0xffffffff, 0x100,
255 	0x9604, 0xffffffff, 0x4,
256 	0x9654, 0xffffffff, 0x100,
257 	0x9604, 0xffffffff, 0x5,
258 	0x9654, 0xffffffff, 0x100,
259 	0x9604, 0xffffffff, 0x6,
260 	0x9654, 0xffffffff, 0x100,
261 	0x9604, 0xffffffff, 0x7,
262 	0x9654, 0xffffffff, 0x100,
263 	0x9604, 0xffffffff, 0x8,
264 	0x9654, 0xffffffff, 0x100,
265 	0x9604, 0xffffffff, 0x9,
266 	0x9654, 0xffffffff, 0x100,
267 	0x9604, 0xffffffff, 0x80000000,
268 	0x9030, 0xffffffff, 0x100,
269 	0x9034, 0xffffffff, 0x100,
270 	0x9038, 0xffffffff, 0x100,
271 	0x903c, 0xffffffff, 0x100,
272 	0x9040, 0xffffffff, 0x100,
273 	0xa200, 0xffffffff, 0x100,
274 	0xa204, 0xffffffff, 0x100,
275 	0xa208, 0xffffffff, 0x100,
276 	0xa20c, 0xffffffff, 0x100,
277 	0x971c, 0xffffffff, 0x100,
278 	0x915c, 0xffffffff, 0x00020001,
279 	0x9160, 0xffffffff, 0x00040003,
280 	0x916c, 0xffffffff, 0x00060005,
281 	0x9170, 0xffffffff, 0x00080007,
282 	0x9174, 0xffffffff, 0x000a0009,
283 	0x9178, 0xffffffff, 0x000c000b,
284 	0x917c, 0xffffffff, 0x000e000d,
285 	0x9180, 0xffffffff, 0x0010000f,
286 	0x918c, 0xffffffff, 0x00120011,
287 	0x9190, 0xffffffff, 0x00140013,
288 	0x9194, 0xffffffff, 0x00020001,
289 	0x9198, 0xffffffff, 0x00040003,
290 	0x919c, 0xffffffff, 0x00060005,
291 	0x91a8, 0xffffffff, 0x00080007,
292 	0x91ac, 0xffffffff, 0x000a0009,
293 	0x91b0, 0xffffffff, 0x000c000b,
294 	0x91b4, 0xffffffff, 0x000e000d,
295 	0x91b8, 0xffffffff, 0x0010000f,
296 	0x91c4, 0xffffffff, 0x00120011,
297 	0x91c8, 0xffffffff, 0x00140013,
298 	0x91cc, 0xffffffff, 0x00020001,
299 	0x91d0, 0xffffffff, 0x00040003,
300 	0x91d4, 0xffffffff, 0x00060005,
301 	0x91e0, 0xffffffff, 0x00080007,
302 	0x91e4, 0xffffffff, 0x000a0009,
303 	0x91e8, 0xffffffff, 0x000c000b,
304 	0x91ec, 0xffffffff, 0x00020001,
305 	0x91f0, 0xffffffff, 0x00040003,
306 	0x91f4, 0xffffffff, 0x00060005,
307 	0x9200, 0xffffffff, 0x00080007,
308 	0x9204, 0xffffffff, 0x000a0009,
309 	0x9208, 0xffffffff, 0x000c000b,
310 	0x920c, 0xffffffff, 0x000e000d,
311 	0x9210, 0xffffffff, 0x0010000f,
312 	0x921c, 0xffffffff, 0x00120011,
313 	0x9220, 0xffffffff, 0x00140013,
314 	0x9224, 0xffffffff, 0x00020001,
315 	0x9228, 0xffffffff, 0x00040003,
316 	0x922c, 0xffffffff, 0x00060005,
317 	0x9238, 0xffffffff, 0x00080007,
318 	0x923c, 0xffffffff, 0x000a0009,
319 	0x9240, 0xffffffff, 0x000c000b,
320 	0x9244, 0xffffffff, 0x000e000d,
321 	0x9248, 0xffffffff, 0x0010000f,
322 	0x9254, 0xffffffff, 0x00120011,
323 	0x9258, 0xffffffff, 0x00140013,
324 	0x925c, 0xffffffff, 0x00020001,
325 	0x9260, 0xffffffff, 0x00040003,
326 	0x9264, 0xffffffff, 0x00060005,
327 	0x9270, 0xffffffff, 0x00080007,
328 	0x9274, 0xffffffff, 0x000a0009,
329 	0x9278, 0xffffffff, 0x000c000b,
330 	0x927c, 0xffffffff, 0x000e000d,
331 	0x9280, 0xffffffff, 0x0010000f,
332 	0x928c, 0xffffffff, 0x00120011,
333 	0x9290, 0xffffffff, 0x00140013,
334 	0x9294, 0xffffffff, 0x00020001,
335 	0x929c, 0xffffffff, 0x00040003,
336 	0x92a0, 0xffffffff, 0x00060005,
337 	0x92a4, 0xffffffff, 0x00080007
338 };
339 
340 static const u32 rv710_golden_registers[] =
341 {
342 	0x3f90, 0x00ff0000, 0x00fc0000,
343 	0x9148, 0x00ff0000, 0x00fc0000,
344 	0x3f94, 0x00ff0000, 0x00fc0000,
345 	0x914c, 0x00ff0000, 0x00fc0000,
346 	0xb4c, 0x00000020, 0x00000020,
347 	0xa180, 0xffffffff, 0x00003f3f
348 };
349 
350 static const u32 rv710_mgcg_init[] =
351 {
352 	0x8bcc, 0xffffffff, 0x13030040,
353 	0x5448, 0xffffffff, 0x100,
354 	0x55e4, 0xffffffff, 0x100,
355 	0x160c, 0xffffffff, 0x100,
356 	0x5644, 0xffffffff, 0x100,
357 	0xc164, 0xffffffff, 0x100,
358 	0x8a18, 0xffffffff, 0x100,
359 	0x897c, 0xffffffff, 0x8000100,
360 	0x8b28, 0xffffffff, 0x3c000100,
361 	0x9144, 0xffffffff, 0x100,
362 	0x9a1c, 0xffffffff, 0x10000,
363 	0x9a50, 0xffffffff, 0x100,
364 	0x9a1c, 0xffffffff, 0x0,
365 	0x9870, 0xffffffff, 0x100,
366 	0x8d58, 0xffffffff, 0x100,
367 	0x9500, 0xffffffff, 0x0,
368 	0x9510, 0xffffffff, 0x100,
369 	0x9500, 0xffffffff, 0x1,
370 	0x9510, 0xffffffff, 0x100,
371 	0x9500, 0xffffffff, 0x8000,
372 	0x9490, 0xffffffff, 0x0,
373 	0x949c, 0xffffffff, 0x100,
374 	0x9490, 0xffffffff, 0x1,
375 	0x949c, 0xffffffff, 0x100,
376 	0x9490, 0xffffffff, 0x8000,
377 	0x9604, 0xffffffff, 0x0,
378 	0x9654, 0xffffffff, 0x100,
379 	0x9604, 0xffffffff, 0x1,
380 	0x9654, 0xffffffff, 0x100,
381 	0x9604, 0xffffffff, 0x80000000,
382 	0x9030, 0xffffffff, 0x100,
383 	0x9034, 0xffffffff, 0x100,
384 	0x9038, 0xffffffff, 0x100,
385 	0x903c, 0xffffffff, 0x100,
386 	0x9040, 0xffffffff, 0x100,
387 	0xa200, 0xffffffff, 0x100,
388 	0xa204, 0xffffffff, 0x100,
389 	0xa208, 0xffffffff, 0x100,
390 	0xa20c, 0xffffffff, 0x100,
391 	0x971c, 0xffffffff, 0x100,
392 	0x915c, 0xffffffff, 0x00020001,
393 	0x9174, 0xffffffff, 0x00000003,
394 	0x9178, 0xffffffff, 0x00050001,
395 	0x917c, 0xffffffff, 0x00030002,
396 	0x918c, 0xffffffff, 0x00000004,
397 	0x9190, 0xffffffff, 0x00070006,
398 	0x9194, 0xffffffff, 0x00050001,
399 	0x9198, 0xffffffff, 0x00030002,
400 	0x91a8, 0xffffffff, 0x00000004,
401 	0x91ac, 0xffffffff, 0x00070006,
402 	0x91e8, 0xffffffff, 0x00000001,
403 	0x9294, 0xffffffff, 0x00000001,
404 	0x929c, 0xffffffff, 0x00000002,
405 	0x92a0, 0xffffffff, 0x00040003,
406 	0x9150, 0xffffffff, 0x4d940000
407 };
408 
409 static const u32 rv730_golden_registers[] =
410 {
411 	0x3f90, 0x00ff0000, 0x00f00000,
412 	0x9148, 0x00ff0000, 0x00f00000,
413 	0x3f94, 0x00ff0000, 0x00f00000,
414 	0x914c, 0x00ff0000, 0x00f00000,
415 	0x900c, 0xffffffff, 0x003b033f,
416 	0xb4c, 0x00000020, 0x00000020,
417 	0xa180, 0xffffffff, 0x00003f3f
418 };
419 
420 static const u32 rv730_mgcg_init[] =
421 {
422 	0x8bcc, 0xffffffff, 0x130300f9,
423 	0x5448, 0xffffffff, 0x100,
424 	0x55e4, 0xffffffff, 0x100,
425 	0x160c, 0xffffffff, 0x100,
426 	0x5644, 0xffffffff, 0x100,
427 	0xc164, 0xffffffff, 0x100,
428 	0x8a18, 0xffffffff, 0x100,
429 	0x897c, 0xffffffff, 0x8000100,
430 	0x8b28, 0xffffffff, 0x3c000100,
431 	0x9144, 0xffffffff, 0x100,
432 	0x9a1c, 0xffffffff, 0x10000,
433 	0x9a50, 0xffffffff, 0x100,
434 	0x9a1c, 0xffffffff, 0x10001,
435 	0x9a50, 0xffffffff, 0x100,
436 	0x9a1c, 0xffffffff, 0x0,
437 	0x9870, 0xffffffff, 0x100,
438 	0x8d58, 0xffffffff, 0x100,
439 	0x9500, 0xffffffff, 0x0,
440 	0x9510, 0xffffffff, 0x100,
441 	0x9500, 0xffffffff, 0x1,
442 	0x9510, 0xffffffff, 0x100,
443 	0x9500, 0xffffffff, 0x2,
444 	0x9510, 0xffffffff, 0x100,
445 	0x9500, 0xffffffff, 0x3,
446 	0x9510, 0xffffffff, 0x100,
447 	0x9500, 0xffffffff, 0x4,
448 	0x9510, 0xffffffff, 0x100,
449 	0x9500, 0xffffffff, 0x5,
450 	0x9510, 0xffffffff, 0x100,
451 	0x9500, 0xffffffff, 0x6,
452 	0x9510, 0xffffffff, 0x100,
453 	0x9500, 0xffffffff, 0x7,
454 	0x9510, 0xffffffff, 0x100,
455 	0x9500, 0xffffffff, 0x8000,
456 	0x9490, 0xffffffff, 0x0,
457 	0x949c, 0xffffffff, 0x100,
458 	0x9490, 0xffffffff, 0x1,
459 	0x949c, 0xffffffff, 0x100,
460 	0x9490, 0xffffffff, 0x2,
461 	0x949c, 0xffffffff, 0x100,
462 	0x9490, 0xffffffff, 0x3,
463 	0x949c, 0xffffffff, 0x100,
464 	0x9490, 0xffffffff, 0x4,
465 	0x949c, 0xffffffff, 0x100,
466 	0x9490, 0xffffffff, 0x5,
467 	0x949c, 0xffffffff, 0x100,
468 	0x9490, 0xffffffff, 0x6,
469 	0x949c, 0xffffffff, 0x100,
470 	0x9490, 0xffffffff, 0x7,
471 	0x949c, 0xffffffff, 0x100,
472 	0x9490, 0xffffffff, 0x8000,
473 	0x9604, 0xffffffff, 0x0,
474 	0x9654, 0xffffffff, 0x100,
475 	0x9604, 0xffffffff, 0x1,
476 	0x9654, 0xffffffff, 0x100,
477 	0x9604, 0xffffffff, 0x2,
478 	0x9654, 0xffffffff, 0x100,
479 	0x9604, 0xffffffff, 0x3,
480 	0x9654, 0xffffffff, 0x100,
481 	0x9604, 0xffffffff, 0x4,
482 	0x9654, 0xffffffff, 0x100,
483 	0x9604, 0xffffffff, 0x5,
484 	0x9654, 0xffffffff, 0x100,
485 	0x9604, 0xffffffff, 0x6,
486 	0x9654, 0xffffffff, 0x100,
487 	0x9604, 0xffffffff, 0x7,
488 	0x9654, 0xffffffff, 0x100,
489 	0x9604, 0xffffffff, 0x80000000,
490 	0x9030, 0xffffffff, 0x100,
491 	0x9034, 0xffffffff, 0x100,
492 	0x9038, 0xffffffff, 0x100,
493 	0x903c, 0xffffffff, 0x100,
494 	0x9040, 0xffffffff, 0x100,
495 	0xa200, 0xffffffff, 0x100,
496 	0xa204, 0xffffffff, 0x100,
497 	0xa208, 0xffffffff, 0x100,
498 	0xa20c, 0xffffffff, 0x100,
499 	0x971c, 0xffffffff, 0x100,
500 	0x915c, 0xffffffff, 0x00020001,
501 	0x916c, 0xffffffff, 0x00040003,
502 	0x9170, 0xffffffff, 0x00000005,
503 	0x9178, 0xffffffff, 0x00050001,
504 	0x917c, 0xffffffff, 0x00030002,
505 	0x918c, 0xffffffff, 0x00000004,
506 	0x9190, 0xffffffff, 0x00070006,
507 	0x9194, 0xffffffff, 0x00050001,
508 	0x9198, 0xffffffff, 0x00030002,
509 	0x91a8, 0xffffffff, 0x00000004,
510 	0x91ac, 0xffffffff, 0x00070006,
511 	0x91b0, 0xffffffff, 0x00050001,
512 	0x91b4, 0xffffffff, 0x00030002,
513 	0x91c4, 0xffffffff, 0x00000004,
514 	0x91c8, 0xffffffff, 0x00070006,
515 	0x91cc, 0xffffffff, 0x00050001,
516 	0x91d0, 0xffffffff, 0x00030002,
517 	0x91e0, 0xffffffff, 0x00000004,
518 	0x91e4, 0xffffffff, 0x00070006,
519 	0x91e8, 0xffffffff, 0x00000001,
520 	0x91ec, 0xffffffff, 0x00050001,
521 	0x91f0, 0xffffffff, 0x00030002,
522 	0x9200, 0xffffffff, 0x00000004,
523 	0x9204, 0xffffffff, 0x00070006,
524 	0x9208, 0xffffffff, 0x00050001,
525 	0x920c, 0xffffffff, 0x00030002,
526 	0x921c, 0xffffffff, 0x00000004,
527 	0x9220, 0xffffffff, 0x00070006,
528 	0x9224, 0xffffffff, 0x00050001,
529 	0x9228, 0xffffffff, 0x00030002,
530 	0x9238, 0xffffffff, 0x00000004,
531 	0x923c, 0xffffffff, 0x00070006,
532 	0x9240, 0xffffffff, 0x00050001,
533 	0x9244, 0xffffffff, 0x00030002,
534 	0x9254, 0xffffffff, 0x00000004,
535 	0x9258, 0xffffffff, 0x00070006,
536 	0x9294, 0xffffffff, 0x00000001,
537 	0x929c, 0xffffffff, 0x00000002,
538 	0x92a0, 0xffffffff, 0x00040003,
539 	0x92a4, 0xffffffff, 0x00000005
540 };
541 
542 static const u32 rv740_golden_registers[] =
543 {
544 	0x88c4, 0xffffffff, 0x00000082,
545 	0x28a50, 0xfffffffc, 0x00000004,
546 	0x2650, 0x00040000, 0,
547 	0x20bc, 0x00040000, 0,
548 	0x733c, 0xffffffff, 0x00000002,
549 	0x7300, 0xffffffff, 0x001000f0,
550 	0x3f90, 0x00ff0000, 0,
551 	0x9148, 0x00ff0000, 0,
552 	0x3f94, 0x00ff0000, 0,
553 	0x914c, 0x00ff0000, 0,
554 	0x240c, 0xffffffff, 0x00000380,
555 	0x8a14, 0x00000007, 0x00000007,
556 	0x8b24, 0xffffffff, 0x00ff0fff,
557 	0x28a4c, 0xffffffff, 0x00004000,
558 	0xa180, 0xffffffff, 0x00003f3f,
559 	0x8d00, 0xffffffff, 0x0e0e003a,
560 	0x8d04, 0xffffffff, 0x013a0e2a,
561 	0x8c00, 0xffffffff, 0xe400000f,
562 	0x8db0, 0xffffffff, 0x98989898,
563 	0x8db4, 0xffffffff, 0x98989898,
564 	0x8db8, 0xffffffff, 0x98989898,
565 	0x8dbc, 0xffffffff, 0x98989898,
566 	0x8dc0, 0xffffffff, 0x98989898,
567 	0x8dc4, 0xffffffff, 0x98989898,
568 	0x8dc8, 0xffffffff, 0x98989898,
569 	0x8dcc, 0xffffffff, 0x98989898,
570 	0x9058, 0xffffffff, 0x0fffc40f,
571 	0x900c, 0xffffffff, 0x003b033f,
572 	0x28350, 0xffffffff, 0,
573 	0x8cf0, 0x1fffffff, 0x08e00420,
574 	0x9508, 0xffffffff, 0x00000002,
575 	0x88c4, 0xffffffff, 0x000000c2,
576 	0x9698, 0x18000000, 0x18000000
577 };
578 
579 static const u32 rv740_mgcg_init[] =
580 {
581 	0x8bcc, 0xffffffff, 0x13030100,
582 	0x5448, 0xffffffff, 0x100,
583 	0x55e4, 0xffffffff, 0x100,
584 	0x160c, 0xffffffff, 0x100,
585 	0x5644, 0xffffffff, 0x100,
586 	0xc164, 0xffffffff, 0x100,
587 	0x8a18, 0xffffffff, 0x100,
588 	0x897c, 0xffffffff, 0x100,
589 	0x8b28, 0xffffffff, 0x100,
590 	0x9144, 0xffffffff, 0x100,
591 	0x9a1c, 0xffffffff, 0x10000,
592 	0x9a50, 0xffffffff, 0x100,
593 	0x9a1c, 0xffffffff, 0x10001,
594 	0x9a50, 0xffffffff, 0x100,
595 	0x9a1c, 0xffffffff, 0x10002,
596 	0x9a50, 0xffffffff, 0x100,
597 	0x9a1c, 0xffffffff, 0x10003,
598 	0x9a50, 0xffffffff, 0x100,
599 	0x9a1c, 0xffffffff, 0x0,
600 	0x9870, 0xffffffff, 0x100,
601 	0x8d58, 0xffffffff, 0x100,
602 	0x9500, 0xffffffff, 0x0,
603 	0x9510, 0xffffffff, 0x100,
604 	0x9500, 0xffffffff, 0x1,
605 	0x9510, 0xffffffff, 0x100,
606 	0x9500, 0xffffffff, 0x2,
607 	0x9510, 0xffffffff, 0x100,
608 	0x9500, 0xffffffff, 0x3,
609 	0x9510, 0xffffffff, 0x100,
610 	0x9500, 0xffffffff, 0x4,
611 	0x9510, 0xffffffff, 0x100,
612 	0x9500, 0xffffffff, 0x5,
613 	0x9510, 0xffffffff, 0x100,
614 	0x9500, 0xffffffff, 0x6,
615 	0x9510, 0xffffffff, 0x100,
616 	0x9500, 0xffffffff, 0x7,
617 	0x9510, 0xffffffff, 0x100,
618 	0x9500, 0xffffffff, 0x8000,
619 	0x9490, 0xffffffff, 0x0,
620 	0x949c, 0xffffffff, 0x100,
621 	0x9490, 0xffffffff, 0x1,
622 	0x949c, 0xffffffff, 0x100,
623 	0x9490, 0xffffffff, 0x2,
624 	0x949c, 0xffffffff, 0x100,
625 	0x9490, 0xffffffff, 0x3,
626 	0x949c, 0xffffffff, 0x100,
627 	0x9490, 0xffffffff, 0x4,
628 	0x949c, 0xffffffff, 0x100,
629 	0x9490, 0xffffffff, 0x5,
630 	0x949c, 0xffffffff, 0x100,
631 	0x9490, 0xffffffff, 0x6,
632 	0x949c, 0xffffffff, 0x100,
633 	0x9490, 0xffffffff, 0x7,
634 	0x949c, 0xffffffff, 0x100,
635 	0x9490, 0xffffffff, 0x8000,
636 	0x9604, 0xffffffff, 0x0,
637 	0x9654, 0xffffffff, 0x100,
638 	0x9604, 0xffffffff, 0x1,
639 	0x9654, 0xffffffff, 0x100,
640 	0x9604, 0xffffffff, 0x2,
641 	0x9654, 0xffffffff, 0x100,
642 	0x9604, 0xffffffff, 0x3,
643 	0x9654, 0xffffffff, 0x100,
644 	0x9604, 0xffffffff, 0x4,
645 	0x9654, 0xffffffff, 0x100,
646 	0x9604, 0xffffffff, 0x5,
647 	0x9654, 0xffffffff, 0x100,
648 	0x9604, 0xffffffff, 0x6,
649 	0x9654, 0xffffffff, 0x100,
650 	0x9604, 0xffffffff, 0x7,
651 	0x9654, 0xffffffff, 0x100,
652 	0x9604, 0xffffffff, 0x80000000,
653 	0x9030, 0xffffffff, 0x100,
654 	0x9034, 0xffffffff, 0x100,
655 	0x9038, 0xffffffff, 0x100,
656 	0x903c, 0xffffffff, 0x100,
657 	0x9040, 0xffffffff, 0x100,
658 	0xa200, 0xffffffff, 0x100,
659 	0xa204, 0xffffffff, 0x100,
660 	0xa208, 0xffffffff, 0x100,
661 	0xa20c, 0xffffffff, 0x100,
662 	0x971c, 0xffffffff, 0x100,
663 	0x915c, 0xffffffff, 0x00020001,
664 	0x9160, 0xffffffff, 0x00040003,
665 	0x916c, 0xffffffff, 0x00060005,
666 	0x9170, 0xffffffff, 0x00080007,
667 	0x9174, 0xffffffff, 0x000a0009,
668 	0x9178, 0xffffffff, 0x000c000b,
669 	0x917c, 0xffffffff, 0x000e000d,
670 	0x9180, 0xffffffff, 0x0010000f,
671 	0x918c, 0xffffffff, 0x00120011,
672 	0x9190, 0xffffffff, 0x00140013,
673 	0x9194, 0xffffffff, 0x00020001,
674 	0x9198, 0xffffffff, 0x00040003,
675 	0x919c, 0xffffffff, 0x00060005,
676 	0x91a8, 0xffffffff, 0x00080007,
677 	0x91ac, 0xffffffff, 0x000a0009,
678 	0x91b0, 0xffffffff, 0x000c000b,
679 	0x91b4, 0xffffffff, 0x000e000d,
680 	0x91b8, 0xffffffff, 0x0010000f,
681 	0x91c4, 0xffffffff, 0x00120011,
682 	0x91c8, 0xffffffff, 0x00140013,
683 	0x91cc, 0xffffffff, 0x00020001,
684 	0x91d0, 0xffffffff, 0x00040003,
685 	0x91d4, 0xffffffff, 0x00060005,
686 	0x91e0, 0xffffffff, 0x00080007,
687 	0x91e4, 0xffffffff, 0x000a0009,
688 	0x91e8, 0xffffffff, 0x000c000b,
689 	0x91ec, 0xffffffff, 0x00020001,
690 	0x91f0, 0xffffffff, 0x00040003,
691 	0x91f4, 0xffffffff, 0x00060005,
692 	0x9200, 0xffffffff, 0x00080007,
693 	0x9204, 0xffffffff, 0x000a0009,
694 	0x9208, 0xffffffff, 0x000c000b,
695 	0x920c, 0xffffffff, 0x000e000d,
696 	0x9210, 0xffffffff, 0x0010000f,
697 	0x921c, 0xffffffff, 0x00120011,
698 	0x9220, 0xffffffff, 0x00140013,
699 	0x9224, 0xffffffff, 0x00020001,
700 	0x9228, 0xffffffff, 0x00040003,
701 	0x922c, 0xffffffff, 0x00060005,
702 	0x9238, 0xffffffff, 0x00080007,
703 	0x923c, 0xffffffff, 0x000a0009,
704 	0x9240, 0xffffffff, 0x000c000b,
705 	0x9244, 0xffffffff, 0x000e000d,
706 	0x9248, 0xffffffff, 0x0010000f,
707 	0x9254, 0xffffffff, 0x00120011,
708 	0x9258, 0xffffffff, 0x00140013,
709 	0x9294, 0xffffffff, 0x00020001,
710 	0x929c, 0xffffffff, 0x00040003,
711 	0x92a0, 0xffffffff, 0x00060005,
712 	0x92a4, 0xffffffff, 0x00080007
713 };
714 
715 static void rv770_init_golden_registers(struct radeon_device *rdev)
716 {
717 	switch (rdev->family) {
718 	case CHIP_RV770:
719 		radeon_program_register_sequence(rdev,
720 						 r7xx_golden_registers,
721 						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
722 		radeon_program_register_sequence(rdev,
723 						 r7xx_golden_dyn_gpr_registers,
724 						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
725 		if (rdev->pdev->device == 0x994e)
726 			radeon_program_register_sequence(rdev,
727 							 rv770ce_golden_registers,
728 							 (const u32)ARRAY_SIZE(rv770ce_golden_registers));
729 		else
730 			radeon_program_register_sequence(rdev,
731 							 rv770_golden_registers,
732 							 (const u32)ARRAY_SIZE(rv770_golden_registers));
733 		radeon_program_register_sequence(rdev,
734 						 rv770_mgcg_init,
735 						 (const u32)ARRAY_SIZE(rv770_mgcg_init));
736 		break;
737 	case CHIP_RV730:
738 		radeon_program_register_sequence(rdev,
739 						 r7xx_golden_registers,
740 						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
741 		radeon_program_register_sequence(rdev,
742 						 r7xx_golden_dyn_gpr_registers,
743 						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
744 		radeon_program_register_sequence(rdev,
745 						 rv730_golden_registers,
746 						 (const u32)ARRAY_SIZE(rv730_golden_registers));
747 		radeon_program_register_sequence(rdev,
748 						 rv730_mgcg_init,
749 						 (const u32)ARRAY_SIZE(rv730_mgcg_init));
750 		break;
751 	case CHIP_RV710:
752 		radeon_program_register_sequence(rdev,
753 						 r7xx_golden_registers,
754 						 (const u32)ARRAY_SIZE(r7xx_golden_registers));
755 		radeon_program_register_sequence(rdev,
756 						 r7xx_golden_dyn_gpr_registers,
757 						 (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
758 		radeon_program_register_sequence(rdev,
759 						 rv710_golden_registers,
760 						 (const u32)ARRAY_SIZE(rv710_golden_registers));
761 		radeon_program_register_sequence(rdev,
762 						 rv710_mgcg_init,
763 						 (const u32)ARRAY_SIZE(rv710_mgcg_init));
764 		break;
765 	case CHIP_RV740:
766 		radeon_program_register_sequence(rdev,
767 						 rv740_golden_registers,
768 						 (const u32)ARRAY_SIZE(rv740_golden_registers));
769 		radeon_program_register_sequence(rdev,
770 						 rv740_mgcg_init,
771 						 (const u32)ARRAY_SIZE(rv740_mgcg_init));
772 		break;
773 	default:
774 		break;
775 	}
776 }
777 
778 #define PCIE_BUS_CLK                10000
779 #define TCLK                        (PCIE_BUS_CLK / 10)
780 
781 /**
782  * rv770_get_xclk - get the xclk
783  *
784  * @rdev: radeon_device pointer
785  *
786  * Returns the reference clock used by the gfx engine
787  * (r7xx-cayman).
788  */
789 u32 rv770_get_xclk(struct radeon_device *rdev)
790 {
791 	u32 reference_clock = rdev->clock.spll.reference_freq;
792 	u32 tmp = RREG32(CG_CLKPIN_CNTL);
793 
794 	if (tmp & MUX_TCLK_TO_XCLK)
795 		return TCLK;
796 
797 	if (tmp & XTALIN_DIVIDE)
798 		return reference_clock / 4;
799 
800 	return reference_clock;
801 }
802 
803 void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
804 {
805 	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
806 	u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
807 	int i;
808 
809 	/* Lock the graphics update lock */
810 	tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
811 	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
812 
813 	/* update the scanout addresses */
814 	if (radeon_crtc->crtc_id) {
815 		WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
816 		WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
817 	} else {
818 		WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
819 		WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
820 	}
821 	WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
822 	       (u32)crtc_base);
823 	WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
824 	       (u32)crtc_base);
825 
826 	/* Wait for update_pending to go high. */
827 	for (i = 0; i < rdev->usec_timeout; i++) {
828 		if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
829 			break;
830 		udelay(1);
831 	}
832 	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
833 
834 	/* Unlock the lock, so double-buffering can take place inside vblank */
835 	tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
836 	WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
837 }
838 
839 bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
840 {
841 	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
842 
843 	/* Return current update_pending status: */
844 	return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
845 		AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
846 }
847 
848 /* get temperature in millidegrees */
849 int rv770_get_temp(struct radeon_device *rdev)
850 {
851 	u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
852 		ASIC_T_SHIFT;
853 	int actual_temp;
854 
855 	if (temp & 0x400)
856 		actual_temp = -256;
857 	else if (temp & 0x200)
858 		actual_temp = 255;
859 	else if (temp & 0x100) {
860 		actual_temp = temp & 0x1ff;
861 		actual_temp |= ~0x1ff;
862 	} else
863 		actual_temp = temp & 0xff;
864 
865 	return (actual_temp * 1000) / 2;
866 }
867 
868 void rv770_pm_misc(struct radeon_device *rdev)
869 {
870 	int req_ps_idx = rdev->pm.requested_power_state_index;
871 	int req_cm_idx = rdev->pm.requested_clock_mode_index;
872 	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
873 	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
874 
875 	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
876 		/* 0xff01 is a flag rather then an actual voltage */
877 		if (voltage->voltage == 0xff01)
878 			return;
879 		if (voltage->voltage != rdev->pm.current_vddc) {
880 			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
881 			rdev->pm.current_vddc = voltage->voltage;
882 			DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
883 		}
884 	}
885 }
886 
887 /*
888  * GART
889  */
890 static int rv770_pcie_gart_enable(struct radeon_device *rdev)
891 {
892 	u32 tmp;
893 	int r, i;
894 
895 	if (rdev->gart.robj == NULL) {
896 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
897 		return -EINVAL;
898 	}
899 	r = radeon_gart_table_vram_pin(rdev);
900 	if (r)
901 		return r;
902 	/* Setup L2 cache */
903 	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
904 				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
905 				EFFECTIVE_L2_QUEUE_SIZE(7));
906 	WREG32(VM_L2_CNTL2, 0);
907 	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
908 	/* Setup TLB control */
909 	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
910 		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
911 		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
912 		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
913 	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
914 	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
915 	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
916 	if (rdev->family == CHIP_RV740)
917 		WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
918 	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
919 	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
920 	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
921 	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
922 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
923 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
924 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
925 	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
926 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
927 	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
928 			(u32)(rdev->dummy_page.addr >> 12));
929 	for (i = 1; i < 7; i++)
930 		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
931 
932 	r600_pcie_gart_tlb_flush(rdev);
933 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
934 		 (unsigned)(rdev->mc.gtt_size >> 20),
935 		 (unsigned long long)rdev->gart.table_addr);
936 	rdev->gart.ready = true;
937 	return 0;
938 }
939 
940 static void rv770_pcie_gart_disable(struct radeon_device *rdev)
941 {
942 	u32 tmp;
943 	int i;
944 
945 	/* Disable all tables */
946 	for (i = 0; i < 7; i++)
947 		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
948 
949 	/* Setup L2 cache */
950 	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
951 				EFFECTIVE_L2_QUEUE_SIZE(7));
952 	WREG32(VM_L2_CNTL2, 0);
953 	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
954 	/* Setup TLB control */
955 	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
956 	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
957 	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
958 	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
959 	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
960 	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
961 	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
962 	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
963 	radeon_gart_table_vram_unpin(rdev);
964 }
965 
966 static void rv770_pcie_gart_fini(struct radeon_device *rdev)
967 {
968 	radeon_gart_fini(rdev);
969 	rv770_pcie_gart_disable(rdev);
970 	radeon_gart_table_vram_free(rdev);
971 }
972 
973 
974 static void rv770_agp_enable(struct radeon_device *rdev)
975 {
976 	u32 tmp;
977 	int i;
978 
979 	/* Setup L2 cache */
980 	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
981 				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
982 				EFFECTIVE_L2_QUEUE_SIZE(7));
983 	WREG32(VM_L2_CNTL2, 0);
984 	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
985 	/* Setup TLB control */
986 	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
987 		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
988 		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
989 		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
990 	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
991 	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
992 	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
993 	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
994 	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
995 	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
996 	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
997 	for (i = 0; i < 7; i++)
998 		WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
999 }
1000 
1001 static void rv770_mc_program(struct radeon_device *rdev)
1002 {
1003 	struct rv515_mc_save save;
1004 	u32 tmp;
1005 	int i, j;
1006 
1007 	/* Initialize HDP */
1008 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1009 		WREG32((0x2c14 + j), 0x00000000);
1010 		WREG32((0x2c18 + j), 0x00000000);
1011 		WREG32((0x2c1c + j), 0x00000000);
1012 		WREG32((0x2c20 + j), 0x00000000);
1013 		WREG32((0x2c24 + j), 0x00000000);
1014 	}
1015 	/* r7xx hw bug.  Read from HDP_DEBUG1 rather
1016 	 * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1017 	 */
1018 	tmp = RREG32(HDP_DEBUG1);
1019 
1020 	rv515_mc_stop(rdev, &save);
1021 	if (r600_mc_wait_for_idle(rdev)) {
1022 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1023 	}
1024 	/* Lockout access through VGA aperture*/
1025 	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1026 	/* Update configuration */
1027 	if (rdev->flags & RADEON_IS_AGP) {
1028 		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1029 			/* VRAM before AGP */
1030 			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1031 				rdev->mc.vram_start >> 12);
1032 			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1033 				rdev->mc.gtt_end >> 12);
1034 		} else {
1035 			/* VRAM after AGP */
1036 			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1037 				rdev->mc.gtt_start >> 12);
1038 			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1039 				rdev->mc.vram_end >> 12);
1040 		}
1041 	} else {
1042 		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1043 			rdev->mc.vram_start >> 12);
1044 		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1045 			rdev->mc.vram_end >> 12);
1046 	}
1047 	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1048 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1049 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1050 	WREG32(MC_VM_FB_LOCATION, tmp);
1051 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1052 	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1053 	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1054 	if (rdev->flags & RADEON_IS_AGP) {
1055 		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1056 		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1057 		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1058 	} else {
1059 		WREG32(MC_VM_AGP_BASE, 0);
1060 		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1061 		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1062 	}
1063 	if (r600_mc_wait_for_idle(rdev)) {
1064 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1065 	}
1066 	rv515_mc_resume(rdev, &save);
1067 	/* we need to own VRAM, so turn off the VGA renderer here
1068 	 * to stop it overwriting our objects */
1069 	rv515_vga_render_disable(rdev);
1070 }
1071 
1072 
1073 /*
1074  * CP.
1075  */
1076 void r700_cp_stop(struct radeon_device *rdev)
1077 {
1078 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1079 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1080 	WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1081 	WREG32(SCRATCH_UMSK, 0);
1082 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1083 }
1084 
1085 static int rv770_cp_load_microcode(struct radeon_device *rdev)
1086 {
1087 	const __be32 *fw_data;
1088 	int i;
1089 
1090 	if (!rdev->me_fw || !rdev->pfp_fw)
1091 		return -EINVAL;
1092 
1093 	r700_cp_stop(rdev);
1094 	WREG32(CP_RB_CNTL,
1095 #ifdef __BIG_ENDIAN
1096 	       BUF_SWAP_32BIT |
1097 #endif
1098 	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1099 
1100 	/* Reset cp */
1101 	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1102 	RREG32(GRBM_SOFT_RESET);
1103 	mdelay(15);
1104 	WREG32(GRBM_SOFT_RESET, 0);
1105 
1106 	fw_data = (const __be32 *)rdev->pfp_fw->data;
1107 	WREG32(CP_PFP_UCODE_ADDR, 0);
1108 	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1109 		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1110 	WREG32(CP_PFP_UCODE_ADDR, 0);
1111 
1112 	fw_data = (const __be32 *)rdev->me_fw->data;
1113 	WREG32(CP_ME_RAM_WADDR, 0);
1114 	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1115 		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1116 
1117 	WREG32(CP_PFP_UCODE_ADDR, 0);
1118 	WREG32(CP_ME_RAM_WADDR, 0);
1119 	WREG32(CP_ME_RAM_RADDR, 0);
1120 	return 0;
1121 }
1122 
1123 void r700_cp_fini(struct radeon_device *rdev)
1124 {
1125 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1126 	r700_cp_stop(rdev);
1127 	radeon_ring_fini(rdev, ring);
1128 	radeon_scratch_free(rdev, ring->rptr_save_reg);
1129 }
1130 
1131 void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1132 {
1133 	u32 tmp, i;
1134 
1135 	if (rdev->flags & RADEON_IS_IGP)
1136 		return;
1137 
1138 	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1139 	tmp &= SCLK_MUX_SEL_MASK;
1140 	tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1141 	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1142 
1143 	for (i = 0; i < rdev->usec_timeout; i++) {
1144 		if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1145 			break;
1146 		udelay(1);
1147 	}
1148 
1149 	tmp &= ~SCLK_MUX_UPDATE;
1150 	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1151 
1152 	tmp = RREG32(MPLL_CNTL_MODE);
1153 	if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1154 		tmp &= ~RV730_MPLL_MCLK_SEL;
1155 	else
1156 		tmp &= ~MPLL_MCLK_SEL;
1157 	WREG32(MPLL_CNTL_MODE, tmp);
1158 }
1159 
1160 /*
1161  * Core functions
1162  */
1163 static void rv770_gpu_init(struct radeon_device *rdev)
1164 {
1165 	int i, j, num_qd_pipes;
1166 	u32 ta_aux_cntl;
1167 	u32 sx_debug_1;
1168 	u32 smx_dc_ctl0;
1169 	u32 db_debug3;
1170 	u32 num_gs_verts_per_thread;
1171 	u32 vgt_gs_per_es;
1172 	u32 gs_prim_buffer_depth = 0;
1173 	u32 sq_ms_fifo_sizes;
1174 	u32 sq_config;
1175 	u32 sq_thread_resource_mgmt;
1176 	u32 hdp_host_path_cntl;
1177 	u32 sq_dyn_gpr_size_simd_ab_0;
1178 	u32 gb_tiling_config = 0;
1179 	u32 cc_gc_shader_pipe_config = 0;
1180 	u32 mc_arb_ramcfg;
1181 	u32 db_debug4, tmp;
1182 	u32 inactive_pipes, shader_pipe_config;
1183 	u32 disabled_rb_mask;
1184 	unsigned active_number;
1185 
1186 	/* setup chip specs */
1187 	rdev->config.rv770.tiling_group_size = 256;
1188 	switch (rdev->family) {
1189 	case CHIP_RV770:
1190 		rdev->config.rv770.max_pipes = 4;
1191 		rdev->config.rv770.max_tile_pipes = 8;
1192 		rdev->config.rv770.max_simds = 10;
1193 		rdev->config.rv770.max_backends = 4;
1194 		rdev->config.rv770.max_gprs = 256;
1195 		rdev->config.rv770.max_threads = 248;
1196 		rdev->config.rv770.max_stack_entries = 512;
1197 		rdev->config.rv770.max_hw_contexts = 8;
1198 		rdev->config.rv770.max_gs_threads = 16 * 2;
1199 		rdev->config.rv770.sx_max_export_size = 128;
1200 		rdev->config.rv770.sx_max_export_pos_size = 16;
1201 		rdev->config.rv770.sx_max_export_smx_size = 112;
1202 		rdev->config.rv770.sq_num_cf_insts = 2;
1203 
1204 		rdev->config.rv770.sx_num_of_sets = 7;
1205 		rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1206 		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1207 		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1208 		break;
1209 	case CHIP_RV730:
1210 		rdev->config.rv770.max_pipes = 2;
1211 		rdev->config.rv770.max_tile_pipes = 4;
1212 		rdev->config.rv770.max_simds = 8;
1213 		rdev->config.rv770.max_backends = 2;
1214 		rdev->config.rv770.max_gprs = 128;
1215 		rdev->config.rv770.max_threads = 248;
1216 		rdev->config.rv770.max_stack_entries = 256;
1217 		rdev->config.rv770.max_hw_contexts = 8;
1218 		rdev->config.rv770.max_gs_threads = 16 * 2;
1219 		rdev->config.rv770.sx_max_export_size = 256;
1220 		rdev->config.rv770.sx_max_export_pos_size = 32;
1221 		rdev->config.rv770.sx_max_export_smx_size = 224;
1222 		rdev->config.rv770.sq_num_cf_insts = 2;
1223 
1224 		rdev->config.rv770.sx_num_of_sets = 7;
1225 		rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1226 		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1227 		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1228 		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1229 			rdev->config.rv770.sx_max_export_pos_size -= 16;
1230 			rdev->config.rv770.sx_max_export_smx_size += 16;
1231 		}
1232 		break;
1233 	case CHIP_RV710:
1234 		rdev->config.rv770.max_pipes = 2;
1235 		rdev->config.rv770.max_tile_pipes = 2;
1236 		rdev->config.rv770.max_simds = 2;
1237 		rdev->config.rv770.max_backends = 1;
1238 		rdev->config.rv770.max_gprs = 256;
1239 		rdev->config.rv770.max_threads = 192;
1240 		rdev->config.rv770.max_stack_entries = 256;
1241 		rdev->config.rv770.max_hw_contexts = 4;
1242 		rdev->config.rv770.max_gs_threads = 8 * 2;
1243 		rdev->config.rv770.sx_max_export_size = 128;
1244 		rdev->config.rv770.sx_max_export_pos_size = 16;
1245 		rdev->config.rv770.sx_max_export_smx_size = 112;
1246 		rdev->config.rv770.sq_num_cf_insts = 1;
1247 
1248 		rdev->config.rv770.sx_num_of_sets = 7;
1249 		rdev->config.rv770.sc_prim_fifo_size = 0x40;
1250 		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1251 		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1252 		break;
1253 	case CHIP_RV740:
1254 		rdev->config.rv770.max_pipes = 4;
1255 		rdev->config.rv770.max_tile_pipes = 4;
1256 		rdev->config.rv770.max_simds = 8;
1257 		rdev->config.rv770.max_backends = 4;
1258 		rdev->config.rv770.max_gprs = 256;
1259 		rdev->config.rv770.max_threads = 248;
1260 		rdev->config.rv770.max_stack_entries = 512;
1261 		rdev->config.rv770.max_hw_contexts = 8;
1262 		rdev->config.rv770.max_gs_threads = 16 * 2;
1263 		rdev->config.rv770.sx_max_export_size = 256;
1264 		rdev->config.rv770.sx_max_export_pos_size = 32;
1265 		rdev->config.rv770.sx_max_export_smx_size = 224;
1266 		rdev->config.rv770.sq_num_cf_insts = 2;
1267 
1268 		rdev->config.rv770.sx_num_of_sets = 7;
1269 		rdev->config.rv770.sc_prim_fifo_size = 0x100;
1270 		rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1271 		rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1272 
1273 		if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1274 			rdev->config.rv770.sx_max_export_pos_size -= 16;
1275 			rdev->config.rv770.sx_max_export_smx_size += 16;
1276 		}
1277 		break;
1278 	default:
1279 		break;
1280 	}
1281 
1282 	/* Initialize HDP */
1283 	j = 0;
1284 	for (i = 0; i < 32; i++) {
1285 		WREG32((0x2c14 + j), 0x00000000);
1286 		WREG32((0x2c18 + j), 0x00000000);
1287 		WREG32((0x2c1c + j), 0x00000000);
1288 		WREG32((0x2c20 + j), 0x00000000);
1289 		WREG32((0x2c24 + j), 0x00000000);
1290 		j += 0x18;
1291 	}
1292 
1293 	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1294 
1295 	/* setup tiling, simd, pipe config */
1296 	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1297 
1298 	shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1299 	inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1300 	for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1301 		if (!(inactive_pipes & tmp)) {
1302 			active_number++;
1303 		}
1304 		tmp <<= 1;
1305 	}
1306 	if (active_number == 1) {
1307 		WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1308 	} else {
1309 		WREG32(SPI_CONFIG_CNTL, 0);
1310 	}
1311 
1312 	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1313 	tmp = rdev->config.rv770.max_simds -
1314 		r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1315 	rdev->config.rv770.active_simds = tmp;
1316 
1317 	switch (rdev->config.rv770.max_tile_pipes) {
1318 	case 1:
1319 	default:
1320 		gb_tiling_config = PIPE_TILING(0);
1321 		break;
1322 	case 2:
1323 		gb_tiling_config = PIPE_TILING(1);
1324 		break;
1325 	case 4:
1326 		gb_tiling_config = PIPE_TILING(2);
1327 		break;
1328 	case 8:
1329 		gb_tiling_config = PIPE_TILING(3);
1330 		break;
1331 	}
1332 	rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1333 
1334 	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1335 	tmp = 0;
1336 	for (i = 0; i < rdev->config.rv770.max_backends; i++)
1337 		tmp |= (1 << i);
1338 	/* if all the backends are disabled, fix it up here */
1339 	if ((disabled_rb_mask & tmp) == tmp) {
1340 		for (i = 0; i < rdev->config.rv770.max_backends; i++)
1341 			disabled_rb_mask &= ~(1 << i);
1342 	}
1343 	tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1344 	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1345 					R7XX_MAX_BACKENDS, disabled_rb_mask);
1346 	gb_tiling_config |= tmp << 16;
1347 	rdev->config.rv770.backend_map = tmp;
1348 
1349 	if (rdev->family == CHIP_RV770)
1350 		gb_tiling_config |= BANK_TILING(1);
1351 	else {
1352 		if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1353 			gb_tiling_config |= BANK_TILING(1);
1354 		else
1355 			gb_tiling_config |= BANK_TILING(0);
1356 	}
1357 	rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1358 	gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1359 	if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1360 		gb_tiling_config |= ROW_TILING(3);
1361 		gb_tiling_config |= SAMPLE_SPLIT(3);
1362 	} else {
1363 		gb_tiling_config |=
1364 			ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1365 		gb_tiling_config |=
1366 			SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1367 	}
1368 
1369 	gb_tiling_config |= BANK_SWAPS(1);
1370 	rdev->config.rv770.tile_config = gb_tiling_config;
1371 
1372 	WREG32(GB_TILING_CONFIG, gb_tiling_config);
1373 	WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1374 	WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1375 	WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1376 	WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1377 	if (rdev->family == CHIP_RV730) {
1378 		WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1379 		WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1380 		WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1381 	}
1382 
1383 	WREG32(CGTS_SYS_TCC_DISABLE, 0);
1384 	WREG32(CGTS_TCC_DISABLE, 0);
1385 	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1386 	WREG32(CGTS_USER_TCC_DISABLE, 0);
1387 
1388 
1389 	num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1390 	WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1391 	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1392 
1393 	/* set HW defaults for 3D engine */
1394 	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1395 				     ROQ_IB2_START(0x2b)));
1396 
1397 	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1398 
1399 	ta_aux_cntl = RREG32(TA_CNTL_AUX);
1400 	WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1401 
1402 	sx_debug_1 = RREG32(SX_DEBUG_1);
1403 	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1404 	WREG32(SX_DEBUG_1, sx_debug_1);
1405 
1406 	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1407 	smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1408 	smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1409 	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1410 
1411 	if (rdev->family != CHIP_RV740)
1412 		WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1413 				       GS_FLUSH_CTL(4) |
1414 				       ACK_FLUSH_CTL(3) |
1415 				       SYNC_FLUSH_CTL));
1416 
1417 	if (rdev->family != CHIP_RV770)
1418 		WREG32(SMX_SAR_CTL0, 0x00003f3f);
1419 
1420 	db_debug3 = RREG32(DB_DEBUG3);
1421 	db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1422 	switch (rdev->family) {
1423 	case CHIP_RV770:
1424 	case CHIP_RV740:
1425 		db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1426 		break;
1427 	case CHIP_RV710:
1428 	case CHIP_RV730:
1429 	default:
1430 		db_debug3 |= DB_CLK_OFF_DELAY(2);
1431 		break;
1432 	}
1433 	WREG32(DB_DEBUG3, db_debug3);
1434 
1435 	if (rdev->family != CHIP_RV770) {
1436 		db_debug4 = RREG32(DB_DEBUG4);
1437 		db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1438 		WREG32(DB_DEBUG4, db_debug4);
1439 	}
1440 
1441 	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1442 					POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1443 					SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1444 
1445 	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1446 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1447 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1448 
1449 	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1450 
1451 	WREG32(VGT_NUM_INSTANCES, 1);
1452 
1453 	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1454 
1455 	WREG32(CP_PERFMON_CNTL, 0);
1456 
1457 	sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1458 			    DONE_FIFO_HIWATER(0xe0) |
1459 			    ALU_UPDATE_FIFO_HIWATER(0x8));
1460 	switch (rdev->family) {
1461 	case CHIP_RV770:
1462 	case CHIP_RV730:
1463 	case CHIP_RV710:
1464 		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1465 		break;
1466 	case CHIP_RV740:
1467 	default:
1468 		sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1469 		break;
1470 	}
1471 	WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1472 
1473 	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1474 	 * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1475 	 */
1476 	sq_config = RREG32(SQ_CONFIG);
1477 	sq_config &= ~(PS_PRIO(3) |
1478 		       VS_PRIO(3) |
1479 		       GS_PRIO(3) |
1480 		       ES_PRIO(3));
1481 	sq_config |= (DX9_CONSTS |
1482 		      VC_ENABLE |
1483 		      EXPORT_SRC_C |
1484 		      PS_PRIO(0) |
1485 		      VS_PRIO(1) |
1486 		      GS_PRIO(2) |
1487 		      ES_PRIO(3));
1488 	if (rdev->family == CHIP_RV710)
1489 		/* no vertex cache */
1490 		sq_config &= ~VC_ENABLE;
1491 
1492 	WREG32(SQ_CONFIG, sq_config);
1493 
1494 	WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1495 					 NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1496 					 NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1497 
1498 	WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1499 					 NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1500 
1501 	sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1502 				   NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1503 				   NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1504 	if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1505 		sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1506 	else
1507 		sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1508 	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1509 
1510 	WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1511 						     NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1512 
1513 	WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1514 						     NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1515 
1516 	sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1517 				     SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1518 				     SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1519 				     SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1520 
1521 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1522 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1523 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1524 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1525 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1526 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1527 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1528 	WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1529 
1530 	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1531 					  FORCE_EOV_MAX_REZ_CNT(255)));
1532 
1533 	if (rdev->family == CHIP_RV710)
1534 		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1535 						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1536 	else
1537 		WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1538 						AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1539 
1540 	switch (rdev->family) {
1541 	case CHIP_RV770:
1542 	case CHIP_RV730:
1543 	case CHIP_RV740:
1544 		gs_prim_buffer_depth = 384;
1545 		break;
1546 	case CHIP_RV710:
1547 		gs_prim_buffer_depth = 128;
1548 		break;
1549 	default:
1550 		break;
1551 	}
1552 
1553 	num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1554 	vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1555 	/* Max value for this is 256 */
1556 	if (vgt_gs_per_es > 256)
1557 		vgt_gs_per_es = 256;
1558 
1559 	WREG32(VGT_ES_PER_GS, 128);
1560 	WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1561 	WREG32(VGT_GS_PER_VS, 2);
1562 
1563 	/* more default values. 2D/3D driver should adjust as needed */
1564 	WREG32(VGT_GS_VERTEX_REUSE, 16);
1565 	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1566 	WREG32(VGT_STRMOUT_EN, 0);
1567 	WREG32(SX_MISC, 0);
1568 	WREG32(PA_SC_MODE_CNTL, 0);
1569 	WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1570 	WREG32(PA_SC_AA_CONFIG, 0);
1571 	WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1572 	WREG32(PA_SC_LINE_STIPPLE, 0);
1573 	WREG32(SPI_INPUT_Z, 0);
1574 	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1575 	WREG32(CB_COLOR7_FRAG, 0);
1576 
1577 	/* clear render buffer base addresses */
1578 	WREG32(CB_COLOR0_BASE, 0);
1579 	WREG32(CB_COLOR1_BASE, 0);
1580 	WREG32(CB_COLOR2_BASE, 0);
1581 	WREG32(CB_COLOR3_BASE, 0);
1582 	WREG32(CB_COLOR4_BASE, 0);
1583 	WREG32(CB_COLOR5_BASE, 0);
1584 	WREG32(CB_COLOR6_BASE, 0);
1585 	WREG32(CB_COLOR7_BASE, 0);
1586 
1587 	WREG32(TCP_CNTL, 0);
1588 
1589 	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1590 	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1591 
1592 	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1593 
1594 	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1595 					  NUM_CLIP_SEQ(3)));
1596 	WREG32(VC_ENHANCE, 0);
1597 }
1598 
1599 void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1600 {
1601 	u64 size_bf, size_af;
1602 
1603 	if (mc->mc_vram_size > 0xE0000000) {
1604 		/* leave room for at least 512M GTT */
1605 		dev_warn(rdev->dev, "limiting VRAM\n");
1606 		mc->real_vram_size = 0xE0000000;
1607 		mc->mc_vram_size = 0xE0000000;
1608 	}
1609 	if (rdev->flags & RADEON_IS_AGP) {
1610 		size_bf = mc->gtt_start;
1611 		size_af = mc->mc_mask - mc->gtt_end;
1612 		if (size_bf > size_af) {
1613 			if (mc->mc_vram_size > size_bf) {
1614 				dev_warn(rdev->dev, "limiting VRAM\n");
1615 				mc->real_vram_size = size_bf;
1616 				mc->mc_vram_size = size_bf;
1617 			}
1618 			mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1619 		} else {
1620 			if (mc->mc_vram_size > size_af) {
1621 				dev_warn(rdev->dev, "limiting VRAM\n");
1622 				mc->real_vram_size = size_af;
1623 				mc->mc_vram_size = size_af;
1624 			}
1625 			mc->vram_start = mc->gtt_end + 1;
1626 		}
1627 		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1628 		dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1629 				mc->mc_vram_size >> 20, mc->vram_start,
1630 				mc->vram_end, mc->real_vram_size >> 20);
1631 	} else {
1632 		radeon_vram_location(rdev, &rdev->mc, 0);
1633 		rdev->mc.gtt_base_align = 0;
1634 		radeon_gtt_location(rdev, mc);
1635 	}
1636 }
1637 
1638 static int rv770_mc_init(struct radeon_device *rdev)
1639 {
1640 	u32 tmp;
1641 	int chansize, numchan;
1642 
1643 	/* Get VRAM informations */
1644 	rdev->mc.vram_is_ddr = true;
1645 	tmp = RREG32(MC_ARB_RAMCFG);
1646 	if (tmp & CHANSIZE_OVERRIDE) {
1647 		chansize = 16;
1648 	} else if (tmp & CHANSIZE_MASK) {
1649 		chansize = 64;
1650 	} else {
1651 		chansize = 32;
1652 	}
1653 	tmp = RREG32(MC_SHARED_CHMAP);
1654 	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1655 	case 0:
1656 	default:
1657 		numchan = 1;
1658 		break;
1659 	case 1:
1660 		numchan = 2;
1661 		break;
1662 	case 2:
1663 		numchan = 4;
1664 		break;
1665 	case 3:
1666 		numchan = 8;
1667 		break;
1668 	}
1669 	rdev->mc.vram_width = numchan * chansize;
1670 	/* Could aper size report 0 ? */
1671 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1672 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1673 	/* Setup GPU memory space */
1674 	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1675 	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1676 	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1677 	r700_vram_gtt_location(rdev, &rdev->mc);
1678 	radeon_update_bandwidth_info(rdev);
1679 
1680 	return 0;
1681 }
1682 
1683 static int rv770_startup(struct radeon_device *rdev)
1684 {
1685 	struct radeon_ring *ring;
1686 	int r;
1687 
1688 	/* enable pcie gen2 link */
1689 	rv770_pcie_gen2_enable(rdev);
1690 
1691 	/* scratch needs to be initialized before MC */
1692 	r = r600_vram_scratch_init(rdev);
1693 	if (r)
1694 		return r;
1695 
1696 	rv770_mc_program(rdev);
1697 
1698 	if (rdev->flags & RADEON_IS_AGP) {
1699 		rv770_agp_enable(rdev);
1700 	} else {
1701 		r = rv770_pcie_gart_enable(rdev);
1702 		if (r)
1703 			return r;
1704 	}
1705 
1706 	rv770_gpu_init(rdev);
1707 
1708 	/* allocate wb buffer */
1709 	r = radeon_wb_init(rdev);
1710 	if (r)
1711 		return r;
1712 
1713 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1714 	if (r) {
1715 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1716 		return r;
1717 	}
1718 
1719 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1720 	if (r) {
1721 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1722 		return r;
1723 	}
1724 
1725 	r = uvd_v2_2_resume(rdev);
1726 	if (!r) {
1727 		r = radeon_fence_driver_start_ring(rdev,
1728 						   R600_RING_TYPE_UVD_INDEX);
1729 		if (r)
1730 			dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
1731 	}
1732 
1733 	if (r)
1734 		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1735 
1736 	/* Enable IRQ */
1737 	if (!rdev->irq.installed) {
1738 		r = radeon_irq_kms_init(rdev);
1739 		if (r)
1740 			return r;
1741 	}
1742 
1743 	r = r600_irq_init(rdev);
1744 	if (r) {
1745 		DRM_ERROR("radeon: IH init failed (%d).\n", r);
1746 		radeon_irq_kms_fini(rdev);
1747 		return r;
1748 	}
1749 	r600_irq_set(rdev);
1750 
1751 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1752 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1753 			     RADEON_CP_PACKET2);
1754 	if (r)
1755 		return r;
1756 
1757 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1758 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1759 			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1760 	if (r)
1761 		return r;
1762 
1763 	r = rv770_cp_load_microcode(rdev);
1764 	if (r)
1765 		return r;
1766 	r = r600_cp_resume(rdev);
1767 	if (r)
1768 		return r;
1769 
1770 	r = r600_dma_resume(rdev);
1771 	if (r)
1772 		return r;
1773 
1774 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1775 	if (ring->ring_size) {
1776 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
1777 				     RADEON_CP_PACKET2);
1778 		if (!r)
1779 			r = uvd_v1_0_init(rdev);
1780 
1781 		if (r)
1782 			DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
1783 	}
1784 
1785 	r = radeon_ib_pool_init(rdev);
1786 	if (r) {
1787 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1788 		return r;
1789 	}
1790 
1791 	r = r600_audio_init(rdev);
1792 	if (r) {
1793 		DRM_ERROR("radeon: audio init failed\n");
1794 		return r;
1795 	}
1796 
1797 	return 0;
1798 }
1799 
1800 int rv770_resume(struct radeon_device *rdev)
1801 {
1802 	int r;
1803 
1804 	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1805 	 * posting will perform necessary task to bring back GPU into good
1806 	 * shape.
1807 	 */
1808 	/* post card */
1809 	atom_asic_init(rdev->mode_info.atom_context);
1810 
1811 	/* init golden registers */
1812 	rv770_init_golden_registers(rdev);
1813 
1814 	if (rdev->pm.pm_method == PM_METHOD_DPM)
1815 		radeon_pm_resume(rdev);
1816 
1817 	rdev->accel_working = true;
1818 	r = rv770_startup(rdev);
1819 	if (r) {
1820 		DRM_ERROR("r600 startup failed on resume\n");
1821 		rdev->accel_working = false;
1822 		return r;
1823 	}
1824 
1825 	return r;
1826 
1827 }
1828 
1829 int rv770_suspend(struct radeon_device *rdev)
1830 {
1831 	radeon_pm_suspend(rdev);
1832 	r600_audio_fini(rdev);
1833 	uvd_v1_0_fini(rdev);
1834 	radeon_uvd_suspend(rdev);
1835 	r700_cp_stop(rdev);
1836 	r600_dma_stop(rdev);
1837 	r600_irq_suspend(rdev);
1838 	radeon_wb_disable(rdev);
1839 	rv770_pcie_gart_disable(rdev);
1840 
1841 	return 0;
1842 }
1843 
1844 /* Plan is to move initialization in that function and use
1845  * helper function so that radeon_device_init pretty much
1846  * do nothing more than calling asic specific function. This
1847  * should also allow to remove a bunch of callback function
1848  * like vram_info.
1849  */
1850 int rv770_init(struct radeon_device *rdev)
1851 {
1852 	int r;
1853 
1854 	/* Read BIOS */
1855 	if (!radeon_get_bios(rdev)) {
1856 		if (ASIC_IS_AVIVO(rdev))
1857 			return -EINVAL;
1858 	}
1859 	/* Must be an ATOMBIOS */
1860 	if (!rdev->is_atom_bios) {
1861 		dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1862 		return -EINVAL;
1863 	}
1864 	r = radeon_atombios_init(rdev);
1865 	if (r)
1866 		return r;
1867 	/* Post card if necessary */
1868 	if (!radeon_card_posted(rdev)) {
1869 		if (!rdev->bios) {
1870 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1871 			return -EINVAL;
1872 		}
1873 		DRM_INFO("GPU not posted. posting now...\n");
1874 		atom_asic_init(rdev->mode_info.atom_context);
1875 	}
1876 	/* init golden registers */
1877 	rv770_init_golden_registers(rdev);
1878 	/* Initialize scratch registers */
1879 	r600_scratch_init(rdev);
1880 	/* Initialize surface registers */
1881 	radeon_surface_init(rdev);
1882 	/* Initialize clocks */
1883 	radeon_get_clock_info(rdev->ddev);
1884 	/* Fence driver */
1885 	r = radeon_fence_driver_init(rdev);
1886 	if (r)
1887 		return r;
1888 	/* initialize AGP */
1889 	if (rdev->flags & RADEON_IS_AGP) {
1890 		r = radeon_agp_init(rdev);
1891 		if (r)
1892 			radeon_agp_disable(rdev);
1893 	}
1894 	r = rv770_mc_init(rdev);
1895 	if (r)
1896 		return r;
1897 	/* Memory manager */
1898 	r = radeon_bo_init(rdev);
1899 	if (r)
1900 		return r;
1901 
1902 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1903 		r = r600_init_microcode(rdev);
1904 		if (r) {
1905 			DRM_ERROR("Failed to load firmware!\n");
1906 			return r;
1907 		}
1908 	}
1909 
1910 	/* Initialize power management */
1911 	radeon_pm_init(rdev);
1912 
1913 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1914 	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1915 
1916 	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1917 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1918 
1919 	r = radeon_uvd_init(rdev);
1920 	if (!r) {
1921 		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1922 		r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
1923 			       4096);
1924 	}
1925 
1926 	rdev->ih.ring_obj = NULL;
1927 	r600_ih_ring_init(rdev, 64 * 1024);
1928 
1929 	r = r600_pcie_gart_init(rdev);
1930 	if (r)
1931 		return r;
1932 
1933 	rdev->accel_working = true;
1934 	r = rv770_startup(rdev);
1935 	if (r) {
1936 		dev_err(rdev->dev, "disabling GPU acceleration\n");
1937 		r700_cp_fini(rdev);
1938 		r600_dma_fini(rdev);
1939 		r600_irq_fini(rdev);
1940 		radeon_wb_fini(rdev);
1941 		radeon_ib_pool_fini(rdev);
1942 		radeon_irq_kms_fini(rdev);
1943 		rv770_pcie_gart_fini(rdev);
1944 		rdev->accel_working = false;
1945 	}
1946 
1947 	return 0;
1948 }
1949 
1950 void rv770_fini(struct radeon_device *rdev)
1951 {
1952 	radeon_pm_fini(rdev);
1953 	r700_cp_fini(rdev);
1954 	r600_dma_fini(rdev);
1955 	r600_irq_fini(rdev);
1956 	radeon_wb_fini(rdev);
1957 	radeon_ib_pool_fini(rdev);
1958 	radeon_irq_kms_fini(rdev);
1959 	uvd_v1_0_fini(rdev);
1960 	radeon_uvd_fini(rdev);
1961 	rv770_pcie_gart_fini(rdev);
1962 	r600_vram_scratch_fini(rdev);
1963 	radeon_gem_fini(rdev);
1964 	radeon_fence_driver_fini(rdev);
1965 	radeon_agp_fini(rdev);
1966 	radeon_bo_fini(rdev);
1967 	radeon_atombios_fini(rdev);
1968 	kfree(rdev->bios);
1969 	rdev->bios = NULL;
1970 }
1971 
1972 static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
1973 {
1974 	u32 link_width_cntl, lanes, speed_cntl, tmp;
1975 	u16 link_cntl2;
1976 
1977 	if (radeon_pcie_gen2 == 0)
1978 		return;
1979 
1980 	if (rdev->flags & RADEON_IS_IGP)
1981 		return;
1982 
1983 	if (!(rdev->flags & RADEON_IS_PCIE))
1984 		return;
1985 
1986 	/* x2 cards have a special sequence */
1987 	if (ASIC_IS_X2(rdev))
1988 		return;
1989 
1990 	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
1991 		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
1992 		return;
1993 
1994 	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
1995 
1996 	/* advertise upconfig capability */
1997 	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1998 	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
1999 	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2000 	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2001 	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2002 		lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2003 		link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2004 				     LC_RECONFIG_ARC_MISSING_ESCAPE);
2005 		link_width_cntl |= lanes | LC_RECONFIG_NOW |
2006 			LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2007 		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2008 	} else {
2009 		link_width_cntl |= LC_UPCONFIGURE_DIS;
2010 		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2011 	}
2012 
2013 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2014 	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2015 	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2016 
2017 		tmp = RREG32(0x541c);
2018 		WREG32(0x541c, tmp | 0x8);
2019 		WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2020 		link_cntl2 = RREG16(0x4088);
2021 		link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2022 		link_cntl2 |= 0x2;
2023 		WREG16(0x4088, link_cntl2);
2024 		WREG32(MM_CFGREGS_CNTL, 0);
2025 
2026 		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2027 		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2028 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2029 
2030 		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2031 		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2032 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2033 
2034 		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2035 		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2036 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2037 
2038 		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2039 		speed_cntl |= LC_GEN2_EN_STRAP;
2040 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2041 
2042 	} else {
2043 		link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2044 		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2045 		if (1)
2046 			link_width_cntl |= LC_UPCONFIGURE_DIS;
2047 		else
2048 			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2049 		WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2050 	}
2051 }
2052