1 /*
2  * Copyright 2013 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  * 	    Roy Spliet <rspliet@eclipso.eu>
24  */
25 #define gt215_ram(p) container_of((p), struct gt215_ram, base)
26 #include "ram.h"
27 #include "ramfuc.h"
28 
29 #include <core/option.h>
30 #include <subdev/bios.h>
31 #include <subdev/bios/M0205.h>
32 #include <subdev/bios/rammap.h>
33 #include <subdev/bios/timing.h>
34 #include <subdev/clk/gt215.h>
35 #include <subdev/gpio.h>
36 
37 struct gt215_ramfuc {
38 	struct ramfuc base;
39 	struct ramfuc_reg r_0x001610;
40 	struct ramfuc_reg r_0x001700;
41 	struct ramfuc_reg r_0x002504;
42 	struct ramfuc_reg r_0x004000;
43 	struct ramfuc_reg r_0x004004;
44 	struct ramfuc_reg r_0x004018;
45 	struct ramfuc_reg r_0x004128;
46 	struct ramfuc_reg r_0x004168;
47 	struct ramfuc_reg r_0x100080;
48 	struct ramfuc_reg r_0x100200;
49 	struct ramfuc_reg r_0x100210;
50 	struct ramfuc_reg r_0x100220[9];
51 	struct ramfuc_reg r_0x100264;
52 	struct ramfuc_reg r_0x1002d0;
53 	struct ramfuc_reg r_0x1002d4;
54 	struct ramfuc_reg r_0x1002dc;
55 	struct ramfuc_reg r_0x10053c;
56 	struct ramfuc_reg r_0x1005a0;
57 	struct ramfuc_reg r_0x1005a4;
58 	struct ramfuc_reg r_0x100700;
59 	struct ramfuc_reg r_0x100714;
60 	struct ramfuc_reg r_0x100718;
61 	struct ramfuc_reg r_0x10071c;
62 	struct ramfuc_reg r_0x100720;
63 	struct ramfuc_reg r_0x100760;
64 	struct ramfuc_reg r_0x1007a0;
65 	struct ramfuc_reg r_0x1007e0;
66 	struct ramfuc_reg r_0x100da0;
67 	struct ramfuc_reg r_0x10f804;
68 	struct ramfuc_reg r_0x1110e0;
69 	struct ramfuc_reg r_0x111100;
70 	struct ramfuc_reg r_0x111104;
71 	struct ramfuc_reg r_0x1111e0;
72 	struct ramfuc_reg r_0x111400;
73 	struct ramfuc_reg r_0x611200;
74 	struct ramfuc_reg r_mr[4];
75 	struct ramfuc_reg r_gpio[4];
76 };
77 
78 struct gt215_ltrain {
79 	enum {
80 		NVA3_TRAIN_UNKNOWN,
81 		NVA3_TRAIN_UNSUPPORTED,
82 		NVA3_TRAIN_ONCE,
83 		NVA3_TRAIN_EXEC,
84 		NVA3_TRAIN_DONE
85 	} state;
86 	u32 r_100720;
87 	u32 r_1111e0;
88 	u32 r_111400;
89 	struct nvkm_mem *mem;
90 };
91 
92 struct gt215_ram {
93 	struct nvkm_ram base;
94 	struct gt215_ramfuc fuc;
95 	struct gt215_ltrain ltrain;
96 };
97 
98 void
99 gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train)
100 {
101 	int i, lo, hi;
102 	u8 median[8], bins[4] = {0, 0, 0, 0}, bin = 0, qty = 0;
103 
104 	for (i = 0; i < 8; i++) {
105 		for (lo = 0; lo < 0x40; lo++) {
106 			if (!(vals[lo] & 0x80000000))
107 				continue;
108 			if (vals[lo] & (0x101 << i))
109 				break;
110 		}
111 
112 		if (lo == 0x40)
113 			return;
114 
115 		for (hi = lo + 1; hi < 0x40; hi++) {
116 			if (!(vals[lo] & 0x80000000))
117 				continue;
118 			if (!(vals[hi] & (0x101 << i))) {
119 				hi--;
120 				break;
121 			}
122 		}
123 
124 		median[i] = ((hi - lo) >> 1) + lo;
125 		bins[(median[i] & 0xf0) >> 4]++;
126 		median[i] += 0x30;
127 	}
128 
129 	/* Find the best value for 0x1111e0 */
130 	for (i = 0; i < 4; i++) {
131 		if (bins[i] > qty) {
132 			bin = i + 3;
133 			qty = bins[i];
134 		}
135 	}
136 
137 	train->r_100720 = 0;
138 	for (i = 0; i < 8; i++) {
139 		median[i] = max(median[i], (u8) (bin << 4));
140 		median[i] = min(median[i], (u8) ((bin << 4) | 0xf));
141 
142 		train->r_100720 |= ((median[i] & 0x0f) << (i << 2));
143 	}
144 
145 	train->r_1111e0 = 0x02000000 | (bin * 0x101);
146 	train->r_111400 = 0x0;
147 }
148 
149 /*
150  * Link training for (at least) DDR3
151  */
152 int
153 gt215_link_train(struct gt215_ram *ram)
154 {
155 	struct gt215_ltrain *train = &ram->ltrain;
156 	struct gt215_ramfuc *fuc = &ram->fuc;
157 	struct nvkm_subdev *subdev = &ram->base.fb->subdev;
158 	struct nvkm_device *device = subdev->device;
159 	struct nvkm_bios *bios = device->bios;
160 	struct nvkm_clk *clk = device->clk;
161 	u32 *result, r1700;
162 	int ret, i;
163 	struct nvbios_M0205T M0205T = { 0 };
164 	u8 ver, hdr, cnt, len, snr, ssz;
165 	unsigned int clk_current;
166 	unsigned long flags;
167 	unsigned long *f = &flags;
168 
169 	if (nvkm_boolopt(device->cfgopt, "NvMemExec", true) != true)
170 		return -ENOSYS;
171 
172 	/* XXX: Multiple partitions? */
173 	result = kmalloc(64 * sizeof(u32), GFP_KERNEL);
174 	if (!result)
175 		return -ENOMEM;
176 
177 	train->state = NVA3_TRAIN_EXEC;
178 
179 	/* Clock speeds for training and back */
180 	nvbios_M0205Tp(bios, &ver, &hdr, &cnt, &len, &snr, &ssz, &M0205T);
181 	if (M0205T.freq == 0) {
182 		kfree(result);
183 		return -ENOENT;
184 	}
185 
186 	clk_current = nvkm_clk_read(clk, nv_clk_src_mem);
187 
188 	ret = gt215_clk_pre(clk, f);
189 	if (ret)
190 		goto out;
191 
192 	/* First: clock up/down */
193 	ret = ram->base.func->calc(&ram->base, (u32) M0205T.freq * 1000);
194 	if (ret)
195 		goto out;
196 
197 	/* Do this *after* calc, eliminates write in script */
198 	nvkm_wr32(device, 0x111400, 0x00000000);
199 	/* XXX: Magic writes that improve train reliability? */
200 	nvkm_mask(device, 0x100674, 0x0000ffff, 0x00000000);
201 	nvkm_mask(device, 0x1005e4, 0x0000ffff, 0x00000000);
202 	nvkm_mask(device, 0x100b0c, 0x000000ff, 0x00000000);
203 	nvkm_wr32(device, 0x100c04, 0x00000400);
204 
205 	/* Now the training script */
206 	r1700 = ram_rd32(fuc, 0x001700);
207 
208 	ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
209 	ram_wr32(fuc, 0x611200, 0x3300);
210 	ram_wait_vblank(fuc);
211 	ram_wait(fuc, 0x611200, 0x00000003, 0x00000000, 500000);
212 	ram_mask(fuc, 0x001610, 0x00000083, 0x00000003);
213 	ram_mask(fuc, 0x100080, 0x00000020, 0x00000000);
214 	ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
215 	ram_wr32(fuc, 0x001700, 0x00000000);
216 
217 	ram_train(fuc);
218 
219 	/* Reset */
220 	ram_mask(fuc, 0x10f804, 0x80000000, 0x80000000);
221 	ram_wr32(fuc, 0x10053c, 0x0);
222 	ram_wr32(fuc, 0x100720, train->r_100720);
223 	ram_wr32(fuc, 0x1111e0, train->r_1111e0);
224 	ram_wr32(fuc, 0x111400, train->r_111400);
225 	ram_nuke(fuc, 0x100080);
226 	ram_mask(fuc, 0x100080, 0x00000020, 0x00000020);
227 	ram_nsec(fuc, 1000);
228 
229 	ram_wr32(fuc, 0x001700, r1700);
230 	ram_mask(fuc, 0x001610, 0x00000083, 0x00000080);
231 	ram_wr32(fuc, 0x611200, 0x3330);
232 	ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
233 
234 	ram_exec(fuc, true);
235 
236 	ram->base.func->calc(&ram->base, clk_current);
237 	ram_exec(fuc, true);
238 
239 	/* Post-processing, avoids flicker */
240 	nvkm_mask(device, 0x616308, 0x10, 0x10);
241 	nvkm_mask(device, 0x616b08, 0x10, 0x10);
242 
243 	gt215_clk_post(clk, f);
244 
245 	ram_train_result(ram->base.fb, result, 64);
246 	for (i = 0; i < 64; i++)
247 		nvkm_debug(subdev, "Train: %08x", result[i]);
248 	gt215_link_train_calc(result, train);
249 
250 	nvkm_debug(subdev, "Train: %08x %08x %08x", train->r_100720,
251 		   train->r_1111e0, train->r_111400);
252 
253 	kfree(result);
254 
255 	train->state = NVA3_TRAIN_DONE;
256 
257 	return ret;
258 
259 out:
260 	if(ret == -EBUSY)
261 		f = NULL;
262 
263 	train->state = NVA3_TRAIN_UNSUPPORTED;
264 
265 	gt215_clk_post(clk, f);
266 	kfree(result);
267 	return ret;
268 }
269 
270 int
271 gt215_link_train_init(struct gt215_ram *ram)
272 {
273 	static const u32 pattern[16] = {
274 		0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
275 		0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
276 		0x33333333, 0x55555555, 0x77777777, 0x66666666,
277 		0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
278 	};
279 	struct gt215_ltrain *train = &ram->ltrain;
280 	struct nvkm_device *device = ram->base.fb->subdev.device;
281 	struct nvkm_bios *bios = device->bios;
282 	struct nvkm_mem *mem;
283 	struct nvbios_M0205E M0205E;
284 	u8 ver, hdr, cnt, len;
285 	u32 r001700;
286 	int ret, i = 0;
287 
288 	train->state = NVA3_TRAIN_UNSUPPORTED;
289 
290 	/* We support type "5"
291 	 * XXX: training pattern table appears to be unused for this routine */
292 	if (!nvbios_M0205Ep(bios, i, &ver, &hdr, &cnt, &len, &M0205E))
293 		return -ENOENT;
294 
295 	if (M0205E.type != 5)
296 		return 0;
297 
298 	train->state = NVA3_TRAIN_ONCE;
299 
300 	ret = ram->base.func->get(&ram->base, 0x8000, 0x10000, 0, 0x800,
301 				  &ram->ltrain.mem);
302 	if (ret)
303 		return ret;
304 
305 	mem = ram->ltrain.mem;
306 
307 	nvkm_wr32(device, 0x100538, 0x10000000 | (mem->offset >> 16));
308 	nvkm_wr32(device, 0x1005a8, 0x0000ffff);
309 	nvkm_mask(device, 0x10f800, 0x00000001, 0x00000001);
310 
311 	for (i = 0; i < 0x30; i++) {
312 		nvkm_wr32(device, 0x10f8c0, (i << 8) | i);
313 		nvkm_wr32(device, 0x10f900, pattern[i % 16]);
314 	}
315 
316 	for (i = 0; i < 0x30; i++) {
317 		nvkm_wr32(device, 0x10f8e0, (i << 8) | i);
318 		nvkm_wr32(device, 0x10f920, pattern[i % 16]);
319 	}
320 
321 	/* And upload the pattern */
322 	r001700 = nvkm_rd32(device, 0x1700);
323 	nvkm_wr32(device, 0x1700, mem->offset >> 16);
324 	for (i = 0; i < 16; i++)
325 		nvkm_wr32(device, 0x700000 + (i << 2), pattern[i]);
326 	for (i = 0; i < 16; i++)
327 		nvkm_wr32(device, 0x700100 + (i << 2), pattern[i]);
328 	nvkm_wr32(device, 0x1700, r001700);
329 
330 	train->r_100720 = nvkm_rd32(device, 0x100720);
331 	train->r_1111e0 = nvkm_rd32(device, 0x1111e0);
332 	train->r_111400 = nvkm_rd32(device, 0x111400);
333 	return 0;
334 }
335 
336 void
337 gt215_link_train_fini(struct gt215_ram *ram)
338 {
339 	if (ram->ltrain.mem)
340 		ram->base.func->put(&ram->base, &ram->ltrain.mem);
341 }
342 
343 /*
344  * RAM reclocking
345  */
346 #define T(t) cfg->timing_10_##t
347 static int
348 gt215_ram_timing_calc(struct gt215_ram *ram, u32 *timing)
349 {
350 	struct nvbios_ramcfg *cfg = &ram->base.target.bios;
351 	struct nvkm_subdev *subdev = &ram->base.fb->subdev;
352 	struct nvkm_device *device = subdev->device;
353 	int tUNK_base, tUNK_40_0, prevCL;
354 	u32 cur2, cur3, cur7, cur8;
355 
356 	cur2 = nvkm_rd32(device, 0x100228);
357 	cur3 = nvkm_rd32(device, 0x10022c);
358 	cur7 = nvkm_rd32(device, 0x10023c);
359 	cur8 = nvkm_rd32(device, 0x100240);
360 
361 
362 	switch ((!T(CWL)) * ram->base.type) {
363 	case NVKM_RAM_TYPE_DDR2:
364 		T(CWL) = T(CL) - 1;
365 		break;
366 	case NVKM_RAM_TYPE_GDDR3:
367 		T(CWL) = ((cur2 & 0xff000000) >> 24) + 1;
368 		break;
369 	}
370 
371 	prevCL = (cur3 & 0x000000ff) + 1;
372 	tUNK_base = ((cur7 & 0x00ff0000) >> 16) - prevCL;
373 
374 	timing[0] = (T(RP) << 24 | T(RAS) << 16 | T(RFC) << 8 | T(RC));
375 	timing[1] = (T(WR) + 1 + T(CWL)) << 24 |
376 		    max_t(u8,T(18), 1) << 16 |
377 		    (T(WTR) + 1 + T(CWL)) << 8 |
378 		    (5 + T(CL) - T(CWL));
379 	timing[2] = (T(CWL) - 1) << 24 |
380 		    (T(RRD) << 16) |
381 		    (T(RCDWR) << 8) |
382 		    T(RCDRD);
383 	timing[3] = (cur3 & 0x00ff0000) |
384 		    (0x30 + T(CL)) << 24 |
385 		    (0xb + T(CL)) << 8 |
386 		    (T(CL) - 1);
387 	timing[4] = T(20) << 24 |
388 		    T(21) << 16 |
389 		    T(13) << 8 |
390 		    T(13);
391 	timing[5] = T(RFC) << 24 |
392 		    max_t(u8,T(RCDRD), T(RCDWR)) << 16 |
393 		    max_t(u8, (T(CWL) + 6), (T(CL) + 2)) << 8 |
394 		    T(RP);
395 	timing[6] = (0x5a + T(CL)) << 16 |
396 		    max_t(u8, 1, (6 - T(CL) + T(CWL))) << 8 |
397 		    (0x50 + T(CL) - T(CWL));
398 	timing[7] = (cur7 & 0xff000000) |
399 		    ((tUNK_base + T(CL)) << 16) |
400 		    0x202;
401 	timing[8] = cur8 & 0xffffff00;
402 
403 	switch (ram->base.type) {
404 	case NVKM_RAM_TYPE_DDR2:
405 	case NVKM_RAM_TYPE_GDDR3:
406 		tUNK_40_0 = prevCL - (cur8 & 0xff);
407 		if (tUNK_40_0 > 0)
408 			timing[8] |= T(CL);
409 		break;
410 	default:
411 		break;
412 	}
413 
414 	nvkm_debug(subdev, "Entry: 220: %08x %08x %08x %08x\n",
415 		   timing[0], timing[1], timing[2], timing[3]);
416 	nvkm_debug(subdev, "  230: %08x %08x %08x %08x\n",
417 		   timing[4], timing[5], timing[6], timing[7]);
418 	nvkm_debug(subdev, "  240: %08x\n", timing[8]);
419 	return 0;
420 }
421 #undef T
422 
423 static void
424 nvkm_sddr2_dll_reset(struct gt215_ramfuc *fuc)
425 {
426 	ram_mask(fuc, mr[0], 0x100, 0x100);
427 	ram_nsec(fuc, 1000);
428 	ram_mask(fuc, mr[0], 0x100, 0x000);
429 	ram_nsec(fuc, 1000);
430 }
431 
432 static void
433 nvkm_sddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)
434 {
435 	u32 mr1_old = ram_rd32(fuc, mr[1]);
436 
437 	if (!(mr1_old & 0x1)) {
438 		ram_wr32(fuc, 0x1002d4, 0x00000001);
439 		ram_wr32(fuc, mr[1], mr[1]);
440 		ram_nsec(fuc, 1000);
441 	}
442 }
443 
444 static void
445 nvkm_gddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)
446 {
447 	u32 mr1_old = ram_rd32(fuc, mr[1]);
448 
449 	if (!(mr1_old & 0x40)) {
450 		ram_wr32(fuc, mr[1], mr[1]);
451 		ram_nsec(fuc, 1000);
452 	}
453 }
454 
455 static void
456 gt215_ram_lock_pll(struct gt215_ramfuc *fuc, struct gt215_clk_info *mclk)
457 {
458 	ram_wr32(fuc, 0x004004, mclk->pll);
459 	ram_mask(fuc, 0x004000, 0x00000001, 0x00000001);
460 	ram_mask(fuc, 0x004000, 0x00000010, 0x00000000);
461 	ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
462 	ram_mask(fuc, 0x004000, 0x00000010, 0x00000010);
463 }
464 
465 static void
466 gt215_ram_gpio(struct gt215_ramfuc *fuc, u8 tag, u32 val)
467 {
468 	struct nvkm_gpio *gpio = fuc->base.fb->subdev.device->gpio;
469 	struct dcb_gpio_func func;
470 	u32 reg, sh, gpio_val;
471 	int ret;
472 
473 	if (nvkm_gpio_get(gpio, 0, tag, DCB_GPIO_UNUSED) != val) {
474 		ret = nvkm_gpio_find(gpio, 0, tag, DCB_GPIO_UNUSED, &func);
475 		if (ret)
476 			return;
477 
478 		reg = func.line >> 3;
479 		sh = (func.line & 0x7) << 2;
480 		gpio_val = ram_rd32(fuc, gpio[reg]);
481 		if (gpio_val & (8 << sh))
482 			val = !val;
483 		if (!(func.log[1] & 1))
484 			val = !val;
485 
486 		ram_mask(fuc, gpio[reg], (0x3 << sh), ((val | 0x2) << sh));
487 		ram_nsec(fuc, 20000);
488 	}
489 }
490 
491 static int
492 gt215_ram_calc(struct nvkm_ram *base, u32 freq)
493 {
494 	struct gt215_ram *ram = gt215_ram(base);
495 	struct gt215_ramfuc *fuc = &ram->fuc;
496 	struct gt215_ltrain *train = &ram->ltrain;
497 	struct nvkm_subdev *subdev = &ram->base.fb->subdev;
498 	struct nvkm_device *device = subdev->device;
499 	struct nvkm_bios *bios = device->bios;
500 	struct gt215_clk_info mclk;
501 	struct nvkm_gpio *gpio = device->gpio;
502 	struct nvkm_ram_data *next;
503 	u8  ver, hdr, cnt, len, strap;
504 	u32 data;
505 	u32 r004018, r100760, r100da0, r111100, ctrl;
506 	u32 unk714, unk718, unk71c;
507 	int ret, i;
508 	u32 timing[9];
509 	bool pll2pll;
510 
511 	next = &ram->base.target;
512 	next->freq = freq;
513 	ram->base.next = next;
514 
515 	if (ram->ltrain.state == NVA3_TRAIN_ONCE)
516 		gt215_link_train(ram);
517 
518 	/* lookup memory config data relevant to the target frequency */
519 	data = nvbios_rammapEm(bios, freq / 1000, &ver, &hdr, &cnt, &len,
520 			       &next->bios);
521 	if (!data || ver != 0x10 || hdr < 0x05) {
522 		nvkm_error(subdev, "invalid/missing rammap entry\n");
523 		return -EINVAL;
524 	}
525 
526 	/* locate specific data set for the attached memory */
527 	strap = nvbios_ramcfg_index(subdev);
528 	if (strap >= cnt) {
529 		nvkm_error(subdev, "invalid ramcfg strap\n");
530 		return -EINVAL;
531 	}
532 
533 	data = nvbios_rammapSp(bios, data, ver, hdr, cnt, len, strap,
534 			       &ver, &hdr, &next->bios);
535 	if (!data || ver != 0x10 || hdr < 0x09) {
536 		nvkm_error(subdev, "invalid/missing ramcfg entry\n");
537 		return -EINVAL;
538 	}
539 
540 	/* lookup memory timings, if bios says they're present */
541 	if (next->bios.ramcfg_timing != 0xff) {
542 		data = nvbios_timingEp(bios, next->bios.ramcfg_timing,
543 				       &ver, &hdr, &cnt, &len,
544 				       &next->bios);
545 		if (!data || ver != 0x10 || hdr < 0x17) {
546 			nvkm_error(subdev, "invalid/missing timing entry\n");
547 			return -EINVAL;
548 		}
549 	}
550 
551 	ret = gt215_pll_info(device->clk, 0x12, 0x4000, freq, &mclk);
552 	if (ret < 0) {
553 		nvkm_error(subdev, "failed mclk calculation\n");
554 		return ret;
555 	}
556 
557 	gt215_ram_timing_calc(ram, timing);
558 
559 	ret = ram_init(fuc, ram->base.fb);
560 	if (ret)
561 		return ret;
562 
563 	/* Determine ram-specific MR values */
564 	ram->base.mr[0] = ram_rd32(fuc, mr[0]);
565 	ram->base.mr[1] = ram_rd32(fuc, mr[1]);
566 	ram->base.mr[2] = ram_rd32(fuc, mr[2]);
567 
568 	switch (ram->base.type) {
569 	case NVKM_RAM_TYPE_DDR2:
570 		ret = nvkm_sddr2_calc(&ram->base);
571 		break;
572 	case NVKM_RAM_TYPE_DDR3:
573 		ret = nvkm_sddr3_calc(&ram->base);
574 		break;
575 	case NVKM_RAM_TYPE_GDDR3:
576 		ret = nvkm_gddr3_calc(&ram->base);
577 		break;
578 	default:
579 		ret = -ENOSYS;
580 		break;
581 	}
582 
583 	if (ret)
584 		return ret;
585 
586 	/* XXX: 750MHz seems rather arbitrary */
587 	if (freq <= 750000) {
588 		r004018 = 0x10000000;
589 		r100760 = 0x22222222;
590 		r100da0 = 0x00000010;
591 	} else {
592 		r004018 = 0x00000000;
593 		r100760 = 0x00000000;
594 		r100da0 = 0x00000000;
595 	}
596 
597 	if (!next->bios.ramcfg_DLLoff)
598 		r004018 |= 0x00004000;
599 
600 	/* pll2pll requires to switch to a safe clock first */
601 	ctrl = ram_rd32(fuc, 0x004000);
602 	pll2pll = (!(ctrl & 0x00000008)) && mclk.pll;
603 
604 	/* Pre, NVIDIA does this outside the script */
605 	if (next->bios.ramcfg_10_02_10) {
606 		ram_mask(fuc, 0x111104, 0x00000600, 0x00000000);
607 	} else {
608 		ram_mask(fuc, 0x111100, 0x40000000, 0x40000000);
609 		ram_mask(fuc, 0x111104, 0x00000180, 0x00000000);
610 	}
611 	/* Always disable this bit during reclock */
612 	ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
613 
614 	/* If switching from non-pll to pll, lock before disabling FB */
615 	if (mclk.pll && !pll2pll) {
616 		ram_mask(fuc, 0x004128, 0x003f3141, mclk.clk | 0x00000101);
617 		gt215_ram_lock_pll(fuc, &mclk);
618 	}
619 
620 	/* Start with disabling some CRTCs and PFIFO? */
621 	ram_wait_vblank(fuc);
622 	ram_wr32(fuc, 0x611200, 0x3300);
623 	ram_mask(fuc, 0x002504, 0x1, 0x1);
624 	ram_nsec(fuc, 10000);
625 	ram_wait(fuc, 0x002504, 0x10, 0x10, 20000); /* XXX: or longer? */
626 	ram_block(fuc);
627 	ram_nsec(fuc, 2000);
628 
629 	if (!next->bios.ramcfg_10_02_10) {
630 		if (ram->base.type == NVKM_RAM_TYPE_GDDR3)
631 			ram_mask(fuc, 0x111100, 0x04020000, 0x00020000);
632 		else
633 			ram_mask(fuc, 0x111100, 0x04020000, 0x04020000);
634 	}
635 
636 	/* If we're disabling the DLL, do it now */
637 	switch (next->bios.ramcfg_DLLoff * ram->base.type) {
638 	case NVKM_RAM_TYPE_DDR3:
639 		nvkm_sddr3_dll_disable(fuc, ram->base.mr);
640 		break;
641 	case NVKM_RAM_TYPE_GDDR3:
642 		nvkm_gddr3_dll_disable(fuc, ram->base.mr);
643 		break;
644 	}
645 
646 	if (next->bios.timing_10_ODT)
647 		gt215_ram_gpio(fuc, 0x2e, 1);
648 
649 	/* Brace RAM for impact */
650 	ram_wr32(fuc, 0x1002d4, 0x00000001);
651 	ram_wr32(fuc, 0x1002d0, 0x00000001);
652 	ram_wr32(fuc, 0x1002d0, 0x00000001);
653 	ram_wr32(fuc, 0x100210, 0x00000000);
654 	ram_wr32(fuc, 0x1002dc, 0x00000001);
655 	ram_nsec(fuc, 2000);
656 
657 	if (device->chipset == 0xa3 && freq <= 500000)
658 		ram_mask(fuc, 0x100700, 0x00000006, 0x00000006);
659 
660 	/* Alter FBVDD/Q, apparently must be done with PLL disabled, thus
661 	 * set it to bypass */
662 	if (nvkm_gpio_get(gpio, 0, 0x18, DCB_GPIO_UNUSED) ==
663 			next->bios.ramcfg_FBVDDQ) {
664 		data = ram_rd32(fuc, 0x004000) & 0x9;
665 
666 		if (data == 0x1)
667 			ram_mask(fuc, 0x004000, 0x8, 0x8);
668 		if (data & 0x1)
669 			ram_mask(fuc, 0x004000, 0x1, 0x0);
670 
671 		gt215_ram_gpio(fuc, 0x18, !next->bios.ramcfg_FBVDDQ);
672 
673 		if (data & 0x1)
674 			ram_mask(fuc, 0x004000, 0x1, 0x1);
675 	}
676 
677 	/* Fiddle with clocks */
678 	/* There's 4 scenario's
679 	 * pll->pll: first switch to a 324MHz clock, set up new PLL, switch
680 	 * clk->pll: Set up new PLL, switch
681 	 * pll->clk: Set up clock, switch
682 	 * clk->clk: Overwrite ctrl and other bits, switch */
683 
684 	/* Switch to regular clock - 324MHz */
685 	if (pll2pll) {
686 		ram_mask(fuc, 0x004000, 0x00000004, 0x00000004);
687 		ram_mask(fuc, 0x004168, 0x003f3141, 0x00083101);
688 		ram_mask(fuc, 0x004000, 0x00000008, 0x00000008);
689 		ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
690 		ram_wr32(fuc, 0x004018, 0x00001000);
691 		gt215_ram_lock_pll(fuc, &mclk);
692 	}
693 
694 	if (mclk.pll) {
695 		ram_mask(fuc, 0x004000, 0x00000105, 0x00000105);
696 		ram_wr32(fuc, 0x004018, 0x00001000 | r004018);
697 		ram_wr32(fuc, 0x100da0, r100da0);
698 	} else {
699 		ram_mask(fuc, 0x004168, 0x003f3141, mclk.clk | 0x00000101);
700 		ram_mask(fuc, 0x004000, 0x00000108, 0x00000008);
701 		ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
702 		ram_wr32(fuc, 0x004018, 0x00009000 | r004018);
703 		ram_wr32(fuc, 0x100da0, r100da0);
704 	}
705 	ram_nsec(fuc, 20000);
706 
707 	if (next->bios.rammap_10_04_08) {
708 		ram_wr32(fuc, 0x1005a0, next->bios.ramcfg_10_06 << 16 |
709 					next->bios.ramcfg_10_05 << 8 |
710 					next->bios.ramcfg_10_05);
711 		ram_wr32(fuc, 0x1005a4, next->bios.ramcfg_10_08 << 8 |
712 					next->bios.ramcfg_10_07);
713 		ram_wr32(fuc, 0x10f804, next->bios.ramcfg_10_09_f0 << 20 |
714 					next->bios.ramcfg_10_03_0f << 16 |
715 					next->bios.ramcfg_10_09_0f |
716 					0x80000000);
717 		ram_mask(fuc, 0x10053c, 0x00001000, 0x00000000);
718 	} else {
719 		if (train->state == NVA3_TRAIN_DONE) {
720 			ram_wr32(fuc, 0x100080, 0x1020);
721 			ram_mask(fuc, 0x111400, 0xffffffff, train->r_111400);
722 			ram_mask(fuc, 0x1111e0, 0xffffffff, train->r_1111e0);
723 			ram_mask(fuc, 0x100720, 0xffffffff, train->r_100720);
724 		}
725 		ram_mask(fuc, 0x10053c, 0x00001000, 0x00001000);
726 		ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
727 		ram_mask(fuc, 0x100760, 0x22222222, r100760);
728 		ram_mask(fuc, 0x1007a0, 0x22222222, r100760);
729 		ram_mask(fuc, 0x1007e0, 0x22222222, r100760);
730 	}
731 
732 	if (device->chipset == 0xa3 && freq > 500000) {
733 		ram_mask(fuc, 0x100700, 0x00000006, 0x00000000);
734 	}
735 
736 	/* Final switch */
737 	if (mclk.pll) {
738 		ram_mask(fuc, 0x1110e0, 0x00088000, 0x00011000);
739 		ram_mask(fuc, 0x004000, 0x00000008, 0x00000000);
740 	}
741 
742 	ram_wr32(fuc, 0x1002dc, 0x00000000);
743 	ram_wr32(fuc, 0x1002d4, 0x00000001);
744 	ram_wr32(fuc, 0x100210, 0x80000000);
745 	ram_nsec(fuc, 2000);
746 
747 	/* Set RAM MR parameters and timings */
748 	for (i = 2; i >= 0; i--) {
749 		if (ram_rd32(fuc, mr[i]) != ram->base.mr[i]) {
750 			ram_wr32(fuc, mr[i], ram->base.mr[i]);
751 			ram_nsec(fuc, 1000);
752 		}
753 	}
754 
755 	ram_wr32(fuc, 0x100220[3], timing[3]);
756 	ram_wr32(fuc, 0x100220[1], timing[1]);
757 	ram_wr32(fuc, 0x100220[6], timing[6]);
758 	ram_wr32(fuc, 0x100220[7], timing[7]);
759 	ram_wr32(fuc, 0x100220[2], timing[2]);
760 	ram_wr32(fuc, 0x100220[4], timing[4]);
761 	ram_wr32(fuc, 0x100220[5], timing[5]);
762 	ram_wr32(fuc, 0x100220[0], timing[0]);
763 	ram_wr32(fuc, 0x100220[8], timing[8]);
764 
765 	/* Misc */
766 	ram_mask(fuc, 0x100200, 0x00001000, !next->bios.ramcfg_10_02_08 << 12);
767 
768 	/* XXX: A lot of "chipset"/"ram type" specific stuff...? */
769 	unk714  = ram_rd32(fuc, 0x100714) & ~0xf0000130;
770 	unk718  = ram_rd32(fuc, 0x100718) & ~0x00000100;
771 	unk71c  = ram_rd32(fuc, 0x10071c) & ~0x00000100;
772 	r111100 = ram_rd32(fuc, 0x111100) & ~0x3a800000;
773 
774 	/* NVA8 seems to skip various bits related to ramcfg_10_02_04 */
775 	if (device->chipset == 0xa8) {
776 		r111100 |= 0x08000000;
777 		if (!next->bios.ramcfg_10_02_04)
778 			unk714  |= 0x00000010;
779 	} else {
780 		if (next->bios.ramcfg_10_02_04) {
781 			switch (ram->base.type) {
782 			case NVKM_RAM_TYPE_DDR2:
783 			case NVKM_RAM_TYPE_DDR3:
784 				r111100 &= ~0x00000020;
785 				if (next->bios.ramcfg_10_02_10)
786 					r111100 |= 0x08000004;
787 				else
788 					r111100 |= 0x00000024;
789 				break;
790 			default:
791 				break;
792 			}
793 		} else {
794 			switch (ram->base.type) {
795 			case NVKM_RAM_TYPE_DDR2:
796 			case NVKM_RAM_TYPE_DDR3:
797 				r111100 &= ~0x00000024;
798 				r111100 |=  0x12800000;
799 
800 				if (next->bios.ramcfg_10_02_10)
801 					r111100 |= 0x08000000;
802 				unk714  |= 0x00000010;
803 				break;
804 			case NVKM_RAM_TYPE_GDDR3:
805 				r111100 |= 0x30000000;
806 				unk714  |= 0x00000020;
807 				break;
808 			default:
809 				break;
810 			}
811 		}
812 	}
813 
814 	unk714 |= (next->bios.ramcfg_10_04_01) << 8;
815 
816 	if (next->bios.ramcfg_10_02_20)
817 		unk714 |= 0xf0000000;
818 	if (next->bios.ramcfg_10_02_02)
819 		unk718 |= 0x00000100;
820 	if (next->bios.ramcfg_10_02_01)
821 		unk71c |= 0x00000100;
822 	if (next->bios.timing_10_24 != 0xff) {
823 		unk718 &= ~0xf0000000;
824 		unk718 |= next->bios.timing_10_24 << 28;
825 	}
826 	if (next->bios.ramcfg_10_02_10)
827 		r111100 &= ~0x04020000;
828 
829 	ram_mask(fuc, 0x100714, 0xffffffff, unk714);
830 	ram_mask(fuc, 0x10071c, 0xffffffff, unk71c);
831 	ram_mask(fuc, 0x100718, 0xffffffff, unk718);
832 	ram_mask(fuc, 0x111100, 0xffffffff, r111100);
833 
834 	if (!next->bios.timing_10_ODT)
835 		gt215_ram_gpio(fuc, 0x2e, 0);
836 
837 	/* Reset DLL */
838 	if (!next->bios.ramcfg_DLLoff)
839 		nvkm_sddr2_dll_reset(fuc);
840 
841 	if (ram->base.type == NVKM_RAM_TYPE_GDDR3) {
842 		ram_nsec(fuc, 31000);
843 	} else {
844 		ram_nsec(fuc, 14000);
845 	}
846 
847 	if (ram->base.type == NVKM_RAM_TYPE_DDR3) {
848 		ram_wr32(fuc, 0x100264, 0x1);
849 		ram_nsec(fuc, 2000);
850 	}
851 
852 	ram_nuke(fuc, 0x100700);
853 	ram_mask(fuc, 0x100700, 0x01000000, 0x01000000);
854 	ram_mask(fuc, 0x100700, 0x01000000, 0x00000000);
855 
856 	/* Re-enable FB */
857 	ram_unblock(fuc);
858 	ram_wr32(fuc, 0x611200, 0x3330);
859 
860 	/* Post fiddlings */
861 	if (next->bios.rammap_10_04_02)
862 		ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
863 	if (next->bios.ramcfg_10_02_10) {
864 		ram_mask(fuc, 0x111104, 0x00000180, 0x00000180);
865 		ram_mask(fuc, 0x111100, 0x40000000, 0x00000000);
866 	} else {
867 		ram_mask(fuc, 0x111104, 0x00000600, 0x00000600);
868 	}
869 
870 	if (mclk.pll) {
871 		ram_mask(fuc, 0x004168, 0x00000001, 0x00000000);
872 		ram_mask(fuc, 0x004168, 0x00000100, 0x00000000);
873 	} else {
874 		ram_mask(fuc, 0x004000, 0x00000001, 0x00000000);
875 		ram_mask(fuc, 0x004128, 0x00000001, 0x00000000);
876 		ram_mask(fuc, 0x004128, 0x00000100, 0x00000000);
877 	}
878 
879 	return 0;
880 }
881 
882 static int
883 gt215_ram_prog(struct nvkm_ram *base)
884 {
885 	struct gt215_ram *ram = gt215_ram(base);
886 	struct gt215_ramfuc *fuc = &ram->fuc;
887 	struct nvkm_device *device = ram->base.fb->subdev.device;
888 	bool exec = nvkm_boolopt(device->cfgopt, "NvMemExec", true);
889 
890 	if (exec) {
891 		nvkm_mask(device, 0x001534, 0x2, 0x2);
892 
893 		ram_exec(fuc, true);
894 
895 		/* Post-processing, avoids flicker */
896 		nvkm_mask(device, 0x002504, 0x1, 0x0);
897 		nvkm_mask(device, 0x001534, 0x2, 0x0);
898 
899 		nvkm_mask(device, 0x616308, 0x10, 0x10);
900 		nvkm_mask(device, 0x616b08, 0x10, 0x10);
901 	} else {
902 		ram_exec(fuc, false);
903 	}
904 	return 0;
905 }
906 
907 static void
908 gt215_ram_tidy(struct nvkm_ram *base)
909 {
910 	struct gt215_ram *ram = gt215_ram(base);
911 	ram_exec(&ram->fuc, false);
912 }
913 
914 static int
915 gt215_ram_init(struct nvkm_ram *base)
916 {
917 	struct gt215_ram *ram = gt215_ram(base);
918 	gt215_link_train_init(ram);
919 	return 0;
920 }
921 
922 static void *
923 gt215_ram_dtor(struct nvkm_ram *base)
924 {
925 	struct gt215_ram *ram = gt215_ram(base);
926 	gt215_link_train_fini(ram);
927 	return ram;
928 }
929 
930 static const struct nvkm_ram_func
931 gt215_ram_func = {
932 	.dtor = gt215_ram_dtor,
933 	.init = gt215_ram_init,
934 	.get = nv50_ram_get,
935 	.put = nv50_ram_put,
936 	.calc = gt215_ram_calc,
937 	.prog = gt215_ram_prog,
938 	.tidy = gt215_ram_tidy,
939 };
940 
941 int
942 gt215_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
943 {
944 	struct gt215_ram *ram;
945 	int ret, i;
946 
947 	if (!(ram = kzalloc(sizeof(*ram), GFP_KERNEL)))
948 		return -ENOMEM;
949 	*pram = &ram->base;
950 
951 	ret = nv50_ram_ctor(&gt215_ram_func, fb, &ram->base);
952 	if (ret)
953 		return ret;
954 
955 	ram->fuc.r_0x001610 = ramfuc_reg(0x001610);
956 	ram->fuc.r_0x001700 = ramfuc_reg(0x001700);
957 	ram->fuc.r_0x002504 = ramfuc_reg(0x002504);
958 	ram->fuc.r_0x004000 = ramfuc_reg(0x004000);
959 	ram->fuc.r_0x004004 = ramfuc_reg(0x004004);
960 	ram->fuc.r_0x004018 = ramfuc_reg(0x004018);
961 	ram->fuc.r_0x004128 = ramfuc_reg(0x004128);
962 	ram->fuc.r_0x004168 = ramfuc_reg(0x004168);
963 	ram->fuc.r_0x100080 = ramfuc_reg(0x100080);
964 	ram->fuc.r_0x100200 = ramfuc_reg(0x100200);
965 	ram->fuc.r_0x100210 = ramfuc_reg(0x100210);
966 	for (i = 0; i < 9; i++)
967 		ram->fuc.r_0x100220[i] = ramfuc_reg(0x100220 + (i * 4));
968 	ram->fuc.r_0x100264 = ramfuc_reg(0x100264);
969 	ram->fuc.r_0x1002d0 = ramfuc_reg(0x1002d0);
970 	ram->fuc.r_0x1002d4 = ramfuc_reg(0x1002d4);
971 	ram->fuc.r_0x1002dc = ramfuc_reg(0x1002dc);
972 	ram->fuc.r_0x10053c = ramfuc_reg(0x10053c);
973 	ram->fuc.r_0x1005a0 = ramfuc_reg(0x1005a0);
974 	ram->fuc.r_0x1005a4 = ramfuc_reg(0x1005a4);
975 	ram->fuc.r_0x100700 = ramfuc_reg(0x100700);
976 	ram->fuc.r_0x100714 = ramfuc_reg(0x100714);
977 	ram->fuc.r_0x100718 = ramfuc_reg(0x100718);
978 	ram->fuc.r_0x10071c = ramfuc_reg(0x10071c);
979 	ram->fuc.r_0x100720 = ramfuc_reg(0x100720);
980 	ram->fuc.r_0x100760 = ramfuc_stride(0x100760, 4, ram->base.part_mask);
981 	ram->fuc.r_0x1007a0 = ramfuc_stride(0x1007a0, 4, ram->base.part_mask);
982 	ram->fuc.r_0x1007e0 = ramfuc_stride(0x1007e0, 4, ram->base.part_mask);
983 	ram->fuc.r_0x100da0 = ramfuc_stride(0x100da0, 4, ram->base.part_mask);
984 	ram->fuc.r_0x10f804 = ramfuc_reg(0x10f804);
985 	ram->fuc.r_0x1110e0 = ramfuc_stride(0x1110e0, 4, ram->base.part_mask);
986 	ram->fuc.r_0x111100 = ramfuc_reg(0x111100);
987 	ram->fuc.r_0x111104 = ramfuc_reg(0x111104);
988 	ram->fuc.r_0x1111e0 = ramfuc_reg(0x1111e0);
989 	ram->fuc.r_0x111400 = ramfuc_reg(0x111400);
990 	ram->fuc.r_0x611200 = ramfuc_reg(0x611200);
991 
992 	if (ram->base.ranks > 1) {
993 		ram->fuc.r_mr[0] = ramfuc_reg2(0x1002c0, 0x1002c8);
994 		ram->fuc.r_mr[1] = ramfuc_reg2(0x1002c4, 0x1002cc);
995 		ram->fuc.r_mr[2] = ramfuc_reg2(0x1002e0, 0x1002e8);
996 		ram->fuc.r_mr[3] = ramfuc_reg2(0x1002e4, 0x1002ec);
997 	} else {
998 		ram->fuc.r_mr[0] = ramfuc_reg(0x1002c0);
999 		ram->fuc.r_mr[1] = ramfuc_reg(0x1002c4);
1000 		ram->fuc.r_mr[2] = ramfuc_reg(0x1002e0);
1001 		ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);
1002 	}
1003 	ram->fuc.r_gpio[0] = ramfuc_reg(0x00e104);
1004 	ram->fuc.r_gpio[1] = ramfuc_reg(0x00e108);
1005 	ram->fuc.r_gpio[2] = ramfuc_reg(0x00e120);
1006 	ram->fuc.r_gpio[3] = ramfuc_reg(0x00e124);
1007 
1008 	return 0;
1009 }
1010