1 /*
2  * Copyright 2013 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  * 	    Roy Spliet <rspliet@eclipso.eu>
24  */
25 #define gt215_ram(p) container_of((p), struct gt215_ram, base)
26 #include "ram.h"
27 #include "ramfuc.h"
28 
29 #include <core/memory.h>
30 #include <core/option.h>
31 #include <subdev/bios.h>
32 #include <subdev/bios/M0205.h>
33 #include <subdev/bios/rammap.h>
34 #include <subdev/bios/timing.h>
35 #include <subdev/clk/gt215.h>
36 #include <subdev/gpio.h>
37 
38 struct gt215_ramfuc {
39 	struct ramfuc base;
40 	struct ramfuc_reg r_0x001610;
41 	struct ramfuc_reg r_0x001700;
42 	struct ramfuc_reg r_0x002504;
43 	struct ramfuc_reg r_0x004000;
44 	struct ramfuc_reg r_0x004004;
45 	struct ramfuc_reg r_0x004018;
46 	struct ramfuc_reg r_0x004128;
47 	struct ramfuc_reg r_0x004168;
48 	struct ramfuc_reg r_0x100080;
49 	struct ramfuc_reg r_0x100200;
50 	struct ramfuc_reg r_0x100210;
51 	struct ramfuc_reg r_0x100220[9];
52 	struct ramfuc_reg r_0x100264;
53 	struct ramfuc_reg r_0x1002d0;
54 	struct ramfuc_reg r_0x1002d4;
55 	struct ramfuc_reg r_0x1002dc;
56 	struct ramfuc_reg r_0x10053c;
57 	struct ramfuc_reg r_0x1005a0;
58 	struct ramfuc_reg r_0x1005a4;
59 	struct ramfuc_reg r_0x100700;
60 	struct ramfuc_reg r_0x100714;
61 	struct ramfuc_reg r_0x100718;
62 	struct ramfuc_reg r_0x10071c;
63 	struct ramfuc_reg r_0x100720;
64 	struct ramfuc_reg r_0x100760;
65 	struct ramfuc_reg r_0x1007a0;
66 	struct ramfuc_reg r_0x1007e0;
67 	struct ramfuc_reg r_0x100da0;
68 	struct ramfuc_reg r_0x10f804;
69 	struct ramfuc_reg r_0x1110e0;
70 	struct ramfuc_reg r_0x111100;
71 	struct ramfuc_reg r_0x111104;
72 	struct ramfuc_reg r_0x1111e0;
73 	struct ramfuc_reg r_0x111400;
74 	struct ramfuc_reg r_0x611200;
75 	struct ramfuc_reg r_mr[4];
76 	struct ramfuc_reg r_gpio[4];
77 };
78 
79 struct gt215_ltrain {
80 	enum {
81 		NVA3_TRAIN_UNKNOWN,
82 		NVA3_TRAIN_UNSUPPORTED,
83 		NVA3_TRAIN_ONCE,
84 		NVA3_TRAIN_EXEC,
85 		NVA3_TRAIN_DONE
86 	} state;
87 	u32 r_100720;
88 	u32 r_1111e0;
89 	u32 r_111400;
90 	struct nvkm_memory *memory;
91 };
92 
93 struct gt215_ram {
94 	struct nvkm_ram base;
95 	struct gt215_ramfuc fuc;
96 	struct gt215_ltrain ltrain;
97 };
98 
99 static void
gt215_link_train_calc(u32 * vals,struct gt215_ltrain * train)100 gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train)
101 {
102 	int i, lo, hi;
103 	u8 median[8], bins[4] = {0, 0, 0, 0}, bin = 0, qty = 0;
104 
105 	for (i = 0; i < 8; i++) {
106 		for (lo = 0; lo < 0x40; lo++) {
107 			if (!(vals[lo] & 0x80000000))
108 				continue;
109 			if (vals[lo] & (0x101 << i))
110 				break;
111 		}
112 
113 		if (lo == 0x40)
114 			return;
115 
116 		for (hi = lo + 1; hi < 0x40; hi++) {
117 			if (!(vals[lo] & 0x80000000))
118 				continue;
119 			if (!(vals[hi] & (0x101 << i))) {
120 				hi--;
121 				break;
122 			}
123 		}
124 
125 		median[i] = ((hi - lo) >> 1) + lo;
126 		bins[(median[i] & 0xf0) >> 4]++;
127 		median[i] += 0x30;
128 	}
129 
130 	/* Find the best value for 0x1111e0 */
131 	for (i = 0; i < 4; i++) {
132 		if (bins[i] > qty) {
133 			bin = i + 3;
134 			qty = bins[i];
135 		}
136 	}
137 
138 	train->r_100720 = 0;
139 	for (i = 0; i < 8; i++) {
140 		median[i] = max(median[i], (u8) (bin << 4));
141 		median[i] = min(median[i], (u8) ((bin << 4) | 0xf));
142 
143 		train->r_100720 |= ((median[i] & 0x0f) << (i << 2));
144 	}
145 
146 	train->r_1111e0 = 0x02000000 | (bin * 0x101);
147 	train->r_111400 = 0x0;
148 }
149 
150 /*
151  * Link training for (at least) DDR3
152  */
153 static int
gt215_link_train(struct gt215_ram * ram)154 gt215_link_train(struct gt215_ram *ram)
155 {
156 	struct gt215_ltrain *train = &ram->ltrain;
157 	struct gt215_ramfuc *fuc = &ram->fuc;
158 	struct nvkm_subdev *subdev = &ram->base.fb->subdev;
159 	struct nvkm_device *device = subdev->device;
160 	struct nvkm_bios *bios = device->bios;
161 	struct nvkm_clk *clk = device->clk;
162 	u32 *result, r1700;
163 	int ret, i;
164 	struct nvbios_M0205T M0205T = { 0 };
165 	u8 ver, hdr, cnt, len, snr, ssz;
166 	unsigned int clk_current;
167 	unsigned long flags;
168 	unsigned long *f = &flags;
169 
170 	if (nvkm_boolopt(device->cfgopt, "NvMemExec", true) != true)
171 		return -ENOSYS;
172 
173 	/* XXX: Multiple partitions? */
174 	result = kmalloc_array(64, sizeof(u32), GFP_KERNEL);
175 	if (!result)
176 		return -ENOMEM;
177 
178 	train->state = NVA3_TRAIN_EXEC;
179 
180 	/* Clock speeds for training and back */
181 	nvbios_M0205Tp(bios, &ver, &hdr, &cnt, &len, &snr, &ssz, &M0205T);
182 	if (M0205T.freq == 0) {
183 		kfree(result);
184 		return -ENOENT;
185 	}
186 
187 	clk_current = nvkm_clk_read(clk, nv_clk_src_mem);
188 
189 	ret = gt215_clk_pre(clk, f);
190 	if (ret)
191 		goto out;
192 
193 	/* First: clock up/down */
194 	ret = ram->base.func->calc(&ram->base, (u32) M0205T.freq * 1000);
195 	if (ret)
196 		goto out;
197 
198 	/* Do this *after* calc, eliminates write in script */
199 	nvkm_wr32(device, 0x111400, 0x00000000);
200 	/* XXX: Magic writes that improve train reliability? */
201 	nvkm_mask(device, 0x100674, 0x0000ffff, 0x00000000);
202 	nvkm_mask(device, 0x1005e4, 0x0000ffff, 0x00000000);
203 	nvkm_mask(device, 0x100b0c, 0x000000ff, 0x00000000);
204 	nvkm_wr32(device, 0x100c04, 0x00000400);
205 
206 	/* Now the training script */
207 	r1700 = ram_rd32(fuc, 0x001700);
208 
209 	ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
210 	ram_wr32(fuc, 0x611200, 0x3300);
211 	ram_wait_vblank(fuc);
212 	ram_wait(fuc, 0x611200, 0x00000003, 0x00000000, 500000);
213 	ram_mask(fuc, 0x001610, 0x00000083, 0x00000003);
214 	ram_mask(fuc, 0x100080, 0x00000020, 0x00000000);
215 	ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
216 	ram_wr32(fuc, 0x001700, 0x00000000);
217 
218 	ram_train(fuc);
219 
220 	/* Reset */
221 	ram_mask(fuc, 0x10f804, 0x80000000, 0x80000000);
222 	ram_wr32(fuc, 0x10053c, 0x0);
223 	ram_wr32(fuc, 0x100720, train->r_100720);
224 	ram_wr32(fuc, 0x1111e0, train->r_1111e0);
225 	ram_wr32(fuc, 0x111400, train->r_111400);
226 	ram_nuke(fuc, 0x100080);
227 	ram_mask(fuc, 0x100080, 0x00000020, 0x00000020);
228 	ram_nsec(fuc, 1000);
229 
230 	ram_wr32(fuc, 0x001700, r1700);
231 	ram_mask(fuc, 0x001610, 0x00000083, 0x00000080);
232 	ram_wr32(fuc, 0x611200, 0x3330);
233 	ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
234 
235 	ram_exec(fuc, true);
236 
237 	ram->base.func->calc(&ram->base, clk_current);
238 	ram_exec(fuc, true);
239 
240 	/* Post-processing, avoids flicker */
241 	nvkm_mask(device, 0x616308, 0x10, 0x10);
242 	nvkm_mask(device, 0x616b08, 0x10, 0x10);
243 
244 	gt215_clk_post(clk, f);
245 
246 	ram_train_result(ram->base.fb, result, 64);
247 	for (i = 0; i < 64; i++)
248 		nvkm_debug(subdev, "Train: %08x", result[i]);
249 	gt215_link_train_calc(result, train);
250 
251 	nvkm_debug(subdev, "Train: %08x %08x %08x", train->r_100720,
252 		   train->r_1111e0, train->r_111400);
253 
254 	kfree(result);
255 
256 	train->state = NVA3_TRAIN_DONE;
257 
258 	return ret;
259 
260 out:
261 	if(ret == -EBUSY)
262 		f = NULL;
263 
264 	train->state = NVA3_TRAIN_UNSUPPORTED;
265 
266 	gt215_clk_post(clk, f);
267 	kfree(result);
268 	return ret;
269 }
270 
271 static int
gt215_link_train_init(struct gt215_ram * ram)272 gt215_link_train_init(struct gt215_ram *ram)
273 {
274 	static const u32 pattern[16] = {
275 		0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
276 		0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
277 		0x33333333, 0x55555555, 0x77777777, 0x66666666,
278 		0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
279 	};
280 	struct gt215_ltrain *train = &ram->ltrain;
281 	struct nvkm_device *device = ram->base.fb->subdev.device;
282 	struct nvkm_bios *bios = device->bios;
283 	struct nvbios_M0205E M0205E;
284 	u8 ver, hdr, cnt, len;
285 	u32 r001700;
286 	u64 addr;
287 	int ret, i = 0;
288 
289 	train->state = NVA3_TRAIN_UNSUPPORTED;
290 
291 	/* We support type "5"
292 	 * XXX: training pattern table appears to be unused for this routine */
293 	if (!nvbios_M0205Ep(bios, i, &ver, &hdr, &cnt, &len, &M0205E))
294 		return -ENOENT;
295 
296 	if (M0205E.type != 5)
297 		return 0;
298 
299 	train->state = NVA3_TRAIN_ONCE;
300 
301 	ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, 0x01, 16, 0x8000,
302 			   true, true, &ram->ltrain.memory);
303 	if (ret)
304 		return ret;
305 
306 	addr = nvkm_memory_addr(ram->ltrain.memory);
307 
308 	nvkm_wr32(device, 0x100538, 0x10000000 | (addr >> 16));
309 	nvkm_wr32(device, 0x1005a8, 0x0000ffff);
310 	nvkm_mask(device, 0x10f800, 0x00000001, 0x00000001);
311 
312 	for (i = 0; i < 0x30; i++) {
313 		nvkm_wr32(device, 0x10f8c0, (i << 8) | i);
314 		nvkm_wr32(device, 0x10f900, pattern[i % 16]);
315 	}
316 
317 	for (i = 0; i < 0x30; i++) {
318 		nvkm_wr32(device, 0x10f8e0, (i << 8) | i);
319 		nvkm_wr32(device, 0x10f920, pattern[i % 16]);
320 	}
321 
322 	/* And upload the pattern */
323 	r001700 = nvkm_rd32(device, 0x1700);
324 	nvkm_wr32(device, 0x1700, addr >> 16);
325 	for (i = 0; i < 16; i++)
326 		nvkm_wr32(device, 0x700000 + (i << 2), pattern[i]);
327 	for (i = 0; i < 16; i++)
328 		nvkm_wr32(device, 0x700100 + (i << 2), pattern[i]);
329 	nvkm_wr32(device, 0x1700, r001700);
330 
331 	train->r_100720 = nvkm_rd32(device, 0x100720);
332 	train->r_1111e0 = nvkm_rd32(device, 0x1111e0);
333 	train->r_111400 = nvkm_rd32(device, 0x111400);
334 	return 0;
335 }
336 
337 static void
gt215_link_train_fini(struct gt215_ram * ram)338 gt215_link_train_fini(struct gt215_ram *ram)
339 {
340 	nvkm_memory_unref(&ram->ltrain.memory);
341 }
342 
343 /*
344  * RAM reclocking
345  */
346 #define T(t) cfg->timing_10_##t
347 static int
gt215_ram_timing_calc(struct gt215_ram * ram,u32 * timing)348 gt215_ram_timing_calc(struct gt215_ram *ram, u32 *timing)
349 {
350 	struct nvbios_ramcfg *cfg = &ram->base.target.bios;
351 	struct nvkm_subdev *subdev = &ram->base.fb->subdev;
352 	struct nvkm_device *device = subdev->device;
353 	int tUNK_base, tUNK_40_0, prevCL;
354 	u32 cur2, cur3, cur7, cur8;
355 
356 	cur2 = nvkm_rd32(device, 0x100228);
357 	cur3 = nvkm_rd32(device, 0x10022c);
358 	cur7 = nvkm_rd32(device, 0x10023c);
359 	cur8 = nvkm_rd32(device, 0x100240);
360 
361 
362 	switch ((!T(CWL)) * ram->base.type) {
363 	case NVKM_RAM_TYPE_DDR2:
364 		T(CWL) = T(CL) - 1;
365 		break;
366 	case NVKM_RAM_TYPE_GDDR3:
367 		T(CWL) = ((cur2 & 0xff000000) >> 24) + 1;
368 		break;
369 	}
370 
371 	prevCL = (cur3 & 0x000000ff) + 1;
372 	tUNK_base = ((cur7 & 0x00ff0000) >> 16) - prevCL;
373 
374 	timing[0] = (T(RP) << 24 | T(RAS) << 16 | T(RFC) << 8 | T(RC));
375 	timing[1] = (T(WR) + 1 + T(CWL)) << 24 |
376 		    max_t(u8,T(18), 1) << 16 |
377 		    (T(WTR) + 1 + T(CWL)) << 8 |
378 		    (5 + T(CL) - T(CWL));
379 	timing[2] = (T(CWL) - 1) << 24 |
380 		    (T(RRD) << 16) |
381 		    (T(RCDWR) << 8) |
382 		    T(RCDRD);
383 	timing[3] = (cur3 & 0x00ff0000) |
384 		    (0x30 + T(CL)) << 24 |
385 		    (0xb + T(CL)) << 8 |
386 		    (T(CL) - 1);
387 	timing[4] = T(20) << 24 |
388 		    T(21) << 16 |
389 		    T(13) << 8 |
390 		    T(13);
391 	timing[5] = T(RFC) << 24 |
392 		    max_t(u8,T(RCDRD), T(RCDWR)) << 16 |
393 		    max_t(u8, (T(CWL) + 6), (T(CL) + 2)) << 8 |
394 		    T(RP);
395 	timing[6] = (0x5a + T(CL)) << 16 |
396 		    max_t(u8, 1, (6 - T(CL) + T(CWL))) << 8 |
397 		    (0x50 + T(CL) - T(CWL));
398 	timing[7] = (cur7 & 0xff000000) |
399 		    ((tUNK_base + T(CL)) << 16) |
400 		    0x202;
401 	timing[8] = cur8 & 0xffffff00;
402 
403 	switch (ram->base.type) {
404 	case NVKM_RAM_TYPE_DDR2:
405 	case NVKM_RAM_TYPE_GDDR3:
406 		tUNK_40_0 = prevCL - (cur8 & 0xff);
407 		if (tUNK_40_0 > 0)
408 			timing[8] |= T(CL);
409 		break;
410 	default:
411 		break;
412 	}
413 
414 	nvkm_debug(subdev, "Entry: 220: %08x %08x %08x %08x\n",
415 		   timing[0], timing[1], timing[2], timing[3]);
416 	nvkm_debug(subdev, "  230: %08x %08x %08x %08x\n",
417 		   timing[4], timing[5], timing[6], timing[7]);
418 	nvkm_debug(subdev, "  240: %08x\n", timing[8]);
419 	return 0;
420 }
421 #undef T
422 
423 static void
nvkm_sddr2_dll_reset(struct gt215_ramfuc * fuc)424 nvkm_sddr2_dll_reset(struct gt215_ramfuc *fuc)
425 {
426 	ram_mask(fuc, mr[0], 0x100, 0x100);
427 	ram_nsec(fuc, 1000);
428 	ram_mask(fuc, mr[0], 0x100, 0x000);
429 	ram_nsec(fuc, 1000);
430 }
431 
432 static void
nvkm_sddr3_dll_disable(struct gt215_ramfuc * fuc,u32 * mr)433 nvkm_sddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)
434 {
435 	u32 mr1_old = ram_rd32(fuc, mr[1]);
436 
437 	if (!(mr1_old & 0x1)) {
438 		ram_wr32(fuc, 0x1002d4, 0x00000001);
439 		ram_wr32(fuc, mr[1], mr[1]);
440 		ram_nsec(fuc, 1000);
441 	}
442 }
443 
444 static void
nvkm_gddr3_dll_disable(struct gt215_ramfuc * fuc,u32 * mr)445 nvkm_gddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)
446 {
447 	u32 mr1_old = ram_rd32(fuc, mr[1]);
448 
449 	if (!(mr1_old & 0x40)) {
450 		ram_wr32(fuc, mr[1], mr[1]);
451 		ram_nsec(fuc, 1000);
452 	}
453 }
454 
455 static void
gt215_ram_lock_pll(struct gt215_ramfuc * fuc,struct gt215_clk_info * mclk)456 gt215_ram_lock_pll(struct gt215_ramfuc *fuc, struct gt215_clk_info *mclk)
457 {
458 	ram_wr32(fuc, 0x004004, mclk->pll);
459 	ram_mask(fuc, 0x004000, 0x00000001, 0x00000001);
460 	ram_mask(fuc, 0x004000, 0x00000010, 0x00000000);
461 	ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
462 	ram_mask(fuc, 0x004000, 0x00000010, 0x00000010);
463 }
464 
465 static void
gt215_ram_gpio(struct gt215_ramfuc * fuc,u8 tag,u32 val)466 gt215_ram_gpio(struct gt215_ramfuc *fuc, u8 tag, u32 val)
467 {
468 	struct nvkm_gpio *gpio = fuc->base.fb->subdev.device->gpio;
469 	struct dcb_gpio_func func;
470 	u32 reg, sh, gpio_val;
471 	int ret;
472 
473 	if (nvkm_gpio_get(gpio, 0, tag, DCB_GPIO_UNUSED) != val) {
474 		ret = nvkm_gpio_find(gpio, 0, tag, DCB_GPIO_UNUSED, &func);
475 		if (ret)
476 			return;
477 
478 		reg = func.line >> 3;
479 		sh = (func.line & 0x7) << 2;
480 		gpio_val = ram_rd32(fuc, gpio[reg]);
481 		if (gpio_val & (8 << sh))
482 			val = !val;
483 		if (!(func.log[1] & 1))
484 			val = !val;
485 
486 		ram_mask(fuc, gpio[reg], (0x3 << sh), ((val | 0x2) << sh));
487 		ram_nsec(fuc, 20000);
488 	}
489 }
490 
491 static int
gt215_ram_calc(struct nvkm_ram * base,u32 freq)492 gt215_ram_calc(struct nvkm_ram *base, u32 freq)
493 {
494 	struct gt215_ram *ram = gt215_ram(base);
495 	struct gt215_ramfuc *fuc = &ram->fuc;
496 	struct gt215_ltrain *train = &ram->ltrain;
497 	struct nvkm_subdev *subdev = &ram->base.fb->subdev;
498 	struct nvkm_device *device = subdev->device;
499 	struct nvkm_bios *bios = device->bios;
500 	struct gt215_clk_info mclk;
501 	struct nvkm_gpio *gpio = device->gpio;
502 	struct nvkm_ram_data *next;
503 	u8  ver, hdr, cnt, len, strap;
504 	u32 data;
505 	u32 r004018, r100760, r100da0, r111100, ctrl;
506 	u32 unk714, unk718, unk71c;
507 	int ret, i;
508 	u32 timing[9];
509 	bool pll2pll;
510 
511 	next = &ram->base.target;
512 	next->freq = freq;
513 	ram->base.next = next;
514 
515 	if (ram->ltrain.state == NVA3_TRAIN_ONCE)
516 		gt215_link_train(ram);
517 
518 	/* lookup memory config data relevant to the target frequency */
519 	data = nvbios_rammapEm(bios, freq / 1000, &ver, &hdr, &cnt, &len,
520 			       &next->bios);
521 	if (!data || ver != 0x10 || hdr < 0x05) {
522 		nvkm_error(subdev, "invalid/missing rammap entry\n");
523 		return -EINVAL;
524 	}
525 
526 	/* locate specific data set for the attached memory */
527 	strap = nvbios_ramcfg_index(subdev);
528 	if (strap >= cnt) {
529 		nvkm_error(subdev, "invalid ramcfg strap\n");
530 		return -EINVAL;
531 	}
532 
533 	data = nvbios_rammapSp(bios, data, ver, hdr, cnt, len, strap,
534 			       &ver, &hdr, &next->bios);
535 	if (!data || ver != 0x10 || hdr < 0x09) {
536 		nvkm_error(subdev, "invalid/missing ramcfg entry\n");
537 		return -EINVAL;
538 	}
539 
540 	/* lookup memory timings, if bios says they're present */
541 	if (next->bios.ramcfg_timing != 0xff) {
542 		data = nvbios_timingEp(bios, next->bios.ramcfg_timing,
543 				       &ver, &hdr, &cnt, &len,
544 				       &next->bios);
545 		if (!data || ver != 0x10 || hdr < 0x17) {
546 			nvkm_error(subdev, "invalid/missing timing entry\n");
547 			return -EINVAL;
548 		}
549 	}
550 
551 	ret = gt215_pll_info(device->clk, 0x12, 0x4000, freq, &mclk);
552 	if (ret < 0) {
553 		nvkm_error(subdev, "failed mclk calculation\n");
554 		return ret;
555 	}
556 
557 	gt215_ram_timing_calc(ram, timing);
558 
559 	ret = ram_init(fuc, ram->base.fb);
560 	if (ret)
561 		return ret;
562 
563 	/* Determine ram-specific MR values */
564 	ram->base.mr[0] = ram_rd32(fuc, mr[0]);
565 	ram->base.mr[1] = ram_rd32(fuc, mr[1]);
566 	ram->base.mr[2] = ram_rd32(fuc, mr[2]);
567 
568 	switch (ram->base.type) {
569 	case NVKM_RAM_TYPE_DDR2:
570 		ret = nvkm_sddr2_calc(&ram->base);
571 		break;
572 	case NVKM_RAM_TYPE_DDR3:
573 		ret = nvkm_sddr3_calc(&ram->base);
574 		break;
575 	case NVKM_RAM_TYPE_GDDR3:
576 		ret = nvkm_gddr3_calc(&ram->base);
577 		break;
578 	default:
579 		ret = -ENOSYS;
580 		break;
581 	}
582 
583 	if (ret)
584 		return ret;
585 
586 	/* XXX: 750MHz seems rather arbitrary */
587 	if (freq <= 750000) {
588 		r004018 = 0x10000000;
589 		r100760 = 0x22222222;
590 		r100da0 = 0x00000010;
591 	} else {
592 		r004018 = 0x00000000;
593 		r100760 = 0x00000000;
594 		r100da0 = 0x00000000;
595 	}
596 
597 	if (!next->bios.ramcfg_DLLoff)
598 		r004018 |= 0x00004000;
599 
600 	/* pll2pll requires to switch to a safe clock first */
601 	ctrl = ram_rd32(fuc, 0x004000);
602 	pll2pll = (!(ctrl & 0x00000008)) && mclk.pll;
603 
604 	/* Pre, NVIDIA does this outside the script */
605 	if (next->bios.ramcfg_10_02_10) {
606 		ram_mask(fuc, 0x111104, 0x00000600, 0x00000000);
607 	} else {
608 		ram_mask(fuc, 0x111100, 0x40000000, 0x40000000);
609 		ram_mask(fuc, 0x111104, 0x00000180, 0x00000000);
610 	}
611 	/* Always disable this bit during reclock */
612 	ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
613 
614 	/* If switching from non-pll to pll, lock before disabling FB */
615 	if (mclk.pll && !pll2pll) {
616 		ram_mask(fuc, 0x004128, 0x003f3141, mclk.clk | 0x00000101);
617 		gt215_ram_lock_pll(fuc, &mclk);
618 	}
619 
620 	/* Start with disabling some CRTCs and PFIFO? */
621 	ram_wait_vblank(fuc);
622 	ram_wr32(fuc, 0x611200, 0x3300);
623 	ram_mask(fuc, 0x002504, 0x1, 0x1);
624 	ram_nsec(fuc, 10000);
625 	ram_wait(fuc, 0x002504, 0x10, 0x10, 20000); /* XXX: or longer? */
626 	ram_block(fuc);
627 	ram_nsec(fuc, 2000);
628 
629 	if (!next->bios.ramcfg_10_02_10) {
630 		if (ram->base.type == NVKM_RAM_TYPE_GDDR3)
631 			ram_mask(fuc, 0x111100, 0x04020000, 0x00020000);
632 		else
633 			ram_mask(fuc, 0x111100, 0x04020000, 0x04020000);
634 	}
635 
636 	/* If we're disabling the DLL, do it now */
637 	switch (next->bios.ramcfg_DLLoff * ram->base.type) {
638 	case NVKM_RAM_TYPE_DDR3:
639 		nvkm_sddr3_dll_disable(fuc, ram->base.mr);
640 		break;
641 	case NVKM_RAM_TYPE_GDDR3:
642 		nvkm_gddr3_dll_disable(fuc, ram->base.mr);
643 		break;
644 	}
645 
646 	if (next->bios.timing_10_ODT)
647 		gt215_ram_gpio(fuc, 0x2e, 1);
648 
649 	/* Brace RAM for impact */
650 	ram_wr32(fuc, 0x1002d4, 0x00000001);
651 	ram_wr32(fuc, 0x1002d0, 0x00000001);
652 	ram_wr32(fuc, 0x1002d0, 0x00000001);
653 	ram_wr32(fuc, 0x100210, 0x00000000);
654 	ram_wr32(fuc, 0x1002dc, 0x00000001);
655 	ram_nsec(fuc, 2000);
656 
657 	if (device->chipset == 0xa3 && freq <= 500000)
658 		ram_mask(fuc, 0x100700, 0x00000006, 0x00000006);
659 
660 	/* Alter FBVDD/Q, apparently must be done with PLL disabled, thus
661 	 * set it to bypass */
662 	if (nvkm_gpio_get(gpio, 0, 0x18, DCB_GPIO_UNUSED) ==
663 			next->bios.ramcfg_FBVDDQ) {
664 		data = ram_rd32(fuc, 0x004000) & 0x9;
665 
666 		if (data == 0x1)
667 			ram_mask(fuc, 0x004000, 0x8, 0x8);
668 		if (data & 0x1)
669 			ram_mask(fuc, 0x004000, 0x1, 0x0);
670 
671 		gt215_ram_gpio(fuc, 0x18, !next->bios.ramcfg_FBVDDQ);
672 
673 		if (data & 0x1)
674 			ram_mask(fuc, 0x004000, 0x1, 0x1);
675 	}
676 
677 	/* Fiddle with clocks */
678 	/* There's 4 scenario's
679 	 * pll->pll: first switch to a 324MHz clock, set up new PLL, switch
680 	 * clk->pll: Set up new PLL, switch
681 	 * pll->clk: Set up clock, switch
682 	 * clk->clk: Overwrite ctrl and other bits, switch */
683 
684 	/* Switch to regular clock - 324MHz */
685 	if (pll2pll) {
686 		ram_mask(fuc, 0x004000, 0x00000004, 0x00000004);
687 		ram_mask(fuc, 0x004168, 0x003f3141, 0x00083101);
688 		ram_mask(fuc, 0x004000, 0x00000008, 0x00000008);
689 		ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
690 		ram_wr32(fuc, 0x004018, 0x00001000);
691 		gt215_ram_lock_pll(fuc, &mclk);
692 	}
693 
694 	if (mclk.pll) {
695 		ram_mask(fuc, 0x004000, 0x00000105, 0x00000105);
696 		ram_wr32(fuc, 0x004018, 0x00001000 | r004018);
697 		ram_wr32(fuc, 0x100da0, r100da0);
698 	} else {
699 		ram_mask(fuc, 0x004168, 0x003f3141, mclk.clk | 0x00000101);
700 		ram_mask(fuc, 0x004000, 0x00000108, 0x00000008);
701 		ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
702 		ram_wr32(fuc, 0x004018, 0x00009000 | r004018);
703 		ram_wr32(fuc, 0x100da0, r100da0);
704 	}
705 	ram_nsec(fuc, 20000);
706 
707 	if (next->bios.rammap_10_04_08) {
708 		ram_wr32(fuc, 0x1005a0, next->bios.ramcfg_10_06 << 16 |
709 					next->bios.ramcfg_10_05 << 8 |
710 					next->bios.ramcfg_10_05);
711 		ram_wr32(fuc, 0x1005a4, next->bios.ramcfg_10_08 << 8 |
712 					next->bios.ramcfg_10_07);
713 		ram_wr32(fuc, 0x10f804, next->bios.ramcfg_10_09_f0 << 20 |
714 					next->bios.ramcfg_10_03_0f << 16 |
715 					next->bios.ramcfg_10_09_0f |
716 					0x80000000);
717 		ram_mask(fuc, 0x10053c, 0x00001000, 0x00000000);
718 	} else {
719 		if (train->state == NVA3_TRAIN_DONE) {
720 			ram_wr32(fuc, 0x100080, 0x1020);
721 			ram_mask(fuc, 0x111400, 0xffffffff, train->r_111400);
722 			ram_mask(fuc, 0x1111e0, 0xffffffff, train->r_1111e0);
723 			ram_mask(fuc, 0x100720, 0xffffffff, train->r_100720);
724 		}
725 		ram_mask(fuc, 0x10053c, 0x00001000, 0x00001000);
726 		ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
727 		ram_mask(fuc, 0x100760, 0x22222222, r100760);
728 		ram_mask(fuc, 0x1007a0, 0x22222222, r100760);
729 		ram_mask(fuc, 0x1007e0, 0x22222222, r100760);
730 	}
731 
732 	if (device->chipset == 0xa3 && freq > 500000) {
733 		ram_mask(fuc, 0x100700, 0x00000006, 0x00000000);
734 	}
735 
736 	/* Final switch */
737 	if (mclk.pll) {
738 		ram_mask(fuc, 0x1110e0, 0x00088000, 0x00011000);
739 		ram_mask(fuc, 0x004000, 0x00000008, 0x00000000);
740 	}
741 
742 	ram_wr32(fuc, 0x1002dc, 0x00000000);
743 	ram_wr32(fuc, 0x1002d4, 0x00000001);
744 	ram_wr32(fuc, 0x100210, 0x80000000);
745 	ram_nsec(fuc, 2000);
746 
747 	/* Set RAM MR parameters and timings */
748 	for (i = 2; i >= 0; i--) {
749 		if (ram_rd32(fuc, mr[i]) != ram->base.mr[i]) {
750 			ram_wr32(fuc, mr[i], ram->base.mr[i]);
751 			ram_nsec(fuc, 1000);
752 		}
753 	}
754 
755 	ram_wr32(fuc, 0x100220[3], timing[3]);
756 	ram_wr32(fuc, 0x100220[1], timing[1]);
757 	ram_wr32(fuc, 0x100220[6], timing[6]);
758 	ram_wr32(fuc, 0x100220[7], timing[7]);
759 	ram_wr32(fuc, 0x100220[2], timing[2]);
760 	ram_wr32(fuc, 0x100220[4], timing[4]);
761 	ram_wr32(fuc, 0x100220[5], timing[5]);
762 	ram_wr32(fuc, 0x100220[0], timing[0]);
763 	ram_wr32(fuc, 0x100220[8], timing[8]);
764 
765 	/* Misc */
766 	ram_mask(fuc, 0x100200, 0x00001000, !next->bios.ramcfg_10_02_08 << 12);
767 
768 	/* XXX: A lot of "chipset"/"ram type" specific stuff...? */
769 	unk714  = ram_rd32(fuc, 0x100714) & ~0xf0000130;
770 	unk718  = ram_rd32(fuc, 0x100718) & ~0x00000100;
771 	unk71c  = ram_rd32(fuc, 0x10071c) & ~0x00000100;
772 	r111100 = ram_rd32(fuc, 0x111100) & ~0x3a800000;
773 
774 	/* NVA8 seems to skip various bits related to ramcfg_10_02_04 */
775 	if (device->chipset == 0xa8) {
776 		r111100 |= 0x08000000;
777 		if (!next->bios.ramcfg_10_02_04)
778 			unk714  |= 0x00000010;
779 	} else {
780 		if (next->bios.ramcfg_10_02_04) {
781 			switch (ram->base.type) {
782 			case NVKM_RAM_TYPE_DDR2:
783 			case NVKM_RAM_TYPE_DDR3:
784 				r111100 &= ~0x00000020;
785 				if (next->bios.ramcfg_10_02_10)
786 					r111100 |= 0x08000004;
787 				else
788 					r111100 |= 0x00000024;
789 				break;
790 			default:
791 				break;
792 			}
793 		} else {
794 			switch (ram->base.type) {
795 			case NVKM_RAM_TYPE_DDR2:
796 			case NVKM_RAM_TYPE_DDR3:
797 				r111100 &= ~0x00000024;
798 				r111100 |=  0x12800000;
799 
800 				if (next->bios.ramcfg_10_02_10)
801 					r111100 |= 0x08000000;
802 				unk714  |= 0x00000010;
803 				break;
804 			case NVKM_RAM_TYPE_GDDR3:
805 				r111100 |= 0x30000000;
806 				unk714  |= 0x00000020;
807 				break;
808 			default:
809 				break;
810 			}
811 		}
812 	}
813 
814 	unk714 |= (next->bios.ramcfg_10_04_01) << 8;
815 
816 	if (next->bios.ramcfg_10_02_20)
817 		unk714 |= 0xf0000000;
818 	if (next->bios.ramcfg_10_02_02)
819 		unk718 |= 0x00000100;
820 	if (next->bios.ramcfg_10_02_01)
821 		unk71c |= 0x00000100;
822 	if (next->bios.timing_10_24 != 0xff) {
823 		unk718 &= ~0xf0000000;
824 		unk718 |= next->bios.timing_10_24 << 28;
825 	}
826 	if (next->bios.ramcfg_10_02_10)
827 		r111100 &= ~0x04020000;
828 
829 	ram_mask(fuc, 0x100714, 0xffffffff, unk714);
830 	ram_mask(fuc, 0x10071c, 0xffffffff, unk71c);
831 	ram_mask(fuc, 0x100718, 0xffffffff, unk718);
832 	ram_mask(fuc, 0x111100, 0xffffffff, r111100);
833 
834 	if (!next->bios.timing_10_ODT)
835 		gt215_ram_gpio(fuc, 0x2e, 0);
836 
837 	/* Reset DLL */
838 	if (!next->bios.ramcfg_DLLoff)
839 		nvkm_sddr2_dll_reset(fuc);
840 
841 	if (ram->base.type == NVKM_RAM_TYPE_GDDR3) {
842 		ram_nsec(fuc, 31000);
843 	} else {
844 		ram_nsec(fuc, 14000);
845 	}
846 
847 	if (ram->base.type == NVKM_RAM_TYPE_DDR3) {
848 		ram_wr32(fuc, 0x100264, 0x1);
849 		ram_nsec(fuc, 2000);
850 	}
851 
852 	ram_nuke(fuc, 0x100700);
853 	ram_mask(fuc, 0x100700, 0x01000000, 0x01000000);
854 	ram_mask(fuc, 0x100700, 0x01000000, 0x00000000);
855 
856 	/* Re-enable FB */
857 	ram_unblock(fuc);
858 	ram_wr32(fuc, 0x611200, 0x3330);
859 
860 	/* Post fiddlings */
861 	if (next->bios.rammap_10_04_02)
862 		ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
863 	if (next->bios.ramcfg_10_02_10) {
864 		ram_mask(fuc, 0x111104, 0x00000180, 0x00000180);
865 		ram_mask(fuc, 0x111100, 0x40000000, 0x00000000);
866 	} else {
867 		ram_mask(fuc, 0x111104, 0x00000600, 0x00000600);
868 	}
869 
870 	if (mclk.pll) {
871 		ram_mask(fuc, 0x004168, 0x00000001, 0x00000000);
872 		ram_mask(fuc, 0x004168, 0x00000100, 0x00000000);
873 	} else {
874 		ram_mask(fuc, 0x004000, 0x00000001, 0x00000000);
875 		ram_mask(fuc, 0x004128, 0x00000001, 0x00000000);
876 		ram_mask(fuc, 0x004128, 0x00000100, 0x00000000);
877 	}
878 
879 	return 0;
880 }
881 
882 static int
gt215_ram_prog(struct nvkm_ram * base)883 gt215_ram_prog(struct nvkm_ram *base)
884 {
885 	struct gt215_ram *ram = gt215_ram(base);
886 	struct gt215_ramfuc *fuc = &ram->fuc;
887 	struct nvkm_device *device = ram->base.fb->subdev.device;
888 	bool exec = nvkm_boolopt(device->cfgopt, "NvMemExec", true);
889 
890 	if (exec) {
891 		nvkm_mask(device, 0x001534, 0x2, 0x2);
892 
893 		ram_exec(fuc, true);
894 
895 		/* Post-processing, avoids flicker */
896 		nvkm_mask(device, 0x002504, 0x1, 0x0);
897 		nvkm_mask(device, 0x001534, 0x2, 0x0);
898 
899 		nvkm_mask(device, 0x616308, 0x10, 0x10);
900 		nvkm_mask(device, 0x616b08, 0x10, 0x10);
901 	} else {
902 		ram_exec(fuc, false);
903 	}
904 	return 0;
905 }
906 
907 static void
gt215_ram_tidy(struct nvkm_ram * base)908 gt215_ram_tidy(struct nvkm_ram *base)
909 {
910 	struct gt215_ram *ram = gt215_ram(base);
911 	ram_exec(&ram->fuc, false);
912 }
913 
914 static int
gt215_ram_init(struct nvkm_ram * base)915 gt215_ram_init(struct nvkm_ram *base)
916 {
917 	struct gt215_ram *ram = gt215_ram(base);
918 	gt215_link_train_init(ram);
919 	return 0;
920 }
921 
922 static void *
gt215_ram_dtor(struct nvkm_ram * base)923 gt215_ram_dtor(struct nvkm_ram *base)
924 {
925 	struct gt215_ram *ram = gt215_ram(base);
926 	gt215_link_train_fini(ram);
927 	return ram;
928 }
929 
930 static const struct nvkm_ram_func
931 gt215_ram_func = {
932 	.dtor = gt215_ram_dtor,
933 	.init = gt215_ram_init,
934 	.calc = gt215_ram_calc,
935 	.prog = gt215_ram_prog,
936 	.tidy = gt215_ram_tidy,
937 };
938 
939 int
gt215_ram_new(struct nvkm_fb * fb,struct nvkm_ram ** pram)940 gt215_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
941 {
942 	struct gt215_ram *ram;
943 	int ret, i;
944 
945 	if (!(ram = kzalloc(sizeof(*ram), GFP_KERNEL)))
946 		return -ENOMEM;
947 	*pram = &ram->base;
948 
949 	ret = nv50_ram_ctor(&gt215_ram_func, fb, &ram->base);
950 	if (ret)
951 		return ret;
952 
953 	ram->fuc.r_0x001610 = ramfuc_reg(0x001610);
954 	ram->fuc.r_0x001700 = ramfuc_reg(0x001700);
955 	ram->fuc.r_0x002504 = ramfuc_reg(0x002504);
956 	ram->fuc.r_0x004000 = ramfuc_reg(0x004000);
957 	ram->fuc.r_0x004004 = ramfuc_reg(0x004004);
958 	ram->fuc.r_0x004018 = ramfuc_reg(0x004018);
959 	ram->fuc.r_0x004128 = ramfuc_reg(0x004128);
960 	ram->fuc.r_0x004168 = ramfuc_reg(0x004168);
961 	ram->fuc.r_0x100080 = ramfuc_reg(0x100080);
962 	ram->fuc.r_0x100200 = ramfuc_reg(0x100200);
963 	ram->fuc.r_0x100210 = ramfuc_reg(0x100210);
964 	for (i = 0; i < 9; i++)
965 		ram->fuc.r_0x100220[i] = ramfuc_reg(0x100220 + (i * 4));
966 	ram->fuc.r_0x100264 = ramfuc_reg(0x100264);
967 	ram->fuc.r_0x1002d0 = ramfuc_reg(0x1002d0);
968 	ram->fuc.r_0x1002d4 = ramfuc_reg(0x1002d4);
969 	ram->fuc.r_0x1002dc = ramfuc_reg(0x1002dc);
970 	ram->fuc.r_0x10053c = ramfuc_reg(0x10053c);
971 	ram->fuc.r_0x1005a0 = ramfuc_reg(0x1005a0);
972 	ram->fuc.r_0x1005a4 = ramfuc_reg(0x1005a4);
973 	ram->fuc.r_0x100700 = ramfuc_reg(0x100700);
974 	ram->fuc.r_0x100714 = ramfuc_reg(0x100714);
975 	ram->fuc.r_0x100718 = ramfuc_reg(0x100718);
976 	ram->fuc.r_0x10071c = ramfuc_reg(0x10071c);
977 	ram->fuc.r_0x100720 = ramfuc_reg(0x100720);
978 	ram->fuc.r_0x100760 = ramfuc_stride(0x100760, 4, ram->base.part_mask);
979 	ram->fuc.r_0x1007a0 = ramfuc_stride(0x1007a0, 4, ram->base.part_mask);
980 	ram->fuc.r_0x1007e0 = ramfuc_stride(0x1007e0, 4, ram->base.part_mask);
981 	ram->fuc.r_0x100da0 = ramfuc_stride(0x100da0, 4, ram->base.part_mask);
982 	ram->fuc.r_0x10f804 = ramfuc_reg(0x10f804);
983 	ram->fuc.r_0x1110e0 = ramfuc_stride(0x1110e0, 4, ram->base.part_mask);
984 	ram->fuc.r_0x111100 = ramfuc_reg(0x111100);
985 	ram->fuc.r_0x111104 = ramfuc_reg(0x111104);
986 	ram->fuc.r_0x1111e0 = ramfuc_reg(0x1111e0);
987 	ram->fuc.r_0x111400 = ramfuc_reg(0x111400);
988 	ram->fuc.r_0x611200 = ramfuc_reg(0x611200);
989 
990 	if (ram->base.ranks > 1) {
991 		ram->fuc.r_mr[0] = ramfuc_reg2(0x1002c0, 0x1002c8);
992 		ram->fuc.r_mr[1] = ramfuc_reg2(0x1002c4, 0x1002cc);
993 		ram->fuc.r_mr[2] = ramfuc_reg2(0x1002e0, 0x1002e8);
994 		ram->fuc.r_mr[3] = ramfuc_reg2(0x1002e4, 0x1002ec);
995 	} else {
996 		ram->fuc.r_mr[0] = ramfuc_reg(0x1002c0);
997 		ram->fuc.r_mr[1] = ramfuc_reg(0x1002c4);
998 		ram->fuc.r_mr[2] = ramfuc_reg(0x1002e0);
999 		ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);
1000 	}
1001 	ram->fuc.r_gpio[0] = ramfuc_reg(0x00e104);
1002 	ram->fuc.r_gpio[1] = ramfuc_reg(0x00e108);
1003 	ram->fuc.r_gpio[2] = ramfuc_reg(0x00e120);
1004 	ram->fuc.r_gpio[3] = ramfuc_reg(0x00e124);
1005 
1006 	return 0;
1007 }
1008