1 /*
2  * Copyright 2007 Matthieu CASTET <castet.matthieu@free.fr>
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragr) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22  * DEALINGS IN THE SOFTWARE.
23  */
24 #include "priv.h"
25 #include "regs.h"
26 
27 #include <core/client.h>
28 #include <engine/fifo.h>
29 #include <engine/fifo/chan.h>
30 #include <subdev/fb.h>
31 
32 struct pipe_state {
33 	u32 pipe_0x0000[0x040/4];
34 	u32 pipe_0x0040[0x010/4];
35 	u32 pipe_0x0200[0x0c0/4];
36 	u32 pipe_0x4400[0x080/4];
37 	u32 pipe_0x6400[0x3b0/4];
38 	u32 pipe_0x6800[0x2f0/4];
39 	u32 pipe_0x6c00[0x030/4];
40 	u32 pipe_0x7000[0x130/4];
41 	u32 pipe_0x7400[0x0c0/4];
42 	u32 pipe_0x7800[0x0c0/4];
43 };
44 
45 static int nv10_gr_ctx_regs[] = {
46 	NV10_PGRAPH_CTX_SWITCH(0),
47 	NV10_PGRAPH_CTX_SWITCH(1),
48 	NV10_PGRAPH_CTX_SWITCH(2),
49 	NV10_PGRAPH_CTX_SWITCH(3),
50 	NV10_PGRAPH_CTX_SWITCH(4),
51 	NV10_PGRAPH_CTX_CACHE(0, 0),
52 	NV10_PGRAPH_CTX_CACHE(0, 1),
53 	NV10_PGRAPH_CTX_CACHE(0, 2),
54 	NV10_PGRAPH_CTX_CACHE(0, 3),
55 	NV10_PGRAPH_CTX_CACHE(0, 4),
56 	NV10_PGRAPH_CTX_CACHE(1, 0),
57 	NV10_PGRAPH_CTX_CACHE(1, 1),
58 	NV10_PGRAPH_CTX_CACHE(1, 2),
59 	NV10_PGRAPH_CTX_CACHE(1, 3),
60 	NV10_PGRAPH_CTX_CACHE(1, 4),
61 	NV10_PGRAPH_CTX_CACHE(2, 0),
62 	NV10_PGRAPH_CTX_CACHE(2, 1),
63 	NV10_PGRAPH_CTX_CACHE(2, 2),
64 	NV10_PGRAPH_CTX_CACHE(2, 3),
65 	NV10_PGRAPH_CTX_CACHE(2, 4),
66 	NV10_PGRAPH_CTX_CACHE(3, 0),
67 	NV10_PGRAPH_CTX_CACHE(3, 1),
68 	NV10_PGRAPH_CTX_CACHE(3, 2),
69 	NV10_PGRAPH_CTX_CACHE(3, 3),
70 	NV10_PGRAPH_CTX_CACHE(3, 4),
71 	NV10_PGRAPH_CTX_CACHE(4, 0),
72 	NV10_PGRAPH_CTX_CACHE(4, 1),
73 	NV10_PGRAPH_CTX_CACHE(4, 2),
74 	NV10_PGRAPH_CTX_CACHE(4, 3),
75 	NV10_PGRAPH_CTX_CACHE(4, 4),
76 	NV10_PGRAPH_CTX_CACHE(5, 0),
77 	NV10_PGRAPH_CTX_CACHE(5, 1),
78 	NV10_PGRAPH_CTX_CACHE(5, 2),
79 	NV10_PGRAPH_CTX_CACHE(5, 3),
80 	NV10_PGRAPH_CTX_CACHE(5, 4),
81 	NV10_PGRAPH_CTX_CACHE(6, 0),
82 	NV10_PGRAPH_CTX_CACHE(6, 1),
83 	NV10_PGRAPH_CTX_CACHE(6, 2),
84 	NV10_PGRAPH_CTX_CACHE(6, 3),
85 	NV10_PGRAPH_CTX_CACHE(6, 4),
86 	NV10_PGRAPH_CTX_CACHE(7, 0),
87 	NV10_PGRAPH_CTX_CACHE(7, 1),
88 	NV10_PGRAPH_CTX_CACHE(7, 2),
89 	NV10_PGRAPH_CTX_CACHE(7, 3),
90 	NV10_PGRAPH_CTX_CACHE(7, 4),
91 	NV10_PGRAPH_CTX_USER,
92 	NV04_PGRAPH_DMA_START_0,
93 	NV04_PGRAPH_DMA_START_1,
94 	NV04_PGRAPH_DMA_LENGTH,
95 	NV04_PGRAPH_DMA_MISC,
96 	NV10_PGRAPH_DMA_PITCH,
97 	NV04_PGRAPH_BOFFSET0,
98 	NV04_PGRAPH_BBASE0,
99 	NV04_PGRAPH_BLIMIT0,
100 	NV04_PGRAPH_BOFFSET1,
101 	NV04_PGRAPH_BBASE1,
102 	NV04_PGRAPH_BLIMIT1,
103 	NV04_PGRAPH_BOFFSET2,
104 	NV04_PGRAPH_BBASE2,
105 	NV04_PGRAPH_BLIMIT2,
106 	NV04_PGRAPH_BOFFSET3,
107 	NV04_PGRAPH_BBASE3,
108 	NV04_PGRAPH_BLIMIT3,
109 	NV04_PGRAPH_BOFFSET4,
110 	NV04_PGRAPH_BBASE4,
111 	NV04_PGRAPH_BLIMIT4,
112 	NV04_PGRAPH_BOFFSET5,
113 	NV04_PGRAPH_BBASE5,
114 	NV04_PGRAPH_BLIMIT5,
115 	NV04_PGRAPH_BPITCH0,
116 	NV04_PGRAPH_BPITCH1,
117 	NV04_PGRAPH_BPITCH2,
118 	NV04_PGRAPH_BPITCH3,
119 	NV04_PGRAPH_BPITCH4,
120 	NV10_PGRAPH_SURFACE,
121 	NV10_PGRAPH_STATE,
122 	NV04_PGRAPH_BSWIZZLE2,
123 	NV04_PGRAPH_BSWIZZLE5,
124 	NV04_PGRAPH_BPIXEL,
125 	NV10_PGRAPH_NOTIFY,
126 	NV04_PGRAPH_PATT_COLOR0,
127 	NV04_PGRAPH_PATT_COLOR1,
128 	NV04_PGRAPH_PATT_COLORRAM, /* 64 values from 0x400900 to 0x4009fc */
129 	0x00400904,
130 	0x00400908,
131 	0x0040090c,
132 	0x00400910,
133 	0x00400914,
134 	0x00400918,
135 	0x0040091c,
136 	0x00400920,
137 	0x00400924,
138 	0x00400928,
139 	0x0040092c,
140 	0x00400930,
141 	0x00400934,
142 	0x00400938,
143 	0x0040093c,
144 	0x00400940,
145 	0x00400944,
146 	0x00400948,
147 	0x0040094c,
148 	0x00400950,
149 	0x00400954,
150 	0x00400958,
151 	0x0040095c,
152 	0x00400960,
153 	0x00400964,
154 	0x00400968,
155 	0x0040096c,
156 	0x00400970,
157 	0x00400974,
158 	0x00400978,
159 	0x0040097c,
160 	0x00400980,
161 	0x00400984,
162 	0x00400988,
163 	0x0040098c,
164 	0x00400990,
165 	0x00400994,
166 	0x00400998,
167 	0x0040099c,
168 	0x004009a0,
169 	0x004009a4,
170 	0x004009a8,
171 	0x004009ac,
172 	0x004009b0,
173 	0x004009b4,
174 	0x004009b8,
175 	0x004009bc,
176 	0x004009c0,
177 	0x004009c4,
178 	0x004009c8,
179 	0x004009cc,
180 	0x004009d0,
181 	0x004009d4,
182 	0x004009d8,
183 	0x004009dc,
184 	0x004009e0,
185 	0x004009e4,
186 	0x004009e8,
187 	0x004009ec,
188 	0x004009f0,
189 	0x004009f4,
190 	0x004009f8,
191 	0x004009fc,
192 	NV04_PGRAPH_PATTERN,	/* 2 values from 0x400808 to 0x40080c */
193 	0x0040080c,
194 	NV04_PGRAPH_PATTERN_SHAPE,
195 	NV03_PGRAPH_MONO_COLOR0,
196 	NV04_PGRAPH_ROP3,
197 	NV04_PGRAPH_CHROMA,
198 	NV04_PGRAPH_BETA_AND,
199 	NV04_PGRAPH_BETA_PREMULT,
200 	0x00400e70,
201 	0x00400e74,
202 	0x00400e78,
203 	0x00400e7c,
204 	0x00400e80,
205 	0x00400e84,
206 	0x00400e88,
207 	0x00400e8c,
208 	0x00400ea0,
209 	0x00400ea4,
210 	0x00400ea8,
211 	0x00400e90,
212 	0x00400e94,
213 	0x00400e98,
214 	0x00400e9c,
215 	NV10_PGRAPH_WINDOWCLIP_HORIZONTAL, /* 8 values from 0x400f00-0x400f1c */
216 	NV10_PGRAPH_WINDOWCLIP_VERTICAL,   /* 8 values from 0x400f20-0x400f3c */
217 	0x00400f04,
218 	0x00400f24,
219 	0x00400f08,
220 	0x00400f28,
221 	0x00400f0c,
222 	0x00400f2c,
223 	0x00400f10,
224 	0x00400f30,
225 	0x00400f14,
226 	0x00400f34,
227 	0x00400f18,
228 	0x00400f38,
229 	0x00400f1c,
230 	0x00400f3c,
231 	NV10_PGRAPH_XFMODE0,
232 	NV10_PGRAPH_XFMODE1,
233 	NV10_PGRAPH_GLOBALSTATE0,
234 	NV10_PGRAPH_GLOBALSTATE1,
235 	NV04_PGRAPH_STORED_FMT,
236 	NV04_PGRAPH_SOURCE_COLOR,
237 	NV03_PGRAPH_ABS_X_RAM,	/* 32 values from 0x400400 to 0x40047c */
238 	NV03_PGRAPH_ABS_Y_RAM,	/* 32 values from 0x400480 to 0x4004fc */
239 	0x00400404,
240 	0x00400484,
241 	0x00400408,
242 	0x00400488,
243 	0x0040040c,
244 	0x0040048c,
245 	0x00400410,
246 	0x00400490,
247 	0x00400414,
248 	0x00400494,
249 	0x00400418,
250 	0x00400498,
251 	0x0040041c,
252 	0x0040049c,
253 	0x00400420,
254 	0x004004a0,
255 	0x00400424,
256 	0x004004a4,
257 	0x00400428,
258 	0x004004a8,
259 	0x0040042c,
260 	0x004004ac,
261 	0x00400430,
262 	0x004004b0,
263 	0x00400434,
264 	0x004004b4,
265 	0x00400438,
266 	0x004004b8,
267 	0x0040043c,
268 	0x004004bc,
269 	0x00400440,
270 	0x004004c0,
271 	0x00400444,
272 	0x004004c4,
273 	0x00400448,
274 	0x004004c8,
275 	0x0040044c,
276 	0x004004cc,
277 	0x00400450,
278 	0x004004d0,
279 	0x00400454,
280 	0x004004d4,
281 	0x00400458,
282 	0x004004d8,
283 	0x0040045c,
284 	0x004004dc,
285 	0x00400460,
286 	0x004004e0,
287 	0x00400464,
288 	0x004004e4,
289 	0x00400468,
290 	0x004004e8,
291 	0x0040046c,
292 	0x004004ec,
293 	0x00400470,
294 	0x004004f0,
295 	0x00400474,
296 	0x004004f4,
297 	0x00400478,
298 	0x004004f8,
299 	0x0040047c,
300 	0x004004fc,
301 	NV03_PGRAPH_ABS_UCLIP_XMIN,
302 	NV03_PGRAPH_ABS_UCLIP_XMAX,
303 	NV03_PGRAPH_ABS_UCLIP_YMIN,
304 	NV03_PGRAPH_ABS_UCLIP_YMAX,
305 	0x00400550,
306 	0x00400558,
307 	0x00400554,
308 	0x0040055c,
309 	NV03_PGRAPH_ABS_UCLIPA_XMIN,
310 	NV03_PGRAPH_ABS_UCLIPA_XMAX,
311 	NV03_PGRAPH_ABS_UCLIPA_YMIN,
312 	NV03_PGRAPH_ABS_UCLIPA_YMAX,
313 	NV03_PGRAPH_ABS_ICLIP_XMAX,
314 	NV03_PGRAPH_ABS_ICLIP_YMAX,
315 	NV03_PGRAPH_XY_LOGIC_MISC0,
316 	NV03_PGRAPH_XY_LOGIC_MISC1,
317 	NV03_PGRAPH_XY_LOGIC_MISC2,
318 	NV03_PGRAPH_XY_LOGIC_MISC3,
319 	NV03_PGRAPH_CLIPX_0,
320 	NV03_PGRAPH_CLIPX_1,
321 	NV03_PGRAPH_CLIPY_0,
322 	NV03_PGRAPH_CLIPY_1,
323 	NV10_PGRAPH_COMBINER0_IN_ALPHA,
324 	NV10_PGRAPH_COMBINER1_IN_ALPHA,
325 	NV10_PGRAPH_COMBINER0_IN_RGB,
326 	NV10_PGRAPH_COMBINER1_IN_RGB,
327 	NV10_PGRAPH_COMBINER_COLOR0,
328 	NV10_PGRAPH_COMBINER_COLOR1,
329 	NV10_PGRAPH_COMBINER0_OUT_ALPHA,
330 	NV10_PGRAPH_COMBINER1_OUT_ALPHA,
331 	NV10_PGRAPH_COMBINER0_OUT_RGB,
332 	NV10_PGRAPH_COMBINER1_OUT_RGB,
333 	NV10_PGRAPH_COMBINER_FINAL0,
334 	NV10_PGRAPH_COMBINER_FINAL1,
335 	0x00400e00,
336 	0x00400e04,
337 	0x00400e08,
338 	0x00400e0c,
339 	0x00400e10,
340 	0x00400e14,
341 	0x00400e18,
342 	0x00400e1c,
343 	0x00400e20,
344 	0x00400e24,
345 	0x00400e28,
346 	0x00400e2c,
347 	0x00400e30,
348 	0x00400e34,
349 	0x00400e38,
350 	0x00400e3c,
351 	NV04_PGRAPH_PASSTHRU_0,
352 	NV04_PGRAPH_PASSTHRU_1,
353 	NV04_PGRAPH_PASSTHRU_2,
354 	NV10_PGRAPH_DIMX_TEXTURE,
355 	NV10_PGRAPH_WDIMX_TEXTURE,
356 	NV10_PGRAPH_DVD_COLORFMT,
357 	NV10_PGRAPH_SCALED_FORMAT,
358 	NV04_PGRAPH_MISC24_0,
359 	NV04_PGRAPH_MISC24_1,
360 	NV04_PGRAPH_MISC24_2,
361 	NV03_PGRAPH_X_MISC,
362 	NV03_PGRAPH_Y_MISC,
363 	NV04_PGRAPH_VALID1,
364 	NV04_PGRAPH_VALID2,
365 };
366 
367 static int nv17_gr_ctx_regs[] = {
368 	NV10_PGRAPH_DEBUG_4,
369 	0x004006b0,
370 	0x00400eac,
371 	0x00400eb0,
372 	0x00400eb4,
373 	0x00400eb8,
374 	0x00400ebc,
375 	0x00400ec0,
376 	0x00400ec4,
377 	0x00400ec8,
378 	0x00400ecc,
379 	0x00400ed0,
380 	0x00400ed4,
381 	0x00400ed8,
382 	0x00400edc,
383 	0x00400ee0,
384 	0x00400a00,
385 	0x00400a04,
386 };
387 
388 #define nv10_gr(p) container_of((p), struct nv10_gr, base)
389 
390 struct nv10_gr {
391 	struct nvkm_gr base;
392 	struct nv10_gr_chan *chan[32];
393 	spinlock_t lock;
394 };
395 
396 #define nv10_gr_chan(p) container_of((p), struct nv10_gr_chan, object)
397 
398 struct nv10_gr_chan {
399 	struct nvkm_object object;
400 	struct nv10_gr *gr;
401 	int chid;
402 	int nv10[ARRAY_SIZE(nv10_gr_ctx_regs)];
403 	int nv17[ARRAY_SIZE(nv17_gr_ctx_regs)];
404 	struct pipe_state pipe_state;
405 	u32 lma_window[4];
406 };
407 
408 
409 /*******************************************************************************
410  * Graphics object classes
411  ******************************************************************************/
412 
413 #define PIPE_SAVE(gr, state, addr)					\
414 	do {								\
415 		int __i;						\
416 		nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, addr);		\
417 		for (__i = 0; __i < ARRAY_SIZE(state); __i++)		\
418 			state[__i] = nvkm_rd32(device, NV10_PGRAPH_PIPE_DATA); \
419 	} while (0)
420 
421 #define PIPE_RESTORE(gr, state, addr)					\
422 	do {								\
423 		int __i;						\
424 		nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, addr);		\
425 		for (__i = 0; __i < ARRAY_SIZE(state); __i++)		\
426 			nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, state[__i]); \
427 	} while (0)
428 
429 static void
430 nv17_gr_mthd_lma_window(struct nv10_gr_chan *chan, u32 mthd, u32 data)
431 {
432 	struct nvkm_device *device = chan->object.engine->subdev.device;
433 	struct nvkm_gr *gr = &chan->gr->base;
434 	struct pipe_state *pipe = &chan->pipe_state;
435 	u32 pipe_0x0040[1], pipe_0x64c0[8], pipe_0x6a80[3], pipe_0x6ab0[3];
436 	u32 xfmode0, xfmode1;
437 	int i;
438 
439 	chan->lma_window[(mthd - 0x1638) / 4] = data;
440 
441 	if (mthd != 0x1644)
442 		return;
443 
444 	nv04_gr_idle(gr);
445 
446 	PIPE_SAVE(device, pipe_0x0040, 0x0040);
447 	PIPE_SAVE(device, pipe->pipe_0x0200, 0x0200);
448 
449 	PIPE_RESTORE(device, chan->lma_window, 0x6790);
450 
451 	nv04_gr_idle(gr);
452 
453 	xfmode0 = nvkm_rd32(device, NV10_PGRAPH_XFMODE0);
454 	xfmode1 = nvkm_rd32(device, NV10_PGRAPH_XFMODE1);
455 
456 	PIPE_SAVE(device, pipe->pipe_0x4400, 0x4400);
457 	PIPE_SAVE(device, pipe_0x64c0, 0x64c0);
458 	PIPE_SAVE(device, pipe_0x6ab0, 0x6ab0);
459 	PIPE_SAVE(device, pipe_0x6a80, 0x6a80);
460 
461 	nv04_gr_idle(gr);
462 
463 	nvkm_wr32(device, NV10_PGRAPH_XFMODE0, 0x10000000);
464 	nvkm_wr32(device, NV10_PGRAPH_XFMODE1, 0x00000000);
465 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x000064c0);
466 	for (i = 0; i < 4; i++)
467 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
468 	for (i = 0; i < 4; i++)
469 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x00000000);
470 
471 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x00006ab0);
472 	for (i = 0; i < 3; i++)
473 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
474 
475 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x00006a80);
476 	for (i = 0; i < 3; i++)
477 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x00000000);
478 
479 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x00000040);
480 	nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x00000008);
481 
482 	PIPE_RESTORE(device, pipe->pipe_0x0200, 0x0200);
483 
484 	nv04_gr_idle(gr);
485 
486 	PIPE_RESTORE(device, pipe_0x0040, 0x0040);
487 
488 	nvkm_wr32(device, NV10_PGRAPH_XFMODE0, xfmode0);
489 	nvkm_wr32(device, NV10_PGRAPH_XFMODE1, xfmode1);
490 
491 	PIPE_RESTORE(device, pipe_0x64c0, 0x64c0);
492 	PIPE_RESTORE(device, pipe_0x6ab0, 0x6ab0);
493 	PIPE_RESTORE(device, pipe_0x6a80, 0x6a80);
494 	PIPE_RESTORE(device, pipe->pipe_0x4400, 0x4400);
495 
496 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x000000c0);
497 	nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x00000000);
498 
499 	nv04_gr_idle(gr);
500 }
501 
502 static void
503 nv17_gr_mthd_lma_enable(struct nv10_gr_chan *chan, u32 mthd, u32 data)
504 {
505 	struct nvkm_device *device = chan->object.engine->subdev.device;
506 	struct nvkm_gr *gr = &chan->gr->base;
507 
508 	nv04_gr_idle(gr);
509 
510 	nvkm_mask(device, NV10_PGRAPH_DEBUG_4, 0x00000100, 0x00000100);
511 	nvkm_mask(device, 0x4006b0, 0x08000000, 0x08000000);
512 }
513 
514 static bool
515 nv17_gr_mthd_celcius(struct nv10_gr_chan *chan, u32 mthd, u32 data)
516 {
517 	void (*func)(struct nv10_gr_chan *, u32, u32);
518 	switch (mthd) {
519 	case 0x1638 ... 0x1644:
520 		     func = nv17_gr_mthd_lma_window; break;
521 	case 0x1658: func = nv17_gr_mthd_lma_enable; break;
522 	default:
523 		return false;
524 	}
525 	func(chan, mthd, data);
526 	return true;
527 }
528 
529 static bool
530 nv10_gr_mthd(struct nv10_gr_chan *chan, u8 class, u32 mthd, u32 data)
531 {
532 	bool (*func)(struct nv10_gr_chan *, u32, u32);
533 	switch (class) {
534 	case 0x99: func = nv17_gr_mthd_celcius; break;
535 	default:
536 		return false;
537 	}
538 	return func(chan, mthd, data);
539 }
540 
541 /*******************************************************************************
542  * PGRAPH context
543  ******************************************************************************/
544 
545 static struct nv10_gr_chan *
546 nv10_gr_channel(struct nv10_gr *gr)
547 {
548 	struct nvkm_device *device = gr->base.engine.subdev.device;
549 	struct nv10_gr_chan *chan = NULL;
550 	if (nvkm_rd32(device, 0x400144) & 0x00010000) {
551 		int chid = nvkm_rd32(device, 0x400148) >> 24;
552 		if (chid < ARRAY_SIZE(gr->chan))
553 			chan = gr->chan[chid];
554 	}
555 	return chan;
556 }
557 
558 static void
559 nv10_gr_save_pipe(struct nv10_gr_chan *chan)
560 {
561 	struct nv10_gr *gr = chan->gr;
562 	struct pipe_state *pipe = &chan->pipe_state;
563 	struct nvkm_device *device = gr->base.engine.subdev.device;
564 
565 	PIPE_SAVE(gr, pipe->pipe_0x4400, 0x4400);
566 	PIPE_SAVE(gr, pipe->pipe_0x0200, 0x0200);
567 	PIPE_SAVE(gr, pipe->pipe_0x6400, 0x6400);
568 	PIPE_SAVE(gr, pipe->pipe_0x6800, 0x6800);
569 	PIPE_SAVE(gr, pipe->pipe_0x6c00, 0x6c00);
570 	PIPE_SAVE(gr, pipe->pipe_0x7000, 0x7000);
571 	PIPE_SAVE(gr, pipe->pipe_0x7400, 0x7400);
572 	PIPE_SAVE(gr, pipe->pipe_0x7800, 0x7800);
573 	PIPE_SAVE(gr, pipe->pipe_0x0040, 0x0040);
574 	PIPE_SAVE(gr, pipe->pipe_0x0000, 0x0000);
575 }
576 
577 static void
578 nv10_gr_load_pipe(struct nv10_gr_chan *chan)
579 {
580 	struct nv10_gr *gr = chan->gr;
581 	struct pipe_state *pipe = &chan->pipe_state;
582 	struct nvkm_device *device = gr->base.engine.subdev.device;
583 	u32 xfmode0, xfmode1;
584 	int i;
585 
586 	nv04_gr_idle(&gr->base);
587 	/* XXX check haiku comments */
588 	xfmode0 = nvkm_rd32(device, NV10_PGRAPH_XFMODE0);
589 	xfmode1 = nvkm_rd32(device, NV10_PGRAPH_XFMODE1);
590 	nvkm_wr32(device, NV10_PGRAPH_XFMODE0, 0x10000000);
591 	nvkm_wr32(device, NV10_PGRAPH_XFMODE1, 0x00000000);
592 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x000064c0);
593 	for (i = 0; i < 4; i++)
594 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
595 	for (i = 0; i < 4; i++)
596 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x00000000);
597 
598 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x00006ab0);
599 	for (i = 0; i < 3; i++)
600 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
601 
602 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x00006a80);
603 	for (i = 0; i < 3; i++)
604 		nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x00000000);
605 
606 	nvkm_wr32(device, NV10_PGRAPH_PIPE_ADDRESS, 0x00000040);
607 	nvkm_wr32(device, NV10_PGRAPH_PIPE_DATA, 0x00000008);
608 
609 
610 	PIPE_RESTORE(gr, pipe->pipe_0x0200, 0x0200);
611 	nv04_gr_idle(&gr->base);
612 
613 	/* restore XFMODE */
614 	nvkm_wr32(device, NV10_PGRAPH_XFMODE0, xfmode0);
615 	nvkm_wr32(device, NV10_PGRAPH_XFMODE1, xfmode1);
616 	PIPE_RESTORE(gr, pipe->pipe_0x6400, 0x6400);
617 	PIPE_RESTORE(gr, pipe->pipe_0x6800, 0x6800);
618 	PIPE_RESTORE(gr, pipe->pipe_0x6c00, 0x6c00);
619 	PIPE_RESTORE(gr, pipe->pipe_0x7000, 0x7000);
620 	PIPE_RESTORE(gr, pipe->pipe_0x7400, 0x7400);
621 	PIPE_RESTORE(gr, pipe->pipe_0x7800, 0x7800);
622 	PIPE_RESTORE(gr, pipe->pipe_0x4400, 0x4400);
623 	PIPE_RESTORE(gr, pipe->pipe_0x0000, 0x0000);
624 	PIPE_RESTORE(gr, pipe->pipe_0x0040, 0x0040);
625 	nv04_gr_idle(&gr->base);
626 }
627 
628 static void
629 nv10_gr_create_pipe(struct nv10_gr_chan *chan)
630 {
631 	struct nv10_gr *gr = chan->gr;
632 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
633 	struct pipe_state *pipe_state = &chan->pipe_state;
634 	u32 *pipe_state_addr;
635 	int i;
636 #define PIPE_INIT(addr) \
637 	do { \
638 		pipe_state_addr = pipe_state->pipe_##addr; \
639 	} while (0)
640 #define PIPE_INIT_END(addr) \
641 	do { \
642 		u32 *__end_addr = pipe_state->pipe_##addr + \
643 				ARRAY_SIZE(pipe_state->pipe_##addr); \
644 		if (pipe_state_addr != __end_addr) \
645 			nvkm_error(subdev, "incomplete pipe init for 0x%x :  %p/%p\n", \
646 				addr, pipe_state_addr, __end_addr); \
647 	} while (0)
648 #define NV_WRITE_PIPE_INIT(value) *(pipe_state_addr++) = value
649 
650 	PIPE_INIT(0x0200);
651 	for (i = 0; i < 48; i++)
652 		NV_WRITE_PIPE_INIT(0x00000000);
653 	PIPE_INIT_END(0x0200);
654 
655 	PIPE_INIT(0x6400);
656 	for (i = 0; i < 211; i++)
657 		NV_WRITE_PIPE_INIT(0x00000000);
658 	NV_WRITE_PIPE_INIT(0x3f800000);
659 	NV_WRITE_PIPE_INIT(0x40000000);
660 	NV_WRITE_PIPE_INIT(0x40000000);
661 	NV_WRITE_PIPE_INIT(0x40000000);
662 	NV_WRITE_PIPE_INIT(0x40000000);
663 	NV_WRITE_PIPE_INIT(0x00000000);
664 	NV_WRITE_PIPE_INIT(0x00000000);
665 	NV_WRITE_PIPE_INIT(0x3f800000);
666 	NV_WRITE_PIPE_INIT(0x00000000);
667 	NV_WRITE_PIPE_INIT(0x3f000000);
668 	NV_WRITE_PIPE_INIT(0x3f000000);
669 	NV_WRITE_PIPE_INIT(0x00000000);
670 	NV_WRITE_PIPE_INIT(0x00000000);
671 	NV_WRITE_PIPE_INIT(0x00000000);
672 	NV_WRITE_PIPE_INIT(0x00000000);
673 	NV_WRITE_PIPE_INIT(0x3f800000);
674 	NV_WRITE_PIPE_INIT(0x00000000);
675 	NV_WRITE_PIPE_INIT(0x00000000);
676 	NV_WRITE_PIPE_INIT(0x00000000);
677 	NV_WRITE_PIPE_INIT(0x00000000);
678 	NV_WRITE_PIPE_INIT(0x00000000);
679 	NV_WRITE_PIPE_INIT(0x3f800000);
680 	NV_WRITE_PIPE_INIT(0x3f800000);
681 	NV_WRITE_PIPE_INIT(0x3f800000);
682 	NV_WRITE_PIPE_INIT(0x3f800000);
683 	PIPE_INIT_END(0x6400);
684 
685 	PIPE_INIT(0x6800);
686 	for (i = 0; i < 162; i++)
687 		NV_WRITE_PIPE_INIT(0x00000000);
688 	NV_WRITE_PIPE_INIT(0x3f800000);
689 	for (i = 0; i < 25; i++)
690 		NV_WRITE_PIPE_INIT(0x00000000);
691 	PIPE_INIT_END(0x6800);
692 
693 	PIPE_INIT(0x6c00);
694 	NV_WRITE_PIPE_INIT(0x00000000);
695 	NV_WRITE_PIPE_INIT(0x00000000);
696 	NV_WRITE_PIPE_INIT(0x00000000);
697 	NV_WRITE_PIPE_INIT(0x00000000);
698 	NV_WRITE_PIPE_INIT(0xbf800000);
699 	NV_WRITE_PIPE_INIT(0x00000000);
700 	NV_WRITE_PIPE_INIT(0x00000000);
701 	NV_WRITE_PIPE_INIT(0x00000000);
702 	NV_WRITE_PIPE_INIT(0x00000000);
703 	NV_WRITE_PIPE_INIT(0x00000000);
704 	NV_WRITE_PIPE_INIT(0x00000000);
705 	NV_WRITE_PIPE_INIT(0x00000000);
706 	PIPE_INIT_END(0x6c00);
707 
708 	PIPE_INIT(0x7000);
709 	NV_WRITE_PIPE_INIT(0x00000000);
710 	NV_WRITE_PIPE_INIT(0x00000000);
711 	NV_WRITE_PIPE_INIT(0x00000000);
712 	NV_WRITE_PIPE_INIT(0x00000000);
713 	NV_WRITE_PIPE_INIT(0x00000000);
714 	NV_WRITE_PIPE_INIT(0x00000000);
715 	NV_WRITE_PIPE_INIT(0x00000000);
716 	NV_WRITE_PIPE_INIT(0x00000000);
717 	NV_WRITE_PIPE_INIT(0x00000000);
718 	NV_WRITE_PIPE_INIT(0x00000000);
719 	NV_WRITE_PIPE_INIT(0x00000000);
720 	NV_WRITE_PIPE_INIT(0x00000000);
721 	NV_WRITE_PIPE_INIT(0x7149f2ca);
722 	NV_WRITE_PIPE_INIT(0x00000000);
723 	NV_WRITE_PIPE_INIT(0x00000000);
724 	NV_WRITE_PIPE_INIT(0x00000000);
725 	NV_WRITE_PIPE_INIT(0x7149f2ca);
726 	NV_WRITE_PIPE_INIT(0x00000000);
727 	NV_WRITE_PIPE_INIT(0x00000000);
728 	NV_WRITE_PIPE_INIT(0x00000000);
729 	NV_WRITE_PIPE_INIT(0x7149f2ca);
730 	NV_WRITE_PIPE_INIT(0x00000000);
731 	NV_WRITE_PIPE_INIT(0x00000000);
732 	NV_WRITE_PIPE_INIT(0x00000000);
733 	NV_WRITE_PIPE_INIT(0x7149f2ca);
734 	NV_WRITE_PIPE_INIT(0x00000000);
735 	NV_WRITE_PIPE_INIT(0x00000000);
736 	NV_WRITE_PIPE_INIT(0x00000000);
737 	NV_WRITE_PIPE_INIT(0x7149f2ca);
738 	NV_WRITE_PIPE_INIT(0x00000000);
739 	NV_WRITE_PIPE_INIT(0x00000000);
740 	NV_WRITE_PIPE_INIT(0x00000000);
741 	NV_WRITE_PIPE_INIT(0x7149f2ca);
742 	NV_WRITE_PIPE_INIT(0x00000000);
743 	NV_WRITE_PIPE_INIT(0x00000000);
744 	NV_WRITE_PIPE_INIT(0x00000000);
745 	NV_WRITE_PIPE_INIT(0x7149f2ca);
746 	NV_WRITE_PIPE_INIT(0x00000000);
747 	NV_WRITE_PIPE_INIT(0x00000000);
748 	NV_WRITE_PIPE_INIT(0x00000000);
749 	NV_WRITE_PIPE_INIT(0x7149f2ca);
750 	for (i = 0; i < 35; i++)
751 		NV_WRITE_PIPE_INIT(0x00000000);
752 	PIPE_INIT_END(0x7000);
753 
754 	PIPE_INIT(0x7400);
755 	for (i = 0; i < 48; i++)
756 		NV_WRITE_PIPE_INIT(0x00000000);
757 	PIPE_INIT_END(0x7400);
758 
759 	PIPE_INIT(0x7800);
760 	for (i = 0; i < 48; i++)
761 		NV_WRITE_PIPE_INIT(0x00000000);
762 	PIPE_INIT_END(0x7800);
763 
764 	PIPE_INIT(0x4400);
765 	for (i = 0; i < 32; i++)
766 		NV_WRITE_PIPE_INIT(0x00000000);
767 	PIPE_INIT_END(0x4400);
768 
769 	PIPE_INIT(0x0000);
770 	for (i = 0; i < 16; i++)
771 		NV_WRITE_PIPE_INIT(0x00000000);
772 	PIPE_INIT_END(0x0000);
773 
774 	PIPE_INIT(0x0040);
775 	for (i = 0; i < 4; i++)
776 		NV_WRITE_PIPE_INIT(0x00000000);
777 	PIPE_INIT_END(0x0040);
778 
779 #undef PIPE_INIT
780 #undef PIPE_INIT_END
781 #undef NV_WRITE_PIPE_INIT
782 }
783 
784 static int
785 nv10_gr_ctx_regs_find_offset(struct nv10_gr *gr, int reg)
786 {
787 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
788 	int i;
789 	for (i = 0; i < ARRAY_SIZE(nv10_gr_ctx_regs); i++) {
790 		if (nv10_gr_ctx_regs[i] == reg)
791 			return i;
792 	}
793 	nvkm_error(subdev, "unknow offset nv10_ctx_regs %d\n", reg);
794 	return -1;
795 }
796 
797 static int
798 nv17_gr_ctx_regs_find_offset(struct nv10_gr *gr, int reg)
799 {
800 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
801 	int i;
802 	for (i = 0; i < ARRAY_SIZE(nv17_gr_ctx_regs); i++) {
803 		if (nv17_gr_ctx_regs[i] == reg)
804 			return i;
805 	}
806 	nvkm_error(subdev, "unknow offset nv17_ctx_regs %d\n", reg);
807 	return -1;
808 }
809 
810 static void
811 nv10_gr_load_dma_vtxbuf(struct nv10_gr_chan *chan, int chid, u32 inst)
812 {
813 	struct nv10_gr *gr = chan->gr;
814 	struct nvkm_device *device = gr->base.engine.subdev.device;
815 	u32 st2, st2_dl, st2_dh, fifo_ptr, fifo[0x60/4];
816 	u32 ctx_user, ctx_switch[5];
817 	int i, subchan = -1;
818 
819 	/* NV10TCL_DMA_VTXBUF (method 0x18c) modifies hidden state
820 	 * that cannot be restored via MMIO. Do it through the FIFO
821 	 * instead.
822 	 */
823 
824 	/* Look for a celsius object */
825 	for (i = 0; i < 8; i++) {
826 		int class = nvkm_rd32(device, NV10_PGRAPH_CTX_CACHE(i, 0)) & 0xfff;
827 
828 		if (class == 0x56 || class == 0x96 || class == 0x99) {
829 			subchan = i;
830 			break;
831 		}
832 	}
833 
834 	if (subchan < 0 || !inst)
835 		return;
836 
837 	/* Save the current ctx object */
838 	ctx_user = nvkm_rd32(device, NV10_PGRAPH_CTX_USER);
839 	for (i = 0; i < 5; i++)
840 		ctx_switch[i] = nvkm_rd32(device, NV10_PGRAPH_CTX_SWITCH(i));
841 
842 	/* Save the FIFO state */
843 	st2 = nvkm_rd32(device, NV10_PGRAPH_FFINTFC_ST2);
844 	st2_dl = nvkm_rd32(device, NV10_PGRAPH_FFINTFC_ST2_DL);
845 	st2_dh = nvkm_rd32(device, NV10_PGRAPH_FFINTFC_ST2_DH);
846 	fifo_ptr = nvkm_rd32(device, NV10_PGRAPH_FFINTFC_FIFO_PTR);
847 
848 	for (i = 0; i < ARRAY_SIZE(fifo); i++)
849 		fifo[i] = nvkm_rd32(device, 0x4007a0 + 4 * i);
850 
851 	/* Switch to the celsius subchannel */
852 	for (i = 0; i < 5; i++)
853 		nvkm_wr32(device, NV10_PGRAPH_CTX_SWITCH(i),
854 			nvkm_rd32(device, NV10_PGRAPH_CTX_CACHE(subchan, i)));
855 	nvkm_mask(device, NV10_PGRAPH_CTX_USER, 0xe000, subchan << 13);
856 
857 	/* Inject NV10TCL_DMA_VTXBUF */
858 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_FIFO_PTR, 0);
859 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_ST2,
860 		0x2c000000 | chid << 20 | subchan << 16 | 0x18c);
861 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_ST2_DL, inst);
862 	nvkm_mask(device, NV10_PGRAPH_CTX_CONTROL, 0, 0x10000);
863 	nvkm_mask(device, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
864 	nvkm_mask(device, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
865 
866 	/* Restore the FIFO state */
867 	for (i = 0; i < ARRAY_SIZE(fifo); i++)
868 		nvkm_wr32(device, 0x4007a0 + 4 * i, fifo[i]);
869 
870 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_FIFO_PTR, fifo_ptr);
871 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_ST2, st2);
872 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_ST2_DL, st2_dl);
873 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_ST2_DH, st2_dh);
874 
875 	/* Restore the current ctx object */
876 	for (i = 0; i < 5; i++)
877 		nvkm_wr32(device, NV10_PGRAPH_CTX_SWITCH(i), ctx_switch[i]);
878 	nvkm_wr32(device, NV10_PGRAPH_CTX_USER, ctx_user);
879 }
880 
881 static int
882 nv10_gr_load_context(struct nv10_gr_chan *chan, int chid)
883 {
884 	struct nv10_gr *gr = chan->gr;
885 	struct nvkm_device *device = gr->base.engine.subdev.device;
886 	u32 inst;
887 	int i;
888 
889 	for (i = 0; i < ARRAY_SIZE(nv10_gr_ctx_regs); i++)
890 		nvkm_wr32(device, nv10_gr_ctx_regs[i], chan->nv10[i]);
891 
892 	if (nv_device(gr)->card_type >= NV_11 &&
893 	    nv_device(gr)->chipset >= 0x17) {
894 		for (i = 0; i < ARRAY_SIZE(nv17_gr_ctx_regs); i++)
895 			nvkm_wr32(device, nv17_gr_ctx_regs[i], chan->nv17[i]);
896 	}
897 
898 	nv10_gr_load_pipe(chan);
899 
900 	inst = nvkm_rd32(device, NV10_PGRAPH_GLOBALSTATE1) & 0xffff;
901 	nv10_gr_load_dma_vtxbuf(chan, chid, inst);
902 
903 	nvkm_wr32(device, NV10_PGRAPH_CTX_CONTROL, 0x10010100);
904 	nvkm_mask(device, NV10_PGRAPH_CTX_USER, 0xff000000, chid << 24);
905 	nvkm_mask(device, NV10_PGRAPH_FFINTFC_ST2, 0x30000000, 0x00000000);
906 	return 0;
907 }
908 
909 static int
910 nv10_gr_unload_context(struct nv10_gr_chan *chan)
911 {
912 	struct nv10_gr *gr = chan->gr;
913 	struct nvkm_device *device = gr->base.engine.subdev.device;
914 	int i;
915 
916 	for (i = 0; i < ARRAY_SIZE(nv10_gr_ctx_regs); i++)
917 		chan->nv10[i] = nvkm_rd32(device, nv10_gr_ctx_regs[i]);
918 
919 	if (nv_device(gr)->card_type >= NV_11 &&
920 	    nv_device(gr)->chipset >= 0x17) {
921 		for (i = 0; i < ARRAY_SIZE(nv17_gr_ctx_regs); i++)
922 			chan->nv17[i] = nvkm_rd32(device, nv17_gr_ctx_regs[i]);
923 	}
924 
925 	nv10_gr_save_pipe(chan);
926 
927 	nvkm_wr32(device, NV10_PGRAPH_CTX_CONTROL, 0x10000000);
928 	nvkm_mask(device, NV10_PGRAPH_CTX_USER, 0xff000000, 0x1f000000);
929 	return 0;
930 }
931 
932 static void
933 nv10_gr_context_switch(struct nv10_gr *gr)
934 {
935 	struct nvkm_device *device = gr->base.engine.subdev.device;
936 	struct nv10_gr_chan *prev = NULL;
937 	struct nv10_gr_chan *next = NULL;
938 	int chid;
939 
940 	nv04_gr_idle(&gr->base);
941 
942 	/* If previous context is valid, we need to save it */
943 	prev = nv10_gr_channel(gr);
944 	if (prev)
945 		nv10_gr_unload_context(prev);
946 
947 	/* load context for next channel */
948 	chid = (nvkm_rd32(device, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
949 	next = gr->chan[chid];
950 	if (next)
951 		nv10_gr_load_context(next, chid);
952 }
953 
954 static int
955 nv10_gr_chan_fini(struct nvkm_object *object, bool suspend)
956 {
957 	struct nv10_gr_chan *chan = nv10_gr_chan(object);
958 	struct nv10_gr *gr = chan->gr;
959 	struct nvkm_device *device = gr->base.engine.subdev.device;
960 	unsigned long flags;
961 
962 	spin_lock_irqsave(&gr->lock, flags);
963 	nvkm_mask(device, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
964 	if (nv10_gr_channel(gr) == chan)
965 		nv10_gr_unload_context(chan);
966 	nvkm_mask(device, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
967 	spin_unlock_irqrestore(&gr->lock, flags);
968 	return 0;
969 }
970 
971 static void *
972 nv10_gr_chan_dtor(struct nvkm_object *object)
973 {
974 	struct nv10_gr_chan *chan = nv10_gr_chan(object);
975 	struct nv10_gr *gr = chan->gr;
976 	unsigned long flags;
977 
978 	spin_lock_irqsave(&gr->lock, flags);
979 	gr->chan[chan->chid] = NULL;
980 	spin_unlock_irqrestore(&gr->lock, flags);
981 	return chan;
982 }
983 
984 static const struct nvkm_object_func
985 nv10_gr_chan = {
986 	.dtor = nv10_gr_chan_dtor,
987 	.fini = nv10_gr_chan_fini,
988 };
989 
990 #define NV_WRITE_CTX(reg, val) do { \
991 	int offset = nv10_gr_ctx_regs_find_offset(gr, reg); \
992 	if (offset > 0) \
993 		chan->nv10[offset] = val; \
994 	} while (0)
995 
996 #define NV17_WRITE_CTX(reg, val) do { \
997 	int offset = nv17_gr_ctx_regs_find_offset(gr, reg); \
998 	if (offset > 0) \
999 		chan->nv17[offset] = val; \
1000 	} while (0)
1001 
1002 static int
1003 nv10_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
1004 		 const struct nvkm_oclass *oclass, struct nvkm_object **pobject)
1005 {
1006 	struct nv10_gr *gr = nv10_gr(base);
1007 	struct nv10_gr_chan *chan;
1008 	struct nvkm_device *device = gr->base.engine.subdev.device;
1009 	unsigned long flags;
1010 
1011 	if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
1012 		return -ENOMEM;
1013 	nvkm_object_ctor(&nv10_gr_chan, oclass, &chan->object);
1014 	chan->gr = gr;
1015 	chan->chid = fifoch->chid;
1016 	*pobject = &chan->object;
1017 
1018 	NV_WRITE_CTX(0x00400e88, 0x08000000);
1019 	NV_WRITE_CTX(0x00400e9c, 0x4b7fffff);
1020 	NV_WRITE_CTX(NV03_PGRAPH_XY_LOGIC_MISC0, 0x0001ffff);
1021 	NV_WRITE_CTX(0x00400e10, 0x00001000);
1022 	NV_WRITE_CTX(0x00400e14, 0x00001000);
1023 	NV_WRITE_CTX(0x00400e30, 0x00080008);
1024 	NV_WRITE_CTX(0x00400e34, 0x00080008);
1025 	if (device->card_type >= NV_11 && device->chipset >= 0x17) {
1026 		/* is it really needed ??? */
1027 		NV17_WRITE_CTX(NV10_PGRAPH_DEBUG_4,
1028 			       nvkm_rd32(device, NV10_PGRAPH_DEBUG_4));
1029 		NV17_WRITE_CTX(0x004006b0, nvkm_rd32(device, 0x004006b0));
1030 		NV17_WRITE_CTX(0x00400eac, 0x0fff0000);
1031 		NV17_WRITE_CTX(0x00400eb0, 0x0fff0000);
1032 		NV17_WRITE_CTX(0x00400ec0, 0x00000080);
1033 		NV17_WRITE_CTX(0x00400ed0, 0x00000080);
1034 	}
1035 	NV_WRITE_CTX(NV10_PGRAPH_CTX_USER, chan->chid << 24);
1036 
1037 	nv10_gr_create_pipe(chan);
1038 
1039 	spin_lock_irqsave(&gr->lock, flags);
1040 	gr->chan[chan->chid] = chan;
1041 	spin_unlock_irqrestore(&gr->lock, flags);
1042 	return 0;
1043 }
1044 
1045 /*******************************************************************************
1046  * PGRAPH engine/subdev functions
1047  ******************************************************************************/
1048 
1049 static void
1050 nv10_gr_tile_prog(struct nvkm_engine *engine, int i)
1051 {
1052 	struct nv10_gr *gr = (void *)engine;
1053 	struct nvkm_device *device = gr->base.engine.subdev.device;
1054 	struct nvkm_fifo *fifo = device->fifo;
1055 	struct nvkm_fb_tile *tile = &device->fb->tile.region[i];
1056 	unsigned long flags;
1057 
1058 	fifo->pause(fifo, &flags);
1059 	nv04_gr_idle(&gr->base);
1060 
1061 	nvkm_wr32(device, NV10_PGRAPH_TLIMIT(i), tile->limit);
1062 	nvkm_wr32(device, NV10_PGRAPH_TSIZE(i), tile->pitch);
1063 	nvkm_wr32(device, NV10_PGRAPH_TILE(i), tile->addr);
1064 
1065 	fifo->start(fifo, &flags);
1066 }
1067 
1068 const struct nvkm_bitfield nv10_gr_intr_name[] = {
1069 	{ NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
1070 	{ NV_PGRAPH_INTR_ERROR,  "ERROR"  },
1071 	{}
1072 };
1073 
1074 const struct nvkm_bitfield nv10_gr_nstatus[] = {
1075 	{ NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
1076 	{ NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
1077 	{ NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
1078 	{ NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
1079 	{}
1080 };
1081 
1082 static void
1083 nv10_gr_intr(struct nvkm_subdev *subdev)
1084 {
1085 	struct nv10_gr *gr = (void *)subdev;
1086 	struct nv10_gr_chan *chan = NULL;
1087 	struct nvkm_device *device = gr->base.engine.subdev.device;
1088 	u32 stat = nvkm_rd32(device, NV03_PGRAPH_INTR);
1089 	u32 nsource = nvkm_rd32(device, NV03_PGRAPH_NSOURCE);
1090 	u32 nstatus = nvkm_rd32(device, NV03_PGRAPH_NSTATUS);
1091 	u32 addr = nvkm_rd32(device, NV04_PGRAPH_TRAPPED_ADDR);
1092 	u32 chid = (addr & 0x01f00000) >> 20;
1093 	u32 subc = (addr & 0x00070000) >> 16;
1094 	u32 mthd = (addr & 0x00001ffc);
1095 	u32 data = nvkm_rd32(device, NV04_PGRAPH_TRAPPED_DATA);
1096 	u32 class = nvkm_rd32(device, 0x400160 + subc * 4) & 0xfff;
1097 	u32 show = stat;
1098 	char msg[128], src[128], sta[128];
1099 	unsigned long flags;
1100 
1101 	spin_lock_irqsave(&gr->lock, flags);
1102 	chan = gr->chan[chid];
1103 
1104 	if (stat & NV_PGRAPH_INTR_ERROR) {
1105 		if (chan && (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD)) {
1106 			if (!nv10_gr_mthd(chan, class, mthd, data))
1107 				show &= ~NV_PGRAPH_INTR_ERROR;
1108 		}
1109 	}
1110 
1111 	if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
1112 		nvkm_wr32(device, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
1113 		stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1114 		show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1115 		nv10_gr_context_switch(gr);
1116 	}
1117 
1118 	nvkm_wr32(device, NV03_PGRAPH_INTR, stat);
1119 	nvkm_wr32(device, NV04_PGRAPH_FIFO, 0x00000001);
1120 
1121 	if (show) {
1122 		nvkm_snprintbf(msg, sizeof(msg), nv10_gr_intr_name, show);
1123 		nvkm_snprintbf(src, sizeof(src), nv04_gr_nsource, nsource);
1124 		nvkm_snprintbf(sta, sizeof(sta), nv10_gr_nstatus, nstatus);
1125 		nvkm_error(subdev, "intr %08x [%s] nsource %08x [%s] "
1126 				   "nstatus %08x [%s] ch %d [%s] subc %d "
1127 				   "class %04x mthd %04x data %08x\n",
1128 			   show, msg, nsource, src, nstatus, sta, chid,
1129 			   chan ? chan->object.client->name : "unknown",
1130 			   subc, class, mthd, data);
1131 	}
1132 
1133 	spin_unlock_irqrestore(&gr->lock, flags);
1134 }
1135 
1136 static const struct nvkm_gr_func
1137 nv10_gr = {
1138 	.chan_new = nv10_gr_chan_new,
1139 	.sclass = {
1140 		{ -1, -1, 0x0012, &nv04_gr_object }, /* beta1 */
1141 		{ -1, -1, 0x0019, &nv04_gr_object }, /* clip */
1142 		{ -1, -1, 0x0030, &nv04_gr_object }, /* null */
1143 		{ -1, -1, 0x0039, &nv04_gr_object }, /* m2mf */
1144 		{ -1, -1, 0x0043, &nv04_gr_object }, /* rop */
1145 		{ -1, -1, 0x0044, &nv04_gr_object }, /* pattern */
1146 		{ -1, -1, 0x004a, &nv04_gr_object }, /* gdi */
1147 		{ -1, -1, 0x0052, &nv04_gr_object }, /* swzsurf */
1148 		{ -1, -1, 0x005f, &nv04_gr_object }, /* blit */
1149 		{ -1, -1, 0x0062, &nv04_gr_object }, /* surf2d */
1150 		{ -1, -1, 0x0072, &nv04_gr_object }, /* beta4 */
1151 		{ -1, -1, 0x0089, &nv04_gr_object }, /* sifm */
1152 		{ -1, -1, 0x008a, &nv04_gr_object }, /* ifc */
1153 		{ -1, -1, 0x009f, &nv04_gr_object }, /* blit */
1154 		{ -1, -1, 0x0093, &nv04_gr_object }, /* surf3d */
1155 		{ -1, -1, 0x0094, &nv04_gr_object }, /* ttri */
1156 		{ -1, -1, 0x0095, &nv04_gr_object }, /* mtri */
1157 		{ -1, -1, 0x0056, &nv04_gr_object }, /* celcius */
1158 		{}
1159 	}
1160 };
1161 
1162 static const struct nvkm_gr_func
1163 nv15_gr = {
1164 	.chan_new = nv10_gr_chan_new,
1165 	.sclass = {
1166 		{ -1, -1, 0x0012, &nv04_gr_object }, /* beta1 */
1167 		{ -1, -1, 0x0019, &nv04_gr_object }, /* clip */
1168 		{ -1, -1, 0x0030, &nv04_gr_object }, /* null */
1169 		{ -1, -1, 0x0039, &nv04_gr_object }, /* m2mf */
1170 		{ -1, -1, 0x0043, &nv04_gr_object }, /* rop */
1171 		{ -1, -1, 0x0044, &nv04_gr_object }, /* pattern */
1172 		{ -1, -1, 0x004a, &nv04_gr_object }, /* gdi */
1173 		{ -1, -1, 0x0052, &nv04_gr_object }, /* swzsurf */
1174 		{ -1, -1, 0x005f, &nv04_gr_object }, /* blit */
1175 		{ -1, -1, 0x0062, &nv04_gr_object }, /* surf2d */
1176 		{ -1, -1, 0x0072, &nv04_gr_object }, /* beta4 */
1177 		{ -1, -1, 0x0089, &nv04_gr_object }, /* sifm */
1178 		{ -1, -1, 0x008a, &nv04_gr_object }, /* ifc */
1179 		{ -1, -1, 0x009f, &nv04_gr_object }, /* blit */
1180 		{ -1, -1, 0x0093, &nv04_gr_object }, /* surf3d */
1181 		{ -1, -1, 0x0094, &nv04_gr_object }, /* ttri */
1182 		{ -1, -1, 0x0095, &nv04_gr_object }, /* mtri */
1183 		{ -1, -1, 0x0096, &nv04_gr_object }, /* celcius */
1184 		{}
1185 	}
1186 };
1187 
1188 
1189 static const struct nvkm_gr_func
1190 nv17_gr = {
1191 	.chan_new = nv10_gr_chan_new,
1192 	.sclass = {
1193 		{ -1, -1, 0x0012, &nv04_gr_object }, /* beta1 */
1194 		{ -1, -1, 0x0019, &nv04_gr_object }, /* clip */
1195 		{ -1, -1, 0x0030, &nv04_gr_object }, /* null */
1196 		{ -1, -1, 0x0039, &nv04_gr_object }, /* m2mf */
1197 		{ -1, -1, 0x0043, &nv04_gr_object }, /* rop */
1198 		{ -1, -1, 0x0044, &nv04_gr_object }, /* pattern */
1199 		{ -1, -1, 0x004a, &nv04_gr_object }, /* gdi */
1200 		{ -1, -1, 0x0052, &nv04_gr_object }, /* swzsurf */
1201 		{ -1, -1, 0x005f, &nv04_gr_object }, /* blit */
1202 		{ -1, -1, 0x0062, &nv04_gr_object }, /* surf2d */
1203 		{ -1, -1, 0x0072, &nv04_gr_object }, /* beta4 */
1204 		{ -1, -1, 0x0089, &nv04_gr_object }, /* sifm */
1205 		{ -1, -1, 0x008a, &nv04_gr_object }, /* ifc */
1206 		{ -1, -1, 0x009f, &nv04_gr_object }, /* blit */
1207 		{ -1, -1, 0x0093, &nv04_gr_object }, /* surf3d */
1208 		{ -1, -1, 0x0094, &nv04_gr_object }, /* ttri */
1209 		{ -1, -1, 0x0095, &nv04_gr_object }, /* mtri */
1210 		{ -1, -1, 0x0099, &nv04_gr_object },
1211 		{}
1212 	}
1213 };
1214 
1215 static int
1216 nv10_gr_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
1217 	     struct nvkm_oclass *oclass, void *data, u32 size,
1218 	     struct nvkm_object **pobject)
1219 {
1220 	struct nv10_gr *gr;
1221 	int ret;
1222 
1223 	ret = nvkm_gr_create(parent, engine, oclass, true, &gr);
1224 	*pobject = nv_object(gr);
1225 	if (ret)
1226 		return ret;
1227 
1228 	nv_subdev(gr)->unit = 0x00001000;
1229 	nv_subdev(gr)->intr = nv10_gr_intr;
1230 
1231 	if (nv_device(gr)->chipset <= 0x10)
1232 		gr->base.func = &nv10_gr;
1233 	else
1234 	if (nv_device(gr)->chipset <  0x17 ||
1235 	    nv_device(gr)->card_type < NV_11)
1236 		gr->base.func = &nv15_gr;
1237 	else
1238 		gr->base.func = &nv17_gr;
1239 
1240 	nv_engine(gr)->tile_prog = nv10_gr_tile_prog;
1241 	spin_lock_init(&gr->lock);
1242 	return 0;
1243 }
1244 
1245 static void
1246 nv10_gr_dtor(struct nvkm_object *object)
1247 {
1248 	struct nv10_gr *gr = (void *)object;
1249 	nvkm_gr_destroy(&gr->base);
1250 }
1251 
1252 static int
1253 nv10_gr_init(struct nvkm_object *object)
1254 {
1255 	struct nvkm_engine *engine = nv_engine(object);
1256 	struct nv10_gr *gr = (void *)engine;
1257 	struct nvkm_device *device = gr->base.engine.subdev.device;
1258 	struct nvkm_fb *fb = device->fb;
1259 	int ret, i;
1260 
1261 	ret = nvkm_gr_init(&gr->base);
1262 	if (ret)
1263 		return ret;
1264 
1265 	nvkm_wr32(device, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
1266 	nvkm_wr32(device, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
1267 
1268 	nvkm_wr32(device, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
1269 	nvkm_wr32(device, NV04_PGRAPH_DEBUG_0, 0x00000000);
1270 	nvkm_wr32(device, NV04_PGRAPH_DEBUG_1, 0x00118700);
1271 	/* nvkm_wr32(device, NV04_PGRAPH_DEBUG_2, 0x24E00810); */ /* 0x25f92ad9 */
1272 	nvkm_wr32(device, NV04_PGRAPH_DEBUG_2, 0x25f92ad9);
1273 	nvkm_wr32(device, NV04_PGRAPH_DEBUG_3, 0x55DE0830 | (1 << 29) | (1 << 31));
1274 
1275 	if (nv_device(gr)->card_type >= NV_11 &&
1276 	    nv_device(gr)->chipset >= 0x17) {
1277 		nvkm_wr32(device, NV10_PGRAPH_DEBUG_4, 0x1f000000);
1278 		nvkm_wr32(device, 0x400a10, 0x03ff3fb6);
1279 		nvkm_wr32(device, 0x400838, 0x002f8684);
1280 		nvkm_wr32(device, 0x40083c, 0x00115f3f);
1281 		nvkm_wr32(device, 0x4006b0, 0x40000020);
1282 	} else {
1283 		nvkm_wr32(device, NV10_PGRAPH_DEBUG_4, 0x00000000);
1284 	}
1285 
1286 	/* Turn all the tiling regions off. */
1287 	for (i = 0; i < fb->tile.regions; i++)
1288 		engine->tile_prog(engine, i);
1289 
1290 	nvkm_wr32(device, NV10_PGRAPH_CTX_SWITCH(0), 0x00000000);
1291 	nvkm_wr32(device, NV10_PGRAPH_CTX_SWITCH(1), 0x00000000);
1292 	nvkm_wr32(device, NV10_PGRAPH_CTX_SWITCH(2), 0x00000000);
1293 	nvkm_wr32(device, NV10_PGRAPH_CTX_SWITCH(3), 0x00000000);
1294 	nvkm_wr32(device, NV10_PGRAPH_CTX_SWITCH(4), 0x00000000);
1295 	nvkm_wr32(device, NV10_PGRAPH_STATE, 0xFFFFFFFF);
1296 
1297 	nvkm_mask(device, NV10_PGRAPH_CTX_USER, 0xff000000, 0x1f000000);
1298 	nvkm_wr32(device, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
1299 	nvkm_wr32(device, NV10_PGRAPH_FFINTFC_ST2, 0x08000000);
1300 	return 0;
1301 }
1302 
1303 static int
1304 nv10_gr_fini(struct nvkm_object *object, bool suspend)
1305 {
1306 	struct nv10_gr *gr = (void *)object;
1307 	return nvkm_gr_fini(&gr->base, suspend);
1308 }
1309 
1310 struct nvkm_oclass
1311 nv10_gr_oclass = {
1312 	.handle = NV_ENGINE(GR, 0x10),
1313 	.ofuncs = &(struct nvkm_ofuncs) {
1314 		.ctor = nv10_gr_ctor,
1315 		.dtor = nv10_gr_dtor,
1316 		.init = nv10_gr_init,
1317 		.fini = nv10_gr_fini,
1318 	},
1319 };
1320