1 /*
2  * Copyright 1993-2003 NVIDIA, Corporation
3  * Copyright 2006 Dave Airlie
4  * Copyright 2007 Maarten Maathuis
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the next
14  * paragraph) shall be included in all copies or substantial portions of the
15  * Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23  * DEALINGS IN THE SOFTWARE.
24  */
25 #include <linux/pm_runtime.h>
26 
27 #include <drm/drmP.h>
28 #include <drm/drm_crtc_helper.h>
29 #include <drm/drm_plane_helper.h>
30 
31 #include "nouveau_drv.h"
32 #include "nouveau_reg.h"
33 #include "nouveau_ttm.h"
34 #include "nouveau_bo.h"
35 #include "nouveau_gem.h"
36 #include "nouveau_encoder.h"
37 #include "nouveau_connector.h"
38 #include "nouveau_crtc.h"
39 #include "hw.h"
40 #include "nvreg.h"
41 #include "nouveau_fbcon.h"
42 #include "disp.h"
43 #include "nouveau_dma.h"
44 
45 #include <subdev/bios/pll.h>
46 #include <subdev/clk.h>
47 
48 static int
49 nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
50 			struct drm_framebuffer *old_fb);
51 
52 static void
53 crtc_wr_cio_state(struct drm_crtc *crtc, struct nv04_crtc_reg *crtcstate, int index)
54 {
55 	NVWriteVgaCrtc(crtc->dev, nouveau_crtc(crtc)->index, index,
56 		       crtcstate->CRTC[index]);
57 }
58 
59 static void nv_crtc_set_digital_vibrance(struct drm_crtc *crtc, int level)
60 {
61 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
62 	struct drm_device *dev = crtc->dev;
63 	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
64 
65 	regp->CRTC[NV_CIO_CRE_CSB] = nv_crtc->saturation = level;
66 	if (nv_crtc->saturation && nv_gf4_disp_arch(crtc->dev)) {
67 		regp->CRTC[NV_CIO_CRE_CSB] = 0x80;
68 		regp->CRTC[NV_CIO_CRE_5B] = nv_crtc->saturation << 2;
69 		crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_5B);
70 	}
71 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_CSB);
72 }
73 
74 static void nv_crtc_set_image_sharpening(struct drm_crtc *crtc, int level)
75 {
76 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
77 	struct drm_device *dev = crtc->dev;
78 	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
79 
80 	nv_crtc->sharpness = level;
81 	if (level < 0)	/* blur is in hw range 0x3f -> 0x20 */
82 		level += 0x40;
83 	regp->ramdac_634 = level;
84 	NVWriteRAMDAC(crtc->dev, nv_crtc->index, NV_PRAMDAC_634, regp->ramdac_634);
85 }
86 
87 #define PLLSEL_VPLL1_MASK				\
88 	(NV_PRAMDAC_PLL_COEFF_SELECT_SOURCE_PROG_VPLL	\
89 	 | NV_PRAMDAC_PLL_COEFF_SELECT_VCLK_RATIO_DB2)
90 #define PLLSEL_VPLL2_MASK				\
91 	(NV_PRAMDAC_PLL_COEFF_SELECT_PLL_SOURCE_VPLL2		\
92 	 | NV_PRAMDAC_PLL_COEFF_SELECT_VCLK2_RATIO_DB2)
93 #define PLLSEL_TV_MASK					\
94 	(NV_PRAMDAC_PLL_COEFF_SELECT_TV_VSCLK1		\
95 	 | NV_PRAMDAC_PLL_COEFF_SELECT_TV_PCLK1		\
96 	 | NV_PRAMDAC_PLL_COEFF_SELECT_TV_VSCLK2	\
97 	 | NV_PRAMDAC_PLL_COEFF_SELECT_TV_PCLK2)
98 
99 /* NV4x 0x40.. pll notes:
100  * gpu pll: 0x4000 + 0x4004
101  * ?gpu? pll: 0x4008 + 0x400c
102  * vpll1: 0x4010 + 0x4014
103  * vpll2: 0x4018 + 0x401c
104  * mpll: 0x4020 + 0x4024
105  * mpll: 0x4038 + 0x403c
106  *
107  * the first register of each pair has some unknown details:
108  * bits 0-7: redirected values from elsewhere? (similar to PLL_SETUP_CONTROL?)
109  * bits 20-23: (mpll) something to do with post divider?
110  * bits 28-31: related to single stage mode? (bit 8/12)
111  */
112 
113 static void nv_crtc_calc_state_ext(struct drm_crtc *crtc, struct drm_display_mode * mode, int dot_clock)
114 {
115 	struct drm_device *dev = crtc->dev;
116 	struct nouveau_drm *drm = nouveau_drm(dev);
117 	struct nvkm_bios *bios = nvxx_bios(&drm->client.device);
118 	struct nvkm_clk *clk = nvxx_clk(&drm->client.device);
119 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
120 	struct nv04_mode_state *state = &nv04_display(dev)->mode_reg;
121 	struct nv04_crtc_reg *regp = &state->crtc_reg[nv_crtc->index];
122 	struct nvkm_pll_vals *pv = &regp->pllvals;
123 	struct nvbios_pll pll_lim;
124 
125 	if (nvbios_pll_parse(bios, nv_crtc->index ? PLL_VPLL1 : PLL_VPLL0,
126 			    &pll_lim))
127 		return;
128 
129 	/* NM2 == 0 is used to determine single stage mode on two stage plls */
130 	pv->NM2 = 0;
131 
132 	/* for newer nv4x the blob uses only the first stage of the vpll below a
133 	 * certain clock.  for a certain nv4b this is 150MHz.  since the max
134 	 * output frequency of the first stage for this card is 300MHz, it is
135 	 * assumed the threshold is given by vco1 maxfreq/2
136 	 */
137 	/* for early nv4x, specifically nv40 and *some* nv43 (devids 0 and 6,
138 	 * not 8, others unknown), the blob always uses both plls.  no problem
139 	 * has yet been observed in allowing the use a single stage pll on all
140 	 * nv43 however.  the behaviour of single stage use is untested on nv40
141 	 */
142 	if (drm->client.device.info.chipset > 0x40 && dot_clock <= (pll_lim.vco1.max_freq / 2))
143 		memset(&pll_lim.vco2, 0, sizeof(pll_lim.vco2));
144 
145 
146 	if (!clk->pll_calc(clk, &pll_lim, dot_clock, pv))
147 		return;
148 
149 	state->pllsel &= PLLSEL_VPLL1_MASK | PLLSEL_VPLL2_MASK | PLLSEL_TV_MASK;
150 
151 	/* The blob uses this always, so let's do the same */
152 	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CURIE)
153 		state->pllsel |= NV_PRAMDAC_PLL_COEFF_SELECT_USE_VPLL2_TRUE;
154 	/* again nv40 and some nv43 act more like nv3x as described above */
155 	if (drm->client.device.info.chipset < 0x41)
156 		state->pllsel |= NV_PRAMDAC_PLL_COEFF_SELECT_SOURCE_PROG_MPLL |
157 				 NV_PRAMDAC_PLL_COEFF_SELECT_SOURCE_PROG_NVPLL;
158 	state->pllsel |= nv_crtc->index ? PLLSEL_VPLL2_MASK : PLLSEL_VPLL1_MASK;
159 
160 	if (pv->NM2)
161 		NV_DEBUG(drm, "vpll: n1 %d n2 %d m1 %d m2 %d log2p %d\n",
162 			 pv->N1, pv->N2, pv->M1, pv->M2, pv->log2P);
163 	else
164 		NV_DEBUG(drm, "vpll: n %d m %d log2p %d\n",
165 			 pv->N1, pv->M1, pv->log2P);
166 
167 	nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.offset);
168 }
169 
170 static void
171 nv_crtc_dpms(struct drm_crtc *crtc, int mode)
172 {
173 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
174 	struct drm_device *dev = crtc->dev;
175 	struct nouveau_drm *drm = nouveau_drm(dev);
176 	unsigned char seq1 = 0, crtc17 = 0;
177 	unsigned char crtc1A;
178 
179 	NV_DEBUG(drm, "Setting dpms mode %d on CRTC %d\n", mode,
180 							nv_crtc->index);
181 
182 	if (nv_crtc->last_dpms == mode) /* Don't do unnecessary mode changes. */
183 		return;
184 
185 	nv_crtc->last_dpms = mode;
186 
187 	if (nv_two_heads(dev))
188 		NVSetOwner(dev, nv_crtc->index);
189 
190 	/* nv4ref indicates these two RPC1 bits inhibit h/v sync */
191 	crtc1A = NVReadVgaCrtc(dev, nv_crtc->index,
192 					NV_CIO_CRE_RPC1_INDEX) & ~0xC0;
193 	switch (mode) {
194 	case DRM_MODE_DPMS_STANDBY:
195 		/* Screen: Off; HSync: Off, VSync: On -- Not Supported */
196 		seq1 = 0x20;
197 		crtc17 = 0x80;
198 		crtc1A |= 0x80;
199 		break;
200 	case DRM_MODE_DPMS_SUSPEND:
201 		/* Screen: Off; HSync: On, VSync: Off -- Not Supported */
202 		seq1 = 0x20;
203 		crtc17 = 0x80;
204 		crtc1A |= 0x40;
205 		break;
206 	case DRM_MODE_DPMS_OFF:
207 		/* Screen: Off; HSync: Off, VSync: Off */
208 		seq1 = 0x20;
209 		crtc17 = 0x00;
210 		crtc1A |= 0xC0;
211 		break;
212 	case DRM_MODE_DPMS_ON:
213 	default:
214 		/* Screen: On; HSync: On, VSync: On */
215 		seq1 = 0x00;
216 		crtc17 = 0x80;
217 		break;
218 	}
219 
220 	NVVgaSeqReset(dev, nv_crtc->index, true);
221 	/* Each head has it's own sequencer, so we can turn it off when we want */
222 	seq1 |= (NVReadVgaSeq(dev, nv_crtc->index, NV_VIO_SR_CLOCK_INDEX) & ~0x20);
223 	NVWriteVgaSeq(dev, nv_crtc->index, NV_VIO_SR_CLOCK_INDEX, seq1);
224 	crtc17 |= (NVReadVgaCrtc(dev, nv_crtc->index, NV_CIO_CR_MODE_INDEX) & ~0x80);
225 	mdelay(10);
226 	NVWriteVgaCrtc(dev, nv_crtc->index, NV_CIO_CR_MODE_INDEX, crtc17);
227 	NVVgaSeqReset(dev, nv_crtc->index, false);
228 
229 	NVWriteVgaCrtc(dev, nv_crtc->index, NV_CIO_CRE_RPC1_INDEX, crtc1A);
230 }
231 
232 static void
233 nv_crtc_mode_set_vga(struct drm_crtc *crtc, struct drm_display_mode *mode)
234 {
235 	struct drm_device *dev = crtc->dev;
236 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
237 	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
238 	struct drm_framebuffer *fb = crtc->primary->fb;
239 
240 	/* Calculate our timings */
241 	int horizDisplay	= (mode->crtc_hdisplay >> 3)		- 1;
242 	int horizStart		= (mode->crtc_hsync_start >> 3) 	+ 1;
243 	int horizEnd		= (mode->crtc_hsync_end >> 3)		+ 1;
244 	int horizTotal		= (mode->crtc_htotal >> 3)		- 5;
245 	int horizBlankStart	= (mode->crtc_hdisplay >> 3)		- 1;
246 	int horizBlankEnd	= (mode->crtc_htotal >> 3)		- 1;
247 	int vertDisplay		= mode->crtc_vdisplay			- 1;
248 	int vertStart		= mode->crtc_vsync_start 		- 1;
249 	int vertEnd		= mode->crtc_vsync_end			- 1;
250 	int vertTotal		= mode->crtc_vtotal 			- 2;
251 	int vertBlankStart	= mode->crtc_vdisplay 			- 1;
252 	int vertBlankEnd	= mode->crtc_vtotal			- 1;
253 
254 	struct drm_encoder *encoder;
255 	bool fp_output = false;
256 
257 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
258 		struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
259 
260 		if (encoder->crtc == crtc &&
261 		    (nv_encoder->dcb->type == DCB_OUTPUT_LVDS ||
262 		     nv_encoder->dcb->type == DCB_OUTPUT_TMDS))
263 			fp_output = true;
264 	}
265 
266 	if (fp_output) {
267 		vertStart = vertTotal - 3;
268 		vertEnd = vertTotal - 2;
269 		vertBlankStart = vertStart;
270 		horizStart = horizTotal - 5;
271 		horizEnd = horizTotal - 2;
272 		horizBlankEnd = horizTotal + 4;
273 #if 0
274 		if (dev->overlayAdaptor && drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
275 			/* This reportedly works around some video overlay bandwidth problems */
276 			horizTotal += 2;
277 #endif
278 	}
279 
280 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
281 		vertTotal |= 1;
282 
283 #if 0
284 	ErrorF("horizDisplay: 0x%X \n", horizDisplay);
285 	ErrorF("horizStart: 0x%X \n", horizStart);
286 	ErrorF("horizEnd: 0x%X \n", horizEnd);
287 	ErrorF("horizTotal: 0x%X \n", horizTotal);
288 	ErrorF("horizBlankStart: 0x%X \n", horizBlankStart);
289 	ErrorF("horizBlankEnd: 0x%X \n", horizBlankEnd);
290 	ErrorF("vertDisplay: 0x%X \n", vertDisplay);
291 	ErrorF("vertStart: 0x%X \n", vertStart);
292 	ErrorF("vertEnd: 0x%X \n", vertEnd);
293 	ErrorF("vertTotal: 0x%X \n", vertTotal);
294 	ErrorF("vertBlankStart: 0x%X \n", vertBlankStart);
295 	ErrorF("vertBlankEnd: 0x%X \n", vertBlankEnd);
296 #endif
297 
298 	/*
299 	* compute correct Hsync & Vsync polarity
300 	*/
301 	if ((mode->flags & (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC))
302 		&& (mode->flags & (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) {
303 
304 		regp->MiscOutReg = 0x23;
305 		if (mode->flags & DRM_MODE_FLAG_NHSYNC)
306 			regp->MiscOutReg |= 0x40;
307 		if (mode->flags & DRM_MODE_FLAG_NVSYNC)
308 			regp->MiscOutReg |= 0x80;
309 	} else {
310 		int vdisplay = mode->vdisplay;
311 		if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
312 			vdisplay *= 2;
313 		if (mode->vscan > 1)
314 			vdisplay *= mode->vscan;
315 		if (vdisplay < 400)
316 			regp->MiscOutReg = 0xA3;	/* +hsync -vsync */
317 		else if (vdisplay < 480)
318 			regp->MiscOutReg = 0x63;	/* -hsync +vsync */
319 		else if (vdisplay < 768)
320 			regp->MiscOutReg = 0xE3;	/* -hsync -vsync */
321 		else
322 			regp->MiscOutReg = 0x23;	/* +hsync +vsync */
323 	}
324 
325 	/*
326 	 * Time Sequencer
327 	 */
328 	regp->Sequencer[NV_VIO_SR_RESET_INDEX] = 0x00;
329 	/* 0x20 disables the sequencer */
330 	if (mode->flags & DRM_MODE_FLAG_CLKDIV2)
331 		regp->Sequencer[NV_VIO_SR_CLOCK_INDEX] = 0x29;
332 	else
333 		regp->Sequencer[NV_VIO_SR_CLOCK_INDEX] = 0x21;
334 	regp->Sequencer[NV_VIO_SR_PLANE_MASK_INDEX] = 0x0F;
335 	regp->Sequencer[NV_VIO_SR_CHAR_MAP_INDEX] = 0x00;
336 	regp->Sequencer[NV_VIO_SR_MEM_MODE_INDEX] = 0x0E;
337 
338 	/*
339 	 * CRTC
340 	 */
341 	regp->CRTC[NV_CIO_CR_HDT_INDEX] = horizTotal;
342 	regp->CRTC[NV_CIO_CR_HDE_INDEX] = horizDisplay;
343 	regp->CRTC[NV_CIO_CR_HBS_INDEX] = horizBlankStart;
344 	regp->CRTC[NV_CIO_CR_HBE_INDEX] = (1 << 7) |
345 					  XLATE(horizBlankEnd, 0, NV_CIO_CR_HBE_4_0);
346 	regp->CRTC[NV_CIO_CR_HRS_INDEX] = horizStart;
347 	regp->CRTC[NV_CIO_CR_HRE_INDEX] = XLATE(horizBlankEnd, 5, NV_CIO_CR_HRE_HBE_5) |
348 					  XLATE(horizEnd, 0, NV_CIO_CR_HRE_4_0);
349 	regp->CRTC[NV_CIO_CR_VDT_INDEX] = vertTotal;
350 	regp->CRTC[NV_CIO_CR_OVL_INDEX] = XLATE(vertStart, 9, NV_CIO_CR_OVL_VRS_9) |
351 					  XLATE(vertDisplay, 9, NV_CIO_CR_OVL_VDE_9) |
352 					  XLATE(vertTotal, 9, NV_CIO_CR_OVL_VDT_9) |
353 					  (1 << 4) |
354 					  XLATE(vertBlankStart, 8, NV_CIO_CR_OVL_VBS_8) |
355 					  XLATE(vertStart, 8, NV_CIO_CR_OVL_VRS_8) |
356 					  XLATE(vertDisplay, 8, NV_CIO_CR_OVL_VDE_8) |
357 					  XLATE(vertTotal, 8, NV_CIO_CR_OVL_VDT_8);
358 	regp->CRTC[NV_CIO_CR_RSAL_INDEX] = 0x00;
359 	regp->CRTC[NV_CIO_CR_CELL_HT_INDEX] = ((mode->flags & DRM_MODE_FLAG_DBLSCAN) ? MASK(NV_CIO_CR_CELL_HT_SCANDBL) : 0) |
360 					      1 << 6 |
361 					      XLATE(vertBlankStart, 9, NV_CIO_CR_CELL_HT_VBS_9);
362 	regp->CRTC[NV_CIO_CR_CURS_ST_INDEX] = 0x00;
363 	regp->CRTC[NV_CIO_CR_CURS_END_INDEX] = 0x00;
364 	regp->CRTC[NV_CIO_CR_SA_HI_INDEX] = 0x00;
365 	regp->CRTC[NV_CIO_CR_SA_LO_INDEX] = 0x00;
366 	regp->CRTC[NV_CIO_CR_TCOFF_HI_INDEX] = 0x00;
367 	regp->CRTC[NV_CIO_CR_TCOFF_LO_INDEX] = 0x00;
368 	regp->CRTC[NV_CIO_CR_VRS_INDEX] = vertStart;
369 	regp->CRTC[NV_CIO_CR_VRE_INDEX] = 1 << 5 | XLATE(vertEnd, 0, NV_CIO_CR_VRE_3_0);
370 	regp->CRTC[NV_CIO_CR_VDE_INDEX] = vertDisplay;
371 	/* framebuffer can be larger than crtc scanout area. */
372 	regp->CRTC[NV_CIO_CR_OFFSET_INDEX] = fb->pitches[0] / 8;
373 	regp->CRTC[NV_CIO_CR_ULINE_INDEX] = 0x00;
374 	regp->CRTC[NV_CIO_CR_VBS_INDEX] = vertBlankStart;
375 	regp->CRTC[NV_CIO_CR_VBE_INDEX] = vertBlankEnd;
376 	regp->CRTC[NV_CIO_CR_MODE_INDEX] = 0x43;
377 	regp->CRTC[NV_CIO_CR_LCOMP_INDEX] = 0xff;
378 
379 	/*
380 	 * Some extended CRTC registers (they are not saved with the rest of the vga regs).
381 	 */
382 
383 	/* framebuffer can be larger than crtc scanout area. */
384 	regp->CRTC[NV_CIO_CRE_RPC0_INDEX] =
385 		XLATE(fb->pitches[0] / 8, 8, NV_CIO_CRE_RPC0_OFFSET_10_8);
386 	regp->CRTC[NV_CIO_CRE_42] =
387 		XLATE(fb->pitches[0] / 8, 11, NV_CIO_CRE_42_OFFSET_11);
388 	regp->CRTC[NV_CIO_CRE_RPC1_INDEX] = mode->crtc_hdisplay < 1280 ?
389 					    MASK(NV_CIO_CRE_RPC1_LARGE) : 0x00;
390 	regp->CRTC[NV_CIO_CRE_LSR_INDEX] = XLATE(horizBlankEnd, 6, NV_CIO_CRE_LSR_HBE_6) |
391 					   XLATE(vertBlankStart, 10, NV_CIO_CRE_LSR_VBS_10) |
392 					   XLATE(vertStart, 10, NV_CIO_CRE_LSR_VRS_10) |
393 					   XLATE(vertDisplay, 10, NV_CIO_CRE_LSR_VDE_10) |
394 					   XLATE(vertTotal, 10, NV_CIO_CRE_LSR_VDT_10);
395 	regp->CRTC[NV_CIO_CRE_HEB__INDEX] = XLATE(horizStart, 8, NV_CIO_CRE_HEB_HRS_8) |
396 					    XLATE(horizBlankStart, 8, NV_CIO_CRE_HEB_HBS_8) |
397 					    XLATE(horizDisplay, 8, NV_CIO_CRE_HEB_HDE_8) |
398 					    XLATE(horizTotal, 8, NV_CIO_CRE_HEB_HDT_8);
399 	regp->CRTC[NV_CIO_CRE_EBR_INDEX] = XLATE(vertBlankStart, 11, NV_CIO_CRE_EBR_VBS_11) |
400 					   XLATE(vertStart, 11, NV_CIO_CRE_EBR_VRS_11) |
401 					   XLATE(vertDisplay, 11, NV_CIO_CRE_EBR_VDE_11) |
402 					   XLATE(vertTotal, 11, NV_CIO_CRE_EBR_VDT_11);
403 
404 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
405 		horizTotal = (horizTotal >> 1) & ~1;
406 		regp->CRTC[NV_CIO_CRE_ILACE__INDEX] = horizTotal;
407 		regp->CRTC[NV_CIO_CRE_HEB__INDEX] |= XLATE(horizTotal, 8, NV_CIO_CRE_HEB_ILC_8);
408 	} else
409 		regp->CRTC[NV_CIO_CRE_ILACE__INDEX] = 0xff;  /* interlace off */
410 
411 	/*
412 	* Graphics Display Controller
413 	*/
414 	regp->Graphics[NV_VIO_GX_SR_INDEX] = 0x00;
415 	regp->Graphics[NV_VIO_GX_SREN_INDEX] = 0x00;
416 	regp->Graphics[NV_VIO_GX_CCOMP_INDEX] = 0x00;
417 	regp->Graphics[NV_VIO_GX_ROP_INDEX] = 0x00;
418 	regp->Graphics[NV_VIO_GX_READ_MAP_INDEX] = 0x00;
419 	regp->Graphics[NV_VIO_GX_MODE_INDEX] = 0x40; /* 256 color mode */
420 	regp->Graphics[NV_VIO_GX_MISC_INDEX] = 0x05; /* map 64k mem + graphic mode */
421 	regp->Graphics[NV_VIO_GX_DONT_CARE_INDEX] = 0x0F;
422 	regp->Graphics[NV_VIO_GX_BIT_MASK_INDEX] = 0xFF;
423 
424 	regp->Attribute[0]  = 0x00; /* standard colormap translation */
425 	regp->Attribute[1]  = 0x01;
426 	regp->Attribute[2]  = 0x02;
427 	regp->Attribute[3]  = 0x03;
428 	regp->Attribute[4]  = 0x04;
429 	regp->Attribute[5]  = 0x05;
430 	regp->Attribute[6]  = 0x06;
431 	regp->Attribute[7]  = 0x07;
432 	regp->Attribute[8]  = 0x08;
433 	regp->Attribute[9]  = 0x09;
434 	regp->Attribute[10] = 0x0A;
435 	regp->Attribute[11] = 0x0B;
436 	regp->Attribute[12] = 0x0C;
437 	regp->Attribute[13] = 0x0D;
438 	regp->Attribute[14] = 0x0E;
439 	regp->Attribute[15] = 0x0F;
440 	regp->Attribute[NV_CIO_AR_MODE_INDEX] = 0x01; /* Enable graphic mode */
441 	/* Non-vga */
442 	regp->Attribute[NV_CIO_AR_OSCAN_INDEX] = 0x00;
443 	regp->Attribute[NV_CIO_AR_PLANE_INDEX] = 0x0F; /* enable all color planes */
444 	regp->Attribute[NV_CIO_AR_HPP_INDEX] = 0x00;
445 	regp->Attribute[NV_CIO_AR_CSEL_INDEX] = 0x00;
446 }
447 
448 /**
449  * Sets up registers for the given mode/adjusted_mode pair.
450  *
451  * The clocks, CRTCs and outputs attached to this CRTC must be off.
452  *
453  * This shouldn't enable any clocks, CRTCs, or outputs, but they should
454  * be easily turned on/off after this.
455  */
456 static void
457 nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
458 {
459 	struct drm_device *dev = crtc->dev;
460 	struct nouveau_drm *drm = nouveau_drm(dev);
461 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
462 	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
463 	struct nv04_crtc_reg *savep = &nv04_display(dev)->saved_reg.crtc_reg[nv_crtc->index];
464 	const struct drm_framebuffer *fb = crtc->primary->fb;
465 	struct drm_encoder *encoder;
466 	bool lvds_output = false, tmds_output = false, tv_output = false,
467 		off_chip_digital = false;
468 
469 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
470 		struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
471 		bool digital = false;
472 
473 		if (encoder->crtc != crtc)
474 			continue;
475 
476 		if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS)
477 			digital = lvds_output = true;
478 		if (nv_encoder->dcb->type == DCB_OUTPUT_TV)
479 			tv_output = true;
480 		if (nv_encoder->dcb->type == DCB_OUTPUT_TMDS)
481 			digital = tmds_output = true;
482 		if (nv_encoder->dcb->location != DCB_LOC_ON_CHIP && digital)
483 			off_chip_digital = true;
484 	}
485 
486 	/* Registers not directly related to the (s)vga mode */
487 
488 	/* What is the meaning of this register? */
489 	/* A few popular values are 0x18, 0x1c, 0x38, 0x3c */
490 	regp->CRTC[NV_CIO_CRE_ENH_INDEX] = savep->CRTC[NV_CIO_CRE_ENH_INDEX] & ~(1<<5);
491 
492 	regp->crtc_eng_ctrl = 0;
493 	/* Except for rare conditions I2C is enabled on the primary crtc */
494 	if (nv_crtc->index == 0)
495 		regp->crtc_eng_ctrl |= NV_CRTC_FSEL_I2C;
496 #if 0
497 	/* Set overlay to desired crtc. */
498 	if (dev->overlayAdaptor) {
499 		NVPortPrivPtr pPriv = GET_OVERLAY_PRIVATE(dev);
500 		if (pPriv->overlayCRTC == nv_crtc->index)
501 			regp->crtc_eng_ctrl |= NV_CRTC_FSEL_OVERLAY;
502 	}
503 #endif
504 
505 	/* ADDRESS_SPACE_PNVM is the same as setting HCUR_ASI */
506 	regp->cursor_cfg = NV_PCRTC_CURSOR_CONFIG_CUR_LINES_64 |
507 			     NV_PCRTC_CURSOR_CONFIG_CUR_PIXELS_64 |
508 			     NV_PCRTC_CURSOR_CONFIG_ADDRESS_SPACE_PNVM;
509 	if (drm->client.device.info.chipset >= 0x11)
510 		regp->cursor_cfg |= NV_PCRTC_CURSOR_CONFIG_CUR_BPP_32;
511 	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
512 		regp->cursor_cfg |= NV_PCRTC_CURSOR_CONFIG_DOUBLE_SCAN_ENABLE;
513 
514 	/* Unblock some timings */
515 	regp->CRTC[NV_CIO_CRE_53] = 0;
516 	regp->CRTC[NV_CIO_CRE_54] = 0;
517 
518 	/* 0x00 is disabled, 0x11 is lvds, 0x22 crt and 0x88 tmds */
519 	if (lvds_output)
520 		regp->CRTC[NV_CIO_CRE_SCRATCH3__INDEX] = 0x11;
521 	else if (tmds_output)
522 		regp->CRTC[NV_CIO_CRE_SCRATCH3__INDEX] = 0x88;
523 	else
524 		regp->CRTC[NV_CIO_CRE_SCRATCH3__INDEX] = 0x22;
525 
526 	/* These values seem to vary */
527 	/* This register seems to be used by the bios to make certain decisions on some G70 cards? */
528 	regp->CRTC[NV_CIO_CRE_SCRATCH4__INDEX] = savep->CRTC[NV_CIO_CRE_SCRATCH4__INDEX];
529 
530 	nv_crtc_set_digital_vibrance(crtc, nv_crtc->saturation);
531 
532 	/* probably a scratch reg, but kept for cargo-cult purposes:
533 	 * bit0: crtc0?, head A
534 	 * bit6: lvds, head A
535 	 * bit7: (only in X), head A
536 	 */
537 	if (nv_crtc->index == 0)
538 		regp->CRTC[NV_CIO_CRE_4B] = savep->CRTC[NV_CIO_CRE_4B] | 0x80;
539 
540 	/* The blob seems to take the current value from crtc 0, add 4 to that
541 	 * and reuse the old value for crtc 1 */
542 	regp->CRTC[NV_CIO_CRE_TVOUT_LATENCY] = nv04_display(dev)->saved_reg.crtc_reg[0].CRTC[NV_CIO_CRE_TVOUT_LATENCY];
543 	if (!nv_crtc->index)
544 		regp->CRTC[NV_CIO_CRE_TVOUT_LATENCY] += 4;
545 
546 	/* the blob sometimes sets |= 0x10 (which is the same as setting |=
547 	 * 1 << 30 on 0x60.830), for no apparent reason */
548 	regp->CRTC[NV_CIO_CRE_59] = off_chip_digital;
549 
550 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
551 		regp->CRTC[0x9f] = off_chip_digital ? 0x11 : 0x1;
552 
553 	regp->crtc_830 = mode->crtc_vdisplay - 3;
554 	regp->crtc_834 = mode->crtc_vdisplay - 1;
555 
556 	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CURIE)
557 		/* This is what the blob does */
558 		regp->crtc_850 = NVReadCRTC(dev, 0, NV_PCRTC_850);
559 
560 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_RANKINE)
561 		regp->gpio_ext = NVReadCRTC(dev, 0, NV_PCRTC_GPIO_EXT);
562 
563 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
564 		regp->crtc_cfg = NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC;
565 	else
566 		regp->crtc_cfg = NV04_PCRTC_CONFIG_START_ADDRESS_HSYNC;
567 
568 	/* Some misc regs */
569 	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CURIE) {
570 		regp->CRTC[NV_CIO_CRE_85] = 0xFF;
571 		regp->CRTC[NV_CIO_CRE_86] = 0x1;
572 	}
573 
574 	regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] = (fb->format->depth + 1) / 8;
575 	/* Enable slaved mode (called MODE_TV in nv4ref.h) */
576 	if (lvds_output || tmds_output || tv_output)
577 		regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] |= (1 << 7);
578 
579 	/* Generic PRAMDAC regs */
580 
581 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS)
582 		/* Only bit that bios and blob set. */
583 		regp->nv10_cursync = (1 << 25);
584 
585 	regp->ramdac_gen_ctrl = NV_PRAMDAC_GENERAL_CONTROL_BPC_8BITS |
586 				NV_PRAMDAC_GENERAL_CONTROL_VGA_STATE_SEL |
587 				NV_PRAMDAC_GENERAL_CONTROL_PIXMIX_ON;
588 	if (fb->format->depth == 16)
589 		regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
590 	if (drm->client.device.info.chipset >= 0x11)
591 		regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_PIPE_LONG;
592 
593 	regp->ramdac_630 = 0; /* turn off green mode (tv test pattern?) */
594 	regp->tv_setup = 0;
595 
596 	nv_crtc_set_image_sharpening(crtc, nv_crtc->sharpness);
597 
598 	/* Some values the blob sets */
599 	regp->ramdac_8c0 = 0x100;
600 	regp->ramdac_a20 = 0x0;
601 	regp->ramdac_a24 = 0xfffff;
602 	regp->ramdac_a34 = 0x1;
603 }
604 
605 static int
606 nv_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb)
607 {
608 	struct nv04_display *disp = nv04_display(crtc->dev);
609 	struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->primary->fb);
610 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
611 	int ret;
612 
613 	ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM, false);
614 	if (ret == 0) {
615 		if (disp->image[nv_crtc->index])
616 			nouveau_bo_unpin(disp->image[nv_crtc->index]);
617 		nouveau_bo_ref(nvfb->nvbo, &disp->image[nv_crtc->index]);
618 	}
619 
620 	return ret;
621 }
622 
623 /**
624  * Sets up registers for the given mode/adjusted_mode pair.
625  *
626  * The clocks, CRTCs and outputs attached to this CRTC must be off.
627  *
628  * This shouldn't enable any clocks, CRTCs, or outputs, but they should
629  * be easily turned on/off after this.
630  */
631 static int
632 nv_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
633 		 struct drm_display_mode *adjusted_mode,
634 		 int x, int y, struct drm_framebuffer *old_fb)
635 {
636 	struct drm_device *dev = crtc->dev;
637 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
638 	struct nouveau_drm *drm = nouveau_drm(dev);
639 	int ret;
640 
641 	NV_DEBUG(drm, "CTRC mode on CRTC %d:\n", nv_crtc->index);
642 	drm_mode_debug_printmodeline(adjusted_mode);
643 
644 	ret = nv_crtc_swap_fbs(crtc, old_fb);
645 	if (ret)
646 		return ret;
647 
648 	/* unlock must come after turning off FP_TG_CONTROL in output_prepare */
649 	nv_lock_vga_crtc_shadow(dev, nv_crtc->index, -1);
650 
651 	nv_crtc_mode_set_vga(crtc, adjusted_mode);
652 	/* calculated in nv04_dfp_prepare, nv40 needs it written before calculating PLLs */
653 	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CURIE)
654 		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, nv04_display(dev)->mode_reg.sel_clk);
655 	nv_crtc_mode_set_regs(crtc, adjusted_mode);
656 	nv_crtc_calc_state_ext(crtc, mode, adjusted_mode->clock);
657 	return 0;
658 }
659 
660 static void nv_crtc_save(struct drm_crtc *crtc)
661 {
662 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
663 	struct drm_device *dev = crtc->dev;
664 	struct nv04_mode_state *state = &nv04_display(dev)->mode_reg;
665 	struct nv04_crtc_reg *crtc_state = &state->crtc_reg[nv_crtc->index];
666 	struct nv04_mode_state *saved = &nv04_display(dev)->saved_reg;
667 	struct nv04_crtc_reg *crtc_saved = &saved->crtc_reg[nv_crtc->index];
668 
669 	if (nv_two_heads(crtc->dev))
670 		NVSetOwner(crtc->dev, nv_crtc->index);
671 
672 	nouveau_hw_save_state(crtc->dev, nv_crtc->index, saved);
673 
674 	/* init some state to saved value */
675 	state->sel_clk = saved->sel_clk & ~(0x5 << 16);
676 	crtc_state->CRTC[NV_CIO_CRE_LCD__INDEX] = crtc_saved->CRTC[NV_CIO_CRE_LCD__INDEX];
677 	state->pllsel = saved->pllsel & ~(PLLSEL_VPLL1_MASK | PLLSEL_VPLL2_MASK | PLLSEL_TV_MASK);
678 	crtc_state->gpio_ext = crtc_saved->gpio_ext;
679 }
680 
681 static void nv_crtc_restore(struct drm_crtc *crtc)
682 {
683 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
684 	struct drm_device *dev = crtc->dev;
685 	int head = nv_crtc->index;
686 	uint8_t saved_cr21 = nv04_display(dev)->saved_reg.crtc_reg[head].CRTC[NV_CIO_CRE_21];
687 
688 	if (nv_two_heads(crtc->dev))
689 		NVSetOwner(crtc->dev, head);
690 
691 	nouveau_hw_load_state(crtc->dev, head, &nv04_display(dev)->saved_reg);
692 	nv_lock_vga_crtc_shadow(crtc->dev, head, saved_cr21);
693 
694 	nv_crtc->last_dpms = NV_DPMS_CLEARED;
695 }
696 
697 static void nv_crtc_prepare(struct drm_crtc *crtc)
698 {
699 	struct drm_device *dev = crtc->dev;
700 	struct nouveau_drm *drm = nouveau_drm(dev);
701 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
702 	const struct drm_crtc_helper_funcs *funcs = crtc->helper_private;
703 
704 	if (nv_two_heads(dev))
705 		NVSetOwner(dev, nv_crtc->index);
706 
707 	drm_crtc_vblank_off(crtc);
708 	funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
709 
710 	NVBlankScreen(dev, nv_crtc->index, true);
711 
712 	/* Some more preparation. */
713 	NVWriteCRTC(dev, nv_crtc->index, NV_PCRTC_CONFIG, NV_PCRTC_CONFIG_START_ADDRESS_NON_VGA);
714 	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CURIE) {
715 		uint32_t reg900 = NVReadRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_900);
716 		NVWriteRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_900, reg900 & ~0x10000);
717 	}
718 }
719 
720 static void nv_crtc_commit(struct drm_crtc *crtc)
721 {
722 	struct drm_device *dev = crtc->dev;
723 	const struct drm_crtc_helper_funcs *funcs = crtc->helper_private;
724 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
725 
726 	nouveau_hw_load_state(dev, nv_crtc->index, &nv04_display(dev)->mode_reg);
727 	nv04_crtc_mode_set_base(crtc, crtc->x, crtc->y, NULL);
728 
729 #ifdef __BIG_ENDIAN
730 	/* turn on LFB swapping */
731 	{
732 		uint8_t tmp = NVReadVgaCrtc(dev, nv_crtc->index, NV_CIO_CRE_RCR);
733 		tmp |= MASK(NV_CIO_CRE_RCR_ENDIAN_BIG);
734 		NVWriteVgaCrtc(dev, nv_crtc->index, NV_CIO_CRE_RCR, tmp);
735 	}
736 #endif
737 
738 	funcs->dpms(crtc, DRM_MODE_DPMS_ON);
739 	drm_crtc_vblank_on(crtc);
740 }
741 
742 static void nv_crtc_destroy(struct drm_crtc *crtc)
743 {
744 	struct nv04_display *disp = nv04_display(crtc->dev);
745 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
746 
747 	if (!nv_crtc)
748 		return;
749 
750 	drm_crtc_cleanup(crtc);
751 
752 	if (disp->image[nv_crtc->index])
753 		nouveau_bo_unpin(disp->image[nv_crtc->index]);
754 	nouveau_bo_ref(NULL, &disp->image[nv_crtc->index]);
755 
756 	nouveau_bo_unmap(nv_crtc->cursor.nvbo);
757 	nouveau_bo_unpin(nv_crtc->cursor.nvbo);
758 	nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo);
759 	kfree(nv_crtc);
760 }
761 
762 static void
763 nv_crtc_gamma_load(struct drm_crtc *crtc)
764 {
765 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
766 	struct drm_device *dev = nv_crtc->base.dev;
767 	struct rgb { uint8_t r, g, b; } __attribute__((packed)) *rgbs;
768 	u16 *r, *g, *b;
769 	int i;
770 
771 	rgbs = (struct rgb *)nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index].DAC;
772 	r = crtc->gamma_store;
773 	g = r + crtc->gamma_size;
774 	b = g + crtc->gamma_size;
775 
776 	for (i = 0; i < 256; i++) {
777 		rgbs[i].r = *r++ >> 8;
778 		rgbs[i].g = *g++ >> 8;
779 		rgbs[i].b = *b++ >> 8;
780 	}
781 
782 	nouveau_hw_load_state_palette(dev, nv_crtc->index, &nv04_display(dev)->mode_reg);
783 }
784 
785 static void
786 nv_crtc_disable(struct drm_crtc *crtc)
787 {
788 	struct nv04_display *disp = nv04_display(crtc->dev);
789 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
790 	if (disp->image[nv_crtc->index])
791 		nouveau_bo_unpin(disp->image[nv_crtc->index]);
792 	nouveau_bo_ref(NULL, &disp->image[nv_crtc->index]);
793 }
794 
795 static int
796 nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
797 		  uint32_t size,
798 		  struct drm_modeset_acquire_ctx *ctx)
799 {
800 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
801 
802 	/* We need to know the depth before we upload, but it's possible to
803 	 * get called before a framebuffer is bound.  If this is the case,
804 	 * mark the lut values as dirty by setting depth==0, and it'll be
805 	 * uploaded on the first mode_set_base()
806 	 */
807 	if (!nv_crtc->base.primary->fb) {
808 		nv_crtc->lut.depth = 0;
809 		return 0;
810 	}
811 
812 	nv_crtc_gamma_load(crtc);
813 
814 	return 0;
815 }
816 
817 static int
818 nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
819 			   struct drm_framebuffer *passed_fb,
820 			   int x, int y, bool atomic)
821 {
822 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
823 	struct drm_device *dev = crtc->dev;
824 	struct nouveau_drm *drm = nouveau_drm(dev);
825 	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
826 	struct drm_framebuffer *drm_fb;
827 	struct nouveau_framebuffer *fb;
828 	int arb_burst, arb_lwm;
829 
830 	NV_DEBUG(drm, "index %d\n", nv_crtc->index);
831 
832 	/* no fb bound */
833 	if (!atomic && !crtc->primary->fb) {
834 		NV_DEBUG(drm, "No FB bound\n");
835 		return 0;
836 	}
837 
838 	/* If atomic, we want to switch to the fb we were passed, so
839 	 * now we update pointers to do that.
840 	 */
841 	if (atomic) {
842 		drm_fb = passed_fb;
843 		fb = nouveau_framebuffer(passed_fb);
844 	} else {
845 		drm_fb = crtc->primary->fb;
846 		fb = nouveau_framebuffer(crtc->primary->fb);
847 	}
848 
849 	nv_crtc->fb.offset = fb->nvbo->bo.offset;
850 
851 	if (nv_crtc->lut.depth != drm_fb->format->depth) {
852 		nv_crtc->lut.depth = drm_fb->format->depth;
853 		nv_crtc_gamma_load(crtc);
854 	}
855 
856 	/* Update the framebuffer format. */
857 	regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] &= ~3;
858 	regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] |= (drm_fb->format->depth + 1) / 8;
859 	regp->ramdac_gen_ctrl &= ~NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
860 	if (drm_fb->format->depth == 16)
861 		regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
862 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_PIXEL_INDEX);
863 	NVWriteRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_GENERAL_CONTROL,
864 		      regp->ramdac_gen_ctrl);
865 
866 	regp->CRTC[NV_CIO_CR_OFFSET_INDEX] = drm_fb->pitches[0] >> 3;
867 	regp->CRTC[NV_CIO_CRE_RPC0_INDEX] =
868 		XLATE(drm_fb->pitches[0] >> 3, 8, NV_CIO_CRE_RPC0_OFFSET_10_8);
869 	regp->CRTC[NV_CIO_CRE_42] =
870 		XLATE(drm_fb->pitches[0] / 8, 11, NV_CIO_CRE_42_OFFSET_11);
871 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_RPC0_INDEX);
872 	crtc_wr_cio_state(crtc, regp, NV_CIO_CR_OFFSET_INDEX);
873 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_42);
874 
875 	/* Update the framebuffer location. */
876 	regp->fb_start = nv_crtc->fb.offset & ~3;
877 	regp->fb_start += (y * drm_fb->pitches[0]) + (x * drm_fb->format->cpp[0]);
878 	nv_set_crtc_base(dev, nv_crtc->index, regp->fb_start);
879 
880 	/* Update the arbitration parameters. */
881 	nouveau_calc_arb(dev, crtc->mode.clock, drm_fb->format->cpp[0] * 8,
882 			 &arb_burst, &arb_lwm);
883 
884 	regp->CRTC[NV_CIO_CRE_FF_INDEX] = arb_burst;
885 	regp->CRTC[NV_CIO_CRE_FFLWM__INDEX] = arb_lwm & 0xff;
886 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FF_INDEX);
887 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FFLWM__INDEX);
888 
889 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_KELVIN) {
890 		regp->CRTC[NV_CIO_CRE_47] = arb_lwm >> 8;
891 		crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_47);
892 	}
893 
894 	return 0;
895 }
896 
897 static int
898 nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
899 			struct drm_framebuffer *old_fb)
900 {
901 	int ret = nv_crtc_swap_fbs(crtc, old_fb);
902 	if (ret)
903 		return ret;
904 	return nv04_crtc_do_mode_set_base(crtc, old_fb, x, y, false);
905 }
906 
907 static int
908 nv04_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
909 			       struct drm_framebuffer *fb,
910 			       int x, int y, enum mode_set_atomic state)
911 {
912 	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
913 	struct drm_device *dev = drm->dev;
914 
915 	if (state == ENTER_ATOMIC_MODE_SET)
916 		nouveau_fbcon_accel_save_disable(dev);
917 	else
918 		nouveau_fbcon_accel_restore(dev);
919 
920 	return nv04_crtc_do_mode_set_base(crtc, fb, x, y, true);
921 }
922 
923 static void nv04_cursor_upload(struct drm_device *dev, struct nouveau_bo *src,
924 			       struct nouveau_bo *dst)
925 {
926 	int width = nv_cursor_width(dev);
927 	uint32_t pixel;
928 	int i, j;
929 
930 	for (i = 0; i < width; i++) {
931 		for (j = 0; j < width; j++) {
932 			pixel = nouveau_bo_rd32(src, i*64 + j);
933 
934 			nouveau_bo_wr16(dst, i*width + j, (pixel & 0x80000000) >> 16
935 				     | (pixel & 0xf80000) >> 9
936 				     | (pixel & 0xf800) >> 6
937 				     | (pixel & 0xf8) >> 3);
938 		}
939 	}
940 }
941 
942 static void nv11_cursor_upload(struct drm_device *dev, struct nouveau_bo *src,
943 			       struct nouveau_bo *dst)
944 {
945 	uint32_t pixel;
946 	int alpha, i;
947 
948 	/* nv11+ supports premultiplied (PM), or non-premultiplied (NPM) alpha
949 	 * cursors (though NPM in combination with fp dithering may not work on
950 	 * nv11, from "nv" driver history)
951 	 * NPM mode needs NV_PCRTC_CURSOR_CONFIG_ALPHA_BLEND set and is what the
952 	 * blob uses, however we get given PM cursors so we use PM mode
953 	 */
954 	for (i = 0; i < 64 * 64; i++) {
955 		pixel = nouveau_bo_rd32(src, i);
956 
957 		/* hw gets unhappy if alpha <= rgb values.  for a PM image "less
958 		 * than" shouldn't happen; fix "equal to" case by adding one to
959 		 * alpha channel (slightly inaccurate, but so is attempting to
960 		 * get back to NPM images, due to limits of integer precision)
961 		 */
962 		alpha = pixel >> 24;
963 		if (alpha > 0 && alpha < 255)
964 			pixel = (pixel & 0x00ffffff) | ((alpha + 1) << 24);
965 
966 #ifdef __BIG_ENDIAN
967 		{
968 			struct nouveau_drm *drm = nouveau_drm(dev);
969 
970 			if (drm->client.device.info.chipset == 0x11) {
971 				pixel = ((pixel & 0x000000ff) << 24) |
972 					((pixel & 0x0000ff00) << 8) |
973 					((pixel & 0x00ff0000) >> 8) |
974 					((pixel & 0xff000000) >> 24);
975 			}
976 		}
977 #endif
978 
979 		nouveau_bo_wr32(dst, i, pixel);
980 	}
981 }
982 
983 static int
984 nv04_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
985 		     uint32_t buffer_handle, uint32_t width, uint32_t height)
986 {
987 	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
988 	struct drm_device *dev = drm->dev;
989 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
990 	struct nouveau_bo *cursor = NULL;
991 	struct drm_gem_object *gem;
992 	int ret = 0;
993 
994 	if (!buffer_handle) {
995 		nv_crtc->cursor.hide(nv_crtc, true);
996 		return 0;
997 	}
998 
999 	if (width != 64 || height != 64)
1000 		return -EINVAL;
1001 
1002 	gem = drm_gem_object_lookup(file_priv, buffer_handle);
1003 	if (!gem)
1004 		return -ENOENT;
1005 	cursor = nouveau_gem_object(gem);
1006 
1007 	ret = nouveau_bo_map(cursor);
1008 	if (ret)
1009 		goto out;
1010 
1011 	if (drm->client.device.info.chipset >= 0x11)
1012 		nv11_cursor_upload(dev, cursor, nv_crtc->cursor.nvbo);
1013 	else
1014 		nv04_cursor_upload(dev, cursor, nv_crtc->cursor.nvbo);
1015 
1016 	nouveau_bo_unmap(cursor);
1017 	nv_crtc->cursor.offset = nv_crtc->cursor.nvbo->bo.offset;
1018 	nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.offset);
1019 	nv_crtc->cursor.show(nv_crtc, true);
1020 out:
1021 	drm_gem_object_put_unlocked(gem);
1022 	return ret;
1023 }
1024 
1025 static int
1026 nv04_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
1027 {
1028 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
1029 
1030 	nv_crtc->cursor.set_pos(nv_crtc, x, y);
1031 	return 0;
1032 }
1033 
1034 static int
1035 nouveau_crtc_set_config(struct drm_mode_set *set,
1036 			struct drm_modeset_acquire_ctx *ctx)
1037 {
1038 	struct drm_device *dev;
1039 	struct nouveau_drm *drm;
1040 	int ret;
1041 	struct drm_crtc *crtc;
1042 	bool active = false;
1043 	if (!set || !set->crtc)
1044 		return -EINVAL;
1045 
1046 	dev = set->crtc->dev;
1047 
1048 	/* get a pm reference here */
1049 	ret = pm_runtime_get_sync(dev->dev);
1050 	if (ret < 0 && ret != -EACCES)
1051 		return ret;
1052 
1053 	ret = drm_crtc_helper_set_config(set, ctx);
1054 
1055 	drm = nouveau_drm(dev);
1056 
1057 	/* if we get here with no crtcs active then we can drop a reference */
1058 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1059 		if (crtc->enabled)
1060 			active = true;
1061 	}
1062 
1063 	pm_runtime_mark_last_busy(dev->dev);
1064 	/* if we have active crtcs and we don't have a power ref,
1065 	   take the current one */
1066 	if (active && !drm->have_disp_power_ref) {
1067 		drm->have_disp_power_ref = true;
1068 		return ret;
1069 	}
1070 	/* if we have no active crtcs, then drop the power ref
1071 	   we got before */
1072 	if (!active && drm->have_disp_power_ref) {
1073 		pm_runtime_put_autosuspend(dev->dev);
1074 		drm->have_disp_power_ref = false;
1075 	}
1076 	/* drop the power reference we got coming in here */
1077 	pm_runtime_put_autosuspend(dev->dev);
1078 	return ret;
1079 }
1080 
1081 struct nv04_page_flip_state {
1082 	struct list_head head;
1083 	struct drm_pending_vblank_event *event;
1084 	struct drm_crtc *crtc;
1085 	int bpp, pitch;
1086 	u64 offset;
1087 };
1088 
1089 static int
1090 nv04_finish_page_flip(struct nouveau_channel *chan,
1091 		      struct nv04_page_flip_state *ps)
1092 {
1093 	struct nouveau_fence_chan *fctx = chan->fence;
1094 	struct nouveau_drm *drm = chan->drm;
1095 	struct drm_device *dev = drm->dev;
1096 	struct nv04_page_flip_state *s;
1097 	unsigned long flags;
1098 
1099 	spin_lock_irqsave(&dev->event_lock, flags);
1100 
1101 	if (list_empty(&fctx->flip)) {
1102 		NV_ERROR(drm, "unexpected pageflip\n");
1103 		spin_unlock_irqrestore(&dev->event_lock, flags);
1104 		return -EINVAL;
1105 	}
1106 
1107 	s = list_first_entry(&fctx->flip, struct nv04_page_flip_state, head);
1108 	if (s->event) {
1109 		drm_crtc_arm_vblank_event(s->crtc, s->event);
1110 	} else {
1111 		/* Give up ownership of vblank for page-flipped crtc */
1112 		drm_crtc_vblank_put(s->crtc);
1113 	}
1114 
1115 	list_del(&s->head);
1116 	if (ps)
1117 		*ps = *s;
1118 	kfree(s);
1119 
1120 	spin_unlock_irqrestore(&dev->event_lock, flags);
1121 	return 0;
1122 }
1123 
1124 int
1125 nv04_flip_complete(struct nvif_notify *notify)
1126 {
1127 	struct nouveau_cli *cli = (void *)notify->object->client;
1128 	struct nouveau_drm *drm = cli->drm;
1129 	struct nouveau_channel *chan = drm->channel;
1130 	struct nv04_page_flip_state state;
1131 
1132 	if (!nv04_finish_page_flip(chan, &state)) {
1133 		nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc),
1134 				 state.offset + state.crtc->y *
1135 				 state.pitch + state.crtc->x *
1136 				 state.bpp / 8);
1137 	}
1138 
1139 	return NVIF_NOTIFY_KEEP;
1140 }
1141 
1142 static int
1143 nv04_page_flip_emit(struct nouveau_channel *chan,
1144 		    struct nouveau_bo *old_bo,
1145 		    struct nouveau_bo *new_bo,
1146 		    struct nv04_page_flip_state *s,
1147 		    struct nouveau_fence **pfence)
1148 {
1149 	struct nouveau_fence_chan *fctx = chan->fence;
1150 	struct nouveau_drm *drm = chan->drm;
1151 	struct drm_device *dev = drm->dev;
1152 	unsigned long flags;
1153 	int ret;
1154 
1155 	/* Queue it to the pending list */
1156 	spin_lock_irqsave(&dev->event_lock, flags);
1157 	list_add_tail(&s->head, &fctx->flip);
1158 	spin_unlock_irqrestore(&dev->event_lock, flags);
1159 
1160 	/* Synchronize with the old framebuffer */
1161 	ret = nouveau_fence_sync(old_bo, chan, false, false);
1162 	if (ret)
1163 		goto fail;
1164 
1165 	/* Emit the pageflip */
1166 	ret = RING_SPACE(chan, 2);
1167 	if (ret)
1168 		goto fail;
1169 
1170 	BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1);
1171 	OUT_RING  (chan, 0x00000000);
1172 	FIRE_RING (chan);
1173 
1174 	ret = nouveau_fence_new(chan, false, pfence);
1175 	if (ret)
1176 		goto fail;
1177 
1178 	return 0;
1179 fail:
1180 	spin_lock_irqsave(&dev->event_lock, flags);
1181 	list_del(&s->head);
1182 	spin_unlock_irqrestore(&dev->event_lock, flags);
1183 	return ret;
1184 }
1185 
1186 static int
1187 nv04_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1188 		    struct drm_pending_vblank_event *event, u32 flags,
1189 		    struct drm_modeset_acquire_ctx *ctx)
1190 {
1191 	const int swap_interval = (flags & DRM_MODE_PAGE_FLIP_ASYNC) ? 0 : 1;
1192 	struct drm_device *dev = crtc->dev;
1193 	struct nouveau_drm *drm = nouveau_drm(dev);
1194 	struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->primary->fb)->nvbo;
1195 	struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo;
1196 	struct nv04_page_flip_state *s;
1197 	struct nouveau_channel *chan;
1198 	struct nouveau_cli *cli;
1199 	struct nouveau_fence *fence;
1200 	struct nv04_display *dispnv04 = nv04_display(dev);
1201 	int head = nouveau_crtc(crtc)->index;
1202 	int ret;
1203 
1204 	chan = drm->channel;
1205 	if (!chan)
1206 		return -ENODEV;
1207 	cli = (void *)chan->user.client;
1208 
1209 	s = kzalloc(sizeof(*s), GFP_KERNEL);
1210 	if (!s)
1211 		return -ENOMEM;
1212 
1213 	if (new_bo != old_bo) {
1214 		ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM, true);
1215 		if (ret)
1216 			goto fail_free;
1217 	}
1218 
1219 	mutex_lock(&cli->mutex);
1220 	ret = ttm_bo_reserve(&new_bo->bo, true, false, NULL);
1221 	if (ret)
1222 		goto fail_unpin;
1223 
1224 	/* synchronise rendering channel with the kernel's channel */
1225 	ret = nouveau_fence_sync(new_bo, chan, false, true);
1226 	if (ret) {
1227 		ttm_bo_unreserve(&new_bo->bo);
1228 		goto fail_unpin;
1229 	}
1230 
1231 	if (new_bo != old_bo) {
1232 		ttm_bo_unreserve(&new_bo->bo);
1233 
1234 		ret = ttm_bo_reserve(&old_bo->bo, true, false, NULL);
1235 		if (ret)
1236 			goto fail_unpin;
1237 	}
1238 
1239 	/* Initialize a page flip struct */
1240 	*s = (struct nv04_page_flip_state)
1241 		{ { }, event, crtc, fb->format->cpp[0] * 8, fb->pitches[0],
1242 		  new_bo->bo.offset };
1243 
1244 	/* Keep vblanks on during flip, for the target crtc of this flip */
1245 	drm_crtc_vblank_get(crtc);
1246 
1247 	/* Emit a page flip */
1248 	if (swap_interval) {
1249 		ret = RING_SPACE(chan, 8);
1250 		if (ret)
1251 			goto fail_unreserve;
1252 
1253 		BEGIN_NV04(chan, NvSubImageBlit, 0x012c, 1);
1254 		OUT_RING  (chan, 0);
1255 		BEGIN_NV04(chan, NvSubImageBlit, 0x0134, 1);
1256 		OUT_RING  (chan, head);
1257 		BEGIN_NV04(chan, NvSubImageBlit, 0x0100, 1);
1258 		OUT_RING  (chan, 0);
1259 		BEGIN_NV04(chan, NvSubImageBlit, 0x0130, 1);
1260 		OUT_RING  (chan, 0);
1261 	}
1262 
1263 	nouveau_bo_ref(new_bo, &dispnv04->image[head]);
1264 
1265 	ret = nv04_page_flip_emit(chan, old_bo, new_bo, s, &fence);
1266 	if (ret)
1267 		goto fail_unreserve;
1268 	mutex_unlock(&cli->mutex);
1269 
1270 	/* Update the crtc struct and cleanup */
1271 	crtc->primary->fb = fb;
1272 
1273 	nouveau_bo_fence(old_bo, fence, false);
1274 	ttm_bo_unreserve(&old_bo->bo);
1275 	if (old_bo != new_bo)
1276 		nouveau_bo_unpin(old_bo);
1277 	nouveau_fence_unref(&fence);
1278 	return 0;
1279 
1280 fail_unreserve:
1281 	drm_crtc_vblank_put(crtc);
1282 	ttm_bo_unreserve(&old_bo->bo);
1283 fail_unpin:
1284 	mutex_unlock(&cli->mutex);
1285 	if (old_bo != new_bo)
1286 		nouveau_bo_unpin(new_bo);
1287 fail_free:
1288 	kfree(s);
1289 	return ret;
1290 }
1291 
1292 static const struct drm_crtc_funcs nv04_crtc_funcs = {
1293 	.cursor_set = nv04_crtc_cursor_set,
1294 	.cursor_move = nv04_crtc_cursor_move,
1295 	.gamma_set = nv_crtc_gamma_set,
1296 	.set_config = nouveau_crtc_set_config,
1297 	.page_flip = nv04_crtc_page_flip,
1298 	.destroy = nv_crtc_destroy,
1299 };
1300 
1301 static const struct drm_crtc_helper_funcs nv04_crtc_helper_funcs = {
1302 	.dpms = nv_crtc_dpms,
1303 	.prepare = nv_crtc_prepare,
1304 	.commit = nv_crtc_commit,
1305 	.mode_set = nv_crtc_mode_set,
1306 	.mode_set_base = nv04_crtc_mode_set_base,
1307 	.mode_set_base_atomic = nv04_crtc_mode_set_base_atomic,
1308 	.disable = nv_crtc_disable,
1309 };
1310 
1311 static const uint32_t modeset_formats[] = {
1312         DRM_FORMAT_XRGB8888,
1313         DRM_FORMAT_RGB565,
1314         DRM_FORMAT_XRGB1555,
1315 };
1316 
1317 static struct drm_plane *
1318 create_primary_plane(struct drm_device *dev)
1319 {
1320         struct drm_plane *primary;
1321         int ret;
1322 
1323         primary = kzalloc(sizeof(*primary), GFP_KERNEL);
1324         if (primary == NULL) {
1325                 DRM_DEBUG_KMS("Failed to allocate primary plane\n");
1326                 return NULL;
1327         }
1328 
1329         /* possible_crtc's will be filled in later by crtc_init */
1330         ret = drm_universal_plane_init(dev, primary, 0,
1331                                        &drm_primary_helper_funcs,
1332                                        modeset_formats,
1333                                        ARRAY_SIZE(modeset_formats), NULL,
1334                                        DRM_PLANE_TYPE_PRIMARY, NULL);
1335         if (ret) {
1336                 kfree(primary);
1337                 primary = NULL;
1338         }
1339 
1340         return primary;
1341 }
1342 
1343 int
1344 nv04_crtc_create(struct drm_device *dev, int crtc_num)
1345 {
1346 	struct nouveau_crtc *nv_crtc;
1347 	int ret;
1348 
1349 	nv_crtc = kzalloc(sizeof(*nv_crtc), GFP_KERNEL);
1350 	if (!nv_crtc)
1351 		return -ENOMEM;
1352 
1353 	nv_crtc->lut.depth = 0;
1354 
1355 	nv_crtc->index = crtc_num;
1356 	nv_crtc->last_dpms = NV_DPMS_CLEARED;
1357 
1358 	nv_crtc->save = nv_crtc_save;
1359 	nv_crtc->restore = nv_crtc_restore;
1360 
1361 	drm_crtc_init_with_planes(dev, &nv_crtc->base,
1362                                   create_primary_plane(dev), NULL,
1363                                   &nv04_crtc_funcs, NULL);
1364 	drm_crtc_helper_add(&nv_crtc->base, &nv04_crtc_helper_funcs);
1365 	drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
1366 
1367 	ret = nouveau_bo_new(&nouveau_drm(dev)->client, 64*64*4, 0x100,
1368 			     TTM_PL_FLAG_VRAM, 0, 0x0000, NULL, NULL,
1369 			     &nv_crtc->cursor.nvbo);
1370 	if (!ret) {
1371 		ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM, false);
1372 		if (!ret) {
1373 			ret = nouveau_bo_map(nv_crtc->cursor.nvbo);
1374 			if (ret)
1375 				nouveau_bo_unpin(nv_crtc->cursor.nvbo);
1376 		}
1377 		if (ret)
1378 			nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo);
1379 	}
1380 
1381 	nv04_cursor_init(nv_crtc);
1382 
1383 	return 0;
1384 }
1385