1 /*
2  * Copyright 2011 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24 #include "disp.h"
25 #include "atom.h"
26 #include "core.h"
27 #include "head.h"
28 #include "wndw.h"
29 #include "handles.h"
30 
31 #include <linux/dma-mapping.h>
32 #include <linux/hdmi.h>
33 #include <linux/component.h>
34 
35 #include <drm/drm_atomic.h>
36 #include <drm/drm_atomic_helper.h>
37 #include <drm/drm_dp_helper.h>
38 #include <drm/drm_edid.h>
39 #include <drm/drm_fb_helper.h>
40 #include <drm/drm_plane_helper.h>
41 #include <drm/drm_probe_helper.h>
42 #include <drm/drm_scdc_helper.h>
43 #include <drm/drm_vblank.h>
44 
45 #include <nvif/push507c.h>
46 
47 #include <nvif/class.h>
48 #include <nvif/cl0002.h>
49 #include <nvif/cl5070.h>
50 #include <nvif/cl507d.h>
51 #include <nvif/event.h>
52 #include <nvif/timer.h>
53 
54 #include <nvhw/class/cl507c.h>
55 #include <nvhw/class/cl507d.h>
56 #include <nvhw/class/cl837d.h>
57 #include <nvhw/class/cl887d.h>
58 #include <nvhw/class/cl907d.h>
59 #include <nvhw/class/cl917d.h>
60 
61 #include "nouveau_drv.h"
62 #include "nouveau_dma.h"
63 #include "nouveau_gem.h"
64 #include "nouveau_connector.h"
65 #include "nouveau_encoder.h"
66 #include "nouveau_fence.h"
67 #include "nouveau_fbcon.h"
68 
69 #include <subdev/bios/dp.h>
70 
71 /******************************************************************************
72  * EVO channel
73  *****************************************************************************/
74 
75 static int
76 nv50_chan_create(struct nvif_device *device, struct nvif_object *disp,
77 		 const s32 *oclass, u8 head, void *data, u32 size,
78 		 struct nv50_chan *chan)
79 {
80 	struct nvif_sclass *sclass;
81 	int ret, i, n;
82 
83 	chan->device = device;
84 
85 	ret = n = nvif_object_sclass_get(disp, &sclass);
86 	if (ret < 0)
87 		return ret;
88 
89 	while (oclass[0]) {
90 		for (i = 0; i < n; i++) {
91 			if (sclass[i].oclass == oclass[0]) {
92 				ret = nvif_object_ctor(disp, "kmsChan", 0,
93 						       oclass[0], data, size,
94 						       &chan->user);
95 				if (ret == 0)
96 					nvif_object_map(&chan->user, NULL, 0);
97 				nvif_object_sclass_put(&sclass);
98 				return ret;
99 			}
100 		}
101 		oclass++;
102 	}
103 
104 	nvif_object_sclass_put(&sclass);
105 	return -ENOSYS;
106 }
107 
108 static void
109 nv50_chan_destroy(struct nv50_chan *chan)
110 {
111 	nvif_object_dtor(&chan->user);
112 }
113 
114 /******************************************************************************
115  * DMA EVO channel
116  *****************************************************************************/
117 
118 void
119 nv50_dmac_destroy(struct nv50_dmac *dmac)
120 {
121 	nvif_object_dtor(&dmac->vram);
122 	nvif_object_dtor(&dmac->sync);
123 
124 	nv50_chan_destroy(&dmac->base);
125 
126 	nvif_mem_dtor(&dmac->_push.mem);
127 }
128 
129 static void
130 nv50_dmac_kick(struct nvif_push *push)
131 {
132 	struct nv50_dmac *dmac = container_of(push, typeof(*dmac), _push);
133 
134 	dmac->cur = push->cur - (u32 *)dmac->_push.mem.object.map.ptr;
135 	if (dmac->put != dmac->cur) {
136 		/* Push buffer fetches are not coherent with BAR1, we need to ensure
137 		 * writes have been flushed right through to VRAM before writing PUT.
138 		 */
139 		if (dmac->push->mem.type & NVIF_MEM_VRAM) {
140 			struct nvif_device *device = dmac->base.device;
141 			nvif_wr32(&device->object, 0x070000, 0x00000001);
142 			nvif_msec(device, 2000,
143 				if (!(nvif_rd32(&device->object, 0x070000) & 0x00000002))
144 					break;
145 			);
146 		}
147 
148 		NVIF_WV32(&dmac->base.user, NV507C, PUT, PTR, dmac->cur);
149 		dmac->put = dmac->cur;
150 	}
151 
152 	push->bgn = push->cur;
153 }
154 
155 static int
156 nv50_dmac_free(struct nv50_dmac *dmac)
157 {
158 	u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR);
159 	if (get > dmac->cur) /* NVIDIA stay 5 away from GET, do the same. */
160 		return get - dmac->cur - 5;
161 	return dmac->max - dmac->cur;
162 }
163 
164 static int
165 nv50_dmac_wind(struct nv50_dmac *dmac)
166 {
167 	/* Wait for GET to depart from the beginning of the push buffer to
168 	 * prevent writing PUT == GET, which would be ignored by HW.
169 	 */
170 	u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR);
171 	if (get == 0) {
172 		/* Corner-case, HW idle, but non-committed work pending. */
173 		if (dmac->put == 0)
174 			nv50_dmac_kick(dmac->push);
175 
176 		if (nvif_msec(dmac->base.device, 2000,
177 			if (NVIF_TV32(&dmac->base.user, NV507C, GET, PTR, >, 0))
178 				break;
179 		) < 0)
180 			return -ETIMEDOUT;
181 	}
182 
183 	PUSH_RSVD(dmac->push, PUSH_JUMP(dmac->push, 0));
184 	dmac->cur = 0;
185 	return 0;
186 }
187 
188 static int
189 nv50_dmac_wait(struct nvif_push *push, u32 size)
190 {
191 	struct nv50_dmac *dmac = container_of(push, typeof(*dmac), _push);
192 	int free;
193 
194 	if (WARN_ON(size > dmac->max))
195 		return -EINVAL;
196 
197 	dmac->cur = push->cur - (u32 *)dmac->_push.mem.object.map.ptr;
198 	if (dmac->cur + size >= dmac->max) {
199 		int ret = nv50_dmac_wind(dmac);
200 		if (ret)
201 			return ret;
202 
203 		push->cur = dmac->_push.mem.object.map.ptr;
204 		push->cur = push->cur + dmac->cur;
205 		nv50_dmac_kick(push);
206 	}
207 
208 	if (nvif_msec(dmac->base.device, 2000,
209 		if ((free = nv50_dmac_free(dmac)) >= size)
210 			break;
211 	) < 0) {
212 		WARN_ON(1);
213 		return -ETIMEDOUT;
214 	}
215 
216 	push->bgn = dmac->_push.mem.object.map.ptr;
217 	push->bgn = push->bgn + dmac->cur;
218 	push->cur = push->bgn;
219 	push->end = push->cur + free;
220 	return 0;
221 }
222 
223 MODULE_PARM_DESC(kms_vram_pushbuf, "Place EVO/NVD push buffers in VRAM (default: auto)");
224 static int nv50_dmac_vram_pushbuf = -1;
225 module_param_named(kms_vram_pushbuf, nv50_dmac_vram_pushbuf, int, 0400);
226 
227 int
228 nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
229 		 const s32 *oclass, u8 head, void *data, u32 size, s64 syncbuf,
230 		 struct nv50_dmac *dmac)
231 {
232 	struct nouveau_cli *cli = (void *)device->object.client;
233 	struct nv50_disp_core_channel_dma_v0 *args = data;
234 	u8 type = NVIF_MEM_COHERENT;
235 	int ret;
236 
237 	mutex_init(&dmac->lock);
238 
239 	/* Pascal added support for 47-bit physical addresses, but some
240 	 * parts of EVO still only accept 40-bit PAs.
241 	 *
242 	 * To avoid issues on systems with large amounts of RAM, and on
243 	 * systems where an IOMMU maps pages at a high address, we need
244 	 * to allocate push buffers in VRAM instead.
245 	 *
246 	 * This appears to match NVIDIA's behaviour on Pascal.
247 	 */
248 	if ((nv50_dmac_vram_pushbuf > 0) ||
249 	    (nv50_dmac_vram_pushbuf < 0 && device->info.family == NV_DEVICE_INFO_V0_PASCAL))
250 		type |= NVIF_MEM_VRAM;
251 
252 	ret = nvif_mem_ctor_map(&cli->mmu, "kmsChanPush", type, 0x1000,
253 				&dmac->_push.mem);
254 	if (ret)
255 		return ret;
256 
257 	dmac->ptr = dmac->_push.mem.object.map.ptr;
258 	dmac->_push.wait = nv50_dmac_wait;
259 	dmac->_push.kick = nv50_dmac_kick;
260 	dmac->push = &dmac->_push;
261 	dmac->push->bgn = dmac->_push.mem.object.map.ptr;
262 	dmac->push->cur = dmac->push->bgn;
263 	dmac->push->end = dmac->push->bgn;
264 	dmac->max = 0x1000/4 - 1;
265 
266 	/* EVO channels are affected by a HW bug where the last 12 DWORDs
267 	 * of the push buffer aren't able to be used safely.
268 	 */
269 	if (disp->oclass < GV100_DISP)
270 		dmac->max -= 12;
271 
272 	args->pushbuf = nvif_handle(&dmac->_push.mem.object);
273 
274 	ret = nv50_chan_create(device, disp, oclass, head, data, size,
275 			       &dmac->base);
276 	if (ret)
277 		return ret;
278 
279 	if (syncbuf < 0)
280 		return 0;
281 
282 	ret = nvif_object_ctor(&dmac->base.user, "kmsSyncCtxDma", NV50_DISP_HANDLE_SYNCBUF,
283 			       NV_DMA_IN_MEMORY,
284 			       &(struct nv_dma_v0) {
285 					.target = NV_DMA_V0_TARGET_VRAM,
286 					.access = NV_DMA_V0_ACCESS_RDWR,
287 					.start = syncbuf + 0x0000,
288 					.limit = syncbuf + 0x0fff,
289 			       }, sizeof(struct nv_dma_v0),
290 			       &dmac->sync);
291 	if (ret)
292 		return ret;
293 
294 	ret = nvif_object_ctor(&dmac->base.user, "kmsVramCtxDma", NV50_DISP_HANDLE_VRAM,
295 			       NV_DMA_IN_MEMORY,
296 			       &(struct nv_dma_v0) {
297 					.target = NV_DMA_V0_TARGET_VRAM,
298 					.access = NV_DMA_V0_ACCESS_RDWR,
299 					.start = 0,
300 					.limit = device->info.ram_user - 1,
301 			       }, sizeof(struct nv_dma_v0),
302 			       &dmac->vram);
303 	if (ret)
304 		return ret;
305 
306 	return ret;
307 }
308 
309 /******************************************************************************
310  * Output path helpers
311  *****************************************************************************/
312 static void
313 nv50_outp_dump_caps(struct nouveau_drm *drm,
314 		    struct nouveau_encoder *outp)
315 {
316 	NV_DEBUG(drm, "%s caps: dp_interlace=%d\n",
317 		 outp->base.base.name, outp->caps.dp_interlace);
318 }
319 
320 static void
321 nv50_outp_release(struct nouveau_encoder *nv_encoder)
322 {
323 	struct nv50_disp *disp = nv50_disp(nv_encoder->base.base.dev);
324 	struct {
325 		struct nv50_disp_mthd_v1 base;
326 	} args = {
327 		.base.version = 1,
328 		.base.method = NV50_DISP_MTHD_V1_RELEASE,
329 		.base.hasht  = nv_encoder->dcb->hasht,
330 		.base.hashm  = nv_encoder->dcb->hashm,
331 	};
332 
333 	nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
334 	nv_encoder->or = -1;
335 	nv_encoder->link = 0;
336 }
337 
338 static int
339 nv50_outp_acquire(struct nouveau_encoder *nv_encoder, bool hda)
340 {
341 	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
342 	struct nv50_disp *disp = nv50_disp(drm->dev);
343 	struct {
344 		struct nv50_disp_mthd_v1 base;
345 		struct nv50_disp_acquire_v0 info;
346 	} args = {
347 		.base.version = 1,
348 		.base.method = NV50_DISP_MTHD_V1_ACQUIRE,
349 		.base.hasht  = nv_encoder->dcb->hasht,
350 		.base.hashm  = nv_encoder->dcb->hashm,
351 		.info.hda = hda,
352 	};
353 	int ret;
354 
355 	ret = nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
356 	if (ret) {
357 		NV_ERROR(drm, "error acquiring output path: %d\n", ret);
358 		return ret;
359 	}
360 
361 	nv_encoder->or = args.info.or;
362 	nv_encoder->link = args.info.link;
363 	return 0;
364 }
365 
366 static int
367 nv50_outp_atomic_check_view(struct drm_encoder *encoder,
368 			    struct drm_crtc_state *crtc_state,
369 			    struct drm_connector_state *conn_state,
370 			    struct drm_display_mode *native_mode)
371 {
372 	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
373 	struct drm_display_mode *mode = &crtc_state->mode;
374 	struct drm_connector *connector = conn_state->connector;
375 	struct nouveau_conn_atom *asyc = nouveau_conn_atom(conn_state);
376 	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
377 
378 	NV_ATOMIC(drm, "%s atomic_check\n", encoder->name);
379 	asyc->scaler.full = false;
380 	if (!native_mode)
381 		return 0;
382 
383 	if (asyc->scaler.mode == DRM_MODE_SCALE_NONE) {
384 		switch (connector->connector_type) {
385 		case DRM_MODE_CONNECTOR_LVDS:
386 		case DRM_MODE_CONNECTOR_eDP:
387 			/* Don't force scaler for EDID modes with
388 			 * same size as the native one (e.g. different
389 			 * refresh rate)
390 			 */
391 			if (mode->hdisplay == native_mode->hdisplay &&
392 			    mode->vdisplay == native_mode->vdisplay &&
393 			    mode->type & DRM_MODE_TYPE_DRIVER)
394 				break;
395 			mode = native_mode;
396 			asyc->scaler.full = true;
397 			break;
398 		default:
399 			break;
400 		}
401 	} else {
402 		mode = native_mode;
403 	}
404 
405 	if (!drm_mode_equal(adjusted_mode, mode)) {
406 		drm_mode_copy(adjusted_mode, mode);
407 		crtc_state->mode_changed = true;
408 	}
409 
410 	return 0;
411 }
412 
413 static int
414 nv50_outp_atomic_check(struct drm_encoder *encoder,
415 		       struct drm_crtc_state *crtc_state,
416 		       struct drm_connector_state *conn_state)
417 {
418 	struct drm_connector *connector = conn_state->connector;
419 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
420 	struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
421 	int ret;
422 
423 	ret = nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
424 					  nv_connector->native_mode);
425 	if (ret)
426 		return ret;
427 
428 	if (crtc_state->mode_changed || crtc_state->connectors_changed)
429 		asyh->or.bpc = connector->display_info.bpc;
430 
431 	return 0;
432 }
433 
434 struct nouveau_connector *
435 nv50_outp_get_new_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp)
436 {
437 	struct drm_connector *connector;
438 	struct drm_connector_state *connector_state;
439 	struct drm_encoder *encoder = to_drm_encoder(outp);
440 	int i;
441 
442 	for_each_new_connector_in_state(state, connector, connector_state, i) {
443 		if (connector_state->best_encoder == encoder)
444 			return nouveau_connector(connector);
445 	}
446 
447 	return NULL;
448 }
449 
450 struct nouveau_connector *
451 nv50_outp_get_old_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp)
452 {
453 	struct drm_connector *connector;
454 	struct drm_connector_state *connector_state;
455 	struct drm_encoder *encoder = to_drm_encoder(outp);
456 	int i;
457 
458 	for_each_old_connector_in_state(state, connector, connector_state, i) {
459 		if (connector_state->best_encoder == encoder)
460 			return nouveau_connector(connector);
461 	}
462 
463 	return NULL;
464 }
465 
466 static struct nouveau_crtc *
467 nv50_outp_get_new_crtc(const struct drm_atomic_state *state, const struct nouveau_encoder *outp)
468 {
469 	struct drm_crtc *crtc;
470 	struct drm_crtc_state *crtc_state;
471 	const u32 mask = drm_encoder_mask(&outp->base.base);
472 	int i;
473 
474 	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
475 		if (crtc_state->encoder_mask & mask)
476 			return nouveau_crtc(crtc);
477 	}
478 
479 	return NULL;
480 }
481 
482 /******************************************************************************
483  * DAC
484  *****************************************************************************/
485 static void
486 nv50_dac_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
487 {
488 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
489 	struct nv50_core *core = nv50_disp(encoder->dev)->core;
490 	const u32 ctrl = NVDEF(NV507D, DAC_SET_CONTROL, OWNER, NONE);
491 
492 	core->func->dac->ctrl(core, nv_encoder->or, ctrl, NULL);
493 	nv_encoder->crtc = NULL;
494 	nv50_outp_release(nv_encoder);
495 }
496 
497 static void
498 nv50_dac_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
499 {
500 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
501 	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
502 	struct nv50_head_atom *asyh =
503 		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base));
504 	struct nv50_core *core = nv50_disp(encoder->dev)->core;
505 	u32 ctrl = 0;
506 
507 	switch (nv_crtc->index) {
508 	case 0: ctrl |= NVDEF(NV507D, DAC_SET_CONTROL, OWNER, HEAD0); break;
509 	case 1: ctrl |= NVDEF(NV507D, DAC_SET_CONTROL, OWNER, HEAD1); break;
510 	case 2: ctrl |= NVDEF(NV907D, DAC_SET_CONTROL, OWNER_MASK, HEAD2); break;
511 	case 3: ctrl |= NVDEF(NV907D, DAC_SET_CONTROL, OWNER_MASK, HEAD3); break;
512 	default:
513 		WARN_ON(1);
514 		break;
515 	}
516 
517 	ctrl |= NVDEF(NV507D, DAC_SET_CONTROL, PROTOCOL, RGB_CRT);
518 
519 	nv50_outp_acquire(nv_encoder, false);
520 
521 	core->func->dac->ctrl(core, nv_encoder->or, ctrl, asyh);
522 	asyh->or.depth = 0;
523 
524 	nv_encoder->crtc = &nv_crtc->base;
525 }
526 
527 static enum drm_connector_status
528 nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
529 {
530 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
531 	struct nv50_disp *disp = nv50_disp(encoder->dev);
532 	struct {
533 		struct nv50_disp_mthd_v1 base;
534 		struct nv50_disp_dac_load_v0 load;
535 	} args = {
536 		.base.version = 1,
537 		.base.method = NV50_DISP_MTHD_V1_DAC_LOAD,
538 		.base.hasht  = nv_encoder->dcb->hasht,
539 		.base.hashm  = nv_encoder->dcb->hashm,
540 	};
541 	int ret;
542 
543 	args.load.data = nouveau_drm(encoder->dev)->vbios.dactestval;
544 	if (args.load.data == 0)
545 		args.load.data = 340;
546 
547 	ret = nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
548 	if (ret || !args.load.load)
549 		return connector_status_disconnected;
550 
551 	return connector_status_connected;
552 }
553 
554 static const struct drm_encoder_helper_funcs
555 nv50_dac_help = {
556 	.atomic_check = nv50_outp_atomic_check,
557 	.atomic_enable = nv50_dac_atomic_enable,
558 	.atomic_disable = nv50_dac_atomic_disable,
559 	.detect = nv50_dac_detect
560 };
561 
562 static void
563 nv50_dac_destroy(struct drm_encoder *encoder)
564 {
565 	drm_encoder_cleanup(encoder);
566 	kfree(encoder);
567 }
568 
569 static const struct drm_encoder_funcs
570 nv50_dac_func = {
571 	.destroy = nv50_dac_destroy,
572 };
573 
574 static int
575 nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
576 {
577 	struct nouveau_drm *drm = nouveau_drm(connector->dev);
578 	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
579 	struct nvkm_i2c_bus *bus;
580 	struct nouveau_encoder *nv_encoder;
581 	struct drm_encoder *encoder;
582 	int type = DRM_MODE_ENCODER_DAC;
583 
584 	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
585 	if (!nv_encoder)
586 		return -ENOMEM;
587 	nv_encoder->dcb = dcbe;
588 
589 	bus = nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
590 	if (bus)
591 		nv_encoder->i2c = &bus->i2c;
592 
593 	encoder = to_drm_encoder(nv_encoder);
594 	encoder->possible_crtcs = dcbe->heads;
595 	encoder->possible_clones = 0;
596 	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type,
597 			 "dac-%04x-%04x", dcbe->hasht, dcbe->hashm);
598 	drm_encoder_helper_add(encoder, &nv50_dac_help);
599 
600 	drm_connector_attach_encoder(connector, encoder);
601 	return 0;
602 }
603 
604 /*
605  * audio component binding for ELD notification
606  */
607 static void
608 nv50_audio_component_eld_notify(struct drm_audio_component *acomp, int port,
609 				int dev_id)
610 {
611 	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
612 		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
613 						 port, dev_id);
614 }
615 
616 static int
617 nv50_audio_component_get_eld(struct device *kdev, int port, int dev_id,
618 			     bool *enabled, unsigned char *buf, int max_bytes)
619 {
620 	struct drm_device *drm_dev = dev_get_drvdata(kdev);
621 	struct nouveau_drm *drm = nouveau_drm(drm_dev);
622 	struct drm_encoder *encoder;
623 	struct nouveau_encoder *nv_encoder;
624 	struct nouveau_crtc *nv_crtc;
625 	int ret = 0;
626 
627 	*enabled = false;
628 
629 	mutex_lock(&drm->audio.lock);
630 
631 	drm_for_each_encoder(encoder, drm->dev) {
632 		struct nouveau_connector *nv_connector = NULL;
633 
634 		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST)
635 			continue; /* TODO */
636 
637 		nv_encoder = nouveau_encoder(encoder);
638 		nv_connector = nouveau_connector(nv_encoder->audio.connector);
639 		nv_crtc = nouveau_crtc(nv_encoder->crtc);
640 
641 		if (!nv_crtc || nv_encoder->or != port || nv_crtc->index != dev_id)
642 			continue;
643 
644 		*enabled = nv_encoder->audio.enabled;
645 		if (*enabled) {
646 			ret = drm_eld_size(nv_connector->base.eld);
647 			memcpy(buf, nv_connector->base.eld,
648 			       min(max_bytes, ret));
649 		}
650 		break;
651 	}
652 
653 	mutex_unlock(&drm->audio.lock);
654 
655 	return ret;
656 }
657 
658 static const struct drm_audio_component_ops nv50_audio_component_ops = {
659 	.get_eld = nv50_audio_component_get_eld,
660 };
661 
662 static int
663 nv50_audio_component_bind(struct device *kdev, struct device *hda_kdev,
664 			  void *data)
665 {
666 	struct drm_device *drm_dev = dev_get_drvdata(kdev);
667 	struct nouveau_drm *drm = nouveau_drm(drm_dev);
668 	struct drm_audio_component *acomp = data;
669 
670 	if (WARN_ON(!device_link_add(hda_kdev, kdev, DL_FLAG_STATELESS)))
671 		return -ENOMEM;
672 
673 	drm_modeset_lock_all(drm_dev);
674 	acomp->ops = &nv50_audio_component_ops;
675 	acomp->dev = kdev;
676 	drm->audio.component = acomp;
677 	drm_modeset_unlock_all(drm_dev);
678 	return 0;
679 }
680 
681 static void
682 nv50_audio_component_unbind(struct device *kdev, struct device *hda_kdev,
683 			    void *data)
684 {
685 	struct drm_device *drm_dev = dev_get_drvdata(kdev);
686 	struct nouveau_drm *drm = nouveau_drm(drm_dev);
687 	struct drm_audio_component *acomp = data;
688 
689 	drm_modeset_lock_all(drm_dev);
690 	drm->audio.component = NULL;
691 	acomp->ops = NULL;
692 	acomp->dev = NULL;
693 	drm_modeset_unlock_all(drm_dev);
694 }
695 
696 static const struct component_ops nv50_audio_component_bind_ops = {
697 	.bind   = nv50_audio_component_bind,
698 	.unbind = nv50_audio_component_unbind,
699 };
700 
701 static void
702 nv50_audio_component_init(struct nouveau_drm *drm)
703 {
704 	if (component_add(drm->dev->dev, &nv50_audio_component_bind_ops))
705 		return;
706 
707 	drm->audio.component_registered = true;
708 	mutex_init(&drm->audio.lock);
709 }
710 
711 static void
712 nv50_audio_component_fini(struct nouveau_drm *drm)
713 {
714 	if (!drm->audio.component_registered)
715 		return;
716 
717 	component_del(drm->dev->dev, &nv50_audio_component_bind_ops);
718 	drm->audio.component_registered = false;
719 	mutex_destroy(&drm->audio.lock);
720 }
721 
722 /******************************************************************************
723  * Audio
724  *****************************************************************************/
725 static void
726 nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
727 {
728 	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
729 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
730 	struct nv50_disp *disp = nv50_disp(encoder->dev);
731 	struct {
732 		struct nv50_disp_mthd_v1 base;
733 		struct nv50_disp_sor_hda_eld_v0 eld;
734 	} args = {
735 		.base.version = 1,
736 		.base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
737 		.base.hasht   = nv_encoder->dcb->hasht,
738 		.base.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
739 				(0x0100 << nv_crtc->index),
740 	};
741 
742 	mutex_lock(&drm->audio.lock);
743 	if (nv_encoder->audio.enabled) {
744 		nv_encoder->audio.enabled = false;
745 		nv_encoder->audio.connector = NULL;
746 		nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
747 	}
748 	mutex_unlock(&drm->audio.lock);
749 
750 	nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or,
751 					nv_crtc->index);
752 }
753 
754 static void
755 nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc,
756 		  struct nouveau_connector *nv_connector, struct drm_atomic_state *state,
757 		  struct drm_display_mode *mode)
758 {
759 	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
760 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
761 	struct nv50_disp *disp = nv50_disp(encoder->dev);
762 	struct __packed {
763 		struct {
764 			struct nv50_disp_mthd_v1 mthd;
765 			struct nv50_disp_sor_hda_eld_v0 eld;
766 		} base;
767 		u8 data[sizeof(nv_connector->base.eld)];
768 	} args = {
769 		.base.mthd.version = 1,
770 		.base.mthd.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
771 		.base.mthd.hasht   = nv_encoder->dcb->hasht,
772 		.base.mthd.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
773 				     (0x0100 << nv_crtc->index),
774 	};
775 
776 	if (!drm_detect_monitor_audio(nv_connector->edid))
777 		return;
778 
779 	mutex_lock(&drm->audio.lock);
780 
781 	memcpy(args.data, nv_connector->base.eld, sizeof(args.data));
782 
783 	nvif_mthd(&disp->disp->object, 0, &args,
784 		  sizeof(args.base) + drm_eld_size(args.data));
785 	nv_encoder->audio.enabled = true;
786 	nv_encoder->audio.connector = &nv_connector->base;
787 
788 	mutex_unlock(&drm->audio.lock);
789 
790 	nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or,
791 					nv_crtc->index);
792 }
793 
794 /******************************************************************************
795  * HDMI
796  *****************************************************************************/
797 static void
798 nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
799 {
800 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
801 	struct nv50_disp *disp = nv50_disp(encoder->dev);
802 	struct {
803 		struct nv50_disp_mthd_v1 base;
804 		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
805 	} args = {
806 		.base.version = 1,
807 		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
808 		.base.hasht  = nv_encoder->dcb->hasht,
809 		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
810 			       (0x0100 << nv_crtc->index),
811 	};
812 
813 	nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
814 }
815 
816 static void
817 nv50_hdmi_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc,
818 		 struct nouveau_connector *nv_connector, struct drm_atomic_state *state,
819 		 struct drm_display_mode *mode)
820 {
821 	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
822 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
823 	struct nv50_disp *disp = nv50_disp(encoder->dev);
824 	struct {
825 		struct nv50_disp_mthd_v1 base;
826 		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
827 		u8 infoframes[2 * 17]; /* two frames, up to 17 bytes each */
828 	} args = {
829 		.base.version = 1,
830 		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
831 		.base.hasht  = nv_encoder->dcb->hasht,
832 		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
833 			       (0x0100 << nv_crtc->index),
834 		.pwr.state = 1,
835 		.pwr.rekey = 56, /* binary driver, and tegra, constant */
836 	};
837 	struct drm_hdmi_info *hdmi;
838 	u32 max_ac_packet;
839 	union hdmi_infoframe avi_frame;
840 	union hdmi_infoframe vendor_frame;
841 	bool high_tmds_clock_ratio = false, scrambling = false;
842 	u8 config;
843 	int ret;
844 	int size;
845 
846 	if (!drm_detect_hdmi_monitor(nv_connector->edid))
847 		return;
848 
849 	hdmi = &nv_connector->base.display_info.hdmi;
850 
851 	ret = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame.avi,
852 						       &nv_connector->base, mode);
853 	if (!ret) {
854 		/* We have an AVI InfoFrame, populate it to the display */
855 		args.pwr.avi_infoframe_length
856 			= hdmi_infoframe_pack(&avi_frame, args.infoframes, 17);
857 	}
858 
859 	ret = drm_hdmi_vendor_infoframe_from_display_mode(&vendor_frame.vendor.hdmi,
860 							  &nv_connector->base, mode);
861 	if (!ret) {
862 		/* We have a Vendor InfoFrame, populate it to the display */
863 		args.pwr.vendor_infoframe_length
864 			= hdmi_infoframe_pack(&vendor_frame,
865 					      args.infoframes
866 					      + args.pwr.avi_infoframe_length,
867 					      17);
868 	}
869 
870 	max_ac_packet  = mode->htotal - mode->hdisplay;
871 	max_ac_packet -= args.pwr.rekey;
872 	max_ac_packet -= 18; /* constant from tegra */
873 	args.pwr.max_ac_packet = max_ac_packet / 32;
874 
875 	if (hdmi->scdc.scrambling.supported) {
876 		high_tmds_clock_ratio = mode->clock > 340000;
877 		scrambling = high_tmds_clock_ratio ||
878 			hdmi->scdc.scrambling.low_rates;
879 	}
880 
881 	args.pwr.scdc =
882 		NV50_DISP_SOR_HDMI_PWR_V0_SCDC_SCRAMBLE * scrambling |
883 		NV50_DISP_SOR_HDMI_PWR_V0_SCDC_DIV_BY_4 * high_tmds_clock_ratio;
884 
885 	size = sizeof(args.base)
886 		+ sizeof(args.pwr)
887 		+ args.pwr.avi_infoframe_length
888 		+ args.pwr.vendor_infoframe_length;
889 	nvif_mthd(&disp->disp->object, 0, &args, size);
890 
891 	nv50_audio_enable(encoder, nv_crtc, nv_connector, state, mode);
892 
893 	/* If SCDC is supported by the downstream monitor, update
894 	 * divider / scrambling settings to what we programmed above.
895 	 */
896 	if (!hdmi->scdc.scrambling.supported)
897 		return;
898 
899 	ret = drm_scdc_readb(nv_encoder->i2c, SCDC_TMDS_CONFIG, &config);
900 	if (ret < 0) {
901 		NV_ERROR(drm, "Failure to read SCDC_TMDS_CONFIG: %d\n", ret);
902 		return;
903 	}
904 	config &= ~(SCDC_TMDS_BIT_CLOCK_RATIO_BY_40 | SCDC_SCRAMBLING_ENABLE);
905 	config |= SCDC_TMDS_BIT_CLOCK_RATIO_BY_40 * high_tmds_clock_ratio;
906 	config |= SCDC_SCRAMBLING_ENABLE * scrambling;
907 	ret = drm_scdc_writeb(nv_encoder->i2c, SCDC_TMDS_CONFIG, config);
908 	if (ret < 0)
909 		NV_ERROR(drm, "Failure to write SCDC_TMDS_CONFIG = 0x%02x: %d\n",
910 			 config, ret);
911 }
912 
913 /******************************************************************************
914  * MST
915  *****************************************************************************/
916 #define nv50_mstm(p) container_of((p), struct nv50_mstm, mgr)
917 #define nv50_mstc(p) container_of((p), struct nv50_mstc, connector)
918 #define nv50_msto(p) container_of((p), struct nv50_msto, encoder)
919 
920 struct nv50_mstc {
921 	struct nv50_mstm *mstm;
922 	struct drm_dp_mst_port *port;
923 	struct drm_connector connector;
924 
925 	struct drm_display_mode *native;
926 	struct edid *edid;
927 };
928 
929 struct nv50_msto {
930 	struct drm_encoder encoder;
931 
932 	/* head is statically assigned on msto creation */
933 	struct nv50_head *head;
934 	struct nv50_mstc *mstc;
935 	bool disabled;
936 };
937 
938 struct nouveau_encoder *nv50_real_outp(struct drm_encoder *encoder)
939 {
940 	struct nv50_msto *msto;
941 
942 	if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST)
943 		return nouveau_encoder(encoder);
944 
945 	msto = nv50_msto(encoder);
946 	if (!msto->mstc)
947 		return NULL;
948 	return msto->mstc->mstm->outp;
949 }
950 
951 static struct drm_dp_payload *
952 nv50_msto_payload(struct nv50_msto *msto)
953 {
954 	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
955 	struct nv50_mstc *mstc = msto->mstc;
956 	struct nv50_mstm *mstm = mstc->mstm;
957 	int vcpi = mstc->port->vcpi.vcpi, i;
958 
959 	WARN_ON(!mutex_is_locked(&mstm->mgr.payload_lock));
960 
961 	NV_ATOMIC(drm, "%s: vcpi %d\n", msto->encoder.name, vcpi);
962 	for (i = 0; i < mstm->mgr.max_payloads; i++) {
963 		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
964 		NV_ATOMIC(drm, "%s: %d: vcpi %d start 0x%02x slots 0x%02x\n",
965 			  mstm->outp->base.base.name, i, payload->vcpi,
966 			  payload->start_slot, payload->num_slots);
967 	}
968 
969 	for (i = 0; i < mstm->mgr.max_payloads; i++) {
970 		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
971 		if (payload->vcpi == vcpi)
972 			return payload;
973 	}
974 
975 	return NULL;
976 }
977 
978 static void
979 nv50_msto_cleanup(struct nv50_msto *msto)
980 {
981 	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
982 	struct nv50_mstc *mstc = msto->mstc;
983 	struct nv50_mstm *mstm = mstc->mstm;
984 
985 	if (!msto->disabled)
986 		return;
987 
988 	NV_ATOMIC(drm, "%s: msto cleanup\n", msto->encoder.name);
989 
990 	drm_dp_mst_deallocate_vcpi(&mstm->mgr, mstc->port);
991 
992 	msto->mstc = NULL;
993 	msto->disabled = false;
994 }
995 
996 static void
997 nv50_msto_prepare(struct nv50_msto *msto)
998 {
999 	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
1000 	struct nv50_mstc *mstc = msto->mstc;
1001 	struct nv50_mstm *mstm = mstc->mstm;
1002 	struct {
1003 		struct nv50_disp_mthd_v1 base;
1004 		struct nv50_disp_sor_dp_mst_vcpi_v0 vcpi;
1005 	} args = {
1006 		.base.version = 1,
1007 		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI,
1008 		.base.hasht  = mstm->outp->dcb->hasht,
1009 		.base.hashm  = (0xf0ff & mstm->outp->dcb->hashm) |
1010 			       (0x0100 << msto->head->base.index),
1011 	};
1012 
1013 	mutex_lock(&mstm->mgr.payload_lock);
1014 
1015 	NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name);
1016 	if (mstc->port->vcpi.vcpi > 0) {
1017 		struct drm_dp_payload *payload = nv50_msto_payload(msto);
1018 		if (payload) {
1019 			args.vcpi.start_slot = payload->start_slot;
1020 			args.vcpi.num_slots = payload->num_slots;
1021 			args.vcpi.pbn = mstc->port->vcpi.pbn;
1022 			args.vcpi.aligned_pbn = mstc->port->vcpi.aligned_pbn;
1023 		}
1024 	}
1025 
1026 	NV_ATOMIC(drm, "%s: %s: %02x %02x %04x %04x\n",
1027 		  msto->encoder.name, msto->head->base.base.name,
1028 		  args.vcpi.start_slot, args.vcpi.num_slots,
1029 		  args.vcpi.pbn, args.vcpi.aligned_pbn);
1030 
1031 	nvif_mthd(&drm->display->disp.object, 0, &args, sizeof(args));
1032 	mutex_unlock(&mstm->mgr.payload_lock);
1033 }
1034 
1035 static int
1036 nv50_msto_atomic_check(struct drm_encoder *encoder,
1037 		       struct drm_crtc_state *crtc_state,
1038 		       struct drm_connector_state *conn_state)
1039 {
1040 	struct drm_atomic_state *state = crtc_state->state;
1041 	struct drm_connector *connector = conn_state->connector;
1042 	struct nv50_mstc *mstc = nv50_mstc(connector);
1043 	struct nv50_mstm *mstm = mstc->mstm;
1044 	struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
1045 	int slots;
1046 	int ret;
1047 
1048 	ret = nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
1049 					  mstc->native);
1050 	if (ret)
1051 		return ret;
1052 
1053 	if (!crtc_state->mode_changed && !crtc_state->connectors_changed)
1054 		return 0;
1055 
1056 	/*
1057 	 * When restoring duplicated states, we need to make sure that the bw
1058 	 * remains the same and avoid recalculating it, as the connector's bpc
1059 	 * may have changed after the state was duplicated
1060 	 */
1061 	if (!state->duplicated) {
1062 		const int clock = crtc_state->adjusted_mode.clock;
1063 
1064 		asyh->or.bpc = connector->display_info.bpc;
1065 		asyh->dp.pbn = drm_dp_calc_pbn_mode(clock, asyh->or.bpc * 3,
1066 						    false);
1067 	}
1068 
1069 	slots = drm_dp_atomic_find_vcpi_slots(state, &mstm->mgr, mstc->port,
1070 					      asyh->dp.pbn, 0);
1071 	if (slots < 0)
1072 		return slots;
1073 
1074 	asyh->dp.tu = slots;
1075 
1076 	return 0;
1077 }
1078 
1079 static u8
1080 nv50_dp_bpc_to_depth(unsigned int bpc)
1081 {
1082 	switch (bpc) {
1083 	case  6: return NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_BPP_18_444;
1084 	case  8: return NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_BPP_24_444;
1085 	case 10:
1086 	default: return NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_BPP_30_444;
1087 	}
1088 }
1089 
1090 static void
1091 nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1092 {
1093 	struct nv50_msto *msto = nv50_msto(encoder);
1094 	struct nv50_head *head = msto->head;
1095 	struct nv50_head_atom *asyh =
1096 		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &head->base.base));
1097 	struct nv50_mstc *mstc = NULL;
1098 	struct nv50_mstm *mstm = NULL;
1099 	struct drm_connector *connector;
1100 	struct drm_connector_list_iter conn_iter;
1101 	u8 proto;
1102 	bool r;
1103 
1104 	drm_connector_list_iter_begin(encoder->dev, &conn_iter);
1105 	drm_for_each_connector_iter(connector, &conn_iter) {
1106 		if (connector->state->best_encoder == &msto->encoder) {
1107 			mstc = nv50_mstc(connector);
1108 			mstm = mstc->mstm;
1109 			break;
1110 		}
1111 	}
1112 	drm_connector_list_iter_end(&conn_iter);
1113 
1114 	if (WARN_ON(!mstc))
1115 		return;
1116 
1117 	r = drm_dp_mst_allocate_vcpi(&mstm->mgr, mstc->port, asyh->dp.pbn, asyh->dp.tu);
1118 	if (!r)
1119 		DRM_DEBUG_KMS("Failed to allocate VCPI\n");
1120 
1121 	if (!mstm->links++)
1122 		nv50_outp_acquire(mstm->outp, false /*XXX: MST audio.*/);
1123 
1124 	if (mstm->outp->link & 1)
1125 		proto = NV917D_SOR_SET_CONTROL_PROTOCOL_DP_A;
1126 	else
1127 		proto = NV917D_SOR_SET_CONTROL_PROTOCOL_DP_B;
1128 
1129 	mstm->outp->update(mstm->outp, head->base.index, asyh, proto,
1130 			   nv50_dp_bpc_to_depth(asyh->or.bpc));
1131 
1132 	msto->mstc = mstc;
1133 	mstm->modified = true;
1134 }
1135 
1136 static void
1137 nv50_msto_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1138 {
1139 	struct nv50_msto *msto = nv50_msto(encoder);
1140 	struct nv50_mstc *mstc = msto->mstc;
1141 	struct nv50_mstm *mstm = mstc->mstm;
1142 
1143 	drm_dp_mst_reset_vcpi_slots(&mstm->mgr, mstc->port);
1144 
1145 	mstm->outp->update(mstm->outp, msto->head->base.index, NULL, 0, 0);
1146 	mstm->modified = true;
1147 	if (!--mstm->links)
1148 		mstm->disabled = true;
1149 	msto->disabled = true;
1150 }
1151 
1152 static const struct drm_encoder_helper_funcs
1153 nv50_msto_help = {
1154 	.atomic_disable = nv50_msto_atomic_disable,
1155 	.atomic_enable = nv50_msto_atomic_enable,
1156 	.atomic_check = nv50_msto_atomic_check,
1157 };
1158 
1159 static void
1160 nv50_msto_destroy(struct drm_encoder *encoder)
1161 {
1162 	struct nv50_msto *msto = nv50_msto(encoder);
1163 	drm_encoder_cleanup(&msto->encoder);
1164 	kfree(msto);
1165 }
1166 
1167 static const struct drm_encoder_funcs
1168 nv50_msto = {
1169 	.destroy = nv50_msto_destroy,
1170 };
1171 
1172 static struct nv50_msto *
1173 nv50_msto_new(struct drm_device *dev, struct nv50_head *head, int id)
1174 {
1175 	struct nv50_msto *msto;
1176 	int ret;
1177 
1178 	msto = kzalloc(sizeof(*msto), GFP_KERNEL);
1179 	if (!msto)
1180 		return ERR_PTR(-ENOMEM);
1181 
1182 	ret = drm_encoder_init(dev, &msto->encoder, &nv50_msto,
1183 			       DRM_MODE_ENCODER_DPMST, "mst-%d", id);
1184 	if (ret) {
1185 		kfree(msto);
1186 		return ERR_PTR(ret);
1187 	}
1188 
1189 	drm_encoder_helper_add(&msto->encoder, &nv50_msto_help);
1190 	msto->encoder.possible_crtcs = drm_crtc_mask(&head->base.base);
1191 	msto->head = head;
1192 	return msto;
1193 }
1194 
1195 static struct drm_encoder *
1196 nv50_mstc_atomic_best_encoder(struct drm_connector *connector,
1197 			      struct drm_atomic_state *state)
1198 {
1199 	struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state,
1200 											 connector);
1201 	struct nv50_mstc *mstc = nv50_mstc(connector);
1202 	struct drm_crtc *crtc = connector_state->crtc;
1203 
1204 	if (!(mstc->mstm->outp->dcb->heads & drm_crtc_mask(crtc)))
1205 		return NULL;
1206 
1207 	return &nv50_head(crtc)->msto->encoder;
1208 }
1209 
1210 static enum drm_mode_status
1211 nv50_mstc_mode_valid(struct drm_connector *connector,
1212 		     struct drm_display_mode *mode)
1213 {
1214 	struct nv50_mstc *mstc = nv50_mstc(connector);
1215 	struct nouveau_encoder *outp = mstc->mstm->outp;
1216 
1217 	/* TODO: calculate the PBN from the dotclock and validate against the
1218 	 * MSTB's max possible PBN
1219 	 */
1220 
1221 	return nv50_dp_mode_valid(connector, outp, mode, NULL);
1222 }
1223 
1224 static int
1225 nv50_mstc_get_modes(struct drm_connector *connector)
1226 {
1227 	struct nv50_mstc *mstc = nv50_mstc(connector);
1228 	int ret = 0;
1229 
1230 	mstc->edid = drm_dp_mst_get_edid(&mstc->connector, mstc->port->mgr, mstc->port);
1231 	drm_connector_update_edid_property(&mstc->connector, mstc->edid);
1232 	if (mstc->edid)
1233 		ret = drm_add_edid_modes(&mstc->connector, mstc->edid);
1234 
1235 	/*
1236 	 * XXX: Since we don't use HDR in userspace quite yet, limit the bpc
1237 	 * to 8 to save bandwidth on the topology. In the future, we'll want
1238 	 * to properly fix this by dynamically selecting the highest possible
1239 	 * bpc that would fit in the topology
1240 	 */
1241 	if (connector->display_info.bpc)
1242 		connector->display_info.bpc =
1243 			clamp(connector->display_info.bpc, 6U, 8U);
1244 	else
1245 		connector->display_info.bpc = 8;
1246 
1247 	if (mstc->native)
1248 		drm_mode_destroy(mstc->connector.dev, mstc->native);
1249 	mstc->native = nouveau_conn_native_mode(&mstc->connector);
1250 	return ret;
1251 }
1252 
1253 static int
1254 nv50_mstc_atomic_check(struct drm_connector *connector,
1255 		       struct drm_atomic_state *state)
1256 {
1257 	struct nv50_mstc *mstc = nv50_mstc(connector);
1258 	struct drm_dp_mst_topology_mgr *mgr = &mstc->mstm->mgr;
1259 	struct drm_connector_state *new_conn_state =
1260 		drm_atomic_get_new_connector_state(state, connector);
1261 	struct drm_connector_state *old_conn_state =
1262 		drm_atomic_get_old_connector_state(state, connector);
1263 	struct drm_crtc_state *crtc_state;
1264 	struct drm_crtc *new_crtc = new_conn_state->crtc;
1265 
1266 	if (!old_conn_state->crtc)
1267 		return 0;
1268 
1269 	/* We only want to free VCPI if this state disables the CRTC on this
1270 	 * connector
1271 	 */
1272 	if (new_crtc) {
1273 		crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
1274 
1275 		if (!crtc_state ||
1276 		    !drm_atomic_crtc_needs_modeset(crtc_state) ||
1277 		    crtc_state->enable)
1278 			return 0;
1279 	}
1280 
1281 	return drm_dp_atomic_release_vcpi_slots(state, mgr, mstc->port);
1282 }
1283 
1284 static int
1285 nv50_mstc_detect(struct drm_connector *connector,
1286 		 struct drm_modeset_acquire_ctx *ctx, bool force)
1287 {
1288 	struct nv50_mstc *mstc = nv50_mstc(connector);
1289 	int ret;
1290 
1291 	if (drm_connector_is_unregistered(connector))
1292 		return connector_status_disconnected;
1293 
1294 	ret = pm_runtime_get_sync(connector->dev->dev);
1295 	if (ret < 0 && ret != -EACCES) {
1296 		pm_runtime_put_autosuspend(connector->dev->dev);
1297 		return connector_status_disconnected;
1298 	}
1299 
1300 	ret = drm_dp_mst_detect_port(connector, ctx, mstc->port->mgr,
1301 				     mstc->port);
1302 	if (ret != connector_status_connected)
1303 		goto out;
1304 
1305 out:
1306 	pm_runtime_mark_last_busy(connector->dev->dev);
1307 	pm_runtime_put_autosuspend(connector->dev->dev);
1308 	return ret;
1309 }
1310 
1311 static const struct drm_connector_helper_funcs
1312 nv50_mstc_help = {
1313 	.get_modes = nv50_mstc_get_modes,
1314 	.mode_valid = nv50_mstc_mode_valid,
1315 	.atomic_best_encoder = nv50_mstc_atomic_best_encoder,
1316 	.atomic_check = nv50_mstc_atomic_check,
1317 	.detect_ctx = nv50_mstc_detect,
1318 };
1319 
1320 static void
1321 nv50_mstc_destroy(struct drm_connector *connector)
1322 {
1323 	struct nv50_mstc *mstc = nv50_mstc(connector);
1324 
1325 	drm_connector_cleanup(&mstc->connector);
1326 	drm_dp_mst_put_port_malloc(mstc->port);
1327 
1328 	kfree(mstc);
1329 }
1330 
1331 static const struct drm_connector_funcs
1332 nv50_mstc = {
1333 	.reset = nouveau_conn_reset,
1334 	.fill_modes = drm_helper_probe_single_connector_modes,
1335 	.destroy = nv50_mstc_destroy,
1336 	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
1337 	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
1338 	.atomic_set_property = nouveau_conn_atomic_set_property,
1339 	.atomic_get_property = nouveau_conn_atomic_get_property,
1340 };
1341 
1342 static int
1343 nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
1344 	      const char *path, struct nv50_mstc **pmstc)
1345 {
1346 	struct drm_device *dev = mstm->outp->base.base.dev;
1347 	struct drm_crtc *crtc;
1348 	struct nv50_mstc *mstc;
1349 	int ret;
1350 
1351 	if (!(mstc = *pmstc = kzalloc(sizeof(*mstc), GFP_KERNEL)))
1352 		return -ENOMEM;
1353 	mstc->mstm = mstm;
1354 	mstc->port = port;
1355 
1356 	ret = drm_connector_init(dev, &mstc->connector, &nv50_mstc,
1357 				 DRM_MODE_CONNECTOR_DisplayPort);
1358 	if (ret) {
1359 		kfree(*pmstc);
1360 		*pmstc = NULL;
1361 		return ret;
1362 	}
1363 
1364 	drm_connector_helper_add(&mstc->connector, &nv50_mstc_help);
1365 
1366 	mstc->connector.funcs->reset(&mstc->connector);
1367 	nouveau_conn_attach_properties(&mstc->connector);
1368 
1369 	drm_for_each_crtc(crtc, dev) {
1370 		if (!(mstm->outp->dcb->heads & drm_crtc_mask(crtc)))
1371 			continue;
1372 
1373 		drm_connector_attach_encoder(&mstc->connector,
1374 					     &nv50_head(crtc)->msto->encoder);
1375 	}
1376 
1377 	drm_object_attach_property(&mstc->connector.base, dev->mode_config.path_property, 0);
1378 	drm_object_attach_property(&mstc->connector.base, dev->mode_config.tile_property, 0);
1379 	drm_connector_set_path_property(&mstc->connector, path);
1380 	drm_dp_mst_get_port_malloc(port);
1381 	return 0;
1382 }
1383 
1384 static void
1385 nv50_mstm_cleanup(struct nv50_mstm *mstm)
1386 {
1387 	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
1388 	struct drm_encoder *encoder;
1389 	int ret;
1390 
1391 	NV_ATOMIC(drm, "%s: mstm cleanup\n", mstm->outp->base.base.name);
1392 	ret = drm_dp_check_act_status(&mstm->mgr);
1393 
1394 	ret = drm_dp_update_payload_part2(&mstm->mgr);
1395 
1396 	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
1397 		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
1398 			struct nv50_msto *msto = nv50_msto(encoder);
1399 			struct nv50_mstc *mstc = msto->mstc;
1400 			if (mstc && mstc->mstm == mstm)
1401 				nv50_msto_cleanup(msto);
1402 		}
1403 	}
1404 
1405 	mstm->modified = false;
1406 }
1407 
1408 static void
1409 nv50_mstm_prepare(struct nv50_mstm *mstm)
1410 {
1411 	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
1412 	struct drm_encoder *encoder;
1413 	int ret;
1414 
1415 	NV_ATOMIC(drm, "%s: mstm prepare\n", mstm->outp->base.base.name);
1416 	ret = drm_dp_update_payload_part1(&mstm->mgr);
1417 
1418 	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
1419 		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
1420 			struct nv50_msto *msto = nv50_msto(encoder);
1421 			struct nv50_mstc *mstc = msto->mstc;
1422 			if (mstc && mstc->mstm == mstm)
1423 				nv50_msto_prepare(msto);
1424 		}
1425 	}
1426 
1427 	if (mstm->disabled) {
1428 		if (!mstm->links)
1429 			nv50_outp_release(mstm->outp);
1430 		mstm->disabled = false;
1431 	}
1432 }
1433 
1434 static struct drm_connector *
1435 nv50_mstm_add_connector(struct drm_dp_mst_topology_mgr *mgr,
1436 			struct drm_dp_mst_port *port, const char *path)
1437 {
1438 	struct nv50_mstm *mstm = nv50_mstm(mgr);
1439 	struct nv50_mstc *mstc;
1440 	int ret;
1441 
1442 	ret = nv50_mstc_new(mstm, port, path, &mstc);
1443 	if (ret)
1444 		return NULL;
1445 
1446 	return &mstc->connector;
1447 }
1448 
1449 static const struct drm_dp_mst_topology_cbs
1450 nv50_mstm = {
1451 	.add_connector = nv50_mstm_add_connector,
1452 };
1453 
1454 bool
1455 nv50_mstm_service(struct nouveau_drm *drm,
1456 		  struct nouveau_connector *nv_connector,
1457 		  struct nv50_mstm *mstm)
1458 {
1459 	struct drm_dp_aux *aux = &nv_connector->aux;
1460 	bool handled = true, ret = true;
1461 	int rc;
1462 	u8 esi[8] = {};
1463 
1464 	while (handled) {
1465 		rc = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8);
1466 		if (rc != 8) {
1467 			ret = false;
1468 			break;
1469 		}
1470 
1471 		drm_dp_mst_hpd_irq(&mstm->mgr, esi, &handled);
1472 		if (!handled)
1473 			break;
1474 
1475 		rc = drm_dp_dpcd_write(aux, DP_SINK_COUNT_ESI + 1, &esi[1],
1476 				       3);
1477 		if (rc != 3) {
1478 			ret = false;
1479 			break;
1480 		}
1481 	}
1482 
1483 	if (!ret)
1484 		NV_DEBUG(drm, "Failed to handle ESI on %s: %d\n",
1485 			 nv_connector->base.name, rc);
1486 
1487 	return ret;
1488 }
1489 
1490 void
1491 nv50_mstm_remove(struct nv50_mstm *mstm)
1492 {
1493 	mstm->is_mst = false;
1494 	drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
1495 }
1496 
1497 static int
1498 nv50_mstm_enable(struct nv50_mstm *mstm, int state)
1499 {
1500 	struct nouveau_encoder *outp = mstm->outp;
1501 	struct {
1502 		struct nv50_disp_mthd_v1 base;
1503 		struct nv50_disp_sor_dp_mst_link_v0 mst;
1504 	} args = {
1505 		.base.version = 1,
1506 		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_LINK,
1507 		.base.hasht = outp->dcb->hasht,
1508 		.base.hashm = outp->dcb->hashm,
1509 		.mst.state = state,
1510 	};
1511 	struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev);
1512 	struct nvif_object *disp = &drm->display->disp.object;
1513 
1514 	return nvif_mthd(disp, 0, &args, sizeof(args));
1515 }
1516 
1517 int
1518 nv50_mstm_detect(struct nouveau_encoder *outp)
1519 {
1520 	struct nv50_mstm *mstm = outp->dp.mstm;
1521 	struct drm_dp_aux *aux;
1522 	int ret;
1523 
1524 	if (!mstm || !mstm->can_mst)
1525 		return 0;
1526 
1527 	aux = mstm->mgr.aux;
1528 
1529 	/* Clear any leftover MST state we didn't set ourselves by first
1530 	 * disabling MST if it was already enabled
1531 	 */
1532 	ret = drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0);
1533 	if (ret < 0)
1534 		return ret;
1535 
1536 	/* And start enabling */
1537 	ret = nv50_mstm_enable(mstm, true);
1538 	if (ret)
1539 		return ret;
1540 
1541 	ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, true);
1542 	if (ret) {
1543 		nv50_mstm_enable(mstm, false);
1544 		return ret;
1545 	}
1546 
1547 	mstm->is_mst = true;
1548 	return 1;
1549 }
1550 
1551 static void
1552 nv50_mstm_fini(struct nouveau_encoder *outp)
1553 {
1554 	struct nv50_mstm *mstm = outp->dp.mstm;
1555 
1556 	if (!mstm)
1557 		return;
1558 
1559 	/* Don't change the MST state of this connector until we've finished
1560 	 * resuming, since we can't safely grab hpd_irq_lock in our resume
1561 	 * path to protect mstm->is_mst without potentially deadlocking
1562 	 */
1563 	mutex_lock(&outp->dp.hpd_irq_lock);
1564 	mstm->suspended = true;
1565 	mutex_unlock(&outp->dp.hpd_irq_lock);
1566 
1567 	if (mstm->is_mst)
1568 		drm_dp_mst_topology_mgr_suspend(&mstm->mgr);
1569 }
1570 
1571 static void
1572 nv50_mstm_init(struct nouveau_encoder *outp, bool runtime)
1573 {
1574 	struct nv50_mstm *mstm = outp->dp.mstm;
1575 	int ret = 0;
1576 
1577 	if (!mstm)
1578 		return;
1579 
1580 	if (mstm->is_mst) {
1581 		ret = drm_dp_mst_topology_mgr_resume(&mstm->mgr, !runtime);
1582 		if (ret == -1)
1583 			nv50_mstm_remove(mstm);
1584 	}
1585 
1586 	mutex_lock(&outp->dp.hpd_irq_lock);
1587 	mstm->suspended = false;
1588 	mutex_unlock(&outp->dp.hpd_irq_lock);
1589 
1590 	if (ret == -1)
1591 		drm_kms_helper_hotplug_event(mstm->mgr.dev);
1592 }
1593 
1594 static void
1595 nv50_mstm_del(struct nv50_mstm **pmstm)
1596 {
1597 	struct nv50_mstm *mstm = *pmstm;
1598 	if (mstm) {
1599 		drm_dp_mst_topology_mgr_destroy(&mstm->mgr);
1600 		kfree(*pmstm);
1601 		*pmstm = NULL;
1602 	}
1603 }
1604 
1605 static int
1606 nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max,
1607 	      int conn_base_id, struct nv50_mstm **pmstm)
1608 {
1609 	const int max_payloads = hweight8(outp->dcb->heads);
1610 	struct drm_device *dev = outp->base.base.dev;
1611 	struct nv50_mstm *mstm;
1612 	int ret;
1613 
1614 	if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL)))
1615 		return -ENOMEM;
1616 	mstm->outp = outp;
1617 	mstm->mgr.cbs = &nv50_mstm;
1618 
1619 	ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev, aux, aux_max,
1620 					   max_payloads, conn_base_id);
1621 	if (ret)
1622 		return ret;
1623 
1624 	return 0;
1625 }
1626 
1627 /******************************************************************************
1628  * SOR
1629  *****************************************************************************/
1630 static void
1631 nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head,
1632 		struct nv50_head_atom *asyh, u8 proto, u8 depth)
1633 {
1634 	struct nv50_disp *disp = nv50_disp(nv_encoder->base.base.dev);
1635 	struct nv50_core *core = disp->core;
1636 
1637 	if (!asyh) {
1638 		nv_encoder->ctrl &= ~BIT(head);
1639 		if (NVDEF_TEST(nv_encoder->ctrl, NV507D, SOR_SET_CONTROL, OWNER, ==, NONE))
1640 			nv_encoder->ctrl = 0;
1641 	} else {
1642 		nv_encoder->ctrl |= NVVAL(NV507D, SOR_SET_CONTROL, PROTOCOL, proto);
1643 		nv_encoder->ctrl |= BIT(head);
1644 		asyh->or.depth = depth;
1645 	}
1646 
1647 	core->func->sor->ctrl(core, nv_encoder->or, nv_encoder->ctrl, asyh);
1648 }
1649 
1650 static void
1651 nv50_sor_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1652 {
1653 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1654 	struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc);
1655 	struct nouveau_connector *nv_connector = nv50_outp_get_old_connector(state, nv_encoder);
1656 	struct drm_dp_aux *aux = &nv_connector->aux;
1657 	u8 pwr;
1658 
1659 	if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
1660 		int ret = drm_dp_dpcd_readb(aux, DP_SET_POWER, &pwr);
1661 
1662 		if (ret == 0) {
1663 			pwr &= ~DP_SET_POWER_MASK;
1664 			pwr |=  DP_SET_POWER_D3;
1665 			drm_dp_dpcd_writeb(aux, DP_SET_POWER, pwr);
1666 		}
1667 	}
1668 
1669 	nv_encoder->update(nv_encoder, nv_crtc->index, NULL, 0, 0);
1670 	nv50_audio_disable(encoder, nv_crtc);
1671 	nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc);
1672 	nv50_outp_release(nv_encoder);
1673 	nv_encoder->crtc = NULL;
1674 }
1675 
1676 static void
1677 nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1678 {
1679 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1680 	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
1681 	struct nv50_head_atom *asyh =
1682 		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base));
1683 	struct drm_display_mode *mode = &asyh->state.adjusted_mode;
1684 	struct {
1685 		struct nv50_disp_mthd_v1 base;
1686 		struct nv50_disp_sor_lvds_script_v0 lvds;
1687 	} lvds = {
1688 		.base.version = 1,
1689 		.base.method  = NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT,
1690 		.base.hasht   = nv_encoder->dcb->hasht,
1691 		.base.hashm   = nv_encoder->dcb->hashm,
1692 	};
1693 	struct nv50_disp *disp = nv50_disp(encoder->dev);
1694 	struct drm_device *dev = encoder->dev;
1695 	struct nouveau_drm *drm = nouveau_drm(dev);
1696 	struct nouveau_connector *nv_connector;
1697 	struct nvbios *bios = &drm->vbios;
1698 	bool hda = false;
1699 	u8 proto = NV507D_SOR_SET_CONTROL_PROTOCOL_CUSTOM;
1700 	u8 depth = NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_DEFAULT;
1701 
1702 	nv_connector = nv50_outp_get_new_connector(state, nv_encoder);
1703 	nv_encoder->crtc = &nv_crtc->base;
1704 
1705 	if ((disp->disp->object.oclass == GT214_DISP ||
1706 	     disp->disp->object.oclass >= GF110_DISP) &&
1707 	    drm_detect_monitor_audio(nv_connector->edid))
1708 		hda = true;
1709 	nv50_outp_acquire(nv_encoder, hda);
1710 
1711 	switch (nv_encoder->dcb->type) {
1712 	case DCB_OUTPUT_TMDS:
1713 		if (nv_encoder->link & 1) {
1714 			proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_A;
1715 			/* Only enable dual-link if:
1716 			 *  - Need to (i.e. rate > 165MHz)
1717 			 *  - DCB says we can
1718 			 *  - Not an HDMI monitor, since there's no dual-link
1719 			 *    on HDMI.
1720 			 */
1721 			if (mode->clock >= 165000 &&
1722 			    nv_encoder->dcb->duallink_possible &&
1723 			    !drm_detect_hdmi_monitor(nv_connector->edid))
1724 				proto = NV507D_SOR_SET_CONTROL_PROTOCOL_DUAL_TMDS;
1725 		} else {
1726 			proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_B;
1727 		}
1728 
1729 		nv50_hdmi_enable(&nv_encoder->base.base, nv_crtc, nv_connector, state, mode);
1730 		break;
1731 	case DCB_OUTPUT_LVDS:
1732 		proto = NV507D_SOR_SET_CONTROL_PROTOCOL_LVDS_CUSTOM;
1733 
1734 		if (bios->fp_no_ddc) {
1735 			if (bios->fp.dual_link)
1736 				lvds.lvds.script |= 0x0100;
1737 			if (bios->fp.if_is_24bit)
1738 				lvds.lvds.script |= 0x0200;
1739 		} else {
1740 			if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
1741 				if (((u8 *)nv_connector->edid)[121] == 2)
1742 					lvds.lvds.script |= 0x0100;
1743 			} else
1744 			if (mode->clock >= bios->fp.duallink_transition_clk) {
1745 				lvds.lvds.script |= 0x0100;
1746 			}
1747 
1748 			if (lvds.lvds.script & 0x0100) {
1749 				if (bios->fp.strapless_is_24bit & 2)
1750 					lvds.lvds.script |= 0x0200;
1751 			} else {
1752 				if (bios->fp.strapless_is_24bit & 1)
1753 					lvds.lvds.script |= 0x0200;
1754 			}
1755 
1756 			if (asyh->or.bpc == 8)
1757 				lvds.lvds.script |= 0x0200;
1758 		}
1759 
1760 		nvif_mthd(&disp->disp->object, 0, &lvds, sizeof(lvds));
1761 		break;
1762 	case DCB_OUTPUT_DP:
1763 		depth = nv50_dp_bpc_to_depth(asyh->or.bpc);
1764 
1765 		if (nv_encoder->link & 1)
1766 			proto = NV887D_SOR_SET_CONTROL_PROTOCOL_DP_A;
1767 		else
1768 			proto = NV887D_SOR_SET_CONTROL_PROTOCOL_DP_B;
1769 
1770 		nv50_audio_enable(encoder, nv_crtc, nv_connector, state, mode);
1771 		break;
1772 	default:
1773 		BUG();
1774 		break;
1775 	}
1776 
1777 	nv_encoder->update(nv_encoder, nv_crtc->index, asyh, proto, depth);
1778 }
1779 
1780 static const struct drm_encoder_helper_funcs
1781 nv50_sor_help = {
1782 	.atomic_check = nv50_outp_atomic_check,
1783 	.atomic_enable = nv50_sor_atomic_enable,
1784 	.atomic_disable = nv50_sor_atomic_disable,
1785 };
1786 
1787 static void
1788 nv50_sor_destroy(struct drm_encoder *encoder)
1789 {
1790 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1791 	nv50_mstm_del(&nv_encoder->dp.mstm);
1792 	drm_encoder_cleanup(encoder);
1793 
1794 	if (nv_encoder->dcb->type == DCB_OUTPUT_DP)
1795 		mutex_destroy(&nv_encoder->dp.hpd_irq_lock);
1796 
1797 	kfree(encoder);
1798 }
1799 
1800 static const struct drm_encoder_funcs
1801 nv50_sor_func = {
1802 	.destroy = nv50_sor_destroy,
1803 };
1804 
1805 static bool nv50_has_mst(struct nouveau_drm *drm)
1806 {
1807 	struct nvkm_bios *bios = nvxx_bios(&drm->client.device);
1808 	u32 data;
1809 	u8 ver, hdr, cnt, len;
1810 
1811 	data = nvbios_dp_table(bios, &ver, &hdr, &cnt, &len);
1812 	return data && ver >= 0x40 && (nvbios_rd08(bios, data + 0x08) & 0x04);
1813 }
1814 
1815 static int
1816 nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
1817 {
1818 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
1819 	struct nouveau_drm *drm = nouveau_drm(connector->dev);
1820 	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
1821 	struct nouveau_encoder *nv_encoder;
1822 	struct drm_encoder *encoder;
1823 	struct nv50_disp *disp = nv50_disp(connector->dev);
1824 	int type, ret;
1825 
1826 	switch (dcbe->type) {
1827 	case DCB_OUTPUT_LVDS: type = DRM_MODE_ENCODER_LVDS; break;
1828 	case DCB_OUTPUT_TMDS:
1829 	case DCB_OUTPUT_DP:
1830 	default:
1831 		type = DRM_MODE_ENCODER_TMDS;
1832 		break;
1833 	}
1834 
1835 	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
1836 	if (!nv_encoder)
1837 		return -ENOMEM;
1838 	nv_encoder->dcb = dcbe;
1839 	nv_encoder->update = nv50_sor_update;
1840 
1841 	encoder = to_drm_encoder(nv_encoder);
1842 	encoder->possible_crtcs = dcbe->heads;
1843 	encoder->possible_clones = 0;
1844 	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type,
1845 			 "sor-%04x-%04x", dcbe->hasht, dcbe->hashm);
1846 	drm_encoder_helper_add(encoder, &nv50_sor_help);
1847 
1848 	drm_connector_attach_encoder(connector, encoder);
1849 
1850 	disp->core->func->sor->get_caps(disp, nv_encoder, ffs(dcbe->or) - 1);
1851 	nv50_outp_dump_caps(drm, nv_encoder);
1852 
1853 	if (dcbe->type == DCB_OUTPUT_DP) {
1854 		struct nvkm_i2c_aux *aux =
1855 			nvkm_i2c_aux_find(i2c, dcbe->i2c_index);
1856 
1857 		mutex_init(&nv_encoder->dp.hpd_irq_lock);
1858 
1859 		if (aux) {
1860 			if (disp->disp->object.oclass < GF110_DISP) {
1861 				/* HW has no support for address-only
1862 				 * transactions, so we're required to
1863 				 * use custom I2C-over-AUX code.
1864 				 */
1865 				nv_encoder->i2c = &aux->i2c;
1866 			} else {
1867 				nv_encoder->i2c = &nv_connector->aux.ddc;
1868 			}
1869 			nv_encoder->aux = aux;
1870 		}
1871 
1872 		if (nv_connector->type != DCB_CONNECTOR_eDP &&
1873 		    nv50_has_mst(drm)) {
1874 			ret = nv50_mstm_new(nv_encoder, &nv_connector->aux,
1875 					    16, nv_connector->base.base.id,
1876 					    &nv_encoder->dp.mstm);
1877 			if (ret)
1878 				return ret;
1879 		}
1880 	} else {
1881 		struct nvkm_i2c_bus *bus =
1882 			nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
1883 		if (bus)
1884 			nv_encoder->i2c = &bus->i2c;
1885 	}
1886 
1887 	return 0;
1888 }
1889 
1890 /******************************************************************************
1891  * PIOR
1892  *****************************************************************************/
1893 static int
1894 nv50_pior_atomic_check(struct drm_encoder *encoder,
1895 		       struct drm_crtc_state *crtc_state,
1896 		       struct drm_connector_state *conn_state)
1897 {
1898 	int ret = nv50_outp_atomic_check(encoder, crtc_state, conn_state);
1899 	if (ret)
1900 		return ret;
1901 	crtc_state->adjusted_mode.clock *= 2;
1902 	return 0;
1903 }
1904 
1905 static void
1906 nv50_pior_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1907 {
1908 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1909 	struct nv50_core *core = nv50_disp(encoder->dev)->core;
1910 	const u32 ctrl = NVDEF(NV507D, PIOR_SET_CONTROL, OWNER, NONE);
1911 
1912 	core->func->pior->ctrl(core, nv_encoder->or, ctrl, NULL);
1913 	nv_encoder->crtc = NULL;
1914 	nv50_outp_release(nv_encoder);
1915 }
1916 
1917 static void
1918 nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
1919 {
1920 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
1921 	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
1922 	struct nv50_head_atom *asyh =
1923 		nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base));
1924 	struct nv50_core *core = nv50_disp(encoder->dev)->core;
1925 	u32 ctrl = 0;
1926 
1927 	switch (nv_crtc->index) {
1928 	case 0: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, OWNER, HEAD0); break;
1929 	case 1: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, OWNER, HEAD1); break;
1930 	default:
1931 		WARN_ON(1);
1932 		break;
1933 	}
1934 
1935 	nv50_outp_acquire(nv_encoder, false);
1936 
1937 	switch (asyh->or.bpc) {
1938 	case 10: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_30_444; break;
1939 	case  8: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_24_444; break;
1940 	case  6: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_18_444; break;
1941 	default: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_DEFAULT; break;
1942 	}
1943 
1944 	switch (nv_encoder->dcb->type) {
1945 	case DCB_OUTPUT_TMDS:
1946 	case DCB_OUTPUT_DP:
1947 		ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC);
1948 		break;
1949 	default:
1950 		BUG();
1951 		break;
1952 	}
1953 
1954 	core->func->pior->ctrl(core, nv_encoder->or, ctrl, asyh);
1955 	nv_encoder->crtc = &nv_crtc->base;
1956 }
1957 
1958 static const struct drm_encoder_helper_funcs
1959 nv50_pior_help = {
1960 	.atomic_check = nv50_pior_atomic_check,
1961 	.atomic_enable = nv50_pior_atomic_enable,
1962 	.atomic_disable = nv50_pior_atomic_disable,
1963 };
1964 
1965 static void
1966 nv50_pior_destroy(struct drm_encoder *encoder)
1967 {
1968 	drm_encoder_cleanup(encoder);
1969 	kfree(encoder);
1970 }
1971 
1972 static const struct drm_encoder_funcs
1973 nv50_pior_func = {
1974 	.destroy = nv50_pior_destroy,
1975 };
1976 
1977 static int
1978 nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
1979 {
1980 	struct drm_device *dev = connector->dev;
1981 	struct nouveau_drm *drm = nouveau_drm(dev);
1982 	struct nv50_disp *disp = nv50_disp(dev);
1983 	struct nvkm_i2c *i2c = nvxx_i2c(&drm->client.device);
1984 	struct nvkm_i2c_bus *bus = NULL;
1985 	struct nvkm_i2c_aux *aux = NULL;
1986 	struct i2c_adapter *ddc;
1987 	struct nouveau_encoder *nv_encoder;
1988 	struct drm_encoder *encoder;
1989 	int type;
1990 
1991 	switch (dcbe->type) {
1992 	case DCB_OUTPUT_TMDS:
1993 		bus  = nvkm_i2c_bus_find(i2c, NVKM_I2C_BUS_EXT(dcbe->extdev));
1994 		ddc  = bus ? &bus->i2c : NULL;
1995 		type = DRM_MODE_ENCODER_TMDS;
1996 		break;
1997 	case DCB_OUTPUT_DP:
1998 		aux  = nvkm_i2c_aux_find(i2c, NVKM_I2C_AUX_EXT(dcbe->extdev));
1999 		ddc  = aux ? &aux->i2c : NULL;
2000 		type = DRM_MODE_ENCODER_TMDS;
2001 		break;
2002 	default:
2003 		return -ENODEV;
2004 	}
2005 
2006 	nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
2007 	if (!nv_encoder)
2008 		return -ENOMEM;
2009 	nv_encoder->dcb = dcbe;
2010 	nv_encoder->i2c = ddc;
2011 	nv_encoder->aux = aux;
2012 
2013 	encoder = to_drm_encoder(nv_encoder);
2014 	encoder->possible_crtcs = dcbe->heads;
2015 	encoder->possible_clones = 0;
2016 	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type,
2017 			 "pior-%04x-%04x", dcbe->hasht, dcbe->hashm);
2018 	drm_encoder_helper_add(encoder, &nv50_pior_help);
2019 
2020 	drm_connector_attach_encoder(connector, encoder);
2021 
2022 	disp->core->func->pior->get_caps(disp, nv_encoder, ffs(dcbe->or) - 1);
2023 	nv50_outp_dump_caps(drm, nv_encoder);
2024 
2025 	return 0;
2026 }
2027 
2028 /******************************************************************************
2029  * Atomic
2030  *****************************************************************************/
2031 
2032 static void
2033 nv50_disp_atomic_commit_core(struct drm_atomic_state *state, u32 *interlock)
2034 {
2035 	struct nouveau_drm *drm = nouveau_drm(state->dev);
2036 	struct nv50_disp *disp = nv50_disp(drm->dev);
2037 	struct nv50_core *core = disp->core;
2038 	struct nv50_mstm *mstm;
2039 	struct drm_encoder *encoder;
2040 
2041 	NV_ATOMIC(drm, "commit core %08x\n", interlock[NV50_DISP_INTERLOCK_BASE]);
2042 
2043 	drm_for_each_encoder(encoder, drm->dev) {
2044 		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
2045 			mstm = nouveau_encoder(encoder)->dp.mstm;
2046 			if (mstm && mstm->modified)
2047 				nv50_mstm_prepare(mstm);
2048 		}
2049 	}
2050 
2051 	core->func->ntfy_init(disp->sync, NV50_DISP_CORE_NTFY);
2052 	core->func->update(core, interlock, true);
2053 	if (core->func->ntfy_wait_done(disp->sync, NV50_DISP_CORE_NTFY,
2054 				       disp->core->chan.base.device))
2055 		NV_ERROR(drm, "core notifier timeout\n");
2056 
2057 	drm_for_each_encoder(encoder, drm->dev) {
2058 		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
2059 			mstm = nouveau_encoder(encoder)->dp.mstm;
2060 			if (mstm && mstm->modified)
2061 				nv50_mstm_cleanup(mstm);
2062 		}
2063 	}
2064 }
2065 
2066 static void
2067 nv50_disp_atomic_commit_wndw(struct drm_atomic_state *state, u32 *interlock)
2068 {
2069 	struct drm_plane_state *new_plane_state;
2070 	struct drm_plane *plane;
2071 	int i;
2072 
2073 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2074 		struct nv50_wndw *wndw = nv50_wndw(plane);
2075 		if (interlock[wndw->interlock.type] & wndw->interlock.data) {
2076 			if (wndw->func->update)
2077 				wndw->func->update(wndw, interlock);
2078 		}
2079 	}
2080 }
2081 
2082 static void
2083 nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
2084 {
2085 	struct drm_device *dev = state->dev;
2086 	struct drm_crtc_state *new_crtc_state, *old_crtc_state;
2087 	struct drm_crtc *crtc;
2088 	struct drm_plane_state *new_plane_state;
2089 	struct drm_plane *plane;
2090 	struct nouveau_drm *drm = nouveau_drm(dev);
2091 	struct nv50_disp *disp = nv50_disp(dev);
2092 	struct nv50_atom *atom = nv50_atom(state);
2093 	struct nv50_core *core = disp->core;
2094 	struct nv50_outp_atom *outp, *outt;
2095 	u32 interlock[NV50_DISP_INTERLOCK__SIZE] = {};
2096 	int i;
2097 	bool flushed = false;
2098 
2099 	NV_ATOMIC(drm, "commit %d %d\n", atom->lock_core, atom->flush_disable);
2100 	nv50_crc_atomic_stop_reporting(state);
2101 	drm_atomic_helper_wait_for_fences(dev, state, false);
2102 	drm_atomic_helper_wait_for_dependencies(state);
2103 	drm_atomic_helper_update_legacy_modeset_state(dev, state);
2104 	drm_atomic_helper_calc_timestamping_constants(state);
2105 
2106 	if (atom->lock_core)
2107 		mutex_lock(&disp->mutex);
2108 
2109 	/* Disable head(s). */
2110 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
2111 		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
2112 		struct nv50_head *head = nv50_head(crtc);
2113 
2114 		NV_ATOMIC(drm, "%s: clr %04x (set %04x)\n", crtc->name,
2115 			  asyh->clr.mask, asyh->set.mask);
2116 
2117 		if (old_crtc_state->active && !new_crtc_state->active) {
2118 			pm_runtime_put_noidle(dev->dev);
2119 			drm_crtc_vblank_off(crtc);
2120 		}
2121 
2122 		if (asyh->clr.mask) {
2123 			nv50_head_flush_clr(head, asyh, atom->flush_disable);
2124 			interlock[NV50_DISP_INTERLOCK_CORE] |= 1;
2125 		}
2126 	}
2127 
2128 	/* Disable plane(s). */
2129 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2130 		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2131 		struct nv50_wndw *wndw = nv50_wndw(plane);
2132 
2133 		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", plane->name,
2134 			  asyw->clr.mask, asyw->set.mask);
2135 		if (!asyw->clr.mask)
2136 			continue;
2137 
2138 		nv50_wndw_flush_clr(wndw, interlock, atom->flush_disable, asyw);
2139 	}
2140 
2141 	/* Disable output path(s). */
2142 	list_for_each_entry(outp, &atom->outp, head) {
2143 		const struct drm_encoder_helper_funcs *help;
2144 		struct drm_encoder *encoder;
2145 
2146 		encoder = outp->encoder;
2147 		help = encoder->helper_private;
2148 
2149 		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", encoder->name,
2150 			  outp->clr.mask, outp->set.mask);
2151 
2152 		if (outp->clr.mask) {
2153 			help->atomic_disable(encoder, state);
2154 			interlock[NV50_DISP_INTERLOCK_CORE] |= 1;
2155 			if (outp->flush_disable) {
2156 				nv50_disp_atomic_commit_wndw(state, interlock);
2157 				nv50_disp_atomic_commit_core(state, interlock);
2158 				memset(interlock, 0x00, sizeof(interlock));
2159 
2160 				flushed = true;
2161 			}
2162 		}
2163 	}
2164 
2165 	/* Flush disable. */
2166 	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
2167 		if (atom->flush_disable) {
2168 			nv50_disp_atomic_commit_wndw(state, interlock);
2169 			nv50_disp_atomic_commit_core(state, interlock);
2170 			memset(interlock, 0x00, sizeof(interlock));
2171 
2172 			flushed = true;
2173 		}
2174 	}
2175 
2176 	if (flushed)
2177 		nv50_crc_atomic_release_notifier_contexts(state);
2178 	nv50_crc_atomic_init_notifier_contexts(state);
2179 
2180 	/* Update output path(s). */
2181 	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
2182 		const struct drm_encoder_helper_funcs *help;
2183 		struct drm_encoder *encoder;
2184 
2185 		encoder = outp->encoder;
2186 		help = encoder->helper_private;
2187 
2188 		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", encoder->name,
2189 			  outp->set.mask, outp->clr.mask);
2190 
2191 		if (outp->set.mask) {
2192 			help->atomic_enable(encoder, state);
2193 			interlock[NV50_DISP_INTERLOCK_CORE] = 1;
2194 		}
2195 
2196 		list_del(&outp->head);
2197 		kfree(outp);
2198 	}
2199 
2200 	/* Update head(s). */
2201 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
2202 		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
2203 		struct nv50_head *head = nv50_head(crtc);
2204 
2205 		NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name,
2206 			  asyh->set.mask, asyh->clr.mask);
2207 
2208 		if (asyh->set.mask) {
2209 			nv50_head_flush_set(head, asyh);
2210 			interlock[NV50_DISP_INTERLOCK_CORE] = 1;
2211 		}
2212 
2213 		if (new_crtc_state->active) {
2214 			if (!old_crtc_state->active) {
2215 				drm_crtc_vblank_on(crtc);
2216 				pm_runtime_get_noresume(dev->dev);
2217 			}
2218 			if (new_crtc_state->event)
2219 				drm_crtc_vblank_get(crtc);
2220 		}
2221 	}
2222 
2223 	/* Update window->head assignment.
2224 	 *
2225 	 * This has to happen in an update that's not interlocked with
2226 	 * any window channels to avoid hitting HW error checks.
2227 	 *
2228 	 *TODO: Proper handling of window ownership (Turing apparently
2229 	 *      supports non-fixed mappings).
2230 	 */
2231 	if (core->assign_windows) {
2232 		core->func->wndw.owner(core);
2233 		nv50_disp_atomic_commit_core(state, interlock);
2234 		core->assign_windows = false;
2235 		interlock[NV50_DISP_INTERLOCK_CORE] = 0;
2236 	}
2237 
2238 	/* Update plane(s). */
2239 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2240 		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2241 		struct nv50_wndw *wndw = nv50_wndw(plane);
2242 
2243 		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", plane->name,
2244 			  asyw->set.mask, asyw->clr.mask);
2245 		if ( !asyw->set.mask &&
2246 		    (!asyw->clr.mask || atom->flush_disable))
2247 			continue;
2248 
2249 		nv50_wndw_flush_set(wndw, interlock, asyw);
2250 	}
2251 
2252 	/* Flush update. */
2253 	nv50_disp_atomic_commit_wndw(state, interlock);
2254 
2255 	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
2256 		if (interlock[NV50_DISP_INTERLOCK_BASE] ||
2257 		    interlock[NV50_DISP_INTERLOCK_OVLY] ||
2258 		    interlock[NV50_DISP_INTERLOCK_WNDW] ||
2259 		    !atom->state.legacy_cursor_update)
2260 			nv50_disp_atomic_commit_core(state, interlock);
2261 		else
2262 			disp->core->func->update(disp->core, interlock, false);
2263 	}
2264 
2265 	if (atom->lock_core)
2266 		mutex_unlock(&disp->mutex);
2267 
2268 	/* Wait for HW to signal completion. */
2269 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2270 		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2271 		struct nv50_wndw *wndw = nv50_wndw(plane);
2272 		int ret = nv50_wndw_wait_armed(wndw, asyw);
2273 		if (ret)
2274 			NV_ERROR(drm, "%s: timeout\n", plane->name);
2275 	}
2276 
2277 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
2278 		if (new_crtc_state->event) {
2279 			unsigned long flags;
2280 			/* Get correct count/ts if racing with vblank irq */
2281 			if (new_crtc_state->active)
2282 				drm_crtc_accurate_vblank_count(crtc);
2283 			spin_lock_irqsave(&crtc->dev->event_lock, flags);
2284 			drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
2285 			spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
2286 
2287 			new_crtc_state->event = NULL;
2288 			if (new_crtc_state->active)
2289 				drm_crtc_vblank_put(crtc);
2290 		}
2291 	}
2292 
2293 	nv50_crc_atomic_start_reporting(state);
2294 	if (!flushed)
2295 		nv50_crc_atomic_release_notifier_contexts(state);
2296 	drm_atomic_helper_commit_hw_done(state);
2297 	drm_atomic_helper_cleanup_planes(dev, state);
2298 	drm_atomic_helper_commit_cleanup_done(state);
2299 	drm_atomic_state_put(state);
2300 
2301 	/* Drop the RPM ref we got from nv50_disp_atomic_commit() */
2302 	pm_runtime_mark_last_busy(dev->dev);
2303 	pm_runtime_put_autosuspend(dev->dev);
2304 }
2305 
2306 static void
2307 nv50_disp_atomic_commit_work(struct work_struct *work)
2308 {
2309 	struct drm_atomic_state *state =
2310 		container_of(work, typeof(*state), commit_work);
2311 	nv50_disp_atomic_commit_tail(state);
2312 }
2313 
2314 static int
2315 nv50_disp_atomic_commit(struct drm_device *dev,
2316 			struct drm_atomic_state *state, bool nonblock)
2317 {
2318 	struct drm_plane_state *new_plane_state;
2319 	struct drm_plane *plane;
2320 	int ret, i;
2321 
2322 	ret = pm_runtime_get_sync(dev->dev);
2323 	if (ret < 0 && ret != -EACCES) {
2324 		pm_runtime_put_autosuspend(dev->dev);
2325 		return ret;
2326 	}
2327 
2328 	ret = drm_atomic_helper_setup_commit(state, nonblock);
2329 	if (ret)
2330 		goto done;
2331 
2332 	INIT_WORK(&state->commit_work, nv50_disp_atomic_commit_work);
2333 
2334 	ret = drm_atomic_helper_prepare_planes(dev, state);
2335 	if (ret)
2336 		goto done;
2337 
2338 	if (!nonblock) {
2339 		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
2340 		if (ret)
2341 			goto err_cleanup;
2342 	}
2343 
2344 	ret = drm_atomic_helper_swap_state(state, true);
2345 	if (ret)
2346 		goto err_cleanup;
2347 
2348 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
2349 		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
2350 		struct nv50_wndw *wndw = nv50_wndw(plane);
2351 
2352 		if (asyw->set.image)
2353 			nv50_wndw_ntfy_enable(wndw, asyw);
2354 	}
2355 
2356 	drm_atomic_state_get(state);
2357 
2358 	/*
2359 	 * Grab another RPM ref for the commit tail, which will release the
2360 	 * ref when it's finished
2361 	 */
2362 	pm_runtime_get_noresume(dev->dev);
2363 
2364 	if (nonblock)
2365 		queue_work(system_unbound_wq, &state->commit_work);
2366 	else
2367 		nv50_disp_atomic_commit_tail(state);
2368 
2369 err_cleanup:
2370 	if (ret)
2371 		drm_atomic_helper_cleanup_planes(dev, state);
2372 done:
2373 	pm_runtime_put_autosuspend(dev->dev);
2374 	return ret;
2375 }
2376 
2377 static struct nv50_outp_atom *
2378 nv50_disp_outp_atomic_add(struct nv50_atom *atom, struct drm_encoder *encoder)
2379 {
2380 	struct nv50_outp_atom *outp;
2381 
2382 	list_for_each_entry(outp, &atom->outp, head) {
2383 		if (outp->encoder == encoder)
2384 			return outp;
2385 	}
2386 
2387 	outp = kzalloc(sizeof(*outp), GFP_KERNEL);
2388 	if (!outp)
2389 		return ERR_PTR(-ENOMEM);
2390 
2391 	list_add(&outp->head, &atom->outp);
2392 	outp->encoder = encoder;
2393 	return outp;
2394 }
2395 
2396 static int
2397 nv50_disp_outp_atomic_check_clr(struct nv50_atom *atom,
2398 				struct drm_connector_state *old_connector_state)
2399 {
2400 	struct drm_encoder *encoder = old_connector_state->best_encoder;
2401 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
2402 	struct drm_crtc *crtc;
2403 	struct nv50_outp_atom *outp;
2404 
2405 	if (!(crtc = old_connector_state->crtc))
2406 		return 0;
2407 
2408 	old_crtc_state = drm_atomic_get_old_crtc_state(&atom->state, crtc);
2409 	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
2410 	if (old_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
2411 		outp = nv50_disp_outp_atomic_add(atom, encoder);
2412 		if (IS_ERR(outp))
2413 			return PTR_ERR(outp);
2414 
2415 		if (outp->encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
2416 			outp->flush_disable = true;
2417 			atom->flush_disable = true;
2418 		}
2419 		outp->clr.ctrl = true;
2420 		atom->lock_core = true;
2421 	}
2422 
2423 	return 0;
2424 }
2425 
2426 static int
2427 nv50_disp_outp_atomic_check_set(struct nv50_atom *atom,
2428 				struct drm_connector_state *connector_state)
2429 {
2430 	struct drm_encoder *encoder = connector_state->best_encoder;
2431 	struct drm_crtc_state *new_crtc_state;
2432 	struct drm_crtc *crtc;
2433 	struct nv50_outp_atom *outp;
2434 
2435 	if (!(crtc = connector_state->crtc))
2436 		return 0;
2437 
2438 	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
2439 	if (new_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
2440 		outp = nv50_disp_outp_atomic_add(atom, encoder);
2441 		if (IS_ERR(outp))
2442 			return PTR_ERR(outp);
2443 
2444 		outp->set.ctrl = true;
2445 		atom->lock_core = true;
2446 	}
2447 
2448 	return 0;
2449 }
2450 
2451 static int
2452 nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
2453 {
2454 	struct nv50_atom *atom = nv50_atom(state);
2455 	struct nv50_core *core = nv50_disp(dev)->core;
2456 	struct drm_connector_state *old_connector_state, *new_connector_state;
2457 	struct drm_connector *connector;
2458 	struct drm_crtc_state *new_crtc_state;
2459 	struct drm_crtc *crtc;
2460 	struct nv50_head *head;
2461 	struct nv50_head_atom *asyh;
2462 	int ret, i;
2463 
2464 	if (core->assign_windows && core->func->head->static_wndw_map) {
2465 		drm_for_each_crtc(crtc, dev) {
2466 			new_crtc_state = drm_atomic_get_crtc_state(state,
2467 								   crtc);
2468 			if (IS_ERR(new_crtc_state))
2469 				return PTR_ERR(new_crtc_state);
2470 
2471 			head = nv50_head(crtc);
2472 			asyh = nv50_head_atom(new_crtc_state);
2473 			core->func->head->static_wndw_map(head, asyh);
2474 		}
2475 	}
2476 
2477 	/* We need to handle colour management on a per-plane basis. */
2478 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
2479 		if (new_crtc_state->color_mgmt_changed) {
2480 			ret = drm_atomic_add_affected_planes(state, crtc);
2481 			if (ret)
2482 				return ret;
2483 		}
2484 	}
2485 
2486 	ret = drm_atomic_helper_check(dev, state);
2487 	if (ret)
2488 		return ret;
2489 
2490 	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
2491 		ret = nv50_disp_outp_atomic_check_clr(atom, old_connector_state);
2492 		if (ret)
2493 			return ret;
2494 
2495 		ret = nv50_disp_outp_atomic_check_set(atom, new_connector_state);
2496 		if (ret)
2497 			return ret;
2498 	}
2499 
2500 	ret = drm_dp_mst_atomic_check(state);
2501 	if (ret)
2502 		return ret;
2503 
2504 	nv50_crc_atomic_check_outp(atom);
2505 
2506 	return 0;
2507 }
2508 
2509 static void
2510 nv50_disp_atomic_state_clear(struct drm_atomic_state *state)
2511 {
2512 	struct nv50_atom *atom = nv50_atom(state);
2513 	struct nv50_outp_atom *outp, *outt;
2514 
2515 	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
2516 		list_del(&outp->head);
2517 		kfree(outp);
2518 	}
2519 
2520 	drm_atomic_state_default_clear(state);
2521 }
2522 
2523 static void
2524 nv50_disp_atomic_state_free(struct drm_atomic_state *state)
2525 {
2526 	struct nv50_atom *atom = nv50_atom(state);
2527 	drm_atomic_state_default_release(&atom->state);
2528 	kfree(atom);
2529 }
2530 
2531 static struct drm_atomic_state *
2532 nv50_disp_atomic_state_alloc(struct drm_device *dev)
2533 {
2534 	struct nv50_atom *atom;
2535 	if (!(atom = kzalloc(sizeof(*atom), GFP_KERNEL)) ||
2536 	    drm_atomic_state_init(dev, &atom->state) < 0) {
2537 		kfree(atom);
2538 		return NULL;
2539 	}
2540 	INIT_LIST_HEAD(&atom->outp);
2541 	return &atom->state;
2542 }
2543 
2544 static const struct drm_mode_config_funcs
2545 nv50_disp_func = {
2546 	.fb_create = nouveau_user_framebuffer_create,
2547 	.output_poll_changed = nouveau_fbcon_output_poll_changed,
2548 	.atomic_check = nv50_disp_atomic_check,
2549 	.atomic_commit = nv50_disp_atomic_commit,
2550 	.atomic_state_alloc = nv50_disp_atomic_state_alloc,
2551 	.atomic_state_clear = nv50_disp_atomic_state_clear,
2552 	.atomic_state_free = nv50_disp_atomic_state_free,
2553 };
2554 
2555 /******************************************************************************
2556  * Init
2557  *****************************************************************************/
2558 
2559 static void
2560 nv50_display_fini(struct drm_device *dev, bool runtime, bool suspend)
2561 {
2562 	struct nouveau_drm *drm = nouveau_drm(dev);
2563 	struct drm_encoder *encoder;
2564 	struct drm_plane *plane;
2565 
2566 	drm_for_each_plane(plane, dev) {
2567 		struct nv50_wndw *wndw = nv50_wndw(plane);
2568 		if (plane->funcs != &nv50_wndw)
2569 			continue;
2570 		nv50_wndw_fini(wndw);
2571 	}
2572 
2573 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
2574 		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST)
2575 			nv50_mstm_fini(nouveau_encoder(encoder));
2576 	}
2577 
2578 	if (!runtime)
2579 		cancel_work_sync(&drm->hpd_work);
2580 }
2581 
2582 static int
2583 nv50_display_init(struct drm_device *dev, bool resume, bool runtime)
2584 {
2585 	struct nv50_core *core = nv50_disp(dev)->core;
2586 	struct drm_encoder *encoder;
2587 	struct drm_plane *plane;
2588 
2589 	if (resume || runtime)
2590 		core->func->init(core);
2591 
2592 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
2593 		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
2594 			struct nouveau_encoder *nv_encoder =
2595 				nouveau_encoder(encoder);
2596 			nv50_mstm_init(nv_encoder, runtime);
2597 		}
2598 	}
2599 
2600 	drm_for_each_plane(plane, dev) {
2601 		struct nv50_wndw *wndw = nv50_wndw(plane);
2602 		if (plane->funcs != &nv50_wndw)
2603 			continue;
2604 		nv50_wndw_init(wndw);
2605 	}
2606 
2607 	return 0;
2608 }
2609 
2610 static void
2611 nv50_display_destroy(struct drm_device *dev)
2612 {
2613 	struct nv50_disp *disp = nv50_disp(dev);
2614 
2615 	nv50_audio_component_fini(nouveau_drm(dev));
2616 
2617 	nvif_object_unmap(&disp->caps);
2618 	nvif_object_dtor(&disp->caps);
2619 	nv50_core_del(&disp->core);
2620 
2621 	nouveau_bo_unmap(disp->sync);
2622 	if (disp->sync)
2623 		nouveau_bo_unpin(disp->sync);
2624 	nouveau_bo_ref(NULL, &disp->sync);
2625 
2626 	nouveau_display(dev)->priv = NULL;
2627 	kfree(disp);
2628 }
2629 
2630 int
2631 nv50_display_create(struct drm_device *dev)
2632 {
2633 	struct nvif_device *device = &nouveau_drm(dev)->client.device;
2634 	struct nouveau_drm *drm = nouveau_drm(dev);
2635 	struct dcb_table *dcb = &drm->vbios.dcb;
2636 	struct drm_connector *connector, *tmp;
2637 	struct nv50_disp *disp;
2638 	struct dcb_output *dcbe;
2639 	int crtcs, ret, i;
2640 	bool has_mst = nv50_has_mst(drm);
2641 
2642 	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
2643 	if (!disp)
2644 		return -ENOMEM;
2645 
2646 	mutex_init(&disp->mutex);
2647 
2648 	nouveau_display(dev)->priv = disp;
2649 	nouveau_display(dev)->dtor = nv50_display_destroy;
2650 	nouveau_display(dev)->init = nv50_display_init;
2651 	nouveau_display(dev)->fini = nv50_display_fini;
2652 	disp->disp = &nouveau_display(dev)->disp;
2653 	dev->mode_config.funcs = &nv50_disp_func;
2654 	dev->mode_config.quirk_addfb_prefer_xbgr_30bpp = true;
2655 	dev->mode_config.normalize_zpos = true;
2656 
2657 	/* small shared memory area we use for notifiers and semaphores */
2658 	ret = nouveau_bo_new(&drm->client, 4096, 0x1000,
2659 			     NOUVEAU_GEM_DOMAIN_VRAM,
2660 			     0, 0x0000, NULL, NULL, &disp->sync);
2661 	if (!ret) {
2662 		ret = nouveau_bo_pin(disp->sync, NOUVEAU_GEM_DOMAIN_VRAM, true);
2663 		if (!ret) {
2664 			ret = nouveau_bo_map(disp->sync);
2665 			if (ret)
2666 				nouveau_bo_unpin(disp->sync);
2667 		}
2668 		if (ret)
2669 			nouveau_bo_ref(NULL, &disp->sync);
2670 	}
2671 
2672 	if (ret)
2673 		goto out;
2674 
2675 	/* allocate master evo channel */
2676 	ret = nv50_core_new(drm, &disp->core);
2677 	if (ret)
2678 		goto out;
2679 
2680 	disp->core->func->init(disp->core);
2681 	if (disp->core->func->caps_init) {
2682 		ret = disp->core->func->caps_init(drm, disp);
2683 		if (ret)
2684 			goto out;
2685 	}
2686 
2687 	/* Assign the correct format modifiers */
2688 	if (disp->disp->object.oclass >= TU102_DISP)
2689 		nouveau_display(dev)->format_modifiers = wndwc57e_modifiers;
2690 	else
2691 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_FERMI)
2692 		nouveau_display(dev)->format_modifiers = disp90xx_modifiers;
2693 	else
2694 		nouveau_display(dev)->format_modifiers = disp50xx_modifiers;
2695 
2696 	/* create crtc objects to represent the hw heads */
2697 	if (disp->disp->object.oclass >= GV100_DISP)
2698 		crtcs = nvif_rd32(&device->object, 0x610060) & 0xff;
2699 	else
2700 	if (disp->disp->object.oclass >= GF110_DISP)
2701 		crtcs = nvif_rd32(&device->object, 0x612004) & 0xf;
2702 	else
2703 		crtcs = 0x3;
2704 
2705 	for (i = 0; i < fls(crtcs); i++) {
2706 		struct nv50_head *head;
2707 
2708 		if (!(crtcs & (1 << i)))
2709 			continue;
2710 
2711 		head = nv50_head_create(dev, i);
2712 		if (IS_ERR(head)) {
2713 			ret = PTR_ERR(head);
2714 			goto out;
2715 		}
2716 
2717 		if (has_mst) {
2718 			head->msto = nv50_msto_new(dev, head, i);
2719 			if (IS_ERR(head->msto)) {
2720 				ret = PTR_ERR(head->msto);
2721 				head->msto = NULL;
2722 				goto out;
2723 			}
2724 
2725 			/*
2726 			 * FIXME: This is a hack to workaround the following
2727 			 * issues:
2728 			 *
2729 			 * https://gitlab.gnome.org/GNOME/mutter/issues/759
2730 			 * https://gitlab.freedesktop.org/xorg/xserver/merge_requests/277
2731 			 *
2732 			 * Once these issues are closed, this should be
2733 			 * removed
2734 			 */
2735 			head->msto->encoder.possible_crtcs = crtcs;
2736 		}
2737 	}
2738 
2739 	/* create encoder/connector objects based on VBIOS DCB table */
2740 	for (i = 0, dcbe = &dcb->entry[0]; i < dcb->entries; i++, dcbe++) {
2741 		connector = nouveau_connector_create(dev, dcbe);
2742 		if (IS_ERR(connector))
2743 			continue;
2744 
2745 		if (dcbe->location == DCB_LOC_ON_CHIP) {
2746 			switch (dcbe->type) {
2747 			case DCB_OUTPUT_TMDS:
2748 			case DCB_OUTPUT_LVDS:
2749 			case DCB_OUTPUT_DP:
2750 				ret = nv50_sor_create(connector, dcbe);
2751 				break;
2752 			case DCB_OUTPUT_ANALOG:
2753 				ret = nv50_dac_create(connector, dcbe);
2754 				break;
2755 			default:
2756 				ret = -ENODEV;
2757 				break;
2758 			}
2759 		} else {
2760 			ret = nv50_pior_create(connector, dcbe);
2761 		}
2762 
2763 		if (ret) {
2764 			NV_WARN(drm, "failed to create encoder %d/%d/%d: %d\n",
2765 				     dcbe->location, dcbe->type,
2766 				     ffs(dcbe->or) - 1, ret);
2767 			ret = 0;
2768 		}
2769 	}
2770 
2771 	/* cull any connectors we created that don't have an encoder */
2772 	list_for_each_entry_safe(connector, tmp, &dev->mode_config.connector_list, head) {
2773 		if (connector->possible_encoders)
2774 			continue;
2775 
2776 		NV_WARN(drm, "%s has no encoders, removing\n",
2777 			connector->name);
2778 		connector->funcs->destroy(connector);
2779 	}
2780 
2781 	/* Disable vblank irqs aggressively for power-saving, safe on nv50+ */
2782 	dev->vblank_disable_immediate = true;
2783 
2784 	nv50_audio_component_init(drm);
2785 
2786 out:
2787 	if (ret)
2788 		nv50_display_destroy(dev);
2789 	return ret;
2790 }
2791 
2792 /******************************************************************************
2793  * Format modifiers
2794  *****************************************************************************/
2795 
2796 /****************************************************************
2797  *            Log2(block height) ----------------------------+  *
2798  *            Page Kind ----------------------------------+  |  *
2799  *            Gob Height/Page Kind Generation ------+     |  |  *
2800  *                          Sector layout -------+  |     |  |  *
2801  *                          Compression ------+  |  |     |  |  */
2802 const u64 disp50xx_modifiers[] = { /*         |  |  |     |  |  */
2803 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 0),
2804 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 1),
2805 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 2),
2806 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 3),
2807 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 4),
2808 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x7a, 5),
2809 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 0),
2810 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 1),
2811 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 2),
2812 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 3),
2813 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 4),
2814 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x78, 5),
2815 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 0),
2816 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 1),
2817 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 2),
2818 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 3),
2819 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 4),
2820 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 1, 0x70, 5),
2821 	DRM_FORMAT_MOD_LINEAR,
2822 	DRM_FORMAT_MOD_INVALID
2823 };
2824 
2825 /****************************************************************
2826  *            Log2(block height) ----------------------------+  *
2827  *            Page Kind ----------------------------------+  |  *
2828  *            Gob Height/Page Kind Generation ------+     |  |  *
2829  *                          Sector layout -------+  |     |  |  *
2830  *                          Compression ------+  |  |     |  |  */
2831 const u64 disp90xx_modifiers[] = { /*         |  |  |     |  |  */
2832 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 0),
2833 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 1),
2834 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 2),
2835 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 3),
2836 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 4),
2837 	DRM_FORMAT_MOD_NVIDIA_BLOCK_LINEAR_2D(0, 1, 0, 0xfe, 5),
2838 	DRM_FORMAT_MOD_LINEAR,
2839 	DRM_FORMAT_MOD_INVALID
2840 };
2841