1 // SPDX-License-Identifier: MIT
2 #include <linux/string.h>
3 #include <drm/drm_crtc.h>
4 #include <drm/drm_atomic_helper.h>
5 #include <drm/drm_vblank.h>
6 #include <drm/drm_vblank_work.h>
7 
8 #include <nvif/class.h>
9 #include <nvif/cl0002.h>
10 #include <nvif/timer.h>
11 
12 #include <nvhw/class/cl907d.h>
13 
14 #include "nouveau_drv.h"
15 #include "core.h"
16 #include "head.h"
17 #include "wndw.h"
18 #include "handles.h"
19 #include "crc.h"
20 
21 static const char * const nv50_crc_sources[] = {
22 	[NV50_CRC_SOURCE_NONE] = "none",
23 	[NV50_CRC_SOURCE_AUTO] = "auto",
24 	[NV50_CRC_SOURCE_RG] = "rg",
25 	[NV50_CRC_SOURCE_OUTP_ACTIVE] = "outp-active",
26 	[NV50_CRC_SOURCE_OUTP_COMPLETE] = "outp-complete",
27 	[NV50_CRC_SOURCE_OUTP_INACTIVE] = "outp-inactive",
28 };
29 
30 static int nv50_crc_parse_source(const char *buf, enum nv50_crc_source *s)
31 {
32 	int i;
33 
34 	if (!buf) {
35 		*s = NV50_CRC_SOURCE_NONE;
36 		return 0;
37 	}
38 
39 	i = match_string(nv50_crc_sources, ARRAY_SIZE(nv50_crc_sources), buf);
40 	if (i < 0)
41 		return i;
42 
43 	*s = i;
44 	return 0;
45 }
46 
47 int
48 nv50_crc_verify_source(struct drm_crtc *crtc, const char *source_name,
49 		       size_t *values_cnt)
50 {
51 	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
52 	enum nv50_crc_source source;
53 
54 	if (nv50_crc_parse_source(source_name, &source) < 0) {
55 		NV_DEBUG(drm, "unknown source %s\n", source_name);
56 		return -EINVAL;
57 	}
58 
59 	*values_cnt = 1;
60 	return 0;
61 }
62 
63 const char *const *nv50_crc_get_sources(struct drm_crtc *crtc, size_t *count)
64 {
65 	*count = ARRAY_SIZE(nv50_crc_sources);
66 	return nv50_crc_sources;
67 }
68 
69 static void
70 nv50_crc_program_ctx(struct nv50_head *head,
71 		     struct nv50_crc_notifier_ctx *ctx)
72 {
73 	struct nv50_disp *disp = nv50_disp(head->base.base.dev);
74 	struct nv50_core *core = disp->core;
75 	u32 interlock[NV50_DISP_INTERLOCK__SIZE] = { 0 };
76 
77 	core->func->crc->set_ctx(head, ctx);
78 	core->func->update(core, interlock, false);
79 }
80 
81 static void nv50_crc_ctx_flip_work(struct kthread_work *base)
82 {
83 	struct drm_vblank_work *work = to_drm_vblank_work(base);
84 	struct nv50_crc *crc = container_of(work, struct nv50_crc, flip_work);
85 	struct nv50_head *head = container_of(crc, struct nv50_head, crc);
86 	struct drm_crtc *crtc = &head->base.base;
87 	struct nv50_disp *disp = nv50_disp(crtc->dev);
88 	u8 new_idx = crc->ctx_idx ^ 1;
89 
90 	/*
91 	 * We don't want to accidentally wait for longer then the vblank, so
92 	 * try again for the next vblank if we don't grab the lock
93 	 */
94 	if (!mutex_trylock(&disp->mutex)) {
95 		DRM_DEV_DEBUG_KMS(crtc->dev->dev,
96 				  "Lock contended, delaying CRC ctx flip for head-%d\n",
97 				  head->base.index);
98 		drm_vblank_work_schedule(work,
99 					 drm_crtc_vblank_count(crtc) + 1,
100 					 true);
101 		return;
102 	}
103 
104 	DRM_DEV_DEBUG_KMS(crtc->dev->dev,
105 			  "Flipping notifier ctx for head %d (%d -> %d)\n",
106 			  drm_crtc_index(crtc), crc->ctx_idx, new_idx);
107 
108 	nv50_crc_program_ctx(head, NULL);
109 	nv50_crc_program_ctx(head, &crc->ctx[new_idx]);
110 	mutex_unlock(&disp->mutex);
111 
112 	spin_lock_irq(&crc->lock);
113 	crc->ctx_changed = true;
114 	spin_unlock_irq(&crc->lock);
115 }
116 
117 static inline void nv50_crc_reset_ctx(struct nv50_crc_notifier_ctx *ctx)
118 {
119 	memset_io(ctx->mem.object.map.ptr, 0, ctx->mem.object.map.size);
120 }
121 
122 static void
123 nv50_crc_get_entries(struct nv50_head *head,
124 		     const struct nv50_crc_func *func,
125 		     enum nv50_crc_source source)
126 {
127 	struct drm_crtc *crtc = &head->base.base;
128 	struct nv50_crc *crc = &head->crc;
129 	u32 output_crc;
130 
131 	while (crc->entry_idx < func->num_entries) {
132 		/*
133 		 * While Nvidia's documentation says CRCs are written on each
134 		 * subsequent vblank after being enabled, in practice they
135 		 * aren't written immediately.
136 		 */
137 		output_crc = func->get_entry(head, &crc->ctx[crc->ctx_idx],
138 					     source, crc->entry_idx);
139 		if (!output_crc)
140 			return;
141 
142 		drm_crtc_add_crc_entry(crtc, true, crc->frame, &output_crc);
143 		crc->frame++;
144 		crc->entry_idx++;
145 	}
146 }
147 
148 void nv50_crc_handle_vblank(struct nv50_head *head)
149 {
150 	struct drm_crtc *crtc = &head->base.base;
151 	struct nv50_crc *crc = &head->crc;
152 	const struct nv50_crc_func *func =
153 		nv50_disp(head->base.base.dev)->core->func->crc;
154 	struct nv50_crc_notifier_ctx *ctx;
155 	bool need_reschedule = false;
156 
157 	if (!func)
158 		return;
159 
160 	/*
161 	 * We don't lose events if we aren't able to report CRCs until the
162 	 * next vblank, so only report CRCs if the locks we need aren't
163 	 * contended to prevent missing an actual vblank event
164 	 */
165 	if (!spin_trylock(&crc->lock))
166 		return;
167 
168 	if (!crc->src)
169 		goto out;
170 
171 	ctx = &crc->ctx[crc->ctx_idx];
172 	if (crc->ctx_changed && func->ctx_finished(head, ctx)) {
173 		nv50_crc_get_entries(head, func, crc->src);
174 
175 		crc->ctx_idx ^= 1;
176 		crc->entry_idx = 0;
177 		crc->ctx_changed = false;
178 
179 		/*
180 		 * Unfortunately when notifier contexts are changed during CRC
181 		 * capture, we will inevitably lose the CRC entry for the
182 		 * frame where the hardware actually latched onto the first
183 		 * UPDATE. According to Nvidia's hardware engineers, there's
184 		 * no workaround for this.
185 		 *
186 		 * Now, we could try to be smart here and calculate the number
187 		 * of missed CRCs based on audit timestamps, but those were
188 		 * removed starting with volta. Since we always flush our
189 		 * updates back-to-back without waiting, we'll just be
190 		 * optimistic and assume we always miss exactly one frame.
191 		 */
192 		DRM_DEV_DEBUG_KMS(head->base.base.dev->dev,
193 				  "Notifier ctx flip for head-%d finished, lost CRC for frame %llu\n",
194 				  head->base.index, crc->frame);
195 		crc->frame++;
196 
197 		nv50_crc_reset_ctx(ctx);
198 		need_reschedule = true;
199 	}
200 
201 	nv50_crc_get_entries(head, func, crc->src);
202 
203 	if (need_reschedule)
204 		drm_vblank_work_schedule(&crc->flip_work,
205 					 drm_crtc_vblank_count(crtc)
206 					 + crc->flip_threshold
207 					 - crc->entry_idx,
208 					 true);
209 
210 out:
211 	spin_unlock(&crc->lock);
212 }
213 
214 static void nv50_crc_wait_ctx_finished(struct nv50_head *head,
215 				       const struct nv50_crc_func *func,
216 				       struct nv50_crc_notifier_ctx *ctx)
217 {
218 	struct drm_device *dev = head->base.base.dev;
219 	struct nouveau_drm *drm = nouveau_drm(dev);
220 	s64 ret;
221 
222 	ret = nvif_msec(&drm->client.device, 50,
223 			if (func->ctx_finished(head, ctx)) break;);
224 	if (ret == -ETIMEDOUT)
225 		NV_ERROR(drm,
226 			 "CRC notifier ctx for head %d not finished after 50ms\n",
227 			 head->base.index);
228 	else if (ret)
229 		NV_ATOMIC(drm,
230 			  "CRC notifier ctx for head-%d finished after %lldns\n",
231 			  head->base.index, ret);
232 }
233 
234 void nv50_crc_atomic_stop_reporting(struct drm_atomic_state *state)
235 {
236 	struct drm_crtc_state *crtc_state;
237 	struct drm_crtc *crtc;
238 	int i;
239 
240 	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
241 		struct nv50_head *head = nv50_head(crtc);
242 		struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
243 		struct nv50_crc *crc = &head->crc;
244 
245 		if (!asyh->clr.crc)
246 			continue;
247 
248 		spin_lock_irq(&crc->lock);
249 		crc->src = NV50_CRC_SOURCE_NONE;
250 		spin_unlock_irq(&crc->lock);
251 
252 		drm_crtc_vblank_put(crtc);
253 		drm_vblank_work_cancel_sync(&crc->flip_work);
254 
255 		NV_ATOMIC(nouveau_drm(crtc->dev),
256 			  "CRC reporting on vblank for head-%d disabled\n",
257 			  head->base.index);
258 
259 		/* CRC generation is still enabled in hw, we'll just report
260 		 * any remaining CRC entries ourselves after it gets disabled
261 		 * in hardware
262 		 */
263 	}
264 }
265 
266 void nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_state *state)
267 {
268 	struct drm_crtc_state *new_crtc_state;
269 	struct drm_crtc *crtc;
270 	int i;
271 
272 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
273 		struct nv50_head *head = nv50_head(crtc);
274 		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
275 		struct nv50_crc *crc = &head->crc;
276 		int i;
277 
278 		if (!asyh->set.crc)
279 			continue;
280 
281 		crc->entry_idx = 0;
282 		crc->ctx_changed = false;
283 		for (i = 0; i < ARRAY_SIZE(crc->ctx); i++)
284 			nv50_crc_reset_ctx(&crc->ctx[i]);
285 	}
286 }
287 
288 void nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_state *state)
289 {
290 	const struct nv50_crc_func *func =
291 		nv50_disp(state->dev)->core->func->crc;
292 	struct drm_crtc_state *new_crtc_state;
293 	struct drm_crtc *crtc;
294 	int i;
295 
296 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
297 		struct nv50_head *head = nv50_head(crtc);
298 		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
299 		struct nv50_crc *crc = &head->crc;
300 		struct nv50_crc_notifier_ctx *ctx = &crc->ctx[crc->ctx_idx];
301 
302 		if (!asyh->clr.crc)
303 			continue;
304 
305 		if (crc->ctx_changed) {
306 			nv50_crc_wait_ctx_finished(head, func, ctx);
307 			ctx = &crc->ctx[crc->ctx_idx ^ 1];
308 		}
309 		nv50_crc_wait_ctx_finished(head, func, ctx);
310 	}
311 }
312 
313 void nv50_crc_atomic_start_reporting(struct drm_atomic_state *state)
314 {
315 	struct drm_crtc_state *crtc_state;
316 	struct drm_crtc *crtc;
317 	int i;
318 
319 	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
320 		struct nv50_head *head = nv50_head(crtc);
321 		struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
322 		struct nv50_crc *crc = &head->crc;
323 		u64 vbl_count;
324 
325 		if (!asyh->set.crc)
326 			continue;
327 
328 		drm_crtc_vblank_get(crtc);
329 
330 		spin_lock_irq(&crc->lock);
331 		vbl_count = drm_crtc_vblank_count(crtc);
332 		crc->frame = vbl_count;
333 		crc->src = asyh->crc.src;
334 		drm_vblank_work_schedule(&crc->flip_work,
335 					 vbl_count + crc->flip_threshold,
336 					 true);
337 		spin_unlock_irq(&crc->lock);
338 
339 		NV_ATOMIC(nouveau_drm(crtc->dev),
340 			  "CRC reporting on vblank for head-%d enabled\n",
341 			  head->base.index);
342 	}
343 }
344 
345 int nv50_crc_atomic_check_head(struct nv50_head *head,
346 			       struct nv50_head_atom *asyh,
347 			       struct nv50_head_atom *armh)
348 {
349 	struct nv50_atom *atom = nv50_atom(asyh->state.state);
350 	struct drm_device *dev = head->base.base.dev;
351 	struct nv50_disp *disp = nv50_disp(dev);
352 	bool changed = armh->crc.src != asyh->crc.src;
353 
354 	if (!armh->crc.src && !asyh->crc.src) {
355 		asyh->set.crc = false;
356 		asyh->clr.crc = false;
357 		return 0;
358 	}
359 
360 	/* While we don't care about entry tags, Volta+ hw always needs the
361 	 * controlling wndw channel programmed to a wndw that's owned by our
362 	 * head
363 	 */
364 	if (asyh->crc.src && disp->disp->object.oclass >= GV100_DISP &&
365 	    !(BIT(asyh->crc.wndw) & asyh->wndw.owned)) {
366 		if (!asyh->wndw.owned) {
367 			/* TODO: once we support flexible channel ownership,
368 			 * we should write some code here to handle attempting
369 			 * to "steal" a plane: e.g. take a plane that is
370 			 * currently not-visible and owned by another head,
371 			 * and reassign it to this head. If we fail to do so,
372 			 * we shuld reject the mode outright as CRC capture
373 			 * then becomes impossible.
374 			 */
375 			NV_ATOMIC(nouveau_drm(dev),
376 				  "No available wndws for CRC readback\n");
377 			return -EINVAL;
378 		}
379 		asyh->crc.wndw = ffs(asyh->wndw.owned) - 1;
380 	}
381 
382 	if (drm_atomic_crtc_needs_modeset(&asyh->state) || changed ||
383 	    armh->crc.wndw != asyh->crc.wndw) {
384 		asyh->clr.crc = armh->crc.src && armh->state.active;
385 		asyh->set.crc = asyh->crc.src && asyh->state.active;
386 		if (changed)
387 			asyh->set.or |= armh->or.crc_raster !=
388 					asyh->or.crc_raster;
389 
390 		if (asyh->clr.crc && asyh->set.crc)
391 			atom->flush_disable = true;
392 	} else {
393 		asyh->set.crc = false;
394 		asyh->clr.crc = false;
395 	}
396 
397 	return 0;
398 }
399 
400 void nv50_crc_atomic_check_outp(struct nv50_atom *atom)
401 {
402 	struct drm_crtc *crtc;
403 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
404 	int i;
405 
406 	if (atom->flush_disable)
407 		return;
408 
409 	for_each_oldnew_crtc_in_state(&atom->state, crtc, old_crtc_state,
410 				      new_crtc_state, i) {
411 		struct nv50_head_atom *armh = nv50_head_atom(old_crtc_state);
412 		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
413 		struct nv50_outp_atom *outp_atom;
414 		struct nouveau_encoder *outp =
415 			nv50_real_outp(nv50_head_atom_get_encoder(armh));
416 		struct drm_encoder *encoder = &outp->base.base;
417 
418 		if (!asyh->clr.crc)
419 			continue;
420 
421 		/*
422 		 * Re-programming ORs can't be done in the same flush as
423 		 * disabling CRCs
424 		 */
425 		list_for_each_entry(outp_atom, &atom->outp, head) {
426 			if (outp_atom->encoder == encoder) {
427 				if (outp_atom->set.mask) {
428 					atom->flush_disable = true;
429 					return;
430 				} else {
431 					break;
432 				}
433 			}
434 		}
435 	}
436 }
437 
438 static enum nv50_crc_source_type
439 nv50_crc_source_type(struct nouveau_encoder *outp,
440 		     enum nv50_crc_source source)
441 {
442 	struct dcb_output *dcbe = outp->dcb;
443 
444 	switch (source) {
445 	case NV50_CRC_SOURCE_NONE: return NV50_CRC_SOURCE_TYPE_NONE;
446 	case NV50_CRC_SOURCE_RG:   return NV50_CRC_SOURCE_TYPE_RG;
447 	default:		   break;
448 	}
449 
450 	if (dcbe->location != DCB_LOC_ON_CHIP)
451 		return NV50_CRC_SOURCE_TYPE_PIOR;
452 
453 	switch (dcbe->type) {
454 	case DCB_OUTPUT_DP:	return NV50_CRC_SOURCE_TYPE_SF;
455 	case DCB_OUTPUT_ANALOG:	return NV50_CRC_SOURCE_TYPE_DAC;
456 	default:		return NV50_CRC_SOURCE_TYPE_SOR;
457 	}
458 }
459 
460 void nv50_crc_atomic_set(struct nv50_head *head,
461 			 struct nv50_head_atom *asyh)
462 {
463 	struct drm_crtc *crtc = &head->base.base;
464 	struct drm_device *dev = crtc->dev;
465 	struct nv50_crc *crc = &head->crc;
466 	const struct nv50_crc_func *func = nv50_disp(dev)->core->func->crc;
467 	struct nouveau_encoder *outp =
468 		nv50_real_outp(nv50_head_atom_get_encoder(asyh));
469 
470 	func->set_src(head, outp->or,
471 		      nv50_crc_source_type(outp, asyh->crc.src),
472 		      &crc->ctx[crc->ctx_idx], asyh->crc.wndw);
473 }
474 
475 void nv50_crc_atomic_clr(struct nv50_head *head)
476 {
477 	const struct nv50_crc_func *func =
478 		nv50_disp(head->base.base.dev)->core->func->crc;
479 
480 	func->set_src(head, 0, NV50_CRC_SOURCE_TYPE_NONE, NULL, 0);
481 }
482 
483 static inline int
484 nv50_crc_raster_type(enum nv50_crc_source source)
485 {
486 	switch (source) {
487 	case NV50_CRC_SOURCE_NONE:
488 	case NV50_CRC_SOURCE_AUTO:
489 	case NV50_CRC_SOURCE_RG:
490 	case NV50_CRC_SOURCE_OUTP_ACTIVE:
491 		return NV907D_HEAD_SET_CONTROL_OUTPUT_RESOURCE_CRC_MODE_ACTIVE_RASTER;
492 	case NV50_CRC_SOURCE_OUTP_COMPLETE:
493 		return NV907D_HEAD_SET_CONTROL_OUTPUT_RESOURCE_CRC_MODE_COMPLETE_RASTER;
494 	case NV50_CRC_SOURCE_OUTP_INACTIVE:
495 		return NV907D_HEAD_SET_CONTROL_OUTPUT_RESOURCE_CRC_MODE_NON_ACTIVE_RASTER;
496 	}
497 
498 	return 0;
499 }
500 
501 /* We handle mapping the memory for CRC notifiers ourselves, since each
502  * notifier needs it's own handle
503  */
504 static inline int
505 nv50_crc_ctx_init(struct nv50_head *head, struct nvif_mmu *mmu,
506 		  struct nv50_crc_notifier_ctx *ctx, size_t len, int idx)
507 {
508 	struct nv50_core *core = nv50_disp(head->base.base.dev)->core;
509 	int ret;
510 
511 	ret = nvif_mem_ctor_map(mmu, "kmsCrcNtfy", NVIF_MEM_VRAM, len, &ctx->mem);
512 	if (ret)
513 		return ret;
514 
515 	ret = nvif_object_ctor(&core->chan.base.user, "kmsCrcNtfyCtxDma",
516 			       NV50_DISP_HANDLE_CRC_CTX(head, idx),
517 			       NV_DMA_IN_MEMORY,
518 			       &(struct nv_dma_v0) {
519 					.target = NV_DMA_V0_TARGET_VRAM,
520 					.access = NV_DMA_V0_ACCESS_RDWR,
521 					.start = ctx->mem.addr,
522 					.limit =  ctx->mem.addr
523 						+ ctx->mem.size - 1,
524 			       }, sizeof(struct nv_dma_v0),
525 			       &ctx->ntfy);
526 	if (ret)
527 		goto fail_fini;
528 
529 	return 0;
530 
531 fail_fini:
532 	nvif_mem_dtor(&ctx->mem);
533 	return ret;
534 }
535 
536 static inline void
537 nv50_crc_ctx_fini(struct nv50_crc_notifier_ctx *ctx)
538 {
539 	nvif_object_dtor(&ctx->ntfy);
540 	nvif_mem_dtor(&ctx->mem);
541 }
542 
543 int nv50_crc_set_source(struct drm_crtc *crtc, const char *source_str)
544 {
545 	struct drm_device *dev = crtc->dev;
546 	struct drm_atomic_state *state;
547 	struct drm_modeset_acquire_ctx ctx;
548 	struct nv50_head *head = nv50_head(crtc);
549 	struct nv50_crc *crc = &head->crc;
550 	const struct nv50_crc_func *func = nv50_disp(dev)->core->func->crc;
551 	struct nvif_mmu *mmu = &nouveau_drm(dev)->client.mmu;
552 	struct nv50_head_atom *asyh;
553 	struct drm_crtc_state *crtc_state;
554 	enum nv50_crc_source source;
555 	int ret = 0, ctx_flags = 0, i;
556 
557 	ret = nv50_crc_parse_source(source_str, &source);
558 	if (ret)
559 		return ret;
560 
561 	/*
562 	 * Since we don't want the user to accidentally interrupt us as we're
563 	 * disabling CRCs
564 	 */
565 	if (source)
566 		ctx_flags |= DRM_MODESET_ACQUIRE_INTERRUPTIBLE;
567 	drm_modeset_acquire_init(&ctx, ctx_flags);
568 
569 	state = drm_atomic_state_alloc(dev);
570 	if (!state) {
571 		ret = -ENOMEM;
572 		goto out_acquire_fini;
573 	}
574 	state->acquire_ctx = &ctx;
575 
576 	if (source) {
577 		for (i = 0; i < ARRAY_SIZE(head->crc.ctx); i++) {
578 			ret = nv50_crc_ctx_init(head, mmu, &crc->ctx[i],
579 						func->notifier_len, i);
580 			if (ret)
581 				goto out_ctx_fini;
582 		}
583 	}
584 
585 retry:
586 	crtc_state = drm_atomic_get_crtc_state(state, &head->base.base);
587 	if (IS_ERR(crtc_state)) {
588 		ret = PTR_ERR(crtc_state);
589 		if (ret == -EDEADLK)
590 			goto deadlock;
591 		else if (ret)
592 			goto out_drop_locks;
593 	}
594 	asyh = nv50_head_atom(crtc_state);
595 	asyh->crc.src = source;
596 	asyh->or.crc_raster = nv50_crc_raster_type(source);
597 
598 	ret = drm_atomic_commit(state);
599 	if (ret == -EDEADLK)
600 		goto deadlock;
601 	else if (ret)
602 		goto out_drop_locks;
603 
604 	if (!source) {
605 		/*
606 		 * If the user specified a custom flip threshold through
607 		 * debugfs, reset it
608 		 */
609 		crc->flip_threshold = func->flip_threshold;
610 	}
611 
612 out_drop_locks:
613 	drm_modeset_drop_locks(&ctx);
614 out_ctx_fini:
615 	if (!source || ret) {
616 		for (i = 0; i < ARRAY_SIZE(crc->ctx); i++)
617 			nv50_crc_ctx_fini(&crc->ctx[i]);
618 	}
619 	drm_atomic_state_put(state);
620 out_acquire_fini:
621 	drm_modeset_acquire_fini(&ctx);
622 	return ret;
623 
624 deadlock:
625 	drm_atomic_state_clear(state);
626 	drm_modeset_backoff(&ctx);
627 	goto retry;
628 }
629 
630 static int
631 nv50_crc_debugfs_flip_threshold_get(struct seq_file *m, void *data)
632 {
633 	struct nv50_head *head = m->private;
634 	struct drm_crtc *crtc = &head->base.base;
635 	struct nv50_crc *crc = &head->crc;
636 	int ret;
637 
638 	ret = drm_modeset_lock_single_interruptible(&crtc->mutex);
639 	if (ret)
640 		return ret;
641 
642 	seq_printf(m, "%d\n", crc->flip_threshold);
643 
644 	drm_modeset_unlock(&crtc->mutex);
645 	return ret;
646 }
647 
648 static int
649 nv50_crc_debugfs_flip_threshold_open(struct inode *inode, struct file *file)
650 {
651 	return single_open(file, nv50_crc_debugfs_flip_threshold_get,
652 			   inode->i_private);
653 }
654 
655 static ssize_t
656 nv50_crc_debugfs_flip_threshold_set(struct file *file,
657 				    const char __user *ubuf, size_t len,
658 				    loff_t *offp)
659 {
660 	struct seq_file *m = file->private_data;
661 	struct nv50_head *head = m->private;
662 	struct nv50_head_atom *armh;
663 	struct drm_crtc *crtc = &head->base.base;
664 	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
665 	struct nv50_crc *crc = &head->crc;
666 	const struct nv50_crc_func *func =
667 		nv50_disp(crtc->dev)->core->func->crc;
668 	int value, ret;
669 
670 	ret = kstrtoint_from_user(ubuf, len, 10, &value);
671 	if (ret)
672 		return ret;
673 
674 	if (value > func->flip_threshold)
675 		return -EINVAL;
676 	else if (value == -1)
677 		value = func->flip_threshold;
678 	else if (value < -1)
679 		return -EINVAL;
680 
681 	ret = drm_modeset_lock_single_interruptible(&crtc->mutex);
682 	if (ret)
683 		return ret;
684 
685 	armh = nv50_head_atom(crtc->state);
686 	if (armh->crc.src) {
687 		ret = -EBUSY;
688 		goto out;
689 	}
690 
691 	NV_DEBUG(drm,
692 		 "Changing CRC flip threshold for next capture on head-%d to %d\n",
693 		 head->base.index, value);
694 	crc->flip_threshold = value;
695 	ret = len;
696 
697 out:
698 	drm_modeset_unlock(&crtc->mutex);
699 	return ret;
700 }
701 
702 static const struct file_operations nv50_crc_flip_threshold_fops = {
703 	.owner = THIS_MODULE,
704 	.open = nv50_crc_debugfs_flip_threshold_open,
705 	.read = seq_read,
706 	.write = nv50_crc_debugfs_flip_threshold_set,
707 	.release = single_release,
708 };
709 
710 int nv50_head_crc_late_register(struct nv50_head *head)
711 {
712 	struct drm_crtc *crtc = &head->base.base;
713 	const struct nv50_crc_func *func =
714 		nv50_disp(crtc->dev)->core->func->crc;
715 	struct dentry *root;
716 
717 	if (!func || !crtc->debugfs_entry)
718 		return 0;
719 
720 	root = debugfs_create_dir("nv_crc", crtc->debugfs_entry);
721 	debugfs_create_file("flip_threshold", 0644, root, head,
722 			    &nv50_crc_flip_threshold_fops);
723 
724 	return 0;
725 }
726 
727 static inline void
728 nv50_crc_init_head(struct nv50_disp *disp, const struct nv50_crc_func *func,
729 		   struct nv50_head *head)
730 {
731 	struct nv50_crc *crc = &head->crc;
732 
733 	crc->flip_threshold = func->flip_threshold;
734 	spin_lock_init(&crc->lock);
735 	drm_vblank_work_init(&crc->flip_work, &head->base.base,
736 			     nv50_crc_ctx_flip_work);
737 }
738 
739 void nv50_crc_init(struct drm_device *dev)
740 {
741 	struct nv50_disp *disp = nv50_disp(dev);
742 	struct drm_crtc *crtc;
743 	const struct nv50_crc_func *func = disp->core->func->crc;
744 
745 	if (!func)
746 		return;
747 
748 	drm_for_each_crtc(crtc, dev)
749 		nv50_crc_init_head(disp, func, nv50_head(crtc));
750 }
751