1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Dave Airlie
30  */
31 #include <linux/seq_file.h>
32 #include <linux/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/kref.h>
35 #include <linux/slab.h>
36 #include <linux/firmware.h>
37 #include <linux/pm_runtime.h>
38 
39 #include <drm/drm_drv.h>
40 #include "amdgpu.h"
41 #include "amdgpu_trace.h"
42 #include "amdgpu_reset.h"
43 
44 /*
45  * Fences mark an event in the GPUs pipeline and are used
46  * for GPU/CPU synchronization.  When the fence is written,
47  * it is expected that all buffers associated with that fence
48  * are no longer in use by the associated ring on the GPU and
49  * that the relevant GPU caches have been flushed.
50  */
51 
52 struct amdgpu_fence {
53 	struct dma_fence base;
54 
55 	/* RB, DMA, etc. */
56 	struct amdgpu_ring		*ring;
57 	ktime_t				start_timestamp;
58 };
59 
60 static struct kmem_cache *amdgpu_fence_slab;
61 
62 int amdgpu_fence_slab_init(void)
63 {
64 	amdgpu_fence_slab = kmem_cache_create(
65 		"amdgpu_fence", sizeof(struct amdgpu_fence), 0,
66 		SLAB_HWCACHE_ALIGN, NULL);
67 	if (!amdgpu_fence_slab)
68 		return -ENOMEM;
69 	return 0;
70 }
71 
72 void amdgpu_fence_slab_fini(void)
73 {
74 	rcu_barrier();
75 	kmem_cache_destroy(amdgpu_fence_slab);
76 }
77 /*
78  * Cast helper
79  */
80 static const struct dma_fence_ops amdgpu_fence_ops;
81 static const struct dma_fence_ops amdgpu_job_fence_ops;
82 static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
83 {
84 	struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
85 
86 	if (__f->base.ops == &amdgpu_fence_ops ||
87 	    __f->base.ops == &amdgpu_job_fence_ops)
88 		return __f;
89 
90 	return NULL;
91 }
92 
93 /**
94  * amdgpu_fence_write - write a fence value
95  *
96  * @ring: ring the fence is associated with
97  * @seq: sequence number to write
98  *
99  * Writes a fence value to memory (all asics).
100  */
101 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
102 {
103 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
104 
105 	if (drv->cpu_addr)
106 		*drv->cpu_addr = cpu_to_le32(seq);
107 }
108 
109 /**
110  * amdgpu_fence_read - read a fence value
111  *
112  * @ring: ring the fence is associated with
113  *
114  * Reads a fence value from memory (all asics).
115  * Returns the value of the fence read from memory.
116  */
117 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
118 {
119 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
120 	u32 seq = 0;
121 
122 	if (drv->cpu_addr)
123 		seq = le32_to_cpu(*drv->cpu_addr);
124 	else
125 		seq = atomic_read(&drv->last_seq);
126 
127 	return seq;
128 }
129 
130 /**
131  * amdgpu_fence_emit - emit a fence on the requested ring
132  *
133  * @ring: ring the fence is associated with
134  * @f: resulting fence object
135  * @job: job the fence is embedded in
136  * @flags: flags to pass into the subordinate .emit_fence() call
137  *
138  * Emits a fence command on the requested ring (all asics).
139  * Returns 0 on success, -ENOMEM on failure.
140  */
141 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,
142 		      unsigned int flags)
143 {
144 	struct amdgpu_device *adev = ring->adev;
145 	struct dma_fence *fence;
146 	struct amdgpu_fence *am_fence;
147 	struct dma_fence __rcu **ptr;
148 	uint32_t seq;
149 	int r;
150 
151 	if (job == NULL) {
152 		/* create a sperate hw fence */
153 		am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);
154 		if (am_fence == NULL)
155 			return -ENOMEM;
156 		fence = &am_fence->base;
157 		am_fence->ring = ring;
158 	} else {
159 		/* take use of job-embedded fence */
160 		fence = &job->hw_fence;
161 	}
162 
163 	seq = ++ring->fence_drv.sync_seq;
164 	if (job && job->job_run_counter) {
165 		/* reinit seq for resubmitted jobs */
166 		fence->seqno = seq;
167 		/* TO be inline with external fence creation and other drivers */
168 		dma_fence_get(fence);
169 	} else {
170 		if (job) {
171 			dma_fence_init(fence, &amdgpu_job_fence_ops,
172 				       &ring->fence_drv.lock,
173 				       adev->fence_context + ring->idx, seq);
174 			/* Against remove in amdgpu_job_{free, free_cb} */
175 			dma_fence_get(fence);
176 		} else {
177 			dma_fence_init(fence, &amdgpu_fence_ops,
178 				       &ring->fence_drv.lock,
179 				       adev->fence_context + ring->idx, seq);
180 		}
181 	}
182 
183 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
184 			       seq, flags | AMDGPU_FENCE_FLAG_INT);
185 	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
186 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
187 	if (unlikely(rcu_dereference_protected(*ptr, 1))) {
188 		struct dma_fence *old;
189 
190 		rcu_read_lock();
191 		old = dma_fence_get_rcu_safe(ptr);
192 		rcu_read_unlock();
193 
194 		if (old) {
195 			r = dma_fence_wait(old, false);
196 			dma_fence_put(old);
197 			if (r)
198 				return r;
199 		}
200 	}
201 
202 	to_amdgpu_fence(fence)->start_timestamp = ktime_get();
203 
204 	/* This function can't be called concurrently anyway, otherwise
205 	 * emitting the fence would mess up the hardware ring buffer.
206 	 */
207 	rcu_assign_pointer(*ptr, dma_fence_get(fence));
208 
209 	*f = fence;
210 
211 	return 0;
212 }
213 
214 /**
215  * amdgpu_fence_emit_polling - emit a fence on the requeste ring
216  *
217  * @ring: ring the fence is associated with
218  * @s: resulting sequence number
219  * @timeout: the timeout for waiting in usecs
220  *
221  * Emits a fence command on the requested ring (all asics).
222  * Used For polling fence.
223  * Returns 0 on success, -ENOMEM on failure.
224  */
225 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
226 			      uint32_t timeout)
227 {
228 	uint32_t seq;
229 	signed long r;
230 
231 	if (!s)
232 		return -EINVAL;
233 
234 	seq = ++ring->fence_drv.sync_seq;
235 	r = amdgpu_fence_wait_polling(ring,
236 				      seq - ring->fence_drv.num_fences_mask,
237 				      timeout);
238 	if (r < 1)
239 		return -ETIMEDOUT;
240 
241 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
242 			       seq, 0);
243 
244 	*s = seq;
245 
246 	return 0;
247 }
248 
249 /**
250  * amdgpu_fence_schedule_fallback - schedule fallback check
251  *
252  * @ring: pointer to struct amdgpu_ring
253  *
254  * Start a timer as fallback to our interrupts.
255  */
256 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
257 {
258 	mod_timer(&ring->fence_drv.fallback_timer,
259 		  jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
260 }
261 
262 /**
263  * amdgpu_fence_process - check for fence activity
264  *
265  * @ring: pointer to struct amdgpu_ring
266  *
267  * Checks the current fence value and calculates the last
268  * signalled fence value. Wakes the fence queue if the
269  * sequence number has increased.
270  *
271  * Returns true if fence was processed
272  */
273 bool amdgpu_fence_process(struct amdgpu_ring *ring)
274 {
275 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
276 	struct amdgpu_device *adev = ring->adev;
277 	uint32_t seq, last_seq;
278 
279 	do {
280 		last_seq = atomic_read(&ring->fence_drv.last_seq);
281 		seq = amdgpu_fence_read(ring);
282 
283 	} while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq);
284 
285 	if (del_timer(&ring->fence_drv.fallback_timer) &&
286 	    seq != ring->fence_drv.sync_seq)
287 		amdgpu_fence_schedule_fallback(ring);
288 
289 	if (unlikely(seq == last_seq))
290 		return false;
291 
292 	last_seq &= drv->num_fences_mask;
293 	seq &= drv->num_fences_mask;
294 
295 	do {
296 		struct dma_fence *fence, **ptr;
297 
298 		++last_seq;
299 		last_seq &= drv->num_fences_mask;
300 		ptr = &drv->fences[last_seq];
301 
302 		/* There is always exactly one thread signaling this fence slot */
303 		fence = rcu_dereference_protected(*ptr, 1);
304 		RCU_INIT_POINTER(*ptr, NULL);
305 
306 		if (!fence)
307 			continue;
308 
309 		dma_fence_signal(fence);
310 		dma_fence_put(fence);
311 		pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
312 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
313 	} while (last_seq != seq);
314 
315 	return true;
316 }
317 
318 /**
319  * amdgpu_fence_fallback - fallback for hardware interrupts
320  *
321  * @t: timer context used to obtain the pointer to ring structure
322  *
323  * Checks for fence activity.
324  */
325 static void amdgpu_fence_fallback(struct timer_list *t)
326 {
327 	struct amdgpu_ring *ring = from_timer(ring, t,
328 					      fence_drv.fallback_timer);
329 
330 	if (amdgpu_fence_process(ring))
331 		DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name);
332 }
333 
334 /**
335  * amdgpu_fence_wait_empty - wait for all fences to signal
336  *
337  * @ring: ring index the fence is associated with
338  *
339  * Wait for all fences on the requested ring to signal (all asics).
340  * Returns 0 if the fences have passed, error for all other cases.
341  */
342 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
343 {
344 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);
345 	struct dma_fence *fence, **ptr;
346 	int r;
347 
348 	if (!seq)
349 		return 0;
350 
351 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
352 	rcu_read_lock();
353 	fence = rcu_dereference(*ptr);
354 	if (!fence || !dma_fence_get_rcu(fence)) {
355 		rcu_read_unlock();
356 		return 0;
357 	}
358 	rcu_read_unlock();
359 
360 	r = dma_fence_wait(fence, false);
361 	dma_fence_put(fence);
362 	return r;
363 }
364 
365 /**
366  * amdgpu_fence_wait_polling - busy wait for givn sequence number
367  *
368  * @ring: ring index the fence is associated with
369  * @wait_seq: sequence number to wait
370  * @timeout: the timeout for waiting in usecs
371  *
372  * Wait for all fences on the requested ring to signal (all asics).
373  * Returns left time if no timeout, 0 or minus if timeout.
374  */
375 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
376 				      uint32_t wait_seq,
377 				      signed long timeout)
378 {
379 
380 	while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {
381 		udelay(2);
382 		timeout -= 2;
383 	}
384 	return timeout > 0 ? timeout : 0;
385 }
386 /**
387  * amdgpu_fence_count_emitted - get the count of emitted fences
388  *
389  * @ring: ring the fence is associated with
390  *
391  * Get the number of fences emitted on the requested ring (all asics).
392  * Returns the number of emitted fences on the ring.  Used by the
393  * dynpm code to ring track activity.
394  */
395 unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
396 {
397 	uint64_t emitted;
398 
399 	/* We are not protected by ring lock when reading the last sequence
400 	 * but it's ok to report slightly wrong fence count here.
401 	 */
402 	emitted = 0x100000000ull;
403 	emitted -= atomic_read(&ring->fence_drv.last_seq);
404 	emitted += READ_ONCE(ring->fence_drv.sync_seq);
405 	return lower_32_bits(emitted);
406 }
407 
408 /**
409  * amdgpu_fence_last_unsignaled_time_us - the time fence emitted until now
410  * @ring: ring the fence is associated with
411  *
412  * Find the earliest fence unsignaled until now, calculate the time delta
413  * between the time fence emitted and now.
414  */
415 u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)
416 {
417 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
418 	struct dma_fence *fence;
419 	uint32_t last_seq, sync_seq;
420 
421 	last_seq = atomic_read(&ring->fence_drv.last_seq);
422 	sync_seq = READ_ONCE(ring->fence_drv.sync_seq);
423 	if (last_seq == sync_seq)
424 		return 0;
425 
426 	++last_seq;
427 	last_seq &= drv->num_fences_mask;
428 	fence = drv->fences[last_seq];
429 	if (!fence)
430 		return 0;
431 
432 	return ktime_us_delta(ktime_get(),
433 		to_amdgpu_fence(fence)->start_timestamp);
434 }
435 
436 /**
437  * amdgpu_fence_update_start_timestamp - update the timestamp of the fence
438  * @ring: ring the fence is associated with
439  * @seq: the fence seq number to update.
440  * @timestamp: the start timestamp to update.
441  *
442  * The function called at the time the fence and related ib is about to
443  * resubmit to gpu in MCBP scenario. Thus we do not consider race condition
444  * with amdgpu_fence_process to modify the same fence.
445  */
446 void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)
447 {
448 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
449 	struct dma_fence *fence;
450 
451 	seq &= drv->num_fences_mask;
452 	fence = drv->fences[seq];
453 	if (!fence)
454 		return;
455 
456 	to_amdgpu_fence(fence)->start_timestamp = timestamp;
457 }
458 
459 /**
460  * amdgpu_fence_driver_start_ring - make the fence driver
461  * ready for use on the requested ring.
462  *
463  * @ring: ring to start the fence driver on
464  * @irq_src: interrupt source to use for this ring
465  * @irq_type: interrupt type to use for this ring
466  *
467  * Make the fence driver ready for processing (all asics).
468  * Not all asics have all rings, so each asic will only
469  * start the fence driver on the rings it has.
470  * Returns 0 for success, errors for failure.
471  */
472 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
473 				   struct amdgpu_irq_src *irq_src,
474 				   unsigned int irq_type)
475 {
476 	struct amdgpu_device *adev = ring->adev;
477 	uint64_t index;
478 
479 	if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {
480 		ring->fence_drv.cpu_addr = ring->fence_cpu_addr;
481 		ring->fence_drv.gpu_addr = ring->fence_gpu_addr;
482 	} else {
483 		/* put fence directly behind firmware */
484 		index = ALIGN(adev->uvd.fw->size, 8);
485 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;
486 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;
487 	}
488 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));
489 
490 	ring->fence_drv.irq_src = irq_src;
491 	ring->fence_drv.irq_type = irq_type;
492 	ring->fence_drv.initialized = true;
493 
494 	DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",
495 		      ring->name, ring->fence_drv.gpu_addr);
496 	return 0;
497 }
498 
499 /**
500  * amdgpu_fence_driver_init_ring - init the fence driver
501  * for the requested ring.
502  *
503  * @ring: ring to init the fence driver on
504  *
505  * Init the fence driver for the requested ring (all asics).
506  * Helper function for amdgpu_fence_driver_init().
507  */
508 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
509 {
510 	struct amdgpu_device *adev = ring->adev;
511 
512 	if (!adev)
513 		return -EINVAL;
514 
515 	if (!is_power_of_2(ring->num_hw_submission))
516 		return -EINVAL;
517 
518 	ring->fence_drv.cpu_addr = NULL;
519 	ring->fence_drv.gpu_addr = 0;
520 	ring->fence_drv.sync_seq = 0;
521 	atomic_set(&ring->fence_drv.last_seq, 0);
522 	ring->fence_drv.initialized = false;
523 
524 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
525 
526 	ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;
527 	spin_lock_init(&ring->fence_drv.lock);
528 	ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),
529 					 GFP_KERNEL);
530 
531 	if (!ring->fence_drv.fences)
532 		return -ENOMEM;
533 
534 	return 0;
535 }
536 
537 /**
538  * amdgpu_fence_driver_sw_init - init the fence driver
539  * for all possible rings.
540  *
541  * @adev: amdgpu device pointer
542  *
543  * Init the fence driver for all possible rings (all asics).
544  * Not all asics have all rings, so each asic will only
545  * start the fence driver on the rings it has using
546  * amdgpu_fence_driver_start_ring().
547  * Returns 0 for success.
548  */
549 int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev)
550 {
551 	return 0;
552 }
553 
554 /**
555  * amdgpu_fence_driver_hw_fini - tear down the fence driver
556  * for all possible rings.
557  *
558  * @adev: amdgpu device pointer
559  *
560  * Tear down the fence driver for all possible rings (all asics).
561  */
562 void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)
563 {
564 	int i, r;
565 
566 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
567 		struct amdgpu_ring *ring = adev->rings[i];
568 
569 		if (!ring || !ring->fence_drv.initialized)
570 			continue;
571 
572 		/* You can't wait for HW to signal if it's gone */
573 		if (!drm_dev_is_unplugged(adev_to_drm(adev)))
574 			r = amdgpu_fence_wait_empty(ring);
575 		else
576 			r = -ENODEV;
577 		/* no need to trigger GPU reset as we are unloading */
578 		if (r)
579 			amdgpu_fence_driver_force_completion(ring);
580 
581 		if (!drm_dev_is_unplugged(adev_to_drm(adev)) &&
582 		    ring->fence_drv.irq_src)
583 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,
584 				       ring->fence_drv.irq_type);
585 
586 		del_timer_sync(&ring->fence_drv.fallback_timer);
587 	}
588 }
589 
590 /* Will either stop and flush handlers for amdgpu interrupt or reanble it */
591 void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop)
592 {
593 	int i;
594 
595 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
596 		struct amdgpu_ring *ring = adev->rings[i];
597 
598 		if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)
599 			continue;
600 
601 		if (stop)
602 			disable_irq(adev->irq.irq);
603 		else
604 			enable_irq(adev->irq.irq);
605 	}
606 }
607 
608 void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev)
609 {
610 	unsigned int i, j;
611 
612 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
613 		struct amdgpu_ring *ring = adev->rings[i];
614 
615 		if (!ring || !ring->fence_drv.initialized)
616 			continue;
617 
618 		/*
619 		 * Notice we check for sched.ops since there's some
620 		 * override on the meaning of sched.ready by amdgpu.
621 		 * The natural check would be sched.ready, which is
622 		 * set as drm_sched_init() finishes...
623 		 */
624 		if (ring->sched.ops)
625 			drm_sched_fini(&ring->sched);
626 
627 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
628 			dma_fence_put(ring->fence_drv.fences[j]);
629 		kfree(ring->fence_drv.fences);
630 		ring->fence_drv.fences = NULL;
631 		ring->fence_drv.initialized = false;
632 	}
633 }
634 
635 /**
636  * amdgpu_fence_driver_hw_init - enable the fence driver
637  * for all possible rings.
638  *
639  * @adev: amdgpu device pointer
640  *
641  * Enable the fence driver for all possible rings (all asics).
642  * Not all asics have all rings, so each asic will only
643  * start the fence driver on the rings it has using
644  * amdgpu_fence_driver_start_ring().
645  * Returns 0 for success.
646  */
647 void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)
648 {
649 	int i;
650 
651 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
652 		struct amdgpu_ring *ring = adev->rings[i];
653 
654 		if (!ring || !ring->fence_drv.initialized)
655 			continue;
656 
657 		/* enable the interrupt */
658 		if (ring->fence_drv.irq_src)
659 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,
660 				       ring->fence_drv.irq_type);
661 	}
662 }
663 
664 /**
665  * amdgpu_fence_driver_clear_job_fences - clear job embedded fences of ring
666  *
667  * @ring: fence of the ring to be cleared
668  *
669  */
670 void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring)
671 {
672 	int i;
673 	struct dma_fence *old, **ptr;
674 
675 	for (i = 0; i <= ring->fence_drv.num_fences_mask; i++) {
676 		ptr = &ring->fence_drv.fences[i];
677 		old = rcu_dereference_protected(*ptr, 1);
678 		if (old && old->ops == &amdgpu_job_fence_ops) {
679 			struct amdgpu_job *job;
680 
681 			/* For non-scheduler bad job, i.e. failed ib test, we need to signal
682 			 * it right here or we won't be able to track them in fence_drv
683 			 * and they will remain unsignaled during sa_bo free.
684 			 */
685 			job = container_of(old, struct amdgpu_job, hw_fence);
686 			if (!job->base.s_fence && !dma_fence_is_signaled(old))
687 				dma_fence_signal(old);
688 			RCU_INIT_POINTER(*ptr, NULL);
689 			dma_fence_put(old);
690 		}
691 	}
692 }
693 
694 /**
695  * amdgpu_fence_driver_set_error - set error code on fences
696  * @ring: the ring which contains the fences
697  * @error: the error code to set
698  *
699  * Set an error code to all the fences pending on the ring.
700  */
701 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)
702 {
703 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
704 	unsigned long flags;
705 
706 	spin_lock_irqsave(&drv->lock, flags);
707 	for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {
708 		struct dma_fence *fence;
709 
710 		fence = rcu_dereference_protected(drv->fences[i],
711 						  lockdep_is_held(&drv->lock));
712 		if (fence && !dma_fence_is_signaled_locked(fence))
713 			dma_fence_set_error(fence, error);
714 	}
715 	spin_unlock_irqrestore(&drv->lock, flags);
716 }
717 
718 /**
719  * amdgpu_fence_driver_force_completion - force signal latest fence of ring
720  *
721  * @ring: fence of the ring to signal
722  *
723  */
724 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
725 {
726 	amdgpu_fence_driver_set_error(ring, -ECANCELED);
727 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
728 	amdgpu_fence_process(ring);
729 }
730 
731 /*
732  * Common fence implementation
733  */
734 
735 static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
736 {
737 	return "amdgpu";
738 }
739 
740 static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
741 {
742 	return (const char *)to_amdgpu_fence(f)->ring->name;
743 }
744 
745 static const char *amdgpu_job_fence_get_timeline_name(struct dma_fence *f)
746 {
747 	struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
748 
749 	return (const char *)to_amdgpu_ring(job->base.sched)->name;
750 }
751 
752 /**
753  * amdgpu_fence_enable_signaling - enable signalling on fence
754  * @f: fence
755  *
756  * This function is called with fence_queue lock held, and adds a callback
757  * to fence_queue that checks if this fence is signaled, and if so it
758  * signals the fence and removes itself.
759  */
760 static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
761 {
762 	if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))
763 		amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring);
764 
765 	return true;
766 }
767 
768 /**
769  * amdgpu_job_fence_enable_signaling - enable signalling on job fence
770  * @f: fence
771  *
772  * This is the simliar function with amdgpu_fence_enable_signaling above, it
773  * only handles the job embedded fence.
774  */
775 static bool amdgpu_job_fence_enable_signaling(struct dma_fence *f)
776 {
777 	struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
778 
779 	if (!timer_pending(&to_amdgpu_ring(job->base.sched)->fence_drv.fallback_timer))
780 		amdgpu_fence_schedule_fallback(to_amdgpu_ring(job->base.sched));
781 
782 	return true;
783 }
784 
785 /**
786  * amdgpu_fence_free - free up the fence memory
787  *
788  * @rcu: RCU callback head
789  *
790  * Free up the fence memory after the RCU grace period.
791  */
792 static void amdgpu_fence_free(struct rcu_head *rcu)
793 {
794 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
795 
796 	/* free fence_slab if it's separated fence*/
797 	kmem_cache_free(amdgpu_fence_slab, to_amdgpu_fence(f));
798 }
799 
800 /**
801  * amdgpu_job_fence_free - free up the job with embedded fence
802  *
803  * @rcu: RCU callback head
804  *
805  * Free up the job with embedded fence after the RCU grace period.
806  */
807 static void amdgpu_job_fence_free(struct rcu_head *rcu)
808 {
809 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
810 
811 	/* free job if fence has a parent job */
812 	kfree(container_of(f, struct amdgpu_job, hw_fence));
813 }
814 
815 /**
816  * amdgpu_fence_release - callback that fence can be freed
817  *
818  * @f: fence
819  *
820  * This function is called when the reference count becomes zero.
821  * It just RCU schedules freeing up the fence.
822  */
823 static void amdgpu_fence_release(struct dma_fence *f)
824 {
825 	call_rcu(&f->rcu, amdgpu_fence_free);
826 }
827 
828 /**
829  * amdgpu_job_fence_release - callback that job embedded fence can be freed
830  *
831  * @f: fence
832  *
833  * This is the simliar function with amdgpu_fence_release above, it
834  * only handles the job embedded fence.
835  */
836 static void amdgpu_job_fence_release(struct dma_fence *f)
837 {
838 	call_rcu(&f->rcu, amdgpu_job_fence_free);
839 }
840 
841 static const struct dma_fence_ops amdgpu_fence_ops = {
842 	.get_driver_name = amdgpu_fence_get_driver_name,
843 	.get_timeline_name = amdgpu_fence_get_timeline_name,
844 	.enable_signaling = amdgpu_fence_enable_signaling,
845 	.release = amdgpu_fence_release,
846 };
847 
848 static const struct dma_fence_ops amdgpu_job_fence_ops = {
849 	.get_driver_name = amdgpu_fence_get_driver_name,
850 	.get_timeline_name = amdgpu_job_fence_get_timeline_name,
851 	.enable_signaling = amdgpu_job_fence_enable_signaling,
852 	.release = amdgpu_job_fence_release,
853 };
854 
855 /*
856  * Fence debugfs
857  */
858 #if defined(CONFIG_DEBUG_FS)
859 static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)
860 {
861 	struct amdgpu_device *adev = m->private;
862 	int i;
863 
864 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
865 		struct amdgpu_ring *ring = adev->rings[i];
866 
867 		if (!ring || !ring->fence_drv.initialized)
868 			continue;
869 
870 		amdgpu_fence_process(ring);
871 
872 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
873 		seq_printf(m, "Last signaled fence          0x%08x\n",
874 			   atomic_read(&ring->fence_drv.last_seq));
875 		seq_printf(m, "Last emitted                 0x%08x\n",
876 			   ring->fence_drv.sync_seq);
877 
878 		if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||
879 		    ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {
880 			seq_printf(m, "Last signaled trailing fence 0x%08x\n",
881 				   le32_to_cpu(*ring->trail_fence_cpu_addr));
882 			seq_printf(m, "Last emitted                 0x%08x\n",
883 				   ring->trail_seq);
884 		}
885 
886 		if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
887 			continue;
888 
889 		/* set in CP_VMID_PREEMPT and preemption occurred */
890 		seq_printf(m, "Last preempted               0x%08x\n",
891 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
892 		/* set in CP_VMID_RESET and reset occurred */
893 		seq_printf(m, "Last reset                   0x%08x\n",
894 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
895 		/* Both preemption and reset occurred */
896 		seq_printf(m, "Last both                    0x%08x\n",
897 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
898 	}
899 	return 0;
900 }
901 
902 /*
903  * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
904  *
905  * Manually trigger a gpu reset at the next fence wait.
906  */
907 static int gpu_recover_get(void *data, u64 *val)
908 {
909 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
910 	struct drm_device *dev = adev_to_drm(adev);
911 	int r;
912 
913 	r = pm_runtime_get_sync(dev->dev);
914 	if (r < 0) {
915 		pm_runtime_put_autosuspend(dev->dev);
916 		return 0;
917 	}
918 
919 	if (amdgpu_reset_domain_schedule(adev->reset_domain, &adev->reset_work))
920 		flush_work(&adev->reset_work);
921 
922 	*val = atomic_read(&adev->reset_domain->reset_res);
923 
924 	pm_runtime_mark_last_busy(dev->dev);
925 	pm_runtime_put_autosuspend(dev->dev);
926 
927 	return 0;
928 }
929 
930 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);
931 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,
932 			 "%lld\n");
933 
934 static void amdgpu_debugfs_reset_work(struct work_struct *work)
935 {
936 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
937 						  reset_work);
938 
939 	struct amdgpu_reset_context reset_context;
940 
941 	memset(&reset_context, 0, sizeof(reset_context));
942 
943 	reset_context.method = AMD_RESET_METHOD_NONE;
944 	reset_context.reset_req_dev = adev;
945 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
946 
947 	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
948 }
949 
950 #endif
951 
952 void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
953 {
954 #if defined(CONFIG_DEBUG_FS)
955 	struct drm_minor *minor = adev_to_drm(adev)->primary;
956 	struct dentry *root = minor->debugfs_root;
957 
958 	debugfs_create_file("amdgpu_fence_info", 0444, root, adev,
959 			    &amdgpu_debugfs_fence_info_fops);
960 
961 	if (!amdgpu_sriov_vf(adev)) {
962 
963 		INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);
964 		debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,
965 				    &amdgpu_debugfs_gpu_recover_fops);
966 	}
967 #endif
968 }
969 
970