1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Dave Airlie
30  */
31 #include <linux/seq_file.h>
32 #include <linux/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/kref.h>
35 #include <linux/slab.h>
36 #include <linux/firmware.h>
37 #include <linux/pm_runtime.h>
38 
39 #include <drm/drm_drv.h>
40 #include "amdgpu.h"
41 #include "amdgpu_trace.h"
42 #include "amdgpu_reset.h"
43 
44 /*
45  * Fences mark an event in the GPUs pipeline and are used
46  * for GPU/CPU synchronization.  When the fence is written,
47  * it is expected that all buffers associated with that fence
48  * are no longer in use by the associated ring on the GPU and
49  * that the relevant GPU caches have been flushed.
50  */
51 
52 struct amdgpu_fence {
53 	struct dma_fence base;
54 
55 	/* RB, DMA, etc. */
56 	struct amdgpu_ring		*ring;
57 	ktime_t				start_timestamp;
58 };
59 
60 static struct kmem_cache *amdgpu_fence_slab;
61 
62 int amdgpu_fence_slab_init(void)
63 {
64 	amdgpu_fence_slab = kmem_cache_create(
65 		"amdgpu_fence", sizeof(struct amdgpu_fence), 0,
66 		SLAB_HWCACHE_ALIGN, NULL);
67 	if (!amdgpu_fence_slab)
68 		return -ENOMEM;
69 	return 0;
70 }
71 
72 void amdgpu_fence_slab_fini(void)
73 {
74 	rcu_barrier();
75 	kmem_cache_destroy(amdgpu_fence_slab);
76 }
77 /*
78  * Cast helper
79  */
80 static const struct dma_fence_ops amdgpu_fence_ops;
81 static const struct dma_fence_ops amdgpu_job_fence_ops;
82 static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
83 {
84 	struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
85 
86 	if (__f->base.ops == &amdgpu_fence_ops ||
87 	    __f->base.ops == &amdgpu_job_fence_ops)
88 		return __f;
89 
90 	return NULL;
91 }
92 
93 /**
94  * amdgpu_fence_write - write a fence value
95  *
96  * @ring: ring the fence is associated with
97  * @seq: sequence number to write
98  *
99  * Writes a fence value to memory (all asics).
100  */
101 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
102 {
103 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
104 
105 	if (drv->cpu_addr)
106 		*drv->cpu_addr = cpu_to_le32(seq);
107 }
108 
109 /**
110  * amdgpu_fence_read - read a fence value
111  *
112  * @ring: ring the fence is associated with
113  *
114  * Reads a fence value from memory (all asics).
115  * Returns the value of the fence read from memory.
116  */
117 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
118 {
119 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
120 	u32 seq = 0;
121 
122 	if (drv->cpu_addr)
123 		seq = le32_to_cpu(*drv->cpu_addr);
124 	else
125 		seq = atomic_read(&drv->last_seq);
126 
127 	return seq;
128 }
129 
130 /**
131  * amdgpu_fence_emit - emit a fence on the requested ring
132  *
133  * @ring: ring the fence is associated with
134  * @f: resulting fence object
135  * @job: job the fence is embedded in
136  * @flags: flags to pass into the subordinate .emit_fence() call
137  *
138  * Emits a fence command on the requested ring (all asics).
139  * Returns 0 on success, -ENOMEM on failure.
140  */
141 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,
142 		      unsigned int flags)
143 {
144 	struct amdgpu_device *adev = ring->adev;
145 	struct dma_fence *fence;
146 	struct amdgpu_fence *am_fence;
147 	struct dma_fence __rcu **ptr;
148 	uint32_t seq;
149 	int r;
150 
151 	if (job == NULL) {
152 		/* create a sperate hw fence */
153 		am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);
154 		if (am_fence == NULL)
155 			return -ENOMEM;
156 		fence = &am_fence->base;
157 		am_fence->ring = ring;
158 	} else {
159 		/* take use of job-embedded fence */
160 		fence = &job->hw_fence;
161 	}
162 
163 	seq = ++ring->fence_drv.sync_seq;
164 	if (job && job->job_run_counter) {
165 		/* reinit seq for resubmitted jobs */
166 		fence->seqno = seq;
167 		/* TO be inline with external fence creation and other drivers */
168 		dma_fence_get(fence);
169 	} else {
170 		if (job) {
171 			dma_fence_init(fence, &amdgpu_job_fence_ops,
172 				       &ring->fence_drv.lock,
173 				       adev->fence_context + ring->idx, seq);
174 			/* Against remove in amdgpu_job_{free, free_cb} */
175 			dma_fence_get(fence);
176 		} else {
177 			dma_fence_init(fence, &amdgpu_fence_ops,
178 				       &ring->fence_drv.lock,
179 				       adev->fence_context + ring->idx, seq);
180 		}
181 	}
182 
183 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
184 			       seq, flags | AMDGPU_FENCE_FLAG_INT);
185 	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
186 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
187 	if (unlikely(rcu_dereference_protected(*ptr, 1))) {
188 		struct dma_fence *old;
189 
190 		rcu_read_lock();
191 		old = dma_fence_get_rcu_safe(ptr);
192 		rcu_read_unlock();
193 
194 		if (old) {
195 			r = dma_fence_wait(old, false);
196 			dma_fence_put(old);
197 			if (r)
198 				return r;
199 		}
200 	}
201 
202 	to_amdgpu_fence(fence)->start_timestamp = ktime_get();
203 
204 	/* This function can't be called concurrently anyway, otherwise
205 	 * emitting the fence would mess up the hardware ring buffer.
206 	 */
207 	rcu_assign_pointer(*ptr, dma_fence_get(fence));
208 
209 	*f = fence;
210 
211 	return 0;
212 }
213 
214 /**
215  * amdgpu_fence_emit_polling - emit a fence on the requeste ring
216  *
217  * @ring: ring the fence is associated with
218  * @s: resulting sequence number
219  * @timeout: the timeout for waiting in usecs
220  *
221  * Emits a fence command on the requested ring (all asics).
222  * Used For polling fence.
223  * Returns 0 on success, -ENOMEM on failure.
224  */
225 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
226 			      uint32_t timeout)
227 {
228 	uint32_t seq;
229 	signed long r;
230 
231 	if (!s)
232 		return -EINVAL;
233 
234 	seq = ++ring->fence_drv.sync_seq;
235 	r = amdgpu_fence_wait_polling(ring,
236 				      seq - ring->fence_drv.num_fences_mask,
237 				      timeout);
238 	if (r < 1)
239 		return -ETIMEDOUT;
240 
241 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
242 			       seq, 0);
243 
244 	*s = seq;
245 
246 	return 0;
247 }
248 
249 /**
250  * amdgpu_fence_schedule_fallback - schedule fallback check
251  *
252  * @ring: pointer to struct amdgpu_ring
253  *
254  * Start a timer as fallback to our interrupts.
255  */
256 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
257 {
258 	mod_timer(&ring->fence_drv.fallback_timer,
259 		  jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
260 }
261 
262 /**
263  * amdgpu_fence_process - check for fence activity
264  *
265  * @ring: pointer to struct amdgpu_ring
266  *
267  * Checks the current fence value and calculates the last
268  * signalled fence value. Wakes the fence queue if the
269  * sequence number has increased.
270  *
271  * Returns true if fence was processed
272  */
273 bool amdgpu_fence_process(struct amdgpu_ring *ring)
274 {
275 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
276 	struct amdgpu_device *adev = ring->adev;
277 	uint32_t seq, last_seq;
278 
279 	do {
280 		last_seq = atomic_read(&ring->fence_drv.last_seq);
281 		seq = amdgpu_fence_read(ring);
282 
283 	} while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq);
284 
285 	if (del_timer(&ring->fence_drv.fallback_timer) &&
286 	    seq != ring->fence_drv.sync_seq)
287 		amdgpu_fence_schedule_fallback(ring);
288 
289 	if (unlikely(seq == last_seq))
290 		return false;
291 
292 	last_seq &= drv->num_fences_mask;
293 	seq &= drv->num_fences_mask;
294 
295 	do {
296 		struct dma_fence *fence, **ptr;
297 
298 		++last_seq;
299 		last_seq &= drv->num_fences_mask;
300 		ptr = &drv->fences[last_seq];
301 
302 		/* There is always exactly one thread signaling this fence slot */
303 		fence = rcu_dereference_protected(*ptr, 1);
304 		RCU_INIT_POINTER(*ptr, NULL);
305 
306 		if (!fence)
307 			continue;
308 
309 		dma_fence_signal(fence);
310 		dma_fence_put(fence);
311 		pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
312 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
313 	} while (last_seq != seq);
314 
315 	return true;
316 }
317 
318 /**
319  * amdgpu_fence_fallback - fallback for hardware interrupts
320  *
321  * @t: timer context used to obtain the pointer to ring structure
322  *
323  * Checks for fence activity.
324  */
325 static void amdgpu_fence_fallback(struct timer_list *t)
326 {
327 	struct amdgpu_ring *ring = from_timer(ring, t,
328 					      fence_drv.fallback_timer);
329 
330 	if (amdgpu_fence_process(ring))
331 		DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name);
332 }
333 
334 /**
335  * amdgpu_fence_wait_empty - wait for all fences to signal
336  *
337  * @ring: ring index the fence is associated with
338  *
339  * Wait for all fences on the requested ring to signal (all asics).
340  * Returns 0 if the fences have passed, error for all other cases.
341  */
342 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
343 {
344 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);
345 	struct dma_fence *fence, **ptr;
346 	int r;
347 
348 	if (!seq)
349 		return 0;
350 
351 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
352 	rcu_read_lock();
353 	fence = rcu_dereference(*ptr);
354 	if (!fence || !dma_fence_get_rcu(fence)) {
355 		rcu_read_unlock();
356 		return 0;
357 	}
358 	rcu_read_unlock();
359 
360 	r = dma_fence_wait(fence, false);
361 	dma_fence_put(fence);
362 	return r;
363 }
364 
365 /**
366  * amdgpu_fence_wait_polling - busy wait for givn sequence number
367  *
368  * @ring: ring index the fence is associated with
369  * @wait_seq: sequence number to wait
370  * @timeout: the timeout for waiting in usecs
371  *
372  * Wait for all fences on the requested ring to signal (all asics).
373  * Returns left time if no timeout, 0 or minus if timeout.
374  */
375 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
376 				      uint32_t wait_seq,
377 				      signed long timeout)
378 {
379 
380 	while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {
381 		udelay(2);
382 		timeout -= 2;
383 	}
384 	return timeout > 0 ? timeout : 0;
385 }
386 /**
387  * amdgpu_fence_count_emitted - get the count of emitted fences
388  *
389  * @ring: ring the fence is associated with
390  *
391  * Get the number of fences emitted on the requested ring (all asics).
392  * Returns the number of emitted fences on the ring.  Used by the
393  * dynpm code to ring track activity.
394  */
395 unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
396 {
397 	uint64_t emitted;
398 
399 	/* We are not protected by ring lock when reading the last sequence
400 	 * but it's ok to report slightly wrong fence count here.
401 	 */
402 	emitted = 0x100000000ull;
403 	emitted -= atomic_read(&ring->fence_drv.last_seq);
404 	emitted += READ_ONCE(ring->fence_drv.sync_seq);
405 	return lower_32_bits(emitted);
406 }
407 
408 /**
409  * amdgpu_fence_last_unsignaled_time_us - the time fence emitted until now
410  * @ring: ring the fence is associated with
411  *
412  * Find the earliest fence unsignaled until now, calculate the time delta
413  * between the time fence emitted and now.
414  */
415 u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)
416 {
417 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
418 	struct dma_fence *fence;
419 	uint32_t last_seq, sync_seq;
420 
421 	last_seq = atomic_read(&ring->fence_drv.last_seq);
422 	sync_seq = READ_ONCE(ring->fence_drv.sync_seq);
423 	if (last_seq == sync_seq)
424 		return 0;
425 
426 	++last_seq;
427 	last_seq &= drv->num_fences_mask;
428 	fence = drv->fences[last_seq];
429 	if (!fence)
430 		return 0;
431 
432 	return ktime_us_delta(ktime_get(),
433 		to_amdgpu_fence(fence)->start_timestamp);
434 }
435 
436 /**
437  * amdgpu_fence_update_start_timestamp - update the timestamp of the fence
438  * @ring: ring the fence is associated with
439  * @seq: the fence seq number to update.
440  * @timestamp: the start timestamp to update.
441  *
442  * The function called at the time the fence and related ib is about to
443  * resubmit to gpu in MCBP scenario. Thus we do not consider race condition
444  * with amdgpu_fence_process to modify the same fence.
445  */
446 void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)
447 {
448 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
449 	struct dma_fence *fence;
450 
451 	seq &= drv->num_fences_mask;
452 	fence = drv->fences[seq];
453 	if (!fence)
454 		return;
455 
456 	to_amdgpu_fence(fence)->start_timestamp = timestamp;
457 }
458 
459 /**
460  * amdgpu_fence_driver_start_ring - make the fence driver
461  * ready for use on the requested ring.
462  *
463  * @ring: ring to start the fence driver on
464  * @irq_src: interrupt source to use for this ring
465  * @irq_type: interrupt type to use for this ring
466  *
467  * Make the fence driver ready for processing (all asics).
468  * Not all asics have all rings, so each asic will only
469  * start the fence driver on the rings it has.
470  * Returns 0 for success, errors for failure.
471  */
472 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
473 				   struct amdgpu_irq_src *irq_src,
474 				   unsigned int irq_type)
475 {
476 	struct amdgpu_device *adev = ring->adev;
477 	uint64_t index;
478 
479 	if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {
480 		ring->fence_drv.cpu_addr = ring->fence_cpu_addr;
481 		ring->fence_drv.gpu_addr = ring->fence_gpu_addr;
482 	} else {
483 		/* put fence directly behind firmware */
484 		index = ALIGN(adev->uvd.fw->size, 8);
485 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;
486 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;
487 	}
488 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));
489 
490 	ring->fence_drv.irq_src = irq_src;
491 	ring->fence_drv.irq_type = irq_type;
492 	ring->fence_drv.initialized = true;
493 
494 	DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",
495 		      ring->name, ring->fence_drv.gpu_addr);
496 	return 0;
497 }
498 
499 /**
500  * amdgpu_fence_driver_init_ring - init the fence driver
501  * for the requested ring.
502  *
503  * @ring: ring to init the fence driver on
504  *
505  * Init the fence driver for the requested ring (all asics).
506  * Helper function for amdgpu_fence_driver_init().
507  */
508 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
509 {
510 	struct amdgpu_device *adev = ring->adev;
511 
512 	if (!adev)
513 		return -EINVAL;
514 
515 	if (!is_power_of_2(ring->num_hw_submission))
516 		return -EINVAL;
517 
518 	ring->fence_drv.cpu_addr = NULL;
519 	ring->fence_drv.gpu_addr = 0;
520 	ring->fence_drv.sync_seq = 0;
521 	atomic_set(&ring->fence_drv.last_seq, 0);
522 	ring->fence_drv.initialized = false;
523 
524 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
525 
526 	ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;
527 	spin_lock_init(&ring->fence_drv.lock);
528 	ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),
529 					 GFP_KERNEL);
530 
531 	if (!ring->fence_drv.fences)
532 		return -ENOMEM;
533 
534 	return 0;
535 }
536 
537 /**
538  * amdgpu_fence_driver_sw_init - init the fence driver
539  * for all possible rings.
540  *
541  * @adev: amdgpu device pointer
542  *
543  * Init the fence driver for all possible rings (all asics).
544  * Not all asics have all rings, so each asic will only
545  * start the fence driver on the rings it has using
546  * amdgpu_fence_driver_start_ring().
547  * Returns 0 for success.
548  */
549 int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev)
550 {
551 	return 0;
552 }
553 
554 /**
555  * amdgpu_fence_need_ring_interrupt_restore - helper function to check whether
556  * fence driver interrupts need to be restored.
557  *
558  * @ring: ring that to be checked
559  *
560  * Interrupts for rings that belong to GFX IP don't need to be restored
561  * when the target power state is s0ix.
562  *
563  * Return true if need to restore interrupts, false otherwise.
564  */
565 static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring)
566 {
567 	struct amdgpu_device *adev = ring->adev;
568 	bool is_gfx_power_domain = false;
569 
570 	switch (ring->funcs->type) {
571 	case AMDGPU_RING_TYPE_SDMA:
572 	/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
573 		if (adev->ip_versions[SDMA0_HWIP][0] >= IP_VERSION(5, 0, 0))
574 			is_gfx_power_domain = true;
575 		break;
576 	case AMDGPU_RING_TYPE_GFX:
577 	case AMDGPU_RING_TYPE_COMPUTE:
578 	case AMDGPU_RING_TYPE_KIQ:
579 	case AMDGPU_RING_TYPE_MES:
580 		is_gfx_power_domain = true;
581 		break;
582 	default:
583 		break;
584 	}
585 
586 	return !(adev->in_s0ix && is_gfx_power_domain);
587 }
588 
589 /**
590  * amdgpu_fence_driver_hw_fini - tear down the fence driver
591  * for all possible rings.
592  *
593  * @adev: amdgpu device pointer
594  *
595  * Tear down the fence driver for all possible rings (all asics).
596  */
597 void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)
598 {
599 	int i, r;
600 
601 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
602 		struct amdgpu_ring *ring = adev->rings[i];
603 
604 		if (!ring || !ring->fence_drv.initialized)
605 			continue;
606 
607 		/* You can't wait for HW to signal if it's gone */
608 		if (!drm_dev_is_unplugged(adev_to_drm(adev)))
609 			r = amdgpu_fence_wait_empty(ring);
610 		else
611 			r = -ENODEV;
612 		/* no need to trigger GPU reset as we are unloading */
613 		if (r)
614 			amdgpu_fence_driver_force_completion(ring);
615 
616 		if (!drm_dev_is_unplugged(adev_to_drm(adev)) &&
617 		    ring->fence_drv.irq_src &&
618 		    amdgpu_fence_need_ring_interrupt_restore(ring))
619 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,
620 				       ring->fence_drv.irq_type);
621 
622 		del_timer_sync(&ring->fence_drv.fallback_timer);
623 	}
624 }
625 
626 /* Will either stop and flush handlers for amdgpu interrupt or reanble it */
627 void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop)
628 {
629 	int i;
630 
631 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
632 		struct amdgpu_ring *ring = adev->rings[i];
633 
634 		if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)
635 			continue;
636 
637 		if (stop)
638 			disable_irq(adev->irq.irq);
639 		else
640 			enable_irq(adev->irq.irq);
641 	}
642 }
643 
644 void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev)
645 {
646 	unsigned int i, j;
647 
648 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
649 		struct amdgpu_ring *ring = adev->rings[i];
650 
651 		if (!ring || !ring->fence_drv.initialized)
652 			continue;
653 
654 		/*
655 		 * Notice we check for sched.ops since there's some
656 		 * override on the meaning of sched.ready by amdgpu.
657 		 * The natural check would be sched.ready, which is
658 		 * set as drm_sched_init() finishes...
659 		 */
660 		if (ring->sched.ops)
661 			drm_sched_fini(&ring->sched);
662 
663 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
664 			dma_fence_put(ring->fence_drv.fences[j]);
665 		kfree(ring->fence_drv.fences);
666 		ring->fence_drv.fences = NULL;
667 		ring->fence_drv.initialized = false;
668 	}
669 }
670 
671 /**
672  * amdgpu_fence_driver_hw_init - enable the fence driver
673  * for all possible rings.
674  *
675  * @adev: amdgpu device pointer
676  *
677  * Enable the fence driver for all possible rings (all asics).
678  * Not all asics have all rings, so each asic will only
679  * start the fence driver on the rings it has using
680  * amdgpu_fence_driver_start_ring().
681  * Returns 0 for success.
682  */
683 void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)
684 {
685 	int i;
686 
687 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
688 		struct amdgpu_ring *ring = adev->rings[i];
689 
690 		if (!ring || !ring->fence_drv.initialized)
691 			continue;
692 
693 		/* enable the interrupt */
694 		if (ring->fence_drv.irq_src &&
695 		    amdgpu_fence_need_ring_interrupt_restore(ring))
696 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,
697 				       ring->fence_drv.irq_type);
698 	}
699 }
700 
701 /**
702  * amdgpu_fence_driver_clear_job_fences - clear job embedded fences of ring
703  *
704  * @ring: fence of the ring to be cleared
705  *
706  */
707 void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring)
708 {
709 	int i;
710 	struct dma_fence *old, **ptr;
711 
712 	for (i = 0; i <= ring->fence_drv.num_fences_mask; i++) {
713 		ptr = &ring->fence_drv.fences[i];
714 		old = rcu_dereference_protected(*ptr, 1);
715 		if (old && old->ops == &amdgpu_job_fence_ops) {
716 			struct amdgpu_job *job;
717 
718 			/* For non-scheduler bad job, i.e. failed ib test, we need to signal
719 			 * it right here or we won't be able to track them in fence_drv
720 			 * and they will remain unsignaled during sa_bo free.
721 			 */
722 			job = container_of(old, struct amdgpu_job, hw_fence);
723 			if (!job->base.s_fence && !dma_fence_is_signaled(old))
724 				dma_fence_signal(old);
725 			RCU_INIT_POINTER(*ptr, NULL);
726 			dma_fence_put(old);
727 		}
728 	}
729 }
730 
731 /**
732  * amdgpu_fence_driver_set_error - set error code on fences
733  * @ring: the ring which contains the fences
734  * @error: the error code to set
735  *
736  * Set an error code to all the fences pending on the ring.
737  */
738 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)
739 {
740 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
741 	unsigned long flags;
742 
743 	spin_lock_irqsave(&drv->lock, flags);
744 	for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {
745 		struct dma_fence *fence;
746 
747 		fence = rcu_dereference_protected(drv->fences[i],
748 						  lockdep_is_held(&drv->lock));
749 		if (fence && !dma_fence_is_signaled_locked(fence))
750 			dma_fence_set_error(fence, error);
751 	}
752 	spin_unlock_irqrestore(&drv->lock, flags);
753 }
754 
755 /**
756  * amdgpu_fence_driver_force_completion - force signal latest fence of ring
757  *
758  * @ring: fence of the ring to signal
759  *
760  */
761 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
762 {
763 	amdgpu_fence_driver_set_error(ring, -ECANCELED);
764 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
765 	amdgpu_fence_process(ring);
766 }
767 
768 /*
769  * Common fence implementation
770  */
771 
772 static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
773 {
774 	return "amdgpu";
775 }
776 
777 static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
778 {
779 	return (const char *)to_amdgpu_fence(f)->ring->name;
780 }
781 
782 static const char *amdgpu_job_fence_get_timeline_name(struct dma_fence *f)
783 {
784 	struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
785 
786 	return (const char *)to_amdgpu_ring(job->base.sched)->name;
787 }
788 
789 /**
790  * amdgpu_fence_enable_signaling - enable signalling on fence
791  * @f: fence
792  *
793  * This function is called with fence_queue lock held, and adds a callback
794  * to fence_queue that checks if this fence is signaled, and if so it
795  * signals the fence and removes itself.
796  */
797 static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
798 {
799 	if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))
800 		amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring);
801 
802 	return true;
803 }
804 
805 /**
806  * amdgpu_job_fence_enable_signaling - enable signalling on job fence
807  * @f: fence
808  *
809  * This is the simliar function with amdgpu_fence_enable_signaling above, it
810  * only handles the job embedded fence.
811  */
812 static bool amdgpu_job_fence_enable_signaling(struct dma_fence *f)
813 {
814 	struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);
815 
816 	if (!timer_pending(&to_amdgpu_ring(job->base.sched)->fence_drv.fallback_timer))
817 		amdgpu_fence_schedule_fallback(to_amdgpu_ring(job->base.sched));
818 
819 	return true;
820 }
821 
822 /**
823  * amdgpu_fence_free - free up the fence memory
824  *
825  * @rcu: RCU callback head
826  *
827  * Free up the fence memory after the RCU grace period.
828  */
829 static void amdgpu_fence_free(struct rcu_head *rcu)
830 {
831 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
832 
833 	/* free fence_slab if it's separated fence*/
834 	kmem_cache_free(amdgpu_fence_slab, to_amdgpu_fence(f));
835 }
836 
837 /**
838  * amdgpu_job_fence_free - free up the job with embedded fence
839  *
840  * @rcu: RCU callback head
841  *
842  * Free up the job with embedded fence after the RCU grace period.
843  */
844 static void amdgpu_job_fence_free(struct rcu_head *rcu)
845 {
846 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
847 
848 	/* free job if fence has a parent job */
849 	kfree(container_of(f, struct amdgpu_job, hw_fence));
850 }
851 
852 /**
853  * amdgpu_fence_release - callback that fence can be freed
854  *
855  * @f: fence
856  *
857  * This function is called when the reference count becomes zero.
858  * It just RCU schedules freeing up the fence.
859  */
860 static void amdgpu_fence_release(struct dma_fence *f)
861 {
862 	call_rcu(&f->rcu, amdgpu_fence_free);
863 }
864 
865 /**
866  * amdgpu_job_fence_release - callback that job embedded fence can be freed
867  *
868  * @f: fence
869  *
870  * This is the simliar function with amdgpu_fence_release above, it
871  * only handles the job embedded fence.
872  */
873 static void amdgpu_job_fence_release(struct dma_fence *f)
874 {
875 	call_rcu(&f->rcu, amdgpu_job_fence_free);
876 }
877 
878 static const struct dma_fence_ops amdgpu_fence_ops = {
879 	.get_driver_name = amdgpu_fence_get_driver_name,
880 	.get_timeline_name = amdgpu_fence_get_timeline_name,
881 	.enable_signaling = amdgpu_fence_enable_signaling,
882 	.release = amdgpu_fence_release,
883 };
884 
885 static const struct dma_fence_ops amdgpu_job_fence_ops = {
886 	.get_driver_name = amdgpu_fence_get_driver_name,
887 	.get_timeline_name = amdgpu_job_fence_get_timeline_name,
888 	.enable_signaling = amdgpu_job_fence_enable_signaling,
889 	.release = amdgpu_job_fence_release,
890 };
891 
892 /*
893  * Fence debugfs
894  */
895 #if defined(CONFIG_DEBUG_FS)
896 static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)
897 {
898 	struct amdgpu_device *adev = m->private;
899 	int i;
900 
901 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
902 		struct amdgpu_ring *ring = adev->rings[i];
903 
904 		if (!ring || !ring->fence_drv.initialized)
905 			continue;
906 
907 		amdgpu_fence_process(ring);
908 
909 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
910 		seq_printf(m, "Last signaled fence          0x%08x\n",
911 			   atomic_read(&ring->fence_drv.last_seq));
912 		seq_printf(m, "Last emitted                 0x%08x\n",
913 			   ring->fence_drv.sync_seq);
914 
915 		if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||
916 		    ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {
917 			seq_printf(m, "Last signaled trailing fence 0x%08x\n",
918 				   le32_to_cpu(*ring->trail_fence_cpu_addr));
919 			seq_printf(m, "Last emitted                 0x%08x\n",
920 				   ring->trail_seq);
921 		}
922 
923 		if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
924 			continue;
925 
926 		/* set in CP_VMID_PREEMPT and preemption occurred */
927 		seq_printf(m, "Last preempted               0x%08x\n",
928 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
929 		/* set in CP_VMID_RESET and reset occurred */
930 		seq_printf(m, "Last reset                   0x%08x\n",
931 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
932 		/* Both preemption and reset occurred */
933 		seq_printf(m, "Last both                    0x%08x\n",
934 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
935 	}
936 	return 0;
937 }
938 
939 /*
940  * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
941  *
942  * Manually trigger a gpu reset at the next fence wait.
943  */
944 static int gpu_recover_get(void *data, u64 *val)
945 {
946 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
947 	struct drm_device *dev = adev_to_drm(adev);
948 	int r;
949 
950 	r = pm_runtime_get_sync(dev->dev);
951 	if (r < 0) {
952 		pm_runtime_put_autosuspend(dev->dev);
953 		return 0;
954 	}
955 
956 	if (amdgpu_reset_domain_schedule(adev->reset_domain, &adev->reset_work))
957 		flush_work(&adev->reset_work);
958 
959 	*val = atomic_read(&adev->reset_domain->reset_res);
960 
961 	pm_runtime_mark_last_busy(dev->dev);
962 	pm_runtime_put_autosuspend(dev->dev);
963 
964 	return 0;
965 }
966 
967 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);
968 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,
969 			 "%lld\n");
970 
971 static void amdgpu_debugfs_reset_work(struct work_struct *work)
972 {
973 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
974 						  reset_work);
975 
976 	struct amdgpu_reset_context reset_context;
977 
978 	memset(&reset_context, 0, sizeof(reset_context));
979 
980 	reset_context.method = AMD_RESET_METHOD_NONE;
981 	reset_context.reset_req_dev = adev;
982 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
983 
984 	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
985 }
986 
987 #endif
988 
989 void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
990 {
991 #if defined(CONFIG_DEBUG_FS)
992 	struct drm_minor *minor = adev_to_drm(adev)->primary;
993 	struct dentry *root = minor->debugfs_root;
994 
995 	debugfs_create_file("amdgpu_fence_info", 0444, root, adev,
996 			    &amdgpu_debugfs_fence_info_fops);
997 
998 	if (!amdgpu_sriov_vf(adev)) {
999 
1000 		INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);
1001 		debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,
1002 				    &amdgpu_debugfs_gpu_recover_fops);
1003 	}
1004 #endif
1005 }
1006 
1007