1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2017 The Linux Foundation. All rights reserved.
3  */
4 
5 #include "msm_gem.h"
6 #include "a5xx_gpu.h"
7 
8 /*
9  * Try to transition the preemption state from old to new. Return
10  * true on success or false if the original state wasn't 'old'
11  */
12 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
13 		enum preempt_state old, enum preempt_state new)
14 {
15 	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
16 		old, new);
17 
18 	return (cur == old);
19 }
20 
21 /*
22  * Force the preemption state to the specified state.  This is used in cases
23  * where the current state is known and won't change
24  */
25 static inline void set_preempt_state(struct a5xx_gpu *gpu,
26 		enum preempt_state new)
27 {
28 	/*
29 	 * preempt_state may be read by other cores trying to trigger a
30 	 * preemption or in the interrupt handler so barriers are needed
31 	 * before...
32 	 */
33 	smp_mb__before_atomic();
34 	atomic_set(&gpu->preempt_state, new);
35 	/* ... and after*/
36 	smp_mb__after_atomic();
37 }
38 
39 /* Write the most recent wptr for the given ring into the hardware */
40 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
41 {
42 	unsigned long flags;
43 	uint32_t wptr;
44 
45 	if (!ring)
46 		return;
47 
48 	spin_lock_irqsave(&ring->preempt_lock, flags);
49 	wptr = get_wptr(ring);
50 	spin_unlock_irqrestore(&ring->preempt_lock, flags);
51 
52 	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
53 }
54 
55 /* Return the highest priority ringbuffer with something in it */
56 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
57 {
58 	unsigned long flags;
59 	int i;
60 
61 	for (i = 0; i < gpu->nr_rings; i++) {
62 		bool empty;
63 		struct msm_ringbuffer *ring = gpu->rb[i];
64 
65 		spin_lock_irqsave(&ring->preempt_lock, flags);
66 		empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
67 		spin_unlock_irqrestore(&ring->preempt_lock, flags);
68 
69 		if (!empty)
70 			return ring;
71 	}
72 
73 	return NULL;
74 }
75 
76 static void a5xx_preempt_timer(struct timer_list *t)
77 {
78 	struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
79 	struct msm_gpu *gpu = &a5xx_gpu->base.base;
80 	struct drm_device *dev = gpu->dev;
81 
82 	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
83 		return;
84 
85 	DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
86 	kthread_queue_work(gpu->worker, &gpu->recover_work);
87 }
88 
89 /* Try to trigger a preemption switch */
90 void a5xx_preempt_trigger(struct msm_gpu *gpu)
91 {
92 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
93 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
94 	unsigned long flags;
95 	struct msm_ringbuffer *ring;
96 
97 	if (gpu->nr_rings == 1)
98 		return;
99 
100 	/*
101 	 * Try to start preemption by moving from NONE to START. If
102 	 * unsuccessful, a preemption is already in flight
103 	 */
104 	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
105 		return;
106 
107 	/* Get the next ring to preempt to */
108 	ring = get_next_ring(gpu);
109 
110 	/*
111 	 * If no ring is populated or the highest priority ring is the current
112 	 * one do nothing except to update the wptr to the latest and greatest
113 	 */
114 	if (!ring || (a5xx_gpu->cur_ring == ring)) {
115 		/*
116 		 * Its possible that while a preemption request is in progress
117 		 * from an irq context, a user context trying to submit might
118 		 * fail to update the write pointer, because it determines
119 		 * that the preempt state is not PREEMPT_NONE.
120 		 *
121 		 * Close the race by introducing an intermediate
122 		 * state PREEMPT_ABORT to let the submit path
123 		 * know that the ringbuffer is not going to change
124 		 * and can safely update the write pointer.
125 		 */
126 
127 		set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
128 		update_wptr(gpu, a5xx_gpu->cur_ring);
129 		set_preempt_state(a5xx_gpu, PREEMPT_NONE);
130 		return;
131 	}
132 
133 	/* Make sure the wptr doesn't update while we're in motion */
134 	spin_lock_irqsave(&ring->preempt_lock, flags);
135 	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
136 	spin_unlock_irqrestore(&ring->preempt_lock, flags);
137 
138 	/* Set the address of the incoming preemption record */
139 	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
140 		a5xx_gpu->preempt_iova[ring->id]);
141 
142 	a5xx_gpu->next_ring = ring;
143 
144 	/* Start a timer to catch a stuck preemption */
145 	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
146 
147 	/* Set the preemption state to triggered */
148 	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
149 
150 	/* Make sure everything is written before hitting the button */
151 	wmb();
152 
153 	/* And actually start the preemption */
154 	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
155 }
156 
157 void a5xx_preempt_irq(struct msm_gpu *gpu)
158 {
159 	uint32_t status;
160 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
161 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
162 	struct drm_device *dev = gpu->dev;
163 
164 	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
165 		return;
166 
167 	/* Delete the preemption watchdog timer */
168 	del_timer(&a5xx_gpu->preempt_timer);
169 
170 	/*
171 	 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
172 	 * firing the interrupt, but there is a non zero chance of a hardware
173 	 * condition or a software race that could set it again before we have a
174 	 * chance to finish. If that happens, log and go for recovery
175 	 */
176 	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
177 	if (unlikely(status)) {
178 		set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
179 		DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
180 			gpu->name);
181 		kthread_queue_work(gpu->worker, &gpu->recover_work);
182 		return;
183 	}
184 
185 	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
186 	a5xx_gpu->next_ring = NULL;
187 
188 	update_wptr(gpu, a5xx_gpu->cur_ring);
189 
190 	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
191 }
192 
193 void a5xx_preempt_hw_init(struct msm_gpu *gpu)
194 {
195 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
196 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
197 	int i;
198 
199 	/* Always come up on rb 0 */
200 	a5xx_gpu->cur_ring = gpu->rb[0];
201 
202 	/* No preemption if we only have one ring */
203 	if (gpu->nr_rings == 1)
204 		return;
205 
206 	for (i = 0; i < gpu->nr_rings; i++) {
207 		a5xx_gpu->preempt[i]->wptr = 0;
208 		a5xx_gpu->preempt[i]->rptr = 0;
209 		a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
210 		a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]);
211 	}
212 
213 	/* Write a 0 to signal that we aren't switching pagetables */
214 	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0);
215 
216 	/* Reset the preemption state */
217 	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
218 }
219 
220 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
221 		struct msm_ringbuffer *ring)
222 {
223 	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
224 	struct msm_gpu *gpu = &adreno_gpu->base;
225 	struct a5xx_preempt_record *ptr;
226 	void *counters;
227 	struct drm_gem_object *bo = NULL, *counters_bo = NULL;
228 	u64 iova = 0, counters_iova = 0;
229 
230 	ptr = msm_gem_kernel_new(gpu->dev,
231 		A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
232 		MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova);
233 
234 	if (IS_ERR(ptr))
235 		return PTR_ERR(ptr);
236 
237 	/* The buffer to store counters needs to be unprivileged */
238 	counters = msm_gem_kernel_new(gpu->dev,
239 		A5XX_PREEMPT_COUNTER_SIZE,
240 		MSM_BO_WC, gpu->aspace, &counters_bo, &counters_iova);
241 	if (IS_ERR(counters)) {
242 		msm_gem_kernel_put(bo, gpu->aspace);
243 		return PTR_ERR(counters);
244 	}
245 
246 	msm_gem_object_set_name(bo, "preempt");
247 	msm_gem_object_set_name(counters_bo, "preempt_counters");
248 
249 	a5xx_gpu->preempt_bo[ring->id] = bo;
250 	a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;
251 	a5xx_gpu->preempt_iova[ring->id] = iova;
252 	a5xx_gpu->preempt[ring->id] = ptr;
253 
254 	/* Set up the defaults on the preemption record */
255 
256 	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
257 	ptr->info = 0;
258 	ptr->data = 0;
259 	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;
260 
261 	ptr->counter = counters_iova;
262 
263 	return 0;
264 }
265 
266 void a5xx_preempt_fini(struct msm_gpu *gpu)
267 {
268 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
269 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
270 	int i;
271 
272 	for (i = 0; i < gpu->nr_rings; i++) {
273 		msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace);
274 		msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->aspace);
275 	}
276 }
277 
278 void a5xx_preempt_init(struct msm_gpu *gpu)
279 {
280 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
281 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
282 	int i;
283 
284 	/* No preemption if we only have one ring */
285 	if (gpu->nr_rings <= 1)
286 		return;
287 
288 	for (i = 0; i < gpu->nr_rings; i++) {
289 		if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
290 			/*
291 			 * On any failure our adventure is over. Clean up and
292 			 * set nr_rings to 1 to force preemption off
293 			 */
294 			a5xx_preempt_fini(gpu);
295 			gpu->nr_rings = 1;
296 
297 			return;
298 		}
299 	}
300 
301 	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
302 }
303