1 /* Copyright (c) 2017 The Linux Foundation. All rights reserved.
2  *
3  * This program is free software; you can redistribute it and/or modify
4  * it under the terms of the GNU General Public License version 2 and
5  * only version 2 as published by the Free Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  */
13 
14 #include "msm_gem.h"
15 #include "a5xx_gpu.h"
16 
17 /*
18  * Try to transition the preemption state from old to new. Return
19  * true on success or false if the original state wasn't 'old'
20  */
21 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
22 		enum preempt_state old, enum preempt_state new)
23 {
24 	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
25 		old, new);
26 
27 	return (cur == old);
28 }
29 
30 /*
31  * Force the preemption state to the specified state.  This is used in cases
32  * where the current state is known and won't change
33  */
34 static inline void set_preempt_state(struct a5xx_gpu *gpu,
35 		enum preempt_state new)
36 {
37 	/*
38 	 * preempt_state may be read by other cores trying to trigger a
39 	 * preemption or in the interrupt handler so barriers are needed
40 	 * before...
41 	 */
42 	smp_mb__before_atomic();
43 	atomic_set(&gpu->preempt_state, new);
44 	/* ... and after*/
45 	smp_mb__after_atomic();
46 }
47 
48 /* Write the most recent wptr for the given ring into the hardware */
49 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
50 {
51 	unsigned long flags;
52 	uint32_t wptr;
53 
54 	if (!ring)
55 		return;
56 
57 	spin_lock_irqsave(&ring->lock, flags);
58 	wptr = get_wptr(ring);
59 	spin_unlock_irqrestore(&ring->lock, flags);
60 
61 	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
62 }
63 
64 /* Return the highest priority ringbuffer with something in it */
65 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
66 {
67 	unsigned long flags;
68 	int i;
69 
70 	for (i = 0; i < gpu->nr_rings; i++) {
71 		bool empty;
72 		struct msm_ringbuffer *ring = gpu->rb[i];
73 
74 		spin_lock_irqsave(&ring->lock, flags);
75 		empty = (get_wptr(ring) == ring->memptrs->rptr);
76 		spin_unlock_irqrestore(&ring->lock, flags);
77 
78 		if (!empty)
79 			return ring;
80 	}
81 
82 	return NULL;
83 }
84 
85 static void a5xx_preempt_timer(struct timer_list *t)
86 {
87 	struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
88 	struct msm_gpu *gpu = &a5xx_gpu->base.base;
89 	struct drm_device *dev = gpu->dev;
90 	struct msm_drm_private *priv = dev->dev_private;
91 
92 	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
93 		return;
94 
95 	DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
96 	queue_work(priv->wq, &gpu->recover_work);
97 }
98 
99 /* Try to trigger a preemption switch */
100 void a5xx_preempt_trigger(struct msm_gpu *gpu)
101 {
102 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
103 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
104 	unsigned long flags;
105 	struct msm_ringbuffer *ring;
106 
107 	if (gpu->nr_rings == 1)
108 		return;
109 
110 	/*
111 	 * Try to start preemption by moving from NONE to START. If
112 	 * unsuccessful, a preemption is already in flight
113 	 */
114 	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
115 		return;
116 
117 	/* Get the next ring to preempt to */
118 	ring = get_next_ring(gpu);
119 
120 	/*
121 	 * If no ring is populated or the highest priority ring is the current
122 	 * one do nothing except to update the wptr to the latest and greatest
123 	 */
124 	if (!ring || (a5xx_gpu->cur_ring == ring)) {
125 		/*
126 		 * Its possible that while a preemption request is in progress
127 		 * from an irq context, a user context trying to submit might
128 		 * fail to update the write pointer, because it determines
129 		 * that the preempt state is not PREEMPT_NONE.
130 		 *
131 		 * Close the race by introducing an intermediate
132 		 * state PREEMPT_ABORT to let the submit path
133 		 * know that the ringbuffer is not going to change
134 		 * and can safely update the write pointer.
135 		 */
136 
137 		set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
138 		update_wptr(gpu, a5xx_gpu->cur_ring);
139 		set_preempt_state(a5xx_gpu, PREEMPT_NONE);
140 		return;
141 	}
142 
143 	/* Make sure the wptr doesn't update while we're in motion */
144 	spin_lock_irqsave(&ring->lock, flags);
145 	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
146 	spin_unlock_irqrestore(&ring->lock, flags);
147 
148 	/* Set the address of the incoming preemption record */
149 	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
150 		REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
151 		a5xx_gpu->preempt_iova[ring->id]);
152 
153 	a5xx_gpu->next_ring = ring;
154 
155 	/* Start a timer to catch a stuck preemption */
156 	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
157 
158 	/* Set the preemption state to triggered */
159 	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
160 
161 	/* Make sure everything is written before hitting the button */
162 	wmb();
163 
164 	/* And actually start the preemption */
165 	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
166 }
167 
168 void a5xx_preempt_irq(struct msm_gpu *gpu)
169 {
170 	uint32_t status;
171 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
172 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
173 	struct drm_device *dev = gpu->dev;
174 	struct msm_drm_private *priv = dev->dev_private;
175 
176 	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
177 		return;
178 
179 	/* Delete the preemption watchdog timer */
180 	del_timer(&a5xx_gpu->preempt_timer);
181 
182 	/*
183 	 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
184 	 * firing the interrupt, but there is a non zero chance of a hardware
185 	 * condition or a software race that could set it again before we have a
186 	 * chance to finish. If that happens, log and go for recovery
187 	 */
188 	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
189 	if (unlikely(status)) {
190 		set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
191 		DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
192 			gpu->name);
193 		queue_work(priv->wq, &gpu->recover_work);
194 		return;
195 	}
196 
197 	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
198 	a5xx_gpu->next_ring = NULL;
199 
200 	update_wptr(gpu, a5xx_gpu->cur_ring);
201 
202 	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
203 }
204 
205 void a5xx_preempt_hw_init(struct msm_gpu *gpu)
206 {
207 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
208 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
209 	int i;
210 
211 	/* Always come up on rb 0 */
212 	a5xx_gpu->cur_ring = gpu->rb[0];
213 
214 	/* No preemption if we only have one ring */
215 	if (gpu->nr_rings == 1)
216 		return;
217 
218 	for (i = 0; i < gpu->nr_rings; i++) {
219 		a5xx_gpu->preempt[i]->wptr = 0;
220 		a5xx_gpu->preempt[i]->rptr = 0;
221 		a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
222 	}
223 
224 	/* Write a 0 to signal that we aren't switching pagetables */
225 	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
226 		REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
227 
228 	/* Reset the preemption state */
229 	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
230 }
231 
232 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
233 		struct msm_ringbuffer *ring)
234 {
235 	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
236 	struct msm_gpu *gpu = &adreno_gpu->base;
237 	struct a5xx_preempt_record *ptr;
238 	struct drm_gem_object *bo = NULL;
239 	u64 iova = 0;
240 
241 	ptr = msm_gem_kernel_new(gpu->dev,
242 		A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
243 		MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);
244 
245 	if (IS_ERR(ptr))
246 		return PTR_ERR(ptr);
247 
248 	msm_gem_object_set_name(bo, "preempt");
249 
250 	a5xx_gpu->preempt_bo[ring->id] = bo;
251 	a5xx_gpu->preempt_iova[ring->id] = iova;
252 	a5xx_gpu->preempt[ring->id] = ptr;
253 
254 	/* Set up the defaults on the preemption record */
255 
256 	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
257 	ptr->info = 0;
258 	ptr->data = 0;
259 	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
260 	ptr->rptr_addr = rbmemptr(ring, rptr);
261 	ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE;
262 
263 	return 0;
264 }
265 
266 void a5xx_preempt_fini(struct msm_gpu *gpu)
267 {
268 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
269 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
270 	int i;
271 
272 	for (i = 0; i < gpu->nr_rings; i++)
273 		msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace, true);
274 }
275 
276 void a5xx_preempt_init(struct msm_gpu *gpu)
277 {
278 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
279 	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
280 	int i;
281 
282 	/* No preemption if we only have one ring */
283 	if (gpu->nr_rings <= 1)
284 		return;
285 
286 	for (i = 0; i < gpu->nr_rings; i++) {
287 		if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
288 			/*
289 			 * On any failure our adventure is over. Clean up and
290 			 * set nr_rings to 1 to force preemption off
291 			 */
292 			a5xx_preempt_fini(gpu);
293 			gpu->nr_rings = 1;
294 
295 			return;
296 		}
297 	}
298 
299 	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
300 }
301