1 // SPDX-License-Identifier: GPL-2.0-only
15 	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,  in try_preempt_state()
25 static inline void set_preempt_state(struct a5xx_gpu *gpu,  in set_preempt_state()  argument
34 	atomic_set(&gpu->preempt_state, new);  in set_preempt_state()
40 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in update_wptr()  argument
48 	spin_lock_irqsave(&ring->preempt_lock, flags);  in update_wptr()
50 	spin_unlock_irqrestore(&ring->preempt_lock, flags);  in update_wptr()
52 	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);  in update_wptr()
56 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)  in get_next_ring()  argument
58 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in get_next_ring()
63 	for (i = 0; i < gpu->nr_rings; i++) {  in get_next_ring()
65 		struct msm_ringbuffer *ring = gpu->rb[i];  in get_next_ring()
67 		spin_lock_irqsave(&ring->preempt_lock, flags);  in get_next_ring()
68 		empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));  in get_next_ring()
69 		if (!empty && ring == a5xx_gpu->cur_ring)  in get_next_ring()
70 			empty = ring->memptrs->fence == a5xx_gpu->last_seqno[i];  in get_next_ring()
71 		spin_unlock_irqrestore(&ring->preempt_lock, flags);  in get_next_ring()
83 	struct msm_gpu *gpu = &a5xx_gpu->base.base;  in a5xx_preempt_timer()  local
84 	struct drm_device *dev = gpu->dev;  in a5xx_preempt_timer()
89 	DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);  in a5xx_preempt_timer()
90 	kthread_queue_work(gpu->worker, &gpu->recover_work);  in a5xx_preempt_timer()
94 void a5xx_preempt_trigger(struct msm_gpu *gpu)  in a5xx_preempt_trigger()  argument
96 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_preempt_trigger()
101 	if (gpu->nr_rings == 1)  in a5xx_preempt_trigger()
109 	spin_lock_irqsave(&a5xx_gpu->preempt_start_lock, flags);  in a5xx_preempt_trigger()
119 	ring = get_next_ring(gpu);  in a5xx_preempt_trigger()
125 	if (!ring || (a5xx_gpu->cur_ring == ring)) {  in a5xx_preempt_trigger()
139 		update_wptr(gpu, a5xx_gpu->cur_ring);  in a5xx_preempt_trigger()
144 	spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);  in a5xx_preempt_trigger()
147 	spin_lock_irqsave(&ring->preempt_lock, flags);  in a5xx_preempt_trigger()
148 	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);  in a5xx_preempt_trigger()
149 	spin_unlock_irqrestore(&ring->preempt_lock, flags);  in a5xx_preempt_trigger()
152 	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,  in a5xx_preempt_trigger()
153 		a5xx_gpu->preempt_iova[ring->id]);  in a5xx_preempt_trigger()
155 	a5xx_gpu->next_ring = ring;  in a5xx_preempt_trigger()
158 	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));  in a5xx_preempt_trigger()
167 	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);  in a5xx_preempt_trigger()
171 	spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);  in a5xx_preempt_trigger()
174 void a5xx_preempt_irq(struct msm_gpu *gpu)  in a5xx_preempt_irq()  argument
177 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_preempt_irq()
179 	struct drm_device *dev = gpu->dev;  in a5xx_preempt_irq()
185 	del_timer(&a5xx_gpu->preempt_timer);  in a5xx_preempt_irq()
193 	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);  in a5xx_preempt_irq()
196 		DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",  in a5xx_preempt_irq()
197 			gpu->name);  in a5xx_preempt_irq()
198 		kthread_queue_work(gpu->worker, &gpu->recover_work);  in a5xx_preempt_irq()
202 	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;  in a5xx_preempt_irq()
203 	a5xx_gpu->next_ring = NULL;  in a5xx_preempt_irq()
205 	update_wptr(gpu, a5xx_gpu->cur_ring);  in a5xx_preempt_irq()
213 	a5xx_preempt_trigger(gpu);  in a5xx_preempt_irq()
216 void a5xx_preempt_hw_init(struct msm_gpu *gpu)  in a5xx_preempt_hw_init()  argument
218 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_preempt_hw_init()
223 	a5xx_gpu->cur_ring = gpu->rb[0];  in a5xx_preempt_hw_init()
226 	if (gpu->nr_rings == 1)  in a5xx_preempt_hw_init()
229 	for (i = 0; i < gpu->nr_rings; i++) {  in a5xx_preempt_hw_init()
230 		a5xx_gpu->preempt[i]->data = 0;  in a5xx_preempt_hw_init()
231 		a5xx_gpu->preempt[i]->info = 0;  in a5xx_preempt_hw_init()
232 		a5xx_gpu->preempt[i]->wptr = 0;  in a5xx_preempt_hw_init()
233 		a5xx_gpu->preempt[i]->rptr = 0;  in a5xx_preempt_hw_init()
234 		a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;  in a5xx_preempt_hw_init()
235 		a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]);  in a5xx_preempt_hw_init()
239 	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0);  in a5xx_preempt_hw_init()
248 	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;  in preempt_init_ring()
249 	struct msm_gpu *gpu = &adreno_gpu->base;  in preempt_init_ring()  local
255 	ptr = msm_gem_kernel_new(gpu->dev,  in preempt_init_ring()
257 		MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova);  in preempt_init_ring()
263 	counters = msm_gem_kernel_new(gpu->dev,  in preempt_init_ring()
265 		MSM_BO_WC, gpu->aspace, &counters_bo, &counters_iova);  in preempt_init_ring()
267 		msm_gem_kernel_put(bo, gpu->aspace);  in preempt_init_ring()
274 	a5xx_gpu->preempt_bo[ring->id] = bo;  in preempt_init_ring()
275 	a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;  in preempt_init_ring()
276 	a5xx_gpu->preempt_iova[ring->id] = iova;  in preempt_init_ring()
277 	a5xx_gpu->preempt[ring->id] = ptr;  in preempt_init_ring()
281 	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;  in preempt_init_ring()
282 	ptr->info = 0;  in preempt_init_ring()
283 	ptr->data = 0;  in preempt_init_ring()
284 	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;  in preempt_init_ring()
286 	ptr->counter = counters_iova;  in preempt_init_ring()
291 void a5xx_preempt_fini(struct msm_gpu *gpu)  in a5xx_preempt_fini()  argument
293 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_preempt_fini()
297 	for (i = 0; i < gpu->nr_rings; i++) {  in a5xx_preempt_fini()
298 		msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace);  in a5xx_preempt_fini()
299 		msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->aspace);  in a5xx_preempt_fini()
303 void a5xx_preempt_init(struct msm_gpu *gpu)  in a5xx_preempt_init()  argument
305 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_preempt_init()
310 	if (gpu->nr_rings <= 1)  in a5xx_preempt_init()
313 	for (i = 0; i < gpu->nr_rings; i++) {  in a5xx_preempt_init()
314 		if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {  in a5xx_preempt_init()
319 			a5xx_preempt_fini(gpu);  in a5xx_preempt_init()
320 			gpu->nr_rings = 1;  in a5xx_preempt_init()
326 	spin_lock_init(&a5xx_gpu->preempt_start_lock);  in a5xx_preempt_init()
327 	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);  in a5xx_preempt_init()