xref: /openbmc/linux/drivers/gpu/drm/lima/lima_sched.c (revision 6c33a6f4) 
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3 
4 #include <linux/kthread.h>
5 #include <linux/slab.h>
6 #include <linux/xarray.h>
7 
8 #include "lima_drv.h"
9 #include "lima_sched.h"
10 #include "lima_vm.h"
11 #include "lima_mmu.h"
12 #include "lima_l2_cache.h"
13 #include "lima_gem.h"
14 
15 struct lima_fence {
16 	struct dma_fence base;
17 	struct lima_sched_pipe *pipe;
18 };
19 
20 static struct kmem_cache *lima_fence_slab;
21 static int lima_fence_slab_refcnt;
22 
23 int lima_sched_slab_init(void)
24 {
25 	if (!lima_fence_slab) {
26 		lima_fence_slab = kmem_cache_create(
27 			"lima_fence", sizeof(struct lima_fence), 0,
28 			SLAB_HWCACHE_ALIGN, NULL);
29 		if (!lima_fence_slab)
30 			return -ENOMEM;
31 	}
32 
33 	lima_fence_slab_refcnt++;
34 	return 0;
35 }
36 
37 void lima_sched_slab_fini(void)
38 {
39 	if (!--lima_fence_slab_refcnt) {
40 		kmem_cache_destroy(lima_fence_slab);
41 		lima_fence_slab = NULL;
42 	}
43 }
44 
45 static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
46 {
47 	return container_of(fence, struct lima_fence, base);
48 }
49 
50 static const char *lima_fence_get_driver_name(struct dma_fence *fence)
51 {
52 	return "lima";
53 }
54 
55 static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
56 {
57 	struct lima_fence *f = to_lima_fence(fence);
58 
59 	return f->pipe->base.name;
60 }
61 
62 static void lima_fence_release_rcu(struct rcu_head *rcu)
63 {
64 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
65 	struct lima_fence *fence = to_lima_fence(f);
66 
67 	kmem_cache_free(lima_fence_slab, fence);
68 }
69 
70 static void lima_fence_release(struct dma_fence *fence)
71 {
72 	struct lima_fence *f = to_lima_fence(fence);
73 
74 	call_rcu(&f->base.rcu, lima_fence_release_rcu);
75 }
76 
77 static const struct dma_fence_ops lima_fence_ops = {
78 	.get_driver_name = lima_fence_get_driver_name,
79 	.get_timeline_name = lima_fence_get_timeline_name,
80 	.release = lima_fence_release,
81 };
82 
83 static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
84 {
85 	struct lima_fence *fence;
86 
87 	fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
88 	if (!fence)
89 		return NULL;
90 
91 	fence->pipe = pipe;
92 	dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
93 		       pipe->fence_context, ++pipe->fence_seqno);
94 
95 	return fence;
96 }
97 
98 static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
99 {
100 	return container_of(job, struct lima_sched_task, base);
101 }
102 
103 static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
104 {
105 	return container_of(sched, struct lima_sched_pipe, base);
106 }
107 
108 int lima_sched_task_init(struct lima_sched_task *task,
109 			 struct lima_sched_context *context,
110 			 struct lima_bo **bos, int num_bos,
111 			 struct lima_vm *vm)
112 {
113 	int err, i;
114 
115 	task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
116 	if (!task->bos)
117 		return -ENOMEM;
118 
119 	for (i = 0; i < num_bos; i++)
120 		drm_gem_object_get(&bos[i]->base.base);
121 
122 	err = drm_sched_job_init(&task->base, &context->base, vm);
123 	if (err) {
124 		kfree(task->bos);
125 		return err;
126 	}
127 
128 	task->num_bos = num_bos;
129 	task->vm = lima_vm_get(vm);
130 
131 	xa_init_flags(&task->deps, XA_FLAGS_ALLOC);
132 
133 	return 0;
134 }
135 
136 void lima_sched_task_fini(struct lima_sched_task *task)
137 {
138 	struct dma_fence *fence;
139 	unsigned long index;
140 	int i;
141 
142 	drm_sched_job_cleanup(&task->base);
143 
144 	xa_for_each(&task->deps, index, fence) {
145 		dma_fence_put(fence);
146 	}
147 	xa_destroy(&task->deps);
148 
149 	if (task->bos) {
150 		for (i = 0; i < task->num_bos; i++)
151 			drm_gem_object_put_unlocked(&task->bos[i]->base.base);
152 		kfree(task->bos);
153 	}
154 
155 	lima_vm_put(task->vm);
156 }
157 
158 int lima_sched_context_init(struct lima_sched_pipe *pipe,
159 			    struct lima_sched_context *context,
160 			    atomic_t *guilty)
161 {
162 	struct drm_gpu_scheduler *sched = &pipe->base;
163 
164 	return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL,
165 				     &sched, 1, guilty);
166 }
167 
168 void lima_sched_context_fini(struct lima_sched_pipe *pipe,
169 			     struct lima_sched_context *context)
170 {
171 	drm_sched_entity_fini(&context->base);
172 }
173 
174 struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *context,
175 						struct lima_sched_task *task)
176 {
177 	struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
178 
179 	drm_sched_entity_push_job(&task->base, &context->base);
180 	return fence;
181 }
182 
183 static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
184 					       struct drm_sched_entity *entity)
185 {
186 	struct lima_sched_task *task = to_lima_task(job);
187 
188 	if (!xa_empty(&task->deps))
189 		return xa_erase(&task->deps, task->last_dep++);
190 
191 	return NULL;
192 }
193 
194 static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
195 {
196 	struct lima_sched_task *task = to_lima_task(job);
197 	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
198 	struct lima_fence *fence;
199 	struct dma_fence *ret;
200 	struct lima_vm *vm = NULL, *last_vm = NULL;
201 	int i;
202 
203 	/* after GPU reset */
204 	if (job->s_fence->finished.error < 0)
205 		return NULL;
206 
207 	fence = lima_fence_create(pipe);
208 	if (!fence)
209 		return NULL;
210 	task->fence = &fence->base;
211 
212 	/* for caller usage of the fence, otherwise irq handler
213 	 * may consume the fence before caller use it
214 	 */
215 	ret = dma_fence_get(task->fence);
216 
217 	pipe->current_task = task;
218 
219 	/* this is needed for MMU to work correctly, otherwise GP/PP
220 	 * will hang or page fault for unknown reason after running for
221 	 * a while.
222 	 *
223 	 * Need to investigate:
224 	 * 1. is it related to TLB
225 	 * 2. how much performance will be affected by L2 cache flush
226 	 * 3. can we reduce the calling of this function because all
227 	 *    GP/PP use the same L2 cache on mali400
228 	 *
229 	 * TODO:
230 	 * 1. move this to task fini to save some wait time?
231 	 * 2. when GP/PP use different l2 cache, need PP wait GP l2
232 	 *    cache flush?
233 	 */
234 	for (i = 0; i < pipe->num_l2_cache; i++)
235 		lima_l2_cache_flush(pipe->l2_cache[i]);
236 
237 	if (task->vm != pipe->current_vm) {
238 		vm = lima_vm_get(task->vm);
239 		last_vm = pipe->current_vm;
240 		pipe->current_vm = task->vm;
241 	}
242 
243 	if (pipe->bcast_mmu)
244 		lima_mmu_switch_vm(pipe->bcast_mmu, vm);
245 	else {
246 		for (i = 0; i < pipe->num_mmu; i++)
247 			lima_mmu_switch_vm(pipe->mmu[i], vm);
248 	}
249 
250 	if (last_vm)
251 		lima_vm_put(last_vm);
252 
253 	pipe->error = false;
254 	pipe->task_run(pipe, task);
255 
256 	return task->fence;
257 }
258 
259 static void lima_sched_timedout_job(struct drm_sched_job *job)
260 {
261 	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
262 	struct lima_sched_task *task = to_lima_task(job);
263 
264 	if (!pipe->error)
265 		DRM_ERROR("lima job timeout\n");
266 
267 	drm_sched_stop(&pipe->base, &task->base);
268 
269 	drm_sched_increase_karma(&task->base);
270 
271 	pipe->task_error(pipe);
272 
273 	if (pipe->bcast_mmu)
274 		lima_mmu_page_fault_resume(pipe->bcast_mmu);
275 	else {
276 		int i;
277 
278 		for (i = 0; i < pipe->num_mmu; i++)
279 			lima_mmu_page_fault_resume(pipe->mmu[i]);
280 	}
281 
282 	if (pipe->current_vm)
283 		lima_vm_put(pipe->current_vm);
284 
285 	pipe->current_vm = NULL;
286 	pipe->current_task = NULL;
287 
288 	drm_sched_resubmit_jobs(&pipe->base);
289 	drm_sched_start(&pipe->base, true);
290 }
291 
292 static void lima_sched_free_job(struct drm_sched_job *job)
293 {
294 	struct lima_sched_task *task = to_lima_task(job);
295 	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
296 	struct lima_vm *vm = task->vm;
297 	struct lima_bo **bos = task->bos;
298 	int i;
299 
300 	dma_fence_put(task->fence);
301 
302 	for (i = 0; i < task->num_bos; i++)
303 		lima_vm_bo_del(vm, bos[i]);
304 
305 	lima_sched_task_fini(task);
306 	kmem_cache_free(pipe->task_slab, task);
307 }
308 
309 static const struct drm_sched_backend_ops lima_sched_ops = {
310 	.dependency = lima_sched_dependency,
311 	.run_job = lima_sched_run_job,
312 	.timedout_job = lima_sched_timedout_job,
313 	.free_job = lima_sched_free_job,
314 };
315 
316 static void lima_sched_recover_work(struct work_struct *work)
317 {
318 	struct lima_sched_pipe *pipe =
319 		container_of(work, struct lima_sched_pipe, recover_work);
320 	int i;
321 
322 	for (i = 0; i < pipe->num_l2_cache; i++)
323 		lima_l2_cache_flush(pipe->l2_cache[i]);
324 
325 	if (pipe->bcast_mmu) {
326 		lima_mmu_flush_tlb(pipe->bcast_mmu);
327 	} else {
328 		for (i = 0; i < pipe->num_mmu; i++)
329 			lima_mmu_flush_tlb(pipe->mmu[i]);
330 	}
331 
332 	if (pipe->task_recover(pipe))
333 		drm_sched_fault(&pipe->base);
334 }
335 
336 int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
337 {
338 	unsigned int timeout = lima_sched_timeout_ms > 0 ?
339 			       lima_sched_timeout_ms : 500;
340 
341 	pipe->fence_context = dma_fence_context_alloc(1);
342 	spin_lock_init(&pipe->fence_lock);
343 
344 	INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
345 
346 	return drm_sched_init(&pipe->base, &lima_sched_ops, 1, 0,
347 			      msecs_to_jiffies(timeout), name);
348 }
349 
350 void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
351 {
352 	drm_sched_fini(&pipe->base);
353 }
354 
355 void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
356 {
357 	struct lima_sched_task *task = pipe->current_task;
358 
359 	if (pipe->error) {
360 		if (task && task->recoverable)
361 			schedule_work(&pipe->recover_work);
362 		else
363 			drm_sched_fault(&pipe->base);
364 	} else {
365 		pipe->task_fini(pipe);
366 		dma_fence_signal(task->fence);
367 	}
368 }
369