1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3 /* Copyright 2019 Collabora ltd. */
4 #include <linux/delay.h>
5 #include <linux/interrupt.h>
6 #include <linux/io.h>
7 #include <linux/iopoll.h>
8 #include <linux/platform_device.h>
9 #include <linux/pm_runtime.h>
10 #include <linux/dma-resv.h>
11 #include <drm/gpu_scheduler.h>
12 #include <drm/panfrost_drm.h>
13 
14 #include "panfrost_device.h"
15 #include "panfrost_devfreq.h"
16 #include "panfrost_job.h"
17 #include "panfrost_features.h"
18 #include "panfrost_issues.h"
19 #include "panfrost_gem.h"
20 #include "panfrost_regs.h"
21 #include "panfrost_gpu.h"
22 #include "panfrost_mmu.h"
23 #include "panfrost_dump.h"
24 
25 #define JOB_TIMEOUT_MS 500
26 
27 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
28 #define job_read(dev, reg) readl(dev->iomem + (reg))
29 
30 struct panfrost_queue_state {
31 	struct drm_gpu_scheduler sched;
32 	u64 fence_context;
33 	u64 emit_seqno;
34 };
35 
36 struct panfrost_job_slot {
37 	struct panfrost_queue_state queue[NUM_JOB_SLOTS];
38 	spinlock_t job_lock;
39 	int irq;
40 };
41 
42 static struct panfrost_job *
43 to_panfrost_job(struct drm_sched_job *sched_job)
44 {
45 	return container_of(sched_job, struct panfrost_job, base);
46 }
47 
48 struct panfrost_fence {
49 	struct dma_fence base;
50 	struct drm_device *dev;
51 	/* panfrost seqno for signaled() test */
52 	u64 seqno;
53 	int queue;
54 };
55 
56 static inline struct panfrost_fence *
57 to_panfrost_fence(struct dma_fence *fence)
58 {
59 	return (struct panfrost_fence *)fence;
60 }
61 
62 static const char *panfrost_fence_get_driver_name(struct dma_fence *fence)
63 {
64 	return "panfrost";
65 }
66 
67 static const char *panfrost_fence_get_timeline_name(struct dma_fence *fence)
68 {
69 	struct panfrost_fence *f = to_panfrost_fence(fence);
70 
71 	switch (f->queue) {
72 	case 0:
73 		return "panfrost-js-0";
74 	case 1:
75 		return "panfrost-js-1";
76 	case 2:
77 		return "panfrost-js-2";
78 	default:
79 		return NULL;
80 	}
81 }
82 
83 static const struct dma_fence_ops panfrost_fence_ops = {
84 	.get_driver_name = panfrost_fence_get_driver_name,
85 	.get_timeline_name = panfrost_fence_get_timeline_name,
86 };
87 
88 static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)
89 {
90 	struct panfrost_fence *fence;
91 	struct panfrost_job_slot *js = pfdev->js;
92 
93 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
94 	if (!fence)
95 		return ERR_PTR(-ENOMEM);
96 
97 	fence->dev = pfdev->ddev;
98 	fence->queue = js_num;
99 	fence->seqno = ++js->queue[js_num].emit_seqno;
100 	dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,
101 		       js->queue[js_num].fence_context, fence->seqno);
102 
103 	return &fence->base;
104 }
105 
106 int panfrost_job_get_slot(struct panfrost_job *job)
107 {
108 	/* JS0: fragment jobs.
109 	 * JS1: vertex/tiler jobs
110 	 * JS2: compute jobs
111 	 */
112 	if (job->requirements & PANFROST_JD_REQ_FS)
113 		return 0;
114 
115 /* Not exposed to userspace yet */
116 #if 0
117 	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {
118 		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&
119 		    (job->pfdev->features.nr_core_groups == 2))
120 			return 2;
121 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))
122 			return 2;
123 	}
124 #endif
125 	return 1;
126 }
127 
128 static void panfrost_job_write_affinity(struct panfrost_device *pfdev,
129 					u32 requirements,
130 					int js)
131 {
132 	u64 affinity;
133 
134 	/*
135 	 * Use all cores for now.
136 	 * Eventually we may need to support tiler only jobs and h/w with
137 	 * multiple (2) coherent core groups
138 	 */
139 	affinity = pfdev->features.shader_present;
140 
141 	job_write(pfdev, JS_AFFINITY_NEXT_LO(js), lower_32_bits(affinity));
142 	job_write(pfdev, JS_AFFINITY_NEXT_HI(js), upper_32_bits(affinity));
143 }
144 
145 static u32
146 panfrost_get_job_chain_flag(const struct panfrost_job *job)
147 {
148 	struct panfrost_fence *f = to_panfrost_fence(job->done_fence);
149 
150 	if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
151 		return 0;
152 
153 	return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0;
154 }
155 
156 static struct panfrost_job *
157 panfrost_dequeue_job(struct panfrost_device *pfdev, int slot)
158 {
159 	struct panfrost_job *job = pfdev->jobs[slot][0];
160 
161 	WARN_ON(!job);
162 	pfdev->jobs[slot][0] = pfdev->jobs[slot][1];
163 	pfdev->jobs[slot][1] = NULL;
164 
165 	return job;
166 }
167 
168 static unsigned int
169 panfrost_enqueue_job(struct panfrost_device *pfdev, int slot,
170 		     struct panfrost_job *job)
171 {
172 	if (WARN_ON(!job))
173 		return 0;
174 
175 	if (!pfdev->jobs[slot][0]) {
176 		pfdev->jobs[slot][0] = job;
177 		return 0;
178 	}
179 
180 	WARN_ON(pfdev->jobs[slot][1]);
181 	pfdev->jobs[slot][1] = job;
182 	WARN_ON(panfrost_get_job_chain_flag(job) ==
183 		panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));
184 	return 1;
185 }
186 
187 static void panfrost_job_hw_submit(struct panfrost_job *job, int js)
188 {
189 	struct panfrost_device *pfdev = job->pfdev;
190 	unsigned int subslot;
191 	u32 cfg;
192 	u64 jc_head = job->jc;
193 	int ret;
194 
195 	panfrost_devfreq_record_busy(&pfdev->pfdevfreq);
196 
197 	ret = pm_runtime_get_sync(pfdev->dev);
198 	if (ret < 0)
199 		return;
200 
201 	if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) {
202 		return;
203 	}
204 
205 	cfg = panfrost_mmu_as_get(pfdev, job->mmu);
206 
207 	job_write(pfdev, JS_HEAD_NEXT_LO(js), lower_32_bits(jc_head));
208 	job_write(pfdev, JS_HEAD_NEXT_HI(js), upper_32_bits(jc_head));
209 
210 	panfrost_job_write_affinity(pfdev, job->requirements, js);
211 
212 	/* start MMU, medium priority, cache clean/flush on end, clean/flush on
213 	 * start */
214 	cfg |= JS_CONFIG_THREAD_PRI(8) |
215 		JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE |
216 		JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE |
217 		panfrost_get_job_chain_flag(job);
218 
219 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
220 		cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
221 
222 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))
223 		cfg |= JS_CONFIG_START_MMU;
224 
225 	job_write(pfdev, JS_CONFIG_NEXT(js), cfg);
226 
227 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
228 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
229 
230 	/* GO ! */
231 
232 	spin_lock(&pfdev->js->job_lock);
233 	subslot = panfrost_enqueue_job(pfdev, js, job);
234 	/* Don't queue the job if a reset is in progress */
235 	if (!atomic_read(&pfdev->reset.pending)) {
236 		job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);
237 		dev_dbg(pfdev->dev,
238 			"JS: Submitting atom %p to js[%d][%d] with head=0x%llx AS %d",
239 			job, js, subslot, jc_head, cfg & 0xf);
240 	}
241 	spin_unlock(&pfdev->js->job_lock);
242 }
243 
244 static int panfrost_acquire_object_fences(struct drm_gem_object **bos,
245 					  int bo_count,
246 					  struct drm_sched_job *job)
247 {
248 	int i, ret;
249 
250 	for (i = 0; i < bo_count; i++) {
251 		ret = dma_resv_reserve_fences(bos[i]->resv, 1);
252 		if (ret)
253 			return ret;
254 
255 		/* panfrost always uses write mode in its current uapi */
256 		ret = drm_sched_job_add_implicit_dependencies(job, bos[i],
257 							      true);
258 		if (ret)
259 			return ret;
260 	}
261 
262 	return 0;
263 }
264 
265 static void panfrost_attach_object_fences(struct drm_gem_object **bos,
266 					  int bo_count,
267 					  struct dma_fence *fence)
268 {
269 	int i;
270 
271 	for (i = 0; i < bo_count; i++)
272 		dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE);
273 }
274 
275 int panfrost_job_push(struct panfrost_job *job)
276 {
277 	struct panfrost_device *pfdev = job->pfdev;
278 	struct ww_acquire_ctx acquire_ctx;
279 	int ret = 0;
280 
281 	ret = drm_gem_lock_reservations(job->bos, job->bo_count,
282 					    &acquire_ctx);
283 	if (ret)
284 		return ret;
285 
286 	mutex_lock(&pfdev->sched_lock);
287 	drm_sched_job_arm(&job->base);
288 
289 	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);
290 
291 	ret = panfrost_acquire_object_fences(job->bos, job->bo_count,
292 					     &job->base);
293 	if (ret) {
294 		mutex_unlock(&pfdev->sched_lock);
295 		goto unlock;
296 	}
297 
298 	kref_get(&job->refcount); /* put by scheduler job completion */
299 
300 	drm_sched_entity_push_job(&job->base);
301 
302 	mutex_unlock(&pfdev->sched_lock);
303 
304 	panfrost_attach_object_fences(job->bos, job->bo_count,
305 				      job->render_done_fence);
306 
307 unlock:
308 	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);
309 
310 	return ret;
311 }
312 
313 static void panfrost_job_cleanup(struct kref *ref)
314 {
315 	struct panfrost_job *job = container_of(ref, struct panfrost_job,
316 						refcount);
317 	unsigned int i;
318 
319 	dma_fence_put(job->done_fence);
320 	dma_fence_put(job->render_done_fence);
321 
322 	if (job->mappings) {
323 		for (i = 0; i < job->bo_count; i++) {
324 			if (!job->mappings[i])
325 				break;
326 
327 			atomic_dec(&job->mappings[i]->obj->gpu_usecount);
328 			panfrost_gem_mapping_put(job->mappings[i]);
329 		}
330 		kvfree(job->mappings);
331 	}
332 
333 	if (job->bos) {
334 		for (i = 0; i < job->bo_count; i++)
335 			drm_gem_object_put(job->bos[i]);
336 
337 		kvfree(job->bos);
338 	}
339 
340 	kfree(job);
341 }
342 
343 void panfrost_job_put(struct panfrost_job *job)
344 {
345 	kref_put(&job->refcount, panfrost_job_cleanup);
346 }
347 
348 static void panfrost_job_free(struct drm_sched_job *sched_job)
349 {
350 	struct panfrost_job *job = to_panfrost_job(sched_job);
351 
352 	drm_sched_job_cleanup(sched_job);
353 
354 	panfrost_job_put(job);
355 }
356 
357 static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job)
358 {
359 	struct panfrost_job *job = to_panfrost_job(sched_job);
360 	struct panfrost_device *pfdev = job->pfdev;
361 	int slot = panfrost_job_get_slot(job);
362 	struct dma_fence *fence = NULL;
363 
364 	if (unlikely(job->base.s_fence->finished.error))
365 		return NULL;
366 
367 	/* Nothing to execute: can happen if the job has finished while
368 	 * we were resetting the GPU.
369 	 */
370 	if (!job->jc)
371 		return NULL;
372 
373 	fence = panfrost_fence_create(pfdev, slot);
374 	if (IS_ERR(fence))
375 		return fence;
376 
377 	if (job->done_fence)
378 		dma_fence_put(job->done_fence);
379 	job->done_fence = dma_fence_get(fence);
380 
381 	panfrost_job_hw_submit(job, slot);
382 
383 	return fence;
384 }
385 
386 void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)
387 {
388 	int j;
389 	u32 irq_mask = 0;
390 
391 	for (j = 0; j < NUM_JOB_SLOTS; j++) {
392 		irq_mask |= MK_JS_MASK(j);
393 	}
394 
395 	job_write(pfdev, JOB_INT_CLEAR, irq_mask);
396 	job_write(pfdev, JOB_INT_MASK, irq_mask);
397 }
398 
399 static void panfrost_job_handle_err(struct panfrost_device *pfdev,
400 				    struct panfrost_job *job,
401 				    unsigned int js)
402 {
403 	u32 js_status = job_read(pfdev, JS_STATUS(js));
404 	const char *exception_name = panfrost_exception_name(js_status);
405 	bool signal_fence = true;
406 
407 	if (!panfrost_exception_is_fault(js_status)) {
408 		dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",
409 			js, exception_name,
410 			job_read(pfdev, JS_HEAD_LO(js)),
411 			job_read(pfdev, JS_TAIL_LO(js)));
412 	} else {
413 		dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
414 			js, exception_name,
415 			job_read(pfdev, JS_HEAD_LO(js)),
416 			job_read(pfdev, JS_TAIL_LO(js)));
417 	}
418 
419 	if (js_status == DRM_PANFROST_EXCEPTION_STOPPED) {
420 		/* Update the job head so we can resume */
421 		job->jc = job_read(pfdev, JS_TAIL_LO(js)) |
422 			  ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32);
423 
424 		/* The job will be resumed, don't signal the fence */
425 		signal_fence = false;
426 	} else if (js_status == DRM_PANFROST_EXCEPTION_TERMINATED) {
427 		/* Job has been hard-stopped, flag it as canceled */
428 		dma_fence_set_error(job->done_fence, -ECANCELED);
429 		job->jc = 0;
430 	} else if (panfrost_exception_is_fault(js_status)) {
431 		/* We might want to provide finer-grained error code based on
432 		 * the exception type, but unconditionally setting to EINVAL
433 		 * is good enough for now.
434 		 */
435 		dma_fence_set_error(job->done_fence, -EINVAL);
436 		job->jc = 0;
437 	}
438 
439 	panfrost_mmu_as_put(pfdev, job->mmu);
440 	panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
441 
442 	if (signal_fence)
443 		dma_fence_signal_locked(job->done_fence);
444 
445 	pm_runtime_put_autosuspend(pfdev->dev);
446 
447 	if (panfrost_exception_needs_reset(pfdev, js_status)) {
448 		atomic_set(&pfdev->reset.pending, 1);
449 		drm_sched_fault(&pfdev->js->queue[js].sched);
450 	}
451 }
452 
453 static void panfrost_job_handle_done(struct panfrost_device *pfdev,
454 				     struct panfrost_job *job)
455 {
456 	/* Set ->jc to 0 to avoid re-submitting an already finished job (can
457 	 * happen when we receive the DONE interrupt while doing a GPU reset).
458 	 */
459 	job->jc = 0;
460 	panfrost_mmu_as_put(pfdev, job->mmu);
461 	panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
462 
463 	dma_fence_signal_locked(job->done_fence);
464 	pm_runtime_put_autosuspend(pfdev->dev);
465 }
466 
467 static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status)
468 {
469 	struct panfrost_job *done[NUM_JOB_SLOTS][2] = {};
470 	struct panfrost_job *failed[NUM_JOB_SLOTS] = {};
471 	u32 js_state = 0, js_events = 0;
472 	unsigned int i, j;
473 
474 	/* First we collect all failed/done jobs. */
475 	while (status) {
476 		u32 js_state_mask = 0;
477 
478 		for (j = 0; j < NUM_JOB_SLOTS; j++) {
479 			if (status & MK_JS_MASK(j))
480 				js_state_mask |= MK_JS_MASK(j);
481 
482 			if (status & JOB_INT_MASK_DONE(j)) {
483 				if (done[j][0])
484 					done[j][1] = panfrost_dequeue_job(pfdev, j);
485 				else
486 					done[j][0] = panfrost_dequeue_job(pfdev, j);
487 			}
488 
489 			if (status & JOB_INT_MASK_ERR(j)) {
490 				/* Cancel the next submission. Will be submitted
491 				 * after we're done handling this failure if
492 				 * there's no reset pending.
493 				 */
494 				job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);
495 				failed[j] = panfrost_dequeue_job(pfdev, j);
496 			}
497 		}
498 
499 		/* JS_STATE is sampled when JOB_INT_CLEAR is written.
500 		 * For each BIT(slot) or BIT(slot + 16) bit written to
501 		 * JOB_INT_CLEAR, the corresponding bits in JS_STATE
502 		 * (BIT(slot) and BIT(slot + 16)) are updated, but this
503 		 * is racy. If we only have one job done at the time we
504 		 * read JOB_INT_RAWSTAT but the second job fails before we
505 		 * clear the status, we end up with a status containing
506 		 * only the DONE bit and consider both jobs as DONE since
507 		 * JS_STATE reports both NEXT and CURRENT as inactive.
508 		 * To prevent that, let's repeat this clear+read steps
509 		 * until status is 0.
510 		 */
511 		job_write(pfdev, JOB_INT_CLEAR, status);
512 		js_state &= ~js_state_mask;
513 		js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask;
514 		js_events |= status;
515 		status = job_read(pfdev, JOB_INT_RAWSTAT);
516 	}
517 
518 	/* Then we handle the dequeued jobs. */
519 	for (j = 0; j < NUM_JOB_SLOTS; j++) {
520 		if (!(js_events & MK_JS_MASK(j)))
521 			continue;
522 
523 		if (failed[j]) {
524 			panfrost_job_handle_err(pfdev, failed[j], j);
525 		} else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {
526 			/* When the current job doesn't fail, the JM dequeues
527 			 * the next job without waiting for an ACK, this means
528 			 * we can have 2 jobs dequeued and only catch the
529 			 * interrupt when the second one is done. If both slots
530 			 * are inactive, but one job remains in pfdev->jobs[j],
531 			 * consider it done. Of course that doesn't apply if a
532 			 * failure happened since we cancelled execution of the
533 			 * job in _NEXT (see above).
534 			 */
535 			if (WARN_ON(!done[j][0]))
536 				done[j][0] = panfrost_dequeue_job(pfdev, j);
537 			else
538 				done[j][1] = panfrost_dequeue_job(pfdev, j);
539 		}
540 
541 		for (i = 0; i < ARRAY_SIZE(done[0]) && done[j][i]; i++)
542 			panfrost_job_handle_done(pfdev, done[j][i]);
543 	}
544 
545 	/* And finally we requeue jobs that were waiting in the second slot
546 	 * and have been stopped if we detected a failure on the first slot.
547 	 */
548 	for (j = 0; j < NUM_JOB_SLOTS; j++) {
549 		if (!(js_events & MK_JS_MASK(j)))
550 			continue;
551 
552 		if (!failed[j] || !pfdev->jobs[j][0])
553 			continue;
554 
555 		if (pfdev->jobs[j][0]->jc == 0) {
556 			/* The job was cancelled, signal the fence now */
557 			struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j);
558 
559 			dma_fence_set_error(canceled->done_fence, -ECANCELED);
560 			panfrost_job_handle_done(pfdev, canceled);
561 		} else if (!atomic_read(&pfdev->reset.pending)) {
562 			/* Requeue the job we removed if no reset is pending */
563 			job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START);
564 		}
565 	}
566 }
567 
568 static void panfrost_job_handle_irqs(struct panfrost_device *pfdev)
569 {
570 	u32 status = job_read(pfdev, JOB_INT_RAWSTAT);
571 
572 	while (status) {
573 		pm_runtime_mark_last_busy(pfdev->dev);
574 
575 		spin_lock(&pfdev->js->job_lock);
576 		panfrost_job_handle_irq(pfdev, status);
577 		spin_unlock(&pfdev->js->job_lock);
578 		status = job_read(pfdev, JOB_INT_RAWSTAT);
579 	}
580 }
581 
582 static u32 panfrost_active_slots(struct panfrost_device *pfdev,
583 				 u32 *js_state_mask, u32 js_state)
584 {
585 	u32 rawstat;
586 
587 	if (!(js_state & *js_state_mask))
588 		return 0;
589 
590 	rawstat = job_read(pfdev, JOB_INT_RAWSTAT);
591 	if (rawstat) {
592 		unsigned int i;
593 
594 		for (i = 0; i < NUM_JOB_SLOTS; i++) {
595 			if (rawstat & MK_JS_MASK(i))
596 				*js_state_mask &= ~MK_JS_MASK(i);
597 		}
598 	}
599 
600 	return js_state & *js_state_mask;
601 }
602 
603 static void
604 panfrost_reset(struct panfrost_device *pfdev,
605 	       struct drm_sched_job *bad)
606 {
607 	u32 js_state, js_state_mask = 0xffffffff;
608 	unsigned int i, j;
609 	bool cookie;
610 	int ret;
611 
612 	if (!atomic_read(&pfdev->reset.pending))
613 		return;
614 
615 	/* Stop the schedulers.
616 	 *
617 	 * FIXME: We temporarily get out of the dma_fence_signalling section
618 	 * because the cleanup path generate lockdep splats when taking locks
619 	 * to release job resources. We should rework the code to follow this
620 	 * pattern:
621 	 *
622 	 *	try_lock
623 	 *	if (locked)
624 	 *		release
625 	 *	else
626 	 *		schedule_work_to_release_later
627 	 */
628 	for (i = 0; i < NUM_JOB_SLOTS; i++)
629 		drm_sched_stop(&pfdev->js->queue[i].sched, bad);
630 
631 	cookie = dma_fence_begin_signalling();
632 
633 	if (bad)
634 		drm_sched_increase_karma(bad);
635 
636 	/* Mask job interrupts and synchronize to make sure we won't be
637 	 * interrupted during our reset.
638 	 */
639 	job_write(pfdev, JOB_INT_MASK, 0);
640 	synchronize_irq(pfdev->js->irq);
641 
642 	for (i = 0; i < NUM_JOB_SLOTS; i++) {
643 		/* Cancel the next job and soft-stop the running job. */
644 		job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
645 		job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP);
646 	}
647 
648 	/* Wait at most 10ms for soft-stops to complete */
649 	ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,
650 				 !panfrost_active_slots(pfdev, &js_state_mask, js_state),
651 				 10, 10000);
652 
653 	if (ret)
654 		dev_err(pfdev->dev, "Soft-stop failed\n");
655 
656 	/* Handle the remaining interrupts before we reset. */
657 	panfrost_job_handle_irqs(pfdev);
658 
659 	/* Remaining interrupts have been handled, but we might still have
660 	 * stuck jobs. Let's make sure the PM counters stay balanced by
661 	 * manually calling pm_runtime_put_noidle() and
662 	 * panfrost_devfreq_record_idle() for each stuck job.
663 	 */
664 	spin_lock(&pfdev->js->job_lock);
665 	for (i = 0; i < NUM_JOB_SLOTS; i++) {
666 		for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) {
667 			pm_runtime_put_noidle(pfdev->dev);
668 			panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
669 		}
670 	}
671 	memset(pfdev->jobs, 0, sizeof(pfdev->jobs));
672 	spin_unlock(&pfdev->js->job_lock);
673 
674 	/* Proceed with reset now. */
675 	panfrost_device_reset(pfdev);
676 
677 	/* panfrost_device_reset() unmasks job interrupts, but we want to
678 	 * keep them masked a bit longer.
679 	 */
680 	job_write(pfdev, JOB_INT_MASK, 0);
681 
682 	/* GPU has been reset, we can clear the reset pending bit. */
683 	atomic_set(&pfdev->reset.pending, 0);
684 
685 	/* Now resubmit jobs that were previously queued but didn't have a
686 	 * chance to finish.
687 	 * FIXME: We temporarily get out of the DMA fence signalling section
688 	 * while resubmitting jobs because the job submission logic will
689 	 * allocate memory with the GFP_KERNEL flag which can trigger memory
690 	 * reclaim and exposes a lock ordering issue.
691 	 */
692 	dma_fence_end_signalling(cookie);
693 	for (i = 0; i < NUM_JOB_SLOTS; i++)
694 		drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
695 	cookie = dma_fence_begin_signalling();
696 
697 	/* Restart the schedulers */
698 	for (i = 0; i < NUM_JOB_SLOTS; i++)
699 		drm_sched_start(&pfdev->js->queue[i].sched, true);
700 
701 	/* Re-enable job interrupts now that everything has been restarted. */
702 	job_write(pfdev, JOB_INT_MASK,
703 		  GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
704 		  GENMASK(NUM_JOB_SLOTS - 1, 0));
705 
706 	dma_fence_end_signalling(cookie);
707 }
708 
709 static enum drm_gpu_sched_stat panfrost_job_timedout(struct drm_sched_job
710 						     *sched_job)
711 {
712 	struct panfrost_job *job = to_panfrost_job(sched_job);
713 	struct panfrost_device *pfdev = job->pfdev;
714 	int js = panfrost_job_get_slot(job);
715 
716 	/*
717 	 * If the GPU managed to complete this jobs fence, the timeout is
718 	 * spurious. Bail out.
719 	 */
720 	if (dma_fence_is_signaled(job->done_fence))
721 		return DRM_GPU_SCHED_STAT_NOMINAL;
722 
723 	dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",
724 		js,
725 		job_read(pfdev, JS_CONFIG(js)),
726 		job_read(pfdev, JS_STATUS(js)),
727 		job_read(pfdev, JS_HEAD_LO(js)),
728 		job_read(pfdev, JS_TAIL_LO(js)),
729 		sched_job);
730 
731 	panfrost_core_dump(job);
732 
733 	atomic_set(&pfdev->reset.pending, 1);
734 	panfrost_reset(pfdev, sched_job);
735 
736 	return DRM_GPU_SCHED_STAT_NOMINAL;
737 }
738 
739 static void panfrost_reset_work(struct work_struct *work)
740 {
741 	struct panfrost_device *pfdev;
742 
743 	pfdev = container_of(work, struct panfrost_device, reset.work);
744 	panfrost_reset(pfdev, NULL);
745 }
746 
747 static const struct drm_sched_backend_ops panfrost_sched_ops = {
748 	.run_job = panfrost_job_run,
749 	.timedout_job = panfrost_job_timedout,
750 	.free_job = panfrost_job_free
751 };
752 
753 static irqreturn_t panfrost_job_irq_handler_thread(int irq, void *data)
754 {
755 	struct panfrost_device *pfdev = data;
756 
757 	panfrost_job_handle_irqs(pfdev);
758 	job_write(pfdev, JOB_INT_MASK,
759 		  GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
760 		  GENMASK(NUM_JOB_SLOTS - 1, 0));
761 	return IRQ_HANDLED;
762 }
763 
764 static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
765 {
766 	struct panfrost_device *pfdev = data;
767 	u32 status = job_read(pfdev, JOB_INT_STAT);
768 
769 	if (!status)
770 		return IRQ_NONE;
771 
772 	job_write(pfdev, JOB_INT_MASK, 0);
773 	return IRQ_WAKE_THREAD;
774 }
775 
776 int panfrost_job_init(struct panfrost_device *pfdev)
777 {
778 	struct panfrost_job_slot *js;
779 	unsigned int nentries = 2;
780 	int ret, j;
781 
782 	/* All GPUs have two entries per queue, but without jobchain
783 	 * disambiguation stopping the right job in the close path is tricky,
784 	 * so let's just advertise one entry in that case.
785 	 */
786 	if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
787 		nentries = 1;
788 
789 	pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
790 	if (!js)
791 		return -ENOMEM;
792 
793 	INIT_WORK(&pfdev->reset.work, panfrost_reset_work);
794 	spin_lock_init(&js->job_lock);
795 
796 	js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
797 	if (js->irq <= 0)
798 		return -ENODEV;
799 
800 	ret = devm_request_threaded_irq(pfdev->dev, js->irq,
801 					panfrost_job_irq_handler,
802 					panfrost_job_irq_handler_thread,
803 					IRQF_SHARED, KBUILD_MODNAME "-job",
804 					pfdev);
805 	if (ret) {
806 		dev_err(pfdev->dev, "failed to request job irq");
807 		return ret;
808 	}
809 
810 	pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0);
811 	if (!pfdev->reset.wq)
812 		return -ENOMEM;
813 
814 	for (j = 0; j < NUM_JOB_SLOTS; j++) {
815 		js->queue[j].fence_context = dma_fence_context_alloc(1);
816 
817 		ret = drm_sched_init(&js->queue[j].sched,
818 				     &panfrost_sched_ops,
819 				     nentries, 0,
820 				     msecs_to_jiffies(JOB_TIMEOUT_MS),
821 				     pfdev->reset.wq,
822 				     NULL, "pan_js", pfdev->dev);
823 		if (ret) {
824 			dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
825 			goto err_sched;
826 		}
827 	}
828 
829 	panfrost_job_enable_interrupts(pfdev);
830 
831 	return 0;
832 
833 err_sched:
834 	for (j--; j >= 0; j--)
835 		drm_sched_fini(&js->queue[j].sched);
836 
837 	destroy_workqueue(pfdev->reset.wq);
838 	return ret;
839 }
840 
841 void panfrost_job_fini(struct panfrost_device *pfdev)
842 {
843 	struct panfrost_job_slot *js = pfdev->js;
844 	int j;
845 
846 	job_write(pfdev, JOB_INT_MASK, 0);
847 
848 	for (j = 0; j < NUM_JOB_SLOTS; j++) {
849 		drm_sched_fini(&js->queue[j].sched);
850 	}
851 
852 	cancel_work_sync(&pfdev->reset.work);
853 	destroy_workqueue(pfdev->reset.wq);
854 }
855 
856 int panfrost_job_open(struct panfrost_file_priv *panfrost_priv)
857 {
858 	struct panfrost_device *pfdev = panfrost_priv->pfdev;
859 	struct panfrost_job_slot *js = pfdev->js;
860 	struct drm_gpu_scheduler *sched;
861 	int ret, i;
862 
863 	for (i = 0; i < NUM_JOB_SLOTS; i++) {
864 		sched = &js->queue[i].sched;
865 		ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i],
866 					    DRM_SCHED_PRIORITY_NORMAL, &sched,
867 					    1, NULL);
868 		if (WARN_ON(ret))
869 			return ret;
870 	}
871 	return 0;
872 }
873 
874 void panfrost_job_close(struct panfrost_file_priv *panfrost_priv)
875 {
876 	struct panfrost_device *pfdev = panfrost_priv->pfdev;
877 	int i;
878 
879 	for (i = 0; i < NUM_JOB_SLOTS; i++)
880 		drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);
881 
882 	/* Kill in-flight jobs */
883 	spin_lock(&pfdev->js->job_lock);
884 	for (i = 0; i < NUM_JOB_SLOTS; i++) {
885 		struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i];
886 		int j;
887 
888 		for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {
889 			struct panfrost_job *job = pfdev->jobs[i][j];
890 			u32 cmd;
891 
892 			if (!job || job->base.entity != entity)
893 				continue;
894 
895 			if (j == 1) {
896 				/* Try to cancel the job before it starts */
897 				job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
898 				/* Reset the job head so it doesn't get restarted if
899 				 * the job in the first slot failed.
900 				 */
901 				job->jc = 0;
902 			}
903 
904 			if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
905 				cmd = panfrost_get_job_chain_flag(job) ?
906 				      JS_COMMAND_HARD_STOP_1 :
907 				      JS_COMMAND_HARD_STOP_0;
908 			} else {
909 				cmd = JS_COMMAND_HARD_STOP;
910 			}
911 
912 			job_write(pfdev, JS_COMMAND(i), cmd);
913 		}
914 	}
915 	spin_unlock(&pfdev->js->job_lock);
916 }
917 
918 int panfrost_job_is_idle(struct panfrost_device *pfdev)
919 {
920 	struct panfrost_job_slot *js = pfdev->js;
921 	int i;
922 
923 	for (i = 0; i < NUM_JOB_SLOTS; i++) {
924 		/* If there are any jobs in the HW queue, we're not idle */
925 		if (atomic_read(&js->queue[i].sched.hw_rq_count))
926 			return false;
927 	}
928 
929 	return true;
930 }
931