xref: /openbmc/linux/drivers/gpu/host1x/intr.c (revision ee7da21a) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Tegra host1x Interrupt Management
4  *
5  * Copyright (c) 2010-2013, NVIDIA Corporation.
6  */
7 
8 #include <linux/clk.h>
9 #include <linux/interrupt.h>
10 #include <linux/slab.h>
11 #include <linux/irq.h>
12 
13 #include <trace/events/host1x.h>
14 #include "channel.h"
15 #include "dev.h"
16 #include "intr.h"
17 
18 /* Wait list management */
19 
20 enum waitlist_state {
21 	WLS_PENDING,
22 	WLS_REMOVED,
23 	WLS_CANCELLED,
24 	WLS_HANDLED
25 };
26 
27 static void waiter_release(struct kref *kref)
28 {
29 	kfree(container_of(kref, struct host1x_waitlist, refcount));
30 }
31 
32 /*
33  * add a waiter to a waiter queue, sorted by threshold
34  * returns true if it was added at the head of the queue
35  */
36 static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
37 				struct list_head *queue)
38 {
39 	struct host1x_waitlist *pos;
40 	u32 thresh = waiter->thresh;
41 
42 	list_for_each_entry_reverse(pos, queue, list)
43 		if ((s32)(pos->thresh - thresh) <= 0) {
44 			list_add(&waiter->list, &pos->list);
45 			return false;
46 		}
47 
48 	list_add(&waiter->list, queue);
49 	return true;
50 }
51 
52 /*
53  * run through a waiter queue for a single sync point ID
54  * and gather all completed waiters into lists by actions
55  */
56 static void remove_completed_waiters(struct list_head *head, u32 sync,
57 			struct list_head completed[HOST1X_INTR_ACTION_COUNT])
58 {
59 	struct list_head *dest;
60 	struct host1x_waitlist *waiter, *next, *prev;
61 
62 	list_for_each_entry_safe(waiter, next, head, list) {
63 		if ((s32)(waiter->thresh - sync) > 0)
64 			break;
65 
66 		dest = completed + waiter->action;
67 
68 		/* consolidate submit cleanups */
69 		if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
70 		    !list_empty(dest)) {
71 			prev = list_entry(dest->prev,
72 					  struct host1x_waitlist, list);
73 			if (prev->data == waiter->data) {
74 				prev->count++;
75 				dest = NULL;
76 			}
77 		}
78 
79 		/* PENDING->REMOVED or CANCELLED->HANDLED */
80 		if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
81 			list_del(&waiter->list);
82 			kref_put(&waiter->refcount, waiter_release);
83 		} else
84 			list_move_tail(&waiter->list, dest);
85 	}
86 }
87 
88 static void reset_threshold_interrupt(struct host1x *host,
89 				      struct list_head *head,
90 				      unsigned int id)
91 {
92 	u32 thresh =
93 		list_first_entry(head, struct host1x_waitlist, list)->thresh;
94 
95 	host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
96 	host1x_hw_intr_enable_syncpt_intr(host, id);
97 }
98 
99 static void action_submit_complete(struct host1x_waitlist *waiter)
100 {
101 	struct host1x_channel *channel = waiter->data;
102 
103 	host1x_cdma_update(&channel->cdma);
104 
105 	/*  Add nr_completed to trace */
106 	trace_host1x_channel_submit_complete(dev_name(channel->dev),
107 					     waiter->count, waiter->thresh);
108 }
109 
110 static void action_wakeup(struct host1x_waitlist *waiter)
111 {
112 	wait_queue_head_t *wq = waiter->data;
113 
114 	wake_up(wq);
115 }
116 
117 static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
118 {
119 	wait_queue_head_t *wq = waiter->data;
120 
121 	wake_up_interruptible(wq);
122 }
123 
124 typedef void (*action_handler)(struct host1x_waitlist *waiter);
125 
126 static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
127 	action_submit_complete,
128 	action_wakeup,
129 	action_wakeup_interruptible,
130 };
131 
132 static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
133 {
134 	struct list_head *head = completed;
135 	unsigned int i;
136 
137 	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
138 		action_handler handler = action_handlers[i];
139 		struct host1x_waitlist *waiter, *next;
140 
141 		list_for_each_entry_safe(waiter, next, head, list) {
142 			list_del(&waiter->list);
143 			handler(waiter);
144 			WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
145 				WLS_REMOVED);
146 			kref_put(&waiter->refcount, waiter_release);
147 		}
148 	}
149 }
150 
151 /*
152  * Remove & handle all waiters that have completed for the given syncpt
153  */
154 static int process_wait_list(struct host1x *host,
155 			     struct host1x_syncpt *syncpt,
156 			     u32 threshold)
157 {
158 	struct list_head completed[HOST1X_INTR_ACTION_COUNT];
159 	unsigned int i;
160 	int empty;
161 
162 	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
163 		INIT_LIST_HEAD(completed + i);
164 
165 	spin_lock(&syncpt->intr.lock);
166 
167 	remove_completed_waiters(&syncpt->intr.wait_head, threshold,
168 				 completed);
169 
170 	empty = list_empty(&syncpt->intr.wait_head);
171 	if (empty)
172 		host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
173 	else
174 		reset_threshold_interrupt(host, &syncpt->intr.wait_head,
175 					  syncpt->id);
176 
177 	spin_unlock(&syncpt->intr.lock);
178 
179 	run_handlers(completed);
180 
181 	return empty;
182 }
183 
184 /*
185  * Sync point threshold interrupt service thread function
186  * Handles sync point threshold triggers, in thread context
187  */
188 
189 static void syncpt_thresh_work(struct work_struct *work)
190 {
191 	struct host1x_syncpt_intr *syncpt_intr =
192 		container_of(work, struct host1x_syncpt_intr, work);
193 	struct host1x_syncpt *syncpt =
194 		container_of(syncpt_intr, struct host1x_syncpt, intr);
195 	unsigned int id = syncpt->id;
196 	struct host1x *host = syncpt->host;
197 
198 	(void)process_wait_list(host, syncpt,
199 				host1x_syncpt_load(host->syncpt + id));
200 }
201 
202 int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
203 			   u32 thresh, enum host1x_intr_action action,
204 			   void *data, struct host1x_waitlist *waiter,
205 			   void **ref)
206 {
207 	int queue_was_empty;
208 
209 	if (waiter == NULL) {
210 		pr_warn("%s: NULL waiter\n", __func__);
211 		return -EINVAL;
212 	}
213 
214 	/* initialize a new waiter */
215 	INIT_LIST_HEAD(&waiter->list);
216 	kref_init(&waiter->refcount);
217 	if (ref)
218 		kref_get(&waiter->refcount);
219 	waiter->thresh = thresh;
220 	waiter->action = action;
221 	atomic_set(&waiter->state, WLS_PENDING);
222 	waiter->data = data;
223 	waiter->count = 1;
224 
225 	spin_lock(&syncpt->intr.lock);
226 
227 	queue_was_empty = list_empty(&syncpt->intr.wait_head);
228 
229 	if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
230 		/* added at head of list - new threshold value */
231 		host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);
232 
233 		/* added as first waiter - enable interrupt */
234 		if (queue_was_empty)
235 			host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
236 	}
237 
238 	if (ref)
239 		*ref = waiter;
240 
241 	spin_unlock(&syncpt->intr.lock);
242 
243 	return 0;
244 }
245 
246 void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref,
247 			 bool flush)
248 {
249 	struct host1x_waitlist *waiter = ref;
250 	struct host1x_syncpt *syncpt;
251 
252 	atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED);
253 
254 	syncpt = host->syncpt + id;
255 
256 	spin_lock(&syncpt->intr.lock);
257 	if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) ==
258 	    WLS_CANCELLED) {
259 		list_del(&waiter->list);
260 		kref_put(&waiter->refcount, waiter_release);
261 	}
262 	spin_unlock(&syncpt->intr.lock);
263 
264 	if (flush) {
265 		/* Wait until any concurrently executing handler has finished. */
266 		while (atomic_read(&waiter->state) != WLS_HANDLED)
267 			schedule();
268 	}
269 
270 	kref_put(&waiter->refcount, waiter_release);
271 }
272 
273 int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
274 {
275 	unsigned int id;
276 	u32 nb_pts = host1x_syncpt_nb_pts(host);
277 
278 	mutex_init(&host->intr_mutex);
279 	host->intr_syncpt_irq = irq_sync;
280 
281 	for (id = 0; id < nb_pts; ++id) {
282 		struct host1x_syncpt *syncpt = host->syncpt + id;
283 
284 		spin_lock_init(&syncpt->intr.lock);
285 		INIT_LIST_HEAD(&syncpt->intr.wait_head);
286 		snprintf(syncpt->intr.thresh_irq_name,
287 			 sizeof(syncpt->intr.thresh_irq_name),
288 			 "host1x_sp_%02u", id);
289 	}
290 
291 	host1x_intr_start(host);
292 
293 	return 0;
294 }
295 
296 void host1x_intr_deinit(struct host1x *host)
297 {
298 	host1x_intr_stop(host);
299 }
300 
301 void host1x_intr_start(struct host1x *host)
302 {
303 	u32 hz = clk_get_rate(host->clk);
304 	int err;
305 
306 	mutex_lock(&host->intr_mutex);
307 	err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
308 					    syncpt_thresh_work);
309 	if (err) {
310 		mutex_unlock(&host->intr_mutex);
311 		return;
312 	}
313 	mutex_unlock(&host->intr_mutex);
314 }
315 
316 void host1x_intr_stop(struct host1x *host)
317 {
318 	unsigned int id;
319 	struct host1x_syncpt *syncpt = host->syncpt;
320 	u32 nb_pts = host1x_syncpt_nb_pts(host);
321 
322 	mutex_lock(&host->intr_mutex);
323 
324 	host1x_hw_intr_disable_all_syncpt_intrs(host);
325 
326 	for (id = 0; id < nb_pts; ++id) {
327 		struct host1x_waitlist *waiter, *next;
328 
329 		list_for_each_entry_safe(waiter, next,
330 			&syncpt[id].intr.wait_head, list) {
331 			if (atomic_cmpxchg(&waiter->state,
332 			    WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
333 				list_del(&waiter->list);
334 				kref_put(&waiter->refcount, waiter_release);
335 			}
336 		}
337 
338 		if (!list_empty(&syncpt[id].intr.wait_head)) {
339 			/* output diagnostics */
340 			mutex_unlock(&host->intr_mutex);
341 			pr_warn("%s cannot stop syncpt intr id=%u\n",
342 				__func__, id);
343 			return;
344 		}
345 	}
346 
347 	host1x_hw_intr_free_syncpt_irq(host);
348 
349 	mutex_unlock(&host->intr_mutex);
350 }
351