1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Interrupt Management 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/interrupt.h> 10 #include <linux/slab.h> 11 #include <linux/irq.h> 12 13 #include <trace/events/host1x.h> 14 #include "channel.h" 15 #include "dev.h" 16 #include "fence.h" 17 #include "intr.h" 18 19 /* Wait list management */ 20 21 enum waitlist_state { 22 WLS_PENDING, 23 WLS_REMOVED, 24 WLS_CANCELLED, 25 WLS_HANDLED 26 }; 27 28 static void waiter_release(struct kref *kref) 29 { 30 kfree(container_of(kref, struct host1x_waitlist, refcount)); 31 } 32 33 /* 34 * add a waiter to a waiter queue, sorted by threshold 35 * returns true if it was added at the head of the queue 36 */ 37 static bool add_waiter_to_queue(struct host1x_waitlist *waiter, 38 struct list_head *queue) 39 { 40 struct host1x_waitlist *pos; 41 u32 thresh = waiter->thresh; 42 43 list_for_each_entry_reverse(pos, queue, list) 44 if ((s32)(pos->thresh - thresh) <= 0) { 45 list_add(&waiter->list, &pos->list); 46 return false; 47 } 48 49 list_add(&waiter->list, queue); 50 return true; 51 } 52 53 /* 54 * run through a waiter queue for a single sync point ID 55 * and gather all completed waiters into lists by actions 56 */ 57 static void remove_completed_waiters(struct list_head *head, u32 sync, 58 struct list_head completed[HOST1X_INTR_ACTION_COUNT]) 59 { 60 struct list_head *dest; 61 struct host1x_waitlist *waiter, *next, *prev; 62 63 list_for_each_entry_safe(waiter, next, head, list) { 64 if ((s32)(waiter->thresh - sync) > 0) 65 break; 66 67 dest = completed + waiter->action; 68 69 /* consolidate submit cleanups */ 70 if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE && 71 !list_empty(dest)) { 72 prev = list_entry(dest->prev, 73 struct host1x_waitlist, list); 74 if (prev->data == waiter->data) { 75 prev->count++; 76 dest = NULL; 77 } 78 } 79 80 /* PENDING->REMOVED or CANCELLED->HANDLED */ 81 if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) { 82 list_del(&waiter->list); 83 kref_put(&waiter->refcount, waiter_release); 84 } else 85 list_move_tail(&waiter->list, dest); 86 } 87 } 88 89 static void reset_threshold_interrupt(struct host1x *host, 90 struct list_head *head, 91 unsigned int id) 92 { 93 u32 thresh = 94 list_first_entry(head, struct host1x_waitlist, list)->thresh; 95 96 host1x_hw_intr_set_syncpt_threshold(host, id, thresh); 97 host1x_hw_intr_enable_syncpt_intr(host, id); 98 } 99 100 static void action_submit_complete(struct host1x_waitlist *waiter) 101 { 102 struct host1x_channel *channel = waiter->data; 103 104 host1x_cdma_update(&channel->cdma); 105 106 /* Add nr_completed to trace */ 107 trace_host1x_channel_submit_complete(dev_name(channel->dev), 108 waiter->count, waiter->thresh); 109 } 110 111 static void action_wakeup(struct host1x_waitlist *waiter) 112 { 113 wait_queue_head_t *wq = waiter->data; 114 115 wake_up(wq); 116 } 117 118 static void action_wakeup_interruptible(struct host1x_waitlist *waiter) 119 { 120 wait_queue_head_t *wq = waiter->data; 121 122 wake_up_interruptible(wq); 123 } 124 125 static void action_signal_fence(struct host1x_waitlist *waiter) 126 { 127 struct host1x_syncpt_fence *f = waiter->data; 128 129 host1x_fence_signal(f); 130 } 131 132 typedef void (*action_handler)(struct host1x_waitlist *waiter); 133 134 static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = { 135 action_submit_complete, 136 action_wakeup, 137 action_wakeup_interruptible, 138 action_signal_fence, 139 }; 140 141 static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT]) 142 { 143 struct list_head *head = completed; 144 unsigned int i; 145 146 for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) { 147 action_handler handler = action_handlers[i]; 148 struct host1x_waitlist *waiter, *next; 149 150 list_for_each_entry_safe(waiter, next, head, list) { 151 list_del(&waiter->list); 152 handler(waiter); 153 WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) != 154 WLS_REMOVED); 155 kref_put(&waiter->refcount, waiter_release); 156 } 157 } 158 } 159 160 /* 161 * Remove & handle all waiters that have completed for the given syncpt 162 */ 163 static int process_wait_list(struct host1x *host, 164 struct host1x_syncpt *syncpt, 165 u32 threshold) 166 { 167 struct list_head completed[HOST1X_INTR_ACTION_COUNT]; 168 unsigned int i; 169 int empty; 170 171 for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i) 172 INIT_LIST_HEAD(completed + i); 173 174 spin_lock(&syncpt->intr.lock); 175 176 remove_completed_waiters(&syncpt->intr.wait_head, threshold, 177 completed); 178 179 empty = list_empty(&syncpt->intr.wait_head); 180 if (empty) 181 host1x_hw_intr_disable_syncpt_intr(host, syncpt->id); 182 else 183 reset_threshold_interrupt(host, &syncpt->intr.wait_head, 184 syncpt->id); 185 186 spin_unlock(&syncpt->intr.lock); 187 188 run_handlers(completed); 189 190 return empty; 191 } 192 193 /* 194 * Sync point threshold interrupt service thread function 195 * Handles sync point threshold triggers, in thread context 196 */ 197 198 static void syncpt_thresh_work(struct work_struct *work) 199 { 200 struct host1x_syncpt_intr *syncpt_intr = 201 container_of(work, struct host1x_syncpt_intr, work); 202 struct host1x_syncpt *syncpt = 203 container_of(syncpt_intr, struct host1x_syncpt, intr); 204 unsigned int id = syncpt->id; 205 struct host1x *host = syncpt->host; 206 207 (void)process_wait_list(host, syncpt, 208 host1x_syncpt_load(host->syncpt + id)); 209 } 210 211 int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt, 212 u32 thresh, enum host1x_intr_action action, 213 void *data, struct host1x_waitlist *waiter, 214 void **ref) 215 { 216 int queue_was_empty; 217 218 if (waiter == NULL) { 219 pr_warn("%s: NULL waiter\n", __func__); 220 return -EINVAL; 221 } 222 223 /* initialize a new waiter */ 224 INIT_LIST_HEAD(&waiter->list); 225 kref_init(&waiter->refcount); 226 if (ref) 227 kref_get(&waiter->refcount); 228 waiter->thresh = thresh; 229 waiter->action = action; 230 atomic_set(&waiter->state, WLS_PENDING); 231 waiter->data = data; 232 waiter->count = 1; 233 234 spin_lock(&syncpt->intr.lock); 235 236 queue_was_empty = list_empty(&syncpt->intr.wait_head); 237 238 if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) { 239 /* added at head of list - new threshold value */ 240 host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh); 241 242 /* added as first waiter - enable interrupt */ 243 if (queue_was_empty) 244 host1x_hw_intr_enable_syncpt_intr(host, syncpt->id); 245 } 246 247 if (ref) 248 *ref = waiter; 249 250 spin_unlock(&syncpt->intr.lock); 251 252 return 0; 253 } 254 255 void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref, 256 bool flush) 257 { 258 struct host1x_waitlist *waiter = ref; 259 struct host1x_syncpt *syncpt; 260 261 atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED); 262 263 syncpt = host->syncpt + id; 264 265 spin_lock(&syncpt->intr.lock); 266 if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) == 267 WLS_CANCELLED) { 268 list_del(&waiter->list); 269 kref_put(&waiter->refcount, waiter_release); 270 } 271 spin_unlock(&syncpt->intr.lock); 272 273 if (flush) { 274 /* Wait until any concurrently executing handler has finished. */ 275 while (atomic_read(&waiter->state) != WLS_HANDLED) 276 schedule(); 277 } 278 279 kref_put(&waiter->refcount, waiter_release); 280 } 281 282 int host1x_intr_init(struct host1x *host, unsigned int irq_sync) 283 { 284 unsigned int id; 285 u32 nb_pts = host1x_syncpt_nb_pts(host); 286 287 mutex_init(&host->intr_mutex); 288 host->intr_syncpt_irq = irq_sync; 289 290 for (id = 0; id < nb_pts; ++id) { 291 struct host1x_syncpt *syncpt = host->syncpt + id; 292 293 spin_lock_init(&syncpt->intr.lock); 294 INIT_LIST_HEAD(&syncpt->intr.wait_head); 295 snprintf(syncpt->intr.thresh_irq_name, 296 sizeof(syncpt->intr.thresh_irq_name), 297 "host1x_sp_%02u", id); 298 } 299 300 host1x_intr_start(host); 301 302 return 0; 303 } 304 305 void host1x_intr_deinit(struct host1x *host) 306 { 307 host1x_intr_stop(host); 308 } 309 310 void host1x_intr_start(struct host1x *host) 311 { 312 u32 hz = clk_get_rate(host->clk); 313 int err; 314 315 mutex_lock(&host->intr_mutex); 316 err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000), 317 syncpt_thresh_work); 318 if (err) { 319 mutex_unlock(&host->intr_mutex); 320 return; 321 } 322 mutex_unlock(&host->intr_mutex); 323 } 324 325 void host1x_intr_stop(struct host1x *host) 326 { 327 unsigned int id; 328 struct host1x_syncpt *syncpt = host->syncpt; 329 u32 nb_pts = host1x_syncpt_nb_pts(host); 330 331 mutex_lock(&host->intr_mutex); 332 333 host1x_hw_intr_disable_all_syncpt_intrs(host); 334 335 for (id = 0; id < nb_pts; ++id) { 336 struct host1x_waitlist *waiter, *next; 337 338 list_for_each_entry_safe(waiter, next, 339 &syncpt[id].intr.wait_head, list) { 340 if (atomic_cmpxchg(&waiter->state, 341 WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) { 342 list_del(&waiter->list); 343 kref_put(&waiter->refcount, waiter_release); 344 } 345 } 346 347 if (!list_empty(&syncpt[id].intr.wait_head)) { 348 /* output diagnostics */ 349 mutex_unlock(&host->intr_mutex); 350 pr_warn("%s cannot stop syncpt intr id=%u\n", 351 __func__, id); 352 return; 353 } 354 } 355 356 host1x_hw_intr_free_syncpt_irq(host); 357 358 mutex_unlock(&host->intr_mutex); 359 } 360