1d87f36a0SRajneesh Bhardwaj // SPDX-License-Identifier: GPL-2.0 OR MIT
22249d558SAndrew Lewycky /*
3d87f36a0SRajneesh Bhardwaj * Copyright 2014-2022 Advanced Micro Devices, Inc.
42249d558SAndrew Lewycky *
52249d558SAndrew Lewycky * Permission is hereby granted, free of charge, to any person obtaining a
62249d558SAndrew Lewycky * copy of this software and associated documentation files (the "Software"),
72249d558SAndrew Lewycky * to deal in the Software without restriction, including without limitation
82249d558SAndrew Lewycky * the rights to use, copy, modify, merge, publish, distribute, sublicense,
92249d558SAndrew Lewycky * and/or sell copies of the Software, and to permit persons to whom the
102249d558SAndrew Lewycky * Software is furnished to do so, subject to the following conditions:
112249d558SAndrew Lewycky *
122249d558SAndrew Lewycky * The above copyright notice and this permission notice shall be included in
132249d558SAndrew Lewycky * all copies or substantial portions of the Software.
142249d558SAndrew Lewycky *
152249d558SAndrew Lewycky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
162249d558SAndrew Lewycky * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
172249d558SAndrew Lewycky * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
182249d558SAndrew Lewycky * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
192249d558SAndrew Lewycky * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
202249d558SAndrew Lewycky * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
212249d558SAndrew Lewycky * OTHER DEALINGS IN THE SOFTWARE.
222249d558SAndrew Lewycky */
232249d558SAndrew Lewycky
242249d558SAndrew Lewycky /*
252249d558SAndrew Lewycky * KFD Interrupts.
262249d558SAndrew Lewycky *
272249d558SAndrew Lewycky * AMD GPUs deliver interrupts by pushing an interrupt description onto the
282249d558SAndrew Lewycky * interrupt ring and then sending an interrupt. KGD receives the interrupt
292249d558SAndrew Lewycky * in ISR and sends us a pointer to each new entry on the interrupt ring.
302249d558SAndrew Lewycky *
312249d558SAndrew Lewycky * We generally can't process interrupt-signaled events from ISR, so we call
322249d558SAndrew Lewycky * out to each interrupt client module (currently only the scheduler) to ask if
332249d558SAndrew Lewycky * each interrupt is interesting. If they return true, then it requires further
342249d558SAndrew Lewycky * processing so we copy it to an internal interrupt ring and call each
352249d558SAndrew Lewycky * interrupt client again from a work-queue.
362249d558SAndrew Lewycky *
372249d558SAndrew Lewycky * There's no acknowledgment for the interrupts we use. The hardware simply
382249d558SAndrew Lewycky * queues a new interrupt each time without waiting.
392249d558SAndrew Lewycky *
402249d558SAndrew Lewycky * The fixed-size internal queue means that it's possible for us to lose
412249d558SAndrew Lewycky * interrupts because we have no back-pressure to the hardware.
422249d558SAndrew Lewycky */
432249d558SAndrew Lewycky
442249d558SAndrew Lewycky #include <linux/slab.h>
452249d558SAndrew Lewycky #include <linux/device.h>
4604ad47bdSAndres Rodriguez #include <linux/kfifo.h>
472249d558SAndrew Lewycky #include "kfd_priv.h"
482249d558SAndrew Lewycky
4927232055SAndres Rodriguez #define KFD_IH_NUM_ENTRIES 8192
502249d558SAndrew Lewycky
512249d558SAndrew Lewycky static void interrupt_wq(struct work_struct *);
522249d558SAndrew Lewycky
kfd_interrupt_init(struct kfd_node * node)53*8dc1db31SMukul Joshi int kfd_interrupt_init(struct kfd_node *node)
542249d558SAndrew Lewycky {
5504ad47bdSAndres Rodriguez int r;
562249d558SAndrew Lewycky
57*8dc1db31SMukul Joshi r = kfifo_alloc(&node->ih_fifo,
58*8dc1db31SMukul Joshi KFD_IH_NUM_ENTRIES * node->kfd->device_info.ih_ring_entry_size,
5904ad47bdSAndres Rodriguez GFP_KERNEL);
6004ad47bdSAndres Rodriguez if (r) {
61*8dc1db31SMukul Joshi dev_err(node->adev->dev, "Failed to allocate IH fifo\n");
6204ad47bdSAndres Rodriguez return r;
6304ad47bdSAndres Rodriguez }
642249d558SAndrew Lewycky
65*8dc1db31SMukul Joshi node->ih_wq = alloc_workqueue("KFD IH", WQ_HIGHPRI, 1);
66*8dc1db31SMukul Joshi if (unlikely(!node->ih_wq)) {
67*8dc1db31SMukul Joshi kfifo_free(&node->ih_fifo);
68*8dc1db31SMukul Joshi dev_err(node->adev->dev, "Failed to allocate KFD IH workqueue\n");
6981de29d8SAllen Pais return -ENOMEM;
7081de29d8SAllen Pais }
71*8dc1db31SMukul Joshi spin_lock_init(&node->interrupt_lock);
722249d558SAndrew Lewycky
73*8dc1db31SMukul Joshi INIT_WORK(&node->interrupt_work, interrupt_wq);
742249d558SAndrew Lewycky
75*8dc1db31SMukul Joshi node->interrupts_active = true;
762249d558SAndrew Lewycky
772249d558SAndrew Lewycky /*
782249d558SAndrew Lewycky * After this function returns, the interrupt will be enabled. This
792249d558SAndrew Lewycky * barrier ensures that the interrupt running on a different processor
802249d558SAndrew Lewycky * sees all the above writes.
812249d558SAndrew Lewycky */
822249d558SAndrew Lewycky smp_wmb();
832249d558SAndrew Lewycky
842249d558SAndrew Lewycky return 0;
852249d558SAndrew Lewycky }
862249d558SAndrew Lewycky
kfd_interrupt_exit(struct kfd_node * node)87*8dc1db31SMukul Joshi void kfd_interrupt_exit(struct kfd_node *node)
882249d558SAndrew Lewycky {
892249d558SAndrew Lewycky /*
902249d558SAndrew Lewycky * Stop the interrupt handler from writing to the ring and scheduling
912249d558SAndrew Lewycky * workqueue items. The spinlock ensures that any interrupt running
922249d558SAndrew Lewycky * after we have unlocked sees interrupts_active = false.
932249d558SAndrew Lewycky */
942249d558SAndrew Lewycky unsigned long flags;
952249d558SAndrew Lewycky
96*8dc1db31SMukul Joshi spin_lock_irqsave(&node->interrupt_lock, flags);
97*8dc1db31SMukul Joshi node->interrupts_active = false;
98*8dc1db31SMukul Joshi spin_unlock_irqrestore(&node->interrupt_lock, flags);
992249d558SAndrew Lewycky
1002249d558SAndrew Lewycky /*
1010f875e3fSAndres Rodriguez * flush_work ensures that there are no outstanding
1022249d558SAndrew Lewycky * work-queue items that will access interrupt_ring. New work items
1032249d558SAndrew Lewycky * can't be created because we stopped interrupt handling above.
1042249d558SAndrew Lewycky */
105*8dc1db31SMukul Joshi flush_workqueue(node->ih_wq);
1062249d558SAndrew Lewycky
107*8dc1db31SMukul Joshi kfifo_free(&node->ih_fifo);
1082249d558SAndrew Lewycky }
1092249d558SAndrew Lewycky
1102249d558SAndrew Lewycky /*
11104ad47bdSAndres Rodriguez * Assumption: single reader/writer. This function is not re-entrant
1122249d558SAndrew Lewycky */
enqueue_ih_ring_entry(struct kfd_node * node,const void * ih_ring_entry)113*8dc1db31SMukul Joshi bool enqueue_ih_ring_entry(struct kfd_node *node, const void *ih_ring_entry)
1142249d558SAndrew Lewycky {
11504ad47bdSAndres Rodriguez int count;
1162249d558SAndrew Lewycky
117*8dc1db31SMukul Joshi count = kfifo_in(&node->ih_fifo, ih_ring_entry,
118*8dc1db31SMukul Joshi node->kfd->device_info.ih_ring_entry_size);
119*8dc1db31SMukul Joshi if (count != node->kfd->device_info.ih_ring_entry_size) {
120*8dc1db31SMukul Joshi dev_dbg_ratelimited(node->adev->dev,
12104ad47bdSAndres Rodriguez "Interrupt ring overflow, dropping interrupt %d\n",
12204ad47bdSAndres Rodriguez count);
1232249d558SAndrew Lewycky return false;
1242249d558SAndrew Lewycky }
1252249d558SAndrew Lewycky
1262249d558SAndrew Lewycky return true;
1272249d558SAndrew Lewycky }
1282249d558SAndrew Lewycky
1292249d558SAndrew Lewycky /*
13004ad47bdSAndres Rodriguez * Assumption: single reader/writer. This function is not re-entrant
1312249d558SAndrew Lewycky */
dequeue_ih_ring_entry(struct kfd_node * node,void * ih_ring_entry)132*8dc1db31SMukul Joshi static bool dequeue_ih_ring_entry(struct kfd_node *node, void *ih_ring_entry)
1332249d558SAndrew Lewycky {
13404ad47bdSAndres Rodriguez int count;
1352249d558SAndrew Lewycky
136*8dc1db31SMukul Joshi count = kfifo_out(&node->ih_fifo, ih_ring_entry,
137*8dc1db31SMukul Joshi node->kfd->device_info.ih_ring_entry_size);
1382249d558SAndrew Lewycky
139*8dc1db31SMukul Joshi WARN_ON(count && count != node->kfd->device_info.ih_ring_entry_size);
1402249d558SAndrew Lewycky
141*8dc1db31SMukul Joshi return count == node->kfd->device_info.ih_ring_entry_size;
1422249d558SAndrew Lewycky }
1432249d558SAndrew Lewycky
interrupt_wq(struct work_struct * work)1442249d558SAndrew Lewycky static void interrupt_wq(struct work_struct *work)
1452249d558SAndrew Lewycky {
146*8dc1db31SMukul Joshi struct kfd_node *dev = container_of(work, struct kfd_node,
1472249d558SAndrew Lewycky interrupt_work);
148af47b390SLaura Abbott uint32_t ih_ring_entry[KFD_MAX_RING_ENTRY_SIZE];
149dcfe584bSYu Zhe unsigned long start_jiffies = jiffies;
1502249d558SAndrew Lewycky
151*8dc1db31SMukul Joshi if (dev->kfd->device_info.ih_ring_entry_size > sizeof(ih_ring_entry)) {
152a0c5fd46SFelix Kuehling dev_err_once(dev->adev->dev, "Ring entry too small\n");
153af47b390SLaura Abbott return;
154af47b390SLaura Abbott }
1552249d558SAndrew Lewycky
1565273e82cSFelix Kuehling while (dequeue_ih_ring_entry(dev, ih_ring_entry)) {
157*8dc1db31SMukul Joshi dev->kfd->device_info.event_interrupt_class->interrupt_wq(dev,
158f3a39818SAndrew Lewycky ih_ring_entry);
159dcfe584bSYu Zhe if (time_is_before_jiffies(start_jiffies + HZ)) {
1605273e82cSFelix Kuehling /* If we spent more than a second processing signals,
1615273e82cSFelix Kuehling * reschedule the worker to avoid soft-lockup warnings
1625273e82cSFelix Kuehling */
1635273e82cSFelix Kuehling queue_work(dev->ih_wq, &dev->interrupt_work);
1645273e82cSFelix Kuehling break;
1655273e82cSFelix Kuehling }
1665273e82cSFelix Kuehling }
1672249d558SAndrew Lewycky }
1682249d558SAndrew Lewycky
interrupt_is_wanted(struct kfd_node * dev,const uint32_t * ih_ring_entry,uint32_t * patched_ihre,bool * flag)169*8dc1db31SMukul Joshi bool interrupt_is_wanted(struct kfd_node *dev,
17058e69886SLan Xiao const uint32_t *ih_ring_entry,
17158e69886SLan Xiao uint32_t *patched_ihre, bool *flag)
1722249d558SAndrew Lewycky {
173f3a39818SAndrew Lewycky /* integer and bitwise OR so there is no boolean short-circuiting */
1748eabaf54SKent Russell unsigned int wanted = 0;
175f3a39818SAndrew Lewycky
176*8dc1db31SMukul Joshi wanted |= dev->kfd->device_info.event_interrupt_class->interrupt_isr(dev,
17758e69886SLan Xiao ih_ring_entry, patched_ihre, flag);
178f3a39818SAndrew Lewycky
179f3a39818SAndrew Lewycky return wanted != 0;
1802249d558SAndrew Lewycky }
181