1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * IRQ subsystem internal functions and variables:
4dbec07baSThomas Gleixner *
5dbec07baSThomas Gleixner * Do not ever include this file from anything else than
6dbec07baSThomas Gleixner * kernel/irq/. Do not even think about using any information outside
7dbec07baSThomas Gleixner * of this file for your non core code.
81da177e4SLinus Torvalds */
9e144710bSThomas Gleixner #include <linux/irqdesc.h>
108f945a33SThomas Gleixner #include <linux/kernel_stat.h>
11be45beb2SJon Hunter #include <linux/pm_runtime.h>
12b2d3d61aSDaniel Lezcano #include <linux/sched/clock.h>
131da177e4SLinus Torvalds
14c1ee6264SThomas Gleixner #ifdef CONFIG_SPARSE_IRQ
15721255b9SShanker Donthineni # define MAX_SPARSE_IRQS INT_MAX
16c1ee6264SThomas Gleixner #else
175e630aa8SShanker Donthineni # define MAX_SPARSE_IRQS NR_IRQS
18c1ee6264SThomas Gleixner #endif
19c1ee6264SThomas Gleixner
20dbec07baSThomas Gleixner #define istate core_internal_state__do_not_mess_with_it
21dbec07baSThomas Gleixner
222329abfaSRusty Russell extern bool noirqdebug;
231da177e4SLinus Torvalds
24e509bd7dSMika Westerberg extern struct irqaction chained_action;
25e509bd7dSMika Westerberg
261535dfacSThomas Gleixner /*
271535dfacSThomas Gleixner * Bits used by threaded handlers:
281535dfacSThomas Gleixner * IRQTF_RUNTHREAD - signals that the interrupt handler thread should run
291535dfacSThomas Gleixner * IRQTF_WARNED - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed
301535dfacSThomas Gleixner * IRQTF_AFFINITY - irq thread is requested to adjust affinity
318d32a307SThomas Gleixner * IRQTF_FORCED_THREAD - irq action is force threaded
328707898eSThomas Pfaff * IRQTF_READY - signals that irq thread is ready
331535dfacSThomas Gleixner */
341535dfacSThomas Gleixner enum {
351535dfacSThomas Gleixner IRQTF_RUNTHREAD,
361535dfacSThomas Gleixner IRQTF_WARNED,
371535dfacSThomas Gleixner IRQTF_AFFINITY,
388d32a307SThomas Gleixner IRQTF_FORCED_THREAD,
398707898eSThomas Pfaff IRQTF_READY,
401535dfacSThomas Gleixner };
411535dfacSThomas Gleixner
42bd062e76SThomas Gleixner /*
43a257954bSJiang Liu * Bit masks for desc->core_internal_state__do_not_mess_with_it
44bd062e76SThomas Gleixner *
45bd062e76SThomas Gleixner * IRQS_AUTODETECT - autodetection in progress
467acdd53eSThomas Gleixner * IRQS_SPURIOUS_DISABLED - was disabled due to spurious interrupt
477acdd53eSThomas Gleixner * detection
486954b75bSThomas Gleixner * IRQS_POLL_INPROGRESS - polling in progress
493d67baecSThomas Gleixner * IRQS_ONESHOT - irq is not unmasked in primary handler
507cc148a3SJames Gowans * IRQS_REPLAY - irq has been resent and will not be resent
517cc148a3SJames Gowans * again until the handler has run and cleared
527cc148a3SJames Gowans * this flag.
53163ef309SThomas Gleixner * IRQS_WAITING - irq is waiting
547cc148a3SJames Gowans * IRQS_PENDING - irq needs to be resent and should be resent
557cc148a3SJames Gowans * at the next available opportunity.
56c531e836SThomas Gleixner * IRQS_SUSPENDED - irq is suspended
57b525903cSJulien Thierry * IRQS_NMI - irq line is used to deliver NMIs
589049e1caSYang Yingliang * IRQS_SYSFS - descriptor has been added to sysfs
59bd062e76SThomas Gleixner */
60bd062e76SThomas Gleixner enum {
61bd062e76SThomas Gleixner IRQS_AUTODETECT = 0x00000001,
627acdd53eSThomas Gleixner IRQS_SPURIOUS_DISABLED = 0x00000002,
636954b75bSThomas Gleixner IRQS_POLL_INPROGRESS = 0x00000008,
643d67baecSThomas Gleixner IRQS_ONESHOT = 0x00000020,
65163ef309SThomas Gleixner IRQS_REPLAY = 0x00000040,
66163ef309SThomas Gleixner IRQS_WAITING = 0x00000080,
672a0d6fb3SThomas Gleixner IRQS_PENDING = 0x00000200,
68c531e836SThomas Gleixner IRQS_SUSPENDED = 0x00000800,
69b2d3d61aSDaniel Lezcano IRQS_TIMINGS = 0x00001000,
70b525903cSJulien Thierry IRQS_NMI = 0x00002000,
719049e1caSYang Yingliang IRQS_SYSFS = 0x00004000,
72bd062e76SThomas Gleixner };
73bd062e76SThomas Gleixner
741ce6068dSThomas Gleixner #include "debug.h"
751ce6068dSThomas Gleixner #include "settings.h"
761ce6068dSThomas Gleixner
77a1ff541aSJiang Liu extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags);
7879ff1cdaSJiang Liu extern void __disable_irq(struct irq_desc *desc);
7979ff1cdaSJiang Liu extern void __enable_irq(struct irq_desc *desc);
800c5d1eb7SDavid Brownell
814cde9c6bSThomas Gleixner #define IRQ_RESEND true
824cde9c6bSThomas Gleixner #define IRQ_NORESEND false
834cde9c6bSThomas Gleixner
844cde9c6bSThomas Gleixner #define IRQ_START_FORCE true
854cde9c6bSThomas Gleixner #define IRQ_START_COND false
864cde9c6bSThomas Gleixner
87c942cee4SThomas Gleixner extern int irq_activate(struct irq_desc *desc);
881beaeacdSThomas Gleixner extern int irq_activate_and_startup(struct irq_desc *desc, bool resend);
894cde9c6bSThomas Gleixner extern int irq_startup(struct irq_desc *desc, bool resend, bool force);
904cde9c6bSThomas Gleixner
9146999238SThomas Gleixner extern void irq_shutdown(struct irq_desc *desc);
924001d8e8SThomas Gleixner extern void irq_shutdown_and_deactivate(struct irq_desc *desc);
9387923470SThomas Gleixner extern void irq_enable(struct irq_desc *desc);
9487923470SThomas Gleixner extern void irq_disable(struct irq_desc *desc);
9531d9d9b6SMarc Zyngier extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu);
9631d9d9b6SMarc Zyngier extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu);
97d4d5e089SThomas Gleixner extern void mask_irq(struct irq_desc *desc);
98d4d5e089SThomas Gleixner extern void unmask_irq(struct irq_desc *desc);
99328a4978SThomas Gleixner extern void unmask_threaded_irq(struct irq_desc *desc);
10046999238SThomas Gleixner
101f63b6a05SThomas Gleixner #ifdef CONFIG_SPARSE_IRQ
irq_mark_irq(unsigned int irq)102f63b6a05SThomas Gleixner static inline void irq_mark_irq(unsigned int irq) { }
103f63b6a05SThomas Gleixner #else
104f63b6a05SThomas Gleixner extern void irq_mark_irq(unsigned int irq);
105f63b6a05SThomas Gleixner #endif
106f63b6a05SThomas Gleixner
10762e04686SThomas Gleixner extern int __irq_get_irqchip_state(struct irq_data *data,
10862e04686SThomas Gleixner enum irqchip_irq_state which,
10962e04686SThomas Gleixner bool *state);
11062e04686SThomas Gleixner
1115320eb42SSebastian Andrzej Siewior irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc);
11271f64340SHuang Shijie irqreturn_t handle_irq_event_percpu(struct irq_desc *desc);
1134912609fSThomas Gleixner irqreturn_t handle_irq_event(struct irq_desc *desc);
1144912609fSThomas Gleixner
115e144710bSThomas Gleixner /* Resending of interrupts :*/
116acd26bcfSThomas Gleixner int check_irq_resend(struct irq_desc *desc, bool inject);
117bc06a9e0SShanker Donthineni void clear_irq_resend(struct irq_desc *desc);
118bc06a9e0SShanker Donthineni void irq_resend_init(struct irq_desc *desc);
119fe200ae4SThomas Gleixner bool irq_wait_for_poll(struct irq_desc *desc);
120a92444c6SThomas Gleixner void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action);
121e144710bSThomas Gleixner
122*e2c12739SVincent Whitchurch void wake_threads_waitq(struct irq_desc *desc);
123*e2c12739SVincent Whitchurch
1241da177e4SLinus Torvalds #ifdef CONFIG_PROC_FS
1252c6927a3SYinghai Lu extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
12613bfe99eSThomas Gleixner extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc);
1271da177e4SLinus Torvalds extern void register_handler_proc(unsigned int irq, struct irqaction *action);
1281da177e4SLinus Torvalds extern void unregister_handler_proc(unsigned int irq, struct irqaction *action);
1291da177e4SLinus Torvalds #else
register_irq_proc(unsigned int irq,struct irq_desc * desc)1302c6927a3SYinghai Lu static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { }
unregister_irq_proc(unsigned int irq,struct irq_desc * desc)13113bfe99eSThomas Gleixner static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { }
register_handler_proc(unsigned int irq,struct irqaction * action)1321da177e4SLinus Torvalds static inline void register_handler_proc(unsigned int irq,
1331da177e4SLinus Torvalds struct irqaction *action) { }
unregister_handler_proc(unsigned int irq,struct irqaction * action)1341da177e4SLinus Torvalds static inline void unregister_handler_proc(unsigned int irq,
1351da177e4SLinus Torvalds struct irqaction *action) { }
1361da177e4SLinus Torvalds #endif
1371da177e4SLinus Torvalds
1389c255583SThomas Gleixner extern bool irq_can_set_affinity_usr(unsigned int irq);
1399c255583SThomas Gleixner
140591d2fb0SThomas Gleixner extern void irq_set_thread_affinity(struct irq_desc *desc);
14157b150ccSYinghai Lu
142818b0f3bSJiang Liu extern int irq_do_set_affinity(struct irq_data *data,
143818b0f3bSJiang Liu const struct cpumask *dest, bool force);
144818b0f3bSJiang Liu
14543564bd9SThomas Gleixner #ifdef CONFIG_SMP
14643564bd9SThomas Gleixner extern int irq_setup_affinity(struct irq_desc *desc);
14743564bd9SThomas Gleixner #else
irq_setup_affinity(struct irq_desc * desc)14843564bd9SThomas Gleixner static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; }
14943564bd9SThomas Gleixner #endif
15043564bd9SThomas Gleixner
15170aedd24SThomas Gleixner /* Inline functions for support of irq chips on slow busses */
chip_bus_lock(struct irq_desc * desc)1523876ec9eSThomas Gleixner static inline void chip_bus_lock(struct irq_desc *desc)
15370aedd24SThomas Gleixner {
1543876ec9eSThomas Gleixner if (unlikely(desc->irq_data.chip->irq_bus_lock))
1553876ec9eSThomas Gleixner desc->irq_data.chip->irq_bus_lock(&desc->irq_data);
15670aedd24SThomas Gleixner }
15770aedd24SThomas Gleixner
chip_bus_sync_unlock(struct irq_desc * desc)1583876ec9eSThomas Gleixner static inline void chip_bus_sync_unlock(struct irq_desc *desc)
15970aedd24SThomas Gleixner {
1603876ec9eSThomas Gleixner if (unlikely(desc->irq_data.chip->irq_bus_sync_unlock))
1613876ec9eSThomas Gleixner desc->irq_data.chip->irq_bus_sync_unlock(&desc->irq_data);
16270aedd24SThomas Gleixner }
16370aedd24SThomas Gleixner
16431d9d9b6SMarc Zyngier #define _IRQ_DESC_CHECK (1 << 0)
16531d9d9b6SMarc Zyngier #define _IRQ_DESC_PERCPU (1 << 1)
16631d9d9b6SMarc Zyngier
16731d9d9b6SMarc Zyngier #define IRQ_GET_DESC_CHECK_GLOBAL (_IRQ_DESC_CHECK)
16831d9d9b6SMarc Zyngier #define IRQ_GET_DESC_CHECK_PERCPU (_IRQ_DESC_CHECK | _IRQ_DESC_PERCPU)
16931d9d9b6SMarc Zyngier
170f944b5a7SDaniel Lezcano #define for_each_action_of_desc(desc, act) \
171163616cfSMasahiro Yamada for (act = desc->action; act; act = act->next)
172f944b5a7SDaniel Lezcano
173d5eb4ad2SThomas Gleixner struct irq_desc *
17431d9d9b6SMarc Zyngier __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
17531d9d9b6SMarc Zyngier unsigned int check);
176d5eb4ad2SThomas Gleixner void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus);
177d5eb4ad2SThomas Gleixner
178d5eb4ad2SThomas Gleixner static inline struct irq_desc *
irq_get_desc_buslock(unsigned int irq,unsigned long * flags,unsigned int check)17931d9d9b6SMarc Zyngier irq_get_desc_buslock(unsigned int irq, unsigned long *flags, unsigned int check)
180d5eb4ad2SThomas Gleixner {
18131d9d9b6SMarc Zyngier return __irq_get_desc_lock(irq, flags, true, check);
182d5eb4ad2SThomas Gleixner }
183d5eb4ad2SThomas Gleixner
184d5eb4ad2SThomas Gleixner static inline void
irq_put_desc_busunlock(struct irq_desc * desc,unsigned long flags)185d5eb4ad2SThomas Gleixner irq_put_desc_busunlock(struct irq_desc *desc, unsigned long flags)
186d5eb4ad2SThomas Gleixner {
187d5eb4ad2SThomas Gleixner __irq_put_desc_unlock(desc, flags, true);
188d5eb4ad2SThomas Gleixner }
189d5eb4ad2SThomas Gleixner
190d5eb4ad2SThomas Gleixner static inline struct irq_desc *
irq_get_desc_lock(unsigned int irq,unsigned long * flags,unsigned int check)19131d9d9b6SMarc Zyngier irq_get_desc_lock(unsigned int irq, unsigned long *flags, unsigned int check)
192d5eb4ad2SThomas Gleixner {
19331d9d9b6SMarc Zyngier return __irq_get_desc_lock(irq, flags, false, check);
194d5eb4ad2SThomas Gleixner }
195d5eb4ad2SThomas Gleixner
196d5eb4ad2SThomas Gleixner static inline void
irq_put_desc_unlock(struct irq_desc * desc,unsigned long flags)197d5eb4ad2SThomas Gleixner irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags)
198d5eb4ad2SThomas Gleixner {
199d5eb4ad2SThomas Gleixner __irq_put_desc_unlock(desc, flags, false);
200d5eb4ad2SThomas Gleixner }
201d5eb4ad2SThomas Gleixner
202b354286eSBoqun Feng #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
203b354286eSBoqun Feng
irqd_get(struct irq_data * d)204087cdfb6SThomas Gleixner static inline unsigned int irqd_get(struct irq_data *d)
205087cdfb6SThomas Gleixner {
206087cdfb6SThomas Gleixner return __irqd_to_state(d);
207087cdfb6SThomas Gleixner }
208087cdfb6SThomas Gleixner
20943f77759SIngo Molnar /*
210f230b6d5SThomas Gleixner * Manipulation functions for irq_data.state
211f230b6d5SThomas Gleixner */
irqd_set_move_pending(struct irq_data * d)212f230b6d5SThomas Gleixner static inline void irqd_set_move_pending(struct irq_data *d)
213f230b6d5SThomas Gleixner {
2140d0b4c86SJiang Liu __irqd_to_state(d) |= IRQD_SETAFFINITY_PENDING;
215f230b6d5SThomas Gleixner }
216f230b6d5SThomas Gleixner
irqd_clr_move_pending(struct irq_data * d)217f230b6d5SThomas Gleixner static inline void irqd_clr_move_pending(struct irq_data *d)
218f230b6d5SThomas Gleixner {
2190d0b4c86SJiang Liu __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING;
220f230b6d5SThomas Gleixner }
221a005677bSThomas Gleixner
irqd_set_managed_shutdown(struct irq_data * d)22254fdf6a0SThomas Gleixner static inline void irqd_set_managed_shutdown(struct irq_data *d)
22354fdf6a0SThomas Gleixner {
22454fdf6a0SThomas Gleixner __irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN;
22554fdf6a0SThomas Gleixner }
22654fdf6a0SThomas Gleixner
irqd_clr_managed_shutdown(struct irq_data * d)22754fdf6a0SThomas Gleixner static inline void irqd_clr_managed_shutdown(struct irq_data *d)
22854fdf6a0SThomas Gleixner {
22954fdf6a0SThomas Gleixner __irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN;
23054fdf6a0SThomas Gleixner }
23154fdf6a0SThomas Gleixner
irqd_clear(struct irq_data * d,unsigned int mask)232a005677bSThomas Gleixner static inline void irqd_clear(struct irq_data *d, unsigned int mask)
233a005677bSThomas Gleixner {
2340d0b4c86SJiang Liu __irqd_to_state(d) &= ~mask;
235a005677bSThomas Gleixner }
236a005677bSThomas Gleixner
irqd_set(struct irq_data * d,unsigned int mask)237a005677bSThomas Gleixner static inline void irqd_set(struct irq_data *d, unsigned int mask)
238a005677bSThomas Gleixner {
2390d0b4c86SJiang Liu __irqd_to_state(d) |= mask;
240a005677bSThomas Gleixner }
241a005677bSThomas Gleixner
irqd_has_set(struct irq_data * d,unsigned int mask)2422bdd1055SThomas Gleixner static inline bool irqd_has_set(struct irq_data *d, unsigned int mask)
2432bdd1055SThomas Gleixner {
2440d0b4c86SJiang Liu return __irqd_to_state(d) & mask;
2452bdd1055SThomas Gleixner }
2468f945a33SThomas Gleixner
irq_state_set_disabled(struct irq_desc * desc)247a696712cSJuergen Gross static inline void irq_state_set_disabled(struct irq_desc *desc)
248a696712cSJuergen Gross {
249a696712cSJuergen Gross irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
250a696712cSJuergen Gross }
251a696712cSJuergen Gross
irq_state_set_masked(struct irq_desc * desc)252a696712cSJuergen Gross static inline void irq_state_set_masked(struct irq_desc *desc)
253a696712cSJuergen Gross {
254a696712cSJuergen Gross irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
255a696712cSJuergen Gross }
256a696712cSJuergen Gross
257b354286eSBoqun Feng #undef __irqd_to_state
258b354286eSBoqun Feng
__kstat_incr_irqs_this_cpu(struct irq_desc * desc)2591136b072SThomas Gleixner static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc)
2608f945a33SThomas Gleixner {
2618f945a33SThomas Gleixner __this_cpu_inc(*desc->kstat_irqs);
2628f945a33SThomas Gleixner __this_cpu_inc(kstat.irqs_sum);
2638f945a33SThomas Gleixner }
264cab303beSThomas Gleixner
kstat_incr_irqs_this_cpu(struct irq_desc * desc)2651136b072SThomas Gleixner static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc)
2661136b072SThomas Gleixner {
2671136b072SThomas Gleixner __kstat_incr_irqs_this_cpu(desc);
2681136b072SThomas Gleixner desc->tot_count++;
2691136b072SThomas Gleixner }
2701136b072SThomas Gleixner
irq_desc_get_node(struct irq_desc * desc)2716783011bSJiang Liu static inline int irq_desc_get_node(struct irq_desc *desc)
2726783011bSJiang Liu {
273449e9caeSJiang Liu return irq_common_data_get_node(&desc->irq_common_data);
2746783011bSJiang Liu }
2756783011bSJiang Liu
irq_desc_is_chained(struct irq_desc * desc)2764717f133SGrygorii Strashko static inline int irq_desc_is_chained(struct irq_desc *desc)
2774717f133SGrygorii Strashko {
2784717f133SGrygorii Strashko return (desc->action && desc->action == &chained_action);
2794717f133SGrygorii Strashko }
2804717f133SGrygorii Strashko
281cab303beSThomas Gleixner #ifdef CONFIG_PM_SLEEP
2829ce7a258SThomas Gleixner bool irq_pm_check_wakeup(struct irq_desc *desc);
283cab303beSThomas Gleixner void irq_pm_install_action(struct irq_desc *desc, struct irqaction *action);
284cab303beSThomas Gleixner void irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action);
285cab303beSThomas Gleixner #else
irq_pm_check_wakeup(struct irq_desc * desc)2869ce7a258SThomas Gleixner static inline bool irq_pm_check_wakeup(struct irq_desc *desc) { return false; }
287cab303beSThomas Gleixner static inline void
irq_pm_install_action(struct irq_desc * desc,struct irqaction * action)288cab303beSThomas Gleixner irq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { }
289cab303beSThomas Gleixner static inline void
irq_pm_remove_action(struct irq_desc * desc,struct irqaction * action)290cab303beSThomas Gleixner irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { }
291cab303beSThomas Gleixner #endif
292f1602039SBartosz Golaszewski
293b2d3d61aSDaniel Lezcano #ifdef CONFIG_IRQ_TIMINGS
294b2d3d61aSDaniel Lezcano
295b2d3d61aSDaniel Lezcano #define IRQ_TIMINGS_SHIFT 5
296b2d3d61aSDaniel Lezcano #define IRQ_TIMINGS_SIZE (1 << IRQ_TIMINGS_SHIFT)
297b2d3d61aSDaniel Lezcano #define IRQ_TIMINGS_MASK (IRQ_TIMINGS_SIZE - 1)
298b2d3d61aSDaniel Lezcano
299b2d3d61aSDaniel Lezcano /**
300b2d3d61aSDaniel Lezcano * struct irq_timings - irq timings storing structure
301b2d3d61aSDaniel Lezcano * @values: a circular buffer of u64 encoded <timestamp,irq> values
302b2d3d61aSDaniel Lezcano * @count: the number of elements in the array
303b2d3d61aSDaniel Lezcano */
304b2d3d61aSDaniel Lezcano struct irq_timings {
305b2d3d61aSDaniel Lezcano u64 values[IRQ_TIMINGS_SIZE];
306b2d3d61aSDaniel Lezcano int count;
307b2d3d61aSDaniel Lezcano };
308b2d3d61aSDaniel Lezcano
309b2d3d61aSDaniel Lezcano DECLARE_PER_CPU(struct irq_timings, irq_timings);
310b2d3d61aSDaniel Lezcano
311e1c92149SDaniel Lezcano extern void irq_timings_free(int irq);
312e1c92149SDaniel Lezcano extern int irq_timings_alloc(int irq);
313e1c92149SDaniel Lezcano
irq_remove_timings(struct irq_desc * desc)314b2d3d61aSDaniel Lezcano static inline void irq_remove_timings(struct irq_desc *desc)
315b2d3d61aSDaniel Lezcano {
316b2d3d61aSDaniel Lezcano desc->istate &= ~IRQS_TIMINGS;
317e1c92149SDaniel Lezcano
318e1c92149SDaniel Lezcano irq_timings_free(irq_desc_get_irq(desc));
319b2d3d61aSDaniel Lezcano }
320b2d3d61aSDaniel Lezcano
irq_setup_timings(struct irq_desc * desc,struct irqaction * act)321b2d3d61aSDaniel Lezcano static inline void irq_setup_timings(struct irq_desc *desc, struct irqaction *act)
322b2d3d61aSDaniel Lezcano {
323e1c92149SDaniel Lezcano int irq = irq_desc_get_irq(desc);
324e1c92149SDaniel Lezcano int ret;
325e1c92149SDaniel Lezcano
326b2d3d61aSDaniel Lezcano /*
327b2d3d61aSDaniel Lezcano * We don't need the measurement because the idle code already
328b2d3d61aSDaniel Lezcano * knows the next expiry event.
329b2d3d61aSDaniel Lezcano */
330b2d3d61aSDaniel Lezcano if (act->flags & __IRQF_TIMER)
331b2d3d61aSDaniel Lezcano return;
332b2d3d61aSDaniel Lezcano
333e1c92149SDaniel Lezcano /*
334e1c92149SDaniel Lezcano * In case the timing allocation fails, we just want to warn,
335e1c92149SDaniel Lezcano * not fail, so letting the system boot anyway.
336e1c92149SDaniel Lezcano */
337e1c92149SDaniel Lezcano ret = irq_timings_alloc(irq);
338e1c92149SDaniel Lezcano if (ret) {
339e1c92149SDaniel Lezcano pr_warn("Failed to allocate irq timing stats for irq%d (%d)",
340e1c92149SDaniel Lezcano irq, ret);
341e1c92149SDaniel Lezcano return;
342e1c92149SDaniel Lezcano }
343e1c92149SDaniel Lezcano
344b2d3d61aSDaniel Lezcano desc->istate |= IRQS_TIMINGS;
345b2d3d61aSDaniel Lezcano }
346b2d3d61aSDaniel Lezcano
347b2d3d61aSDaniel Lezcano extern void irq_timings_enable(void);
348b2d3d61aSDaniel Lezcano extern void irq_timings_disable(void);
349b2d3d61aSDaniel Lezcano
350b2d3d61aSDaniel Lezcano DECLARE_STATIC_KEY_FALSE(irq_timing_enabled);
351b2d3d61aSDaniel Lezcano
352b2d3d61aSDaniel Lezcano /*
353b2d3d61aSDaniel Lezcano * The interrupt number and the timestamp are encoded into a single
354b2d3d61aSDaniel Lezcano * u64 variable to optimize the size.
355b2d3d61aSDaniel Lezcano * 48 bit time stamp and 16 bit IRQ number is way sufficient.
356b2d3d61aSDaniel Lezcano * Who cares an IRQ after 78 hours of idle time?
357b2d3d61aSDaniel Lezcano */
irq_timing_encode(u64 timestamp,int irq)358b2d3d61aSDaniel Lezcano static inline u64 irq_timing_encode(u64 timestamp, int irq)
359b2d3d61aSDaniel Lezcano {
360b2d3d61aSDaniel Lezcano return (timestamp << 16) | irq;
361b2d3d61aSDaniel Lezcano }
362b2d3d61aSDaniel Lezcano
irq_timing_decode(u64 value,u64 * timestamp)363b2d3d61aSDaniel Lezcano static inline int irq_timing_decode(u64 value, u64 *timestamp)
364b2d3d61aSDaniel Lezcano {
365b2d3d61aSDaniel Lezcano *timestamp = value >> 16;
366b2d3d61aSDaniel Lezcano return value & U16_MAX;
367b2d3d61aSDaniel Lezcano }
368b2d3d61aSDaniel Lezcano
irq_timings_push(u64 ts,int irq)369df025e47SDaniel Lezcano static __always_inline void irq_timings_push(u64 ts, int irq)
370df025e47SDaniel Lezcano {
371df025e47SDaniel Lezcano struct irq_timings *timings = this_cpu_ptr(&irq_timings);
372df025e47SDaniel Lezcano
373df025e47SDaniel Lezcano timings->values[timings->count & IRQ_TIMINGS_MASK] =
374df025e47SDaniel Lezcano irq_timing_encode(ts, irq);
375df025e47SDaniel Lezcano
376df025e47SDaniel Lezcano timings->count++;
377df025e47SDaniel Lezcano }
378df025e47SDaniel Lezcano
379b2d3d61aSDaniel Lezcano /*
380b2d3d61aSDaniel Lezcano * The function record_irq_time is only called in one place in the
381b2d3d61aSDaniel Lezcano * interrupts handler. We want this function always inline so the code
382b2d3d61aSDaniel Lezcano * inside is embedded in the function and the static key branching
383b2d3d61aSDaniel Lezcano * code can act at the higher level. Without the explicit
384b2d3d61aSDaniel Lezcano * __always_inline we can end up with a function call and a small
385b2d3d61aSDaniel Lezcano * overhead in the hotpath for nothing.
386b2d3d61aSDaniel Lezcano */
record_irq_time(struct irq_desc * desc)387b2d3d61aSDaniel Lezcano static __always_inline void record_irq_time(struct irq_desc *desc)
388b2d3d61aSDaniel Lezcano {
389b2d3d61aSDaniel Lezcano if (!static_branch_likely(&irq_timing_enabled))
390b2d3d61aSDaniel Lezcano return;
391b2d3d61aSDaniel Lezcano
392df025e47SDaniel Lezcano if (desc->istate & IRQS_TIMINGS)
393df025e47SDaniel Lezcano irq_timings_push(local_clock(), irq_desc_get_irq(desc));
394b2d3d61aSDaniel Lezcano }
395b2d3d61aSDaniel Lezcano #else
irq_remove_timings(struct irq_desc * desc)396b2d3d61aSDaniel Lezcano static inline void irq_remove_timings(struct irq_desc *desc) {}
irq_setup_timings(struct irq_desc * desc,struct irqaction * act)397b2d3d61aSDaniel Lezcano static inline void irq_setup_timings(struct irq_desc *desc,
398b2d3d61aSDaniel Lezcano struct irqaction *act) {};
record_irq_time(struct irq_desc * desc)399b2d3d61aSDaniel Lezcano static inline void record_irq_time(struct irq_desc *desc) {}
400b2d3d61aSDaniel Lezcano #endif /* CONFIG_IRQ_TIMINGS */
401b2d3d61aSDaniel Lezcano
402b2d3d61aSDaniel Lezcano
403f1602039SBartosz Golaszewski #ifdef CONFIG_GENERIC_IRQ_CHIP
404f1602039SBartosz Golaszewski void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
405f1602039SBartosz Golaszewski int num_ct, unsigned int irq_base,
406f1602039SBartosz Golaszewski void __iomem *reg_base, irq_flow_handler_t handler);
407f1602039SBartosz Golaszewski #else
408f1602039SBartosz Golaszewski static inline void
irq_init_generic_chip(struct irq_chip_generic * gc,const char * name,int num_ct,unsigned int irq_base,void __iomem * reg_base,irq_flow_handler_t handler)409f1602039SBartosz Golaszewski irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
410f1602039SBartosz Golaszewski int num_ct, unsigned int irq_base,
411f1602039SBartosz Golaszewski void __iomem *reg_base, irq_flow_handler_t handler) { }
412f1602039SBartosz Golaszewski #endif /* CONFIG_GENERIC_IRQ_CHIP */
413087cdfb6SThomas Gleixner
414137221dfSChristoph Hellwig #ifdef CONFIG_GENERIC_PENDING_IRQ
irq_can_move_pcntxt(struct irq_data * data)415137221dfSChristoph Hellwig static inline bool irq_can_move_pcntxt(struct irq_data *data)
416137221dfSChristoph Hellwig {
417137221dfSChristoph Hellwig return irqd_can_move_in_process_context(data);
418137221dfSChristoph Hellwig }
irq_move_pending(struct irq_data * data)419137221dfSChristoph Hellwig static inline bool irq_move_pending(struct irq_data *data)
420137221dfSChristoph Hellwig {
421137221dfSChristoph Hellwig return irqd_is_setaffinity_pending(data);
422137221dfSChristoph Hellwig }
423137221dfSChristoph Hellwig static inline void
irq_copy_pending(struct irq_desc * desc,const struct cpumask * mask)424137221dfSChristoph Hellwig irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
425137221dfSChristoph Hellwig {
426137221dfSChristoph Hellwig cpumask_copy(desc->pending_mask, mask);
427137221dfSChristoph Hellwig }
428137221dfSChristoph Hellwig static inline void
irq_get_pending(struct cpumask * mask,struct irq_desc * desc)429137221dfSChristoph Hellwig irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
430137221dfSChristoph Hellwig {
431137221dfSChristoph Hellwig cpumask_copy(mask, desc->pending_mask);
432137221dfSChristoph Hellwig }
irq_desc_get_pending_mask(struct irq_desc * desc)433f0383c24SThomas Gleixner static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
434f0383c24SThomas Gleixner {
435f0383c24SThomas Gleixner return desc->pending_mask;
436f0383c24SThomas Gleixner }
handle_enforce_irqctx(struct irq_data * data)437c16816acSThomas Gleixner static inline bool handle_enforce_irqctx(struct irq_data *data)
438c16816acSThomas Gleixner {
439c16816acSThomas Gleixner return irqd_is_handle_enforce_irqctx(data);
440c16816acSThomas Gleixner }
44136d84fb4SThomas Gleixner bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear);
442137221dfSChristoph Hellwig #else /* CONFIG_GENERIC_PENDING_IRQ */
irq_can_move_pcntxt(struct irq_data * data)443137221dfSChristoph Hellwig static inline bool irq_can_move_pcntxt(struct irq_data *data)
444137221dfSChristoph Hellwig {
445137221dfSChristoph Hellwig return true;
446137221dfSChristoph Hellwig }
irq_move_pending(struct irq_data * data)447137221dfSChristoph Hellwig static inline bool irq_move_pending(struct irq_data *data)
448137221dfSChristoph Hellwig {
449137221dfSChristoph Hellwig return false;
450137221dfSChristoph Hellwig }
451137221dfSChristoph Hellwig static inline void
irq_copy_pending(struct irq_desc * desc,const struct cpumask * mask)452137221dfSChristoph Hellwig irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
453137221dfSChristoph Hellwig {
454137221dfSChristoph Hellwig }
455137221dfSChristoph Hellwig static inline void
irq_get_pending(struct cpumask * mask,struct irq_desc * desc)456137221dfSChristoph Hellwig irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
457137221dfSChristoph Hellwig {
458137221dfSChristoph Hellwig }
irq_desc_get_pending_mask(struct irq_desc * desc)459f0383c24SThomas Gleixner static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
460f0383c24SThomas Gleixner {
461f0383c24SThomas Gleixner return NULL;
462f0383c24SThomas Gleixner }
irq_fixup_move_pending(struct irq_desc * desc,bool fclear)46336d84fb4SThomas Gleixner static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear)
46436d84fb4SThomas Gleixner {
46536d84fb4SThomas Gleixner return false;
46636d84fb4SThomas Gleixner }
handle_enforce_irqctx(struct irq_data * data)467c16816acSThomas Gleixner static inline bool handle_enforce_irqctx(struct irq_data *data)
468c16816acSThomas Gleixner {
469c16816acSThomas Gleixner return false;
470c16816acSThomas Gleixner }
471f0383c24SThomas Gleixner #endif /* !CONFIG_GENERIC_PENDING_IRQ */
472137221dfSChristoph Hellwig
473457f6d35SThomas Gleixner #if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY)
irq_domain_activate_irq(struct irq_data * data,bool reserve)474702cb0a0SThomas Gleixner static inline int irq_domain_activate_irq(struct irq_data *data, bool reserve)
475457f6d35SThomas Gleixner {
476457f6d35SThomas Gleixner irqd_set_activated(data);
477bb9b428aSThomas Gleixner return 0;
478457f6d35SThomas Gleixner }
irq_domain_deactivate_irq(struct irq_data * data)479457f6d35SThomas Gleixner static inline void irq_domain_deactivate_irq(struct irq_data *data)
480457f6d35SThomas Gleixner {
481457f6d35SThomas Gleixner irqd_clr_activated(data);
482457f6d35SThomas Gleixner }
483457f6d35SThomas Gleixner #endif
484457f6d35SThomas Gleixner
irqd_get_parent_data(struct irq_data * irqd)48513b90cadSThomas Gleixner static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd)
48613b90cadSThomas Gleixner {
48713b90cadSThomas Gleixner #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
48813b90cadSThomas Gleixner return irqd->parent_data;
48913b90cadSThomas Gleixner #else
49013b90cadSThomas Gleixner return NULL;
49113b90cadSThomas Gleixner #endif
49213b90cadSThomas Gleixner }
49313b90cadSThomas Gleixner
494087cdfb6SThomas Gleixner #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
495c2ce34c0SThomas Gleixner #include <linux/debugfs.h>
496c2ce34c0SThomas Gleixner
497087cdfb6SThomas Gleixner void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc);
irq_remove_debugfs_entry(struct irq_desc * desc)498c2ce34c0SThomas Gleixner static inline void irq_remove_debugfs_entry(struct irq_desc *desc)
499c2ce34c0SThomas Gleixner {
500c2ce34c0SThomas Gleixner debugfs_remove(desc->debugfs_file);
50107557ccbSThomas Gleixner kfree(desc->dev_name);
502c2ce34c0SThomas Gleixner }
50307557ccbSThomas Gleixner void irq_debugfs_copy_devname(int irq, struct device *dev);
504087cdfb6SThomas Gleixner # ifdef CONFIG_IRQ_DOMAIN
505087cdfb6SThomas Gleixner void irq_domain_debugfs_init(struct dentry *root);
506087cdfb6SThomas Gleixner # else
irq_domain_debugfs_init(struct dentry * root)507e5682b4eSSebastian Ott static inline void irq_domain_debugfs_init(struct dentry *root)
508e5682b4eSSebastian Ott {
509e5682b4eSSebastian Ott }
510087cdfb6SThomas Gleixner # endif
511087cdfb6SThomas Gleixner #else /* CONFIG_GENERIC_IRQ_DEBUGFS */
irq_add_debugfs_entry(unsigned int irq,struct irq_desc * d)512087cdfb6SThomas Gleixner static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d)
513087cdfb6SThomas Gleixner {
514087cdfb6SThomas Gleixner }
irq_remove_debugfs_entry(struct irq_desc * d)515087cdfb6SThomas Gleixner static inline void irq_remove_debugfs_entry(struct irq_desc *d)
516087cdfb6SThomas Gleixner {
517087cdfb6SThomas Gleixner }
irq_debugfs_copy_devname(int irq,struct device * dev)51807557ccbSThomas Gleixner static inline void irq_debugfs_copy_devname(int irq, struct device *dev)
51907557ccbSThomas Gleixner {
52007557ccbSThomas Gleixner }
521087cdfb6SThomas Gleixner #endif /* CONFIG_GENERIC_IRQ_DEBUGFS */
522