xref: /openbmc/linux/kernel/irq/manage.c (revision 4b357daed698c95d6b5eacc1c3c4afa206071ba2)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  * linux/kernel/irq/manage.c
31da177e4SLinus Torvalds  *
4a34db9b2SIngo Molnar  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
5a34db9b2SIngo Molnar  * Copyright (C) 2005-2006 Thomas Gleixner
61da177e4SLinus Torvalds  *
71da177e4SLinus Torvalds  * This file contains driver APIs to the irq subsystem.
81da177e4SLinus Torvalds  */
91da177e4SLinus Torvalds 
1097fd75b7SAndrew Morton #define pr_fmt(fmt) "genirq: " fmt
1197fd75b7SAndrew Morton 
121da177e4SLinus Torvalds #include <linux/irq.h>
133aa551c9SThomas Gleixner #include <linux/kthread.h>
141da177e4SLinus Torvalds #include <linux/module.h>
151da177e4SLinus Torvalds #include <linux/random.h>
161da177e4SLinus Torvalds #include <linux/interrupt.h>
171aeb272cSRobert P. J. Day #include <linux/slab.h>
183aa551c9SThomas Gleixner #include <linux/sched.h>
198bd75c77SClark Williams #include <linux/sched/rt.h>
204d1d61a6SOleg Nesterov #include <linux/task_work.h>
211da177e4SLinus Torvalds 
221da177e4SLinus Torvalds #include "internals.h"
231da177e4SLinus Torvalds 
248d32a307SThomas Gleixner #ifdef CONFIG_IRQ_FORCED_THREADING
258d32a307SThomas Gleixner __read_mostly bool force_irqthreads;
268d32a307SThomas Gleixner 
278d32a307SThomas Gleixner static int __init setup_forced_irqthreads(char *arg)
288d32a307SThomas Gleixner {
298d32a307SThomas Gleixner 	force_irqthreads = true;
308d32a307SThomas Gleixner 	return 0;
318d32a307SThomas Gleixner }
328d32a307SThomas Gleixner early_param("threadirqs", setup_forced_irqthreads);
338d32a307SThomas Gleixner #endif
348d32a307SThomas Gleixner 
3518258f72SThomas Gleixner static void __synchronize_hardirq(struct irq_desc *desc)
361da177e4SLinus Torvalds {
3732f4125eSThomas Gleixner 	bool inprogress;
381da177e4SLinus Torvalds 
39a98ce5c6SHerbert Xu 	do {
40a98ce5c6SHerbert Xu 		unsigned long flags;
41a98ce5c6SHerbert Xu 
42a98ce5c6SHerbert Xu 		/*
43a98ce5c6SHerbert Xu 		 * Wait until we're out of the critical section.  This might
44a98ce5c6SHerbert Xu 		 * give the wrong answer due to the lack of memory barriers.
45a98ce5c6SHerbert Xu 		 */
4632f4125eSThomas Gleixner 		while (irqd_irq_inprogress(&desc->irq_data))
471da177e4SLinus Torvalds 			cpu_relax();
48a98ce5c6SHerbert Xu 
49a98ce5c6SHerbert Xu 		/* Ok, that indicated we're done: double-check carefully. */
50239007b8SThomas Gleixner 		raw_spin_lock_irqsave(&desc->lock, flags);
5132f4125eSThomas Gleixner 		inprogress = irqd_irq_inprogress(&desc->irq_data);
52239007b8SThomas Gleixner 		raw_spin_unlock_irqrestore(&desc->lock, flags);
53a98ce5c6SHerbert Xu 
54a98ce5c6SHerbert Xu 		/* Oops, that failed? */
5532f4125eSThomas Gleixner 	} while (inprogress);
5618258f72SThomas Gleixner }
573aa551c9SThomas Gleixner 
5818258f72SThomas Gleixner /**
5918258f72SThomas Gleixner  *	synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
6018258f72SThomas Gleixner  *	@irq: interrupt number to wait for
6118258f72SThomas Gleixner  *
6218258f72SThomas Gleixner  *	This function waits for any pending hard IRQ handlers for this
6318258f72SThomas Gleixner  *	interrupt to complete before returning. If you use this
6418258f72SThomas Gleixner  *	function while holding a resource the IRQ handler may need you
6518258f72SThomas Gleixner  *	will deadlock. It does not take associated threaded handlers
6618258f72SThomas Gleixner  *	into account.
6718258f72SThomas Gleixner  *
6818258f72SThomas Gleixner  *	Do not use this for shutdown scenarios where you must be sure
6918258f72SThomas Gleixner  *	that all parts (hardirq and threaded handler) have completed.
7018258f72SThomas Gleixner  *
7102cea395SPeter Zijlstra  *	Returns: false if a threaded handler is active.
7202cea395SPeter Zijlstra  *
7318258f72SThomas Gleixner  *	This function may be called - with care - from IRQ context.
743aa551c9SThomas Gleixner  */
7502cea395SPeter Zijlstra bool synchronize_hardirq(unsigned int irq)
7618258f72SThomas Gleixner {
7718258f72SThomas Gleixner 	struct irq_desc *desc = irq_to_desc(irq);
7818258f72SThomas Gleixner 
7902cea395SPeter Zijlstra 	if (desc) {
8018258f72SThomas Gleixner 		__synchronize_hardirq(desc);
8102cea395SPeter Zijlstra 		return !atomic_read(&desc->threads_active);
8202cea395SPeter Zijlstra 	}
8302cea395SPeter Zijlstra 
8402cea395SPeter Zijlstra 	return true;
8518258f72SThomas Gleixner }
8618258f72SThomas Gleixner EXPORT_SYMBOL(synchronize_hardirq);
8718258f72SThomas Gleixner 
8818258f72SThomas Gleixner /**
8918258f72SThomas Gleixner  *	synchronize_irq - wait for pending IRQ handlers (on other CPUs)
9018258f72SThomas Gleixner  *	@irq: interrupt number to wait for
9118258f72SThomas Gleixner  *
9218258f72SThomas Gleixner  *	This function waits for any pending IRQ handlers for this interrupt
9318258f72SThomas Gleixner  *	to complete before returning. If you use this function while
9418258f72SThomas Gleixner  *	holding a resource the IRQ handler may need you will deadlock.
9518258f72SThomas Gleixner  *
9618258f72SThomas Gleixner  *	This function may be called - with care - from IRQ context.
9718258f72SThomas Gleixner  */
9818258f72SThomas Gleixner void synchronize_irq(unsigned int irq)
9918258f72SThomas Gleixner {
10018258f72SThomas Gleixner 	struct irq_desc *desc = irq_to_desc(irq);
10118258f72SThomas Gleixner 
10218258f72SThomas Gleixner 	if (desc) {
10318258f72SThomas Gleixner 		__synchronize_hardirq(desc);
10418258f72SThomas Gleixner 		/*
10518258f72SThomas Gleixner 		 * We made sure that no hardirq handler is
10618258f72SThomas Gleixner 		 * running. Now verify that no threaded handlers are
10718258f72SThomas Gleixner 		 * active.
10818258f72SThomas Gleixner 		 */
10918258f72SThomas Gleixner 		wait_event(desc->wait_for_threads,
11018258f72SThomas Gleixner 			   !atomic_read(&desc->threads_active));
11118258f72SThomas Gleixner 	}
1121da177e4SLinus Torvalds }
1131da177e4SLinus Torvalds EXPORT_SYMBOL(synchronize_irq);
1141da177e4SLinus Torvalds 
1153aa551c9SThomas Gleixner #ifdef CONFIG_SMP
1163aa551c9SThomas Gleixner cpumask_var_t irq_default_affinity;
1173aa551c9SThomas Gleixner 
118e019c249SJiang Liu static int __irq_can_set_affinity(struct irq_desc *desc)
119e019c249SJiang Liu {
120e019c249SJiang Liu 	if (!desc || !irqd_can_balance(&desc->irq_data) ||
121e019c249SJiang Liu 	    !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
122e019c249SJiang Liu 		return 0;
123e019c249SJiang Liu 	return 1;
124e019c249SJiang Liu }
125e019c249SJiang Liu 
126771ee3b0SThomas Gleixner /**
127771ee3b0SThomas Gleixner  *	irq_can_set_affinity - Check if the affinity of a given irq can be set
128771ee3b0SThomas Gleixner  *	@irq:		Interrupt to check
129771ee3b0SThomas Gleixner  *
130771ee3b0SThomas Gleixner  */
131771ee3b0SThomas Gleixner int irq_can_set_affinity(unsigned int irq)
132771ee3b0SThomas Gleixner {
133e019c249SJiang Liu 	return __irq_can_set_affinity(irq_to_desc(irq));
134771ee3b0SThomas Gleixner }
135771ee3b0SThomas Gleixner 
136591d2fb0SThomas Gleixner /**
137591d2fb0SThomas Gleixner  *	irq_set_thread_affinity - Notify irq threads to adjust affinity
138591d2fb0SThomas Gleixner  *	@desc:		irq descriptor which has affitnity changed
139591d2fb0SThomas Gleixner  *
140591d2fb0SThomas Gleixner  *	We just set IRQTF_AFFINITY and delegate the affinity setting
141591d2fb0SThomas Gleixner  *	to the interrupt thread itself. We can not call
142591d2fb0SThomas Gleixner  *	set_cpus_allowed_ptr() here as we hold desc->lock and this
143591d2fb0SThomas Gleixner  *	code can be called from hard interrupt context.
144591d2fb0SThomas Gleixner  */
145591d2fb0SThomas Gleixner void irq_set_thread_affinity(struct irq_desc *desc)
1463aa551c9SThomas Gleixner {
147f944b5a7SDaniel Lezcano 	struct irqaction *action;
1483aa551c9SThomas Gleixner 
149f944b5a7SDaniel Lezcano 	for_each_action_of_desc(desc, action)
1503aa551c9SThomas Gleixner 		if (action->thread)
151591d2fb0SThomas Gleixner 			set_bit(IRQTF_AFFINITY, &action->thread_flags);
1523aa551c9SThomas Gleixner }
1533aa551c9SThomas Gleixner 
1541fa46f1fSThomas Gleixner #ifdef CONFIG_GENERIC_PENDING_IRQ
1550ef5ca1eSThomas Gleixner static inline bool irq_can_move_pcntxt(struct irq_data *data)
1561fa46f1fSThomas Gleixner {
1570ef5ca1eSThomas Gleixner 	return irqd_can_move_in_process_context(data);
1581fa46f1fSThomas Gleixner }
1590ef5ca1eSThomas Gleixner static inline bool irq_move_pending(struct irq_data *data)
1601fa46f1fSThomas Gleixner {
1610ef5ca1eSThomas Gleixner 	return irqd_is_setaffinity_pending(data);
1621fa46f1fSThomas Gleixner }
1631fa46f1fSThomas Gleixner static inline void
1641fa46f1fSThomas Gleixner irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
1651fa46f1fSThomas Gleixner {
1661fa46f1fSThomas Gleixner 	cpumask_copy(desc->pending_mask, mask);
1671fa46f1fSThomas Gleixner }
1681fa46f1fSThomas Gleixner static inline void
1691fa46f1fSThomas Gleixner irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
1701fa46f1fSThomas Gleixner {
1711fa46f1fSThomas Gleixner 	cpumask_copy(mask, desc->pending_mask);
1721fa46f1fSThomas Gleixner }
1731fa46f1fSThomas Gleixner #else
1740ef5ca1eSThomas Gleixner static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
175cd22c0e4SThomas Gleixner static inline bool irq_move_pending(struct irq_data *data) { return false; }
1761fa46f1fSThomas Gleixner static inline void
1771fa46f1fSThomas Gleixner irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
1781fa46f1fSThomas Gleixner static inline void
1791fa46f1fSThomas Gleixner irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
1801fa46f1fSThomas Gleixner #endif
1811fa46f1fSThomas Gleixner 
182818b0f3bSJiang Liu int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
183818b0f3bSJiang Liu 			bool force)
184818b0f3bSJiang Liu {
185818b0f3bSJiang Liu 	struct irq_desc *desc = irq_data_to_desc(data);
186818b0f3bSJiang Liu 	struct irq_chip *chip = irq_data_get_irq_chip(data);
187818b0f3bSJiang Liu 	int ret;
188818b0f3bSJiang Liu 
18901f8fa4fSThomas Gleixner 	ret = chip->irq_set_affinity(data, mask, force);
190818b0f3bSJiang Liu 	switch (ret) {
191818b0f3bSJiang Liu 	case IRQ_SET_MASK_OK:
1922cb62547SJiang Liu 	case IRQ_SET_MASK_OK_DONE:
1939df872faSJiang Liu 		cpumask_copy(desc->irq_common_data.affinity, mask);
194818b0f3bSJiang Liu 	case IRQ_SET_MASK_OK_NOCOPY:
195818b0f3bSJiang Liu 		irq_set_thread_affinity(desc);
196818b0f3bSJiang Liu 		ret = 0;
197818b0f3bSJiang Liu 	}
198818b0f3bSJiang Liu 
199818b0f3bSJiang Liu 	return ret;
200818b0f3bSJiang Liu }
201818b0f3bSJiang Liu 
20201f8fa4fSThomas Gleixner int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
20301f8fa4fSThomas Gleixner 			    bool force)
204c2d0c555SDavid Daney {
205c2d0c555SDavid Daney 	struct irq_chip *chip = irq_data_get_irq_chip(data);
206c2d0c555SDavid Daney 	struct irq_desc *desc = irq_data_to_desc(data);
207c2d0c555SDavid Daney 	int ret = 0;
208c2d0c555SDavid Daney 
209c2d0c555SDavid Daney 	if (!chip || !chip->irq_set_affinity)
210c2d0c555SDavid Daney 		return -EINVAL;
211c2d0c555SDavid Daney 
2120ef5ca1eSThomas Gleixner 	if (irq_can_move_pcntxt(data)) {
21301f8fa4fSThomas Gleixner 		ret = irq_do_set_affinity(data, mask, force);
214c2d0c555SDavid Daney 	} else {
215c2d0c555SDavid Daney 		irqd_set_move_pending(data);
216c2d0c555SDavid Daney 		irq_copy_pending(desc, mask);
217c2d0c555SDavid Daney 	}
218c2d0c555SDavid Daney 
219c2d0c555SDavid Daney 	if (desc->affinity_notify) {
220c2d0c555SDavid Daney 		kref_get(&desc->affinity_notify->kref);
221c2d0c555SDavid Daney 		schedule_work(&desc->affinity_notify->work);
222c2d0c555SDavid Daney 	}
223c2d0c555SDavid Daney 	irqd_set(data, IRQD_AFFINITY_SET);
224c2d0c555SDavid Daney 
225c2d0c555SDavid Daney 	return ret;
226c2d0c555SDavid Daney }
227c2d0c555SDavid Daney 
22801f8fa4fSThomas Gleixner int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
229771ee3b0SThomas Gleixner {
23008678b08SYinghai Lu 	struct irq_desc *desc = irq_to_desc(irq);
231f6d87f4bSThomas Gleixner 	unsigned long flags;
232c2d0c555SDavid Daney 	int ret;
233771ee3b0SThomas Gleixner 
234c2d0c555SDavid Daney 	if (!desc)
235771ee3b0SThomas Gleixner 		return -EINVAL;
236771ee3b0SThomas Gleixner 
237239007b8SThomas Gleixner 	raw_spin_lock_irqsave(&desc->lock, flags);
23801f8fa4fSThomas Gleixner 	ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
239239007b8SThomas Gleixner 	raw_spin_unlock_irqrestore(&desc->lock, flags);
2401fa46f1fSThomas Gleixner 	return ret;
241771ee3b0SThomas Gleixner }
242771ee3b0SThomas Gleixner 
243e7a297b0SPeter P Waskiewicz Jr int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
244e7a297b0SPeter P Waskiewicz Jr {
245e7a297b0SPeter P Waskiewicz Jr 	unsigned long flags;
24631d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
247e7a297b0SPeter P Waskiewicz Jr 
248e7a297b0SPeter P Waskiewicz Jr 	if (!desc)
249e7a297b0SPeter P Waskiewicz Jr 		return -EINVAL;
250e7a297b0SPeter P Waskiewicz Jr 	desc->affinity_hint = m;
25102725e74SThomas Gleixner 	irq_put_desc_unlock(desc, flags);
252e2e64a93SJesse Brandeburg 	/* set the initial affinity to prevent every interrupt being on CPU0 */
2534fe7ffb7SJesse Brandeburg 	if (m)
254e2e64a93SJesse Brandeburg 		__irq_set_affinity(irq, m, false);
255e7a297b0SPeter P Waskiewicz Jr 	return 0;
256e7a297b0SPeter P Waskiewicz Jr }
257e7a297b0SPeter P Waskiewicz Jr EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
258e7a297b0SPeter P Waskiewicz Jr 
259cd7eab44SBen Hutchings static void irq_affinity_notify(struct work_struct *work)
260cd7eab44SBen Hutchings {
261cd7eab44SBen Hutchings 	struct irq_affinity_notify *notify =
262cd7eab44SBen Hutchings 		container_of(work, struct irq_affinity_notify, work);
263cd7eab44SBen Hutchings 	struct irq_desc *desc = irq_to_desc(notify->irq);
264cd7eab44SBen Hutchings 	cpumask_var_t cpumask;
265cd7eab44SBen Hutchings 	unsigned long flags;
266cd7eab44SBen Hutchings 
2671fa46f1fSThomas Gleixner 	if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
268cd7eab44SBen Hutchings 		goto out;
269cd7eab44SBen Hutchings 
270cd7eab44SBen Hutchings 	raw_spin_lock_irqsave(&desc->lock, flags);
2710ef5ca1eSThomas Gleixner 	if (irq_move_pending(&desc->irq_data))
2721fa46f1fSThomas Gleixner 		irq_get_pending(cpumask, desc);
273cd7eab44SBen Hutchings 	else
2749df872faSJiang Liu 		cpumask_copy(cpumask, desc->irq_common_data.affinity);
275cd7eab44SBen Hutchings 	raw_spin_unlock_irqrestore(&desc->lock, flags);
276cd7eab44SBen Hutchings 
277cd7eab44SBen Hutchings 	notify->notify(notify, cpumask);
278cd7eab44SBen Hutchings 
279cd7eab44SBen Hutchings 	free_cpumask_var(cpumask);
280cd7eab44SBen Hutchings out:
281cd7eab44SBen Hutchings 	kref_put(&notify->kref, notify->release);
282cd7eab44SBen Hutchings }
283cd7eab44SBen Hutchings 
284cd7eab44SBen Hutchings /**
285cd7eab44SBen Hutchings  *	irq_set_affinity_notifier - control notification of IRQ affinity changes
286cd7eab44SBen Hutchings  *	@irq:		Interrupt for which to enable/disable notification
287cd7eab44SBen Hutchings  *	@notify:	Context for notification, or %NULL to disable
288cd7eab44SBen Hutchings  *			notification.  Function pointers must be initialised;
289cd7eab44SBen Hutchings  *			the other fields will be initialised by this function.
290cd7eab44SBen Hutchings  *
291cd7eab44SBen Hutchings  *	Must be called in process context.  Notification may only be enabled
292cd7eab44SBen Hutchings  *	after the IRQ is allocated and must be disabled before the IRQ is
293cd7eab44SBen Hutchings  *	freed using free_irq().
294cd7eab44SBen Hutchings  */
295cd7eab44SBen Hutchings int
296cd7eab44SBen Hutchings irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
297cd7eab44SBen Hutchings {
298cd7eab44SBen Hutchings 	struct irq_desc *desc = irq_to_desc(irq);
299cd7eab44SBen Hutchings 	struct irq_affinity_notify *old_notify;
300cd7eab44SBen Hutchings 	unsigned long flags;
301cd7eab44SBen Hutchings 
302cd7eab44SBen Hutchings 	/* The release function is promised process context */
303cd7eab44SBen Hutchings 	might_sleep();
304cd7eab44SBen Hutchings 
305cd7eab44SBen Hutchings 	if (!desc)
306cd7eab44SBen Hutchings 		return -EINVAL;
307cd7eab44SBen Hutchings 
308cd7eab44SBen Hutchings 	/* Complete initialisation of *notify */
309cd7eab44SBen Hutchings 	if (notify) {
310cd7eab44SBen Hutchings 		notify->irq = irq;
311cd7eab44SBen Hutchings 		kref_init(&notify->kref);
312cd7eab44SBen Hutchings 		INIT_WORK(&notify->work, irq_affinity_notify);
313cd7eab44SBen Hutchings 	}
314cd7eab44SBen Hutchings 
315cd7eab44SBen Hutchings 	raw_spin_lock_irqsave(&desc->lock, flags);
316cd7eab44SBen Hutchings 	old_notify = desc->affinity_notify;
317cd7eab44SBen Hutchings 	desc->affinity_notify = notify;
318cd7eab44SBen Hutchings 	raw_spin_unlock_irqrestore(&desc->lock, flags);
319cd7eab44SBen Hutchings 
320cd7eab44SBen Hutchings 	if (old_notify)
321cd7eab44SBen Hutchings 		kref_put(&old_notify->kref, old_notify->release);
322cd7eab44SBen Hutchings 
323cd7eab44SBen Hutchings 	return 0;
324cd7eab44SBen Hutchings }
325cd7eab44SBen Hutchings EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
326cd7eab44SBen Hutchings 
32718404756SMax Krasnyansky #ifndef CONFIG_AUTO_IRQ_AFFINITY
32818404756SMax Krasnyansky /*
32918404756SMax Krasnyansky  * Generic version of the affinity autoselector.
33018404756SMax Krasnyansky  */
331a8a98eacSJiang Liu static int setup_affinity(struct irq_desc *desc, struct cpumask *mask)
33218404756SMax Krasnyansky {
333569bda8dSThomas Gleixner 	struct cpumask *set = irq_default_affinity;
3346783011bSJiang Liu 	int node = irq_desc_get_node(desc);
335569bda8dSThomas Gleixner 
336b008207cSThomas Gleixner 	/* Excludes PER_CPU and NO_BALANCE interrupts */
337e019c249SJiang Liu 	if (!__irq_can_set_affinity(desc))
33818404756SMax Krasnyansky 		return 0;
33918404756SMax Krasnyansky 
340f6d87f4bSThomas Gleixner 	/*
341f6d87f4bSThomas Gleixner 	 * Preserve an userspace affinity setup, but make sure that
342f6d87f4bSThomas Gleixner 	 * one of the targets is online.
343f6d87f4bSThomas Gleixner 	 */
3442bdd1055SThomas Gleixner 	if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
3459df872faSJiang Liu 		if (cpumask_intersects(desc->irq_common_data.affinity,
346569bda8dSThomas Gleixner 				       cpu_online_mask))
3479df872faSJiang Liu 			set = desc->irq_common_data.affinity;
3480c6f8a8bSThomas Gleixner 		else
3492bdd1055SThomas Gleixner 			irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
3502bdd1055SThomas Gleixner 	}
35118404756SMax Krasnyansky 
3523b8249e7SThomas Gleixner 	cpumask_and(mask, cpu_online_mask, set);
353241fc640SPrarit Bhargava 	if (node != NUMA_NO_NODE) {
354241fc640SPrarit Bhargava 		const struct cpumask *nodemask = cpumask_of_node(node);
355241fc640SPrarit Bhargava 
356241fc640SPrarit Bhargava 		/* make sure at least one of the cpus in nodemask is online */
357241fc640SPrarit Bhargava 		if (cpumask_intersects(mask, nodemask))
358241fc640SPrarit Bhargava 			cpumask_and(mask, mask, nodemask);
359241fc640SPrarit Bhargava 	}
360818b0f3bSJiang Liu 	irq_do_set_affinity(&desc->irq_data, mask, false);
36118404756SMax Krasnyansky 	return 0;
36218404756SMax Krasnyansky }
363f6d87f4bSThomas Gleixner #else
364a8a98eacSJiang Liu /* Wrapper for ALPHA specific affinity selector magic */
365a8a98eacSJiang Liu static inline int setup_affinity(struct irq_desc *d, struct cpumask *mask)
366f6d87f4bSThomas Gleixner {
367a8a98eacSJiang Liu 	return irq_select_affinity(irq_desc_get_irq(d));
368f6d87f4bSThomas Gleixner }
36918404756SMax Krasnyansky #endif
37018404756SMax Krasnyansky 
371f6d87f4bSThomas Gleixner /*
372f6d87f4bSThomas Gleixner  * Called when affinity is set via /proc/irq
373f6d87f4bSThomas Gleixner  */
3743b8249e7SThomas Gleixner int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
375f6d87f4bSThomas Gleixner {
376f6d87f4bSThomas Gleixner 	struct irq_desc *desc = irq_to_desc(irq);
377f6d87f4bSThomas Gleixner 	unsigned long flags;
378f6d87f4bSThomas Gleixner 	int ret;
379f6d87f4bSThomas Gleixner 
380239007b8SThomas Gleixner 	raw_spin_lock_irqsave(&desc->lock, flags);
381a8a98eacSJiang Liu 	ret = setup_affinity(desc, mask);
382239007b8SThomas Gleixner 	raw_spin_unlock_irqrestore(&desc->lock, flags);
383f6d87f4bSThomas Gleixner 	return ret;
384f6d87f4bSThomas Gleixner }
385f6d87f4bSThomas Gleixner 
386f6d87f4bSThomas Gleixner #else
3873b8249e7SThomas Gleixner static inline int
388a8a98eacSJiang Liu setup_affinity(struct irq_desc *desc, struct cpumask *mask)
389f6d87f4bSThomas Gleixner {
390f6d87f4bSThomas Gleixner 	return 0;
391f6d87f4bSThomas Gleixner }
3921da177e4SLinus Torvalds #endif
3931da177e4SLinus Torvalds 
394fcf1ae2fSFeng Wu /**
395fcf1ae2fSFeng Wu  *	irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
396fcf1ae2fSFeng Wu  *	@irq: interrupt number to set affinity
397fcf1ae2fSFeng Wu  *	@vcpu_info: vCPU specific data
398fcf1ae2fSFeng Wu  *
399fcf1ae2fSFeng Wu  *	This function uses the vCPU specific data to set the vCPU
400fcf1ae2fSFeng Wu  *	affinity for an irq. The vCPU specific data is passed from
401fcf1ae2fSFeng Wu  *	outside, such as KVM. One example code path is as below:
402fcf1ae2fSFeng Wu  *	KVM -> IOMMU -> irq_set_vcpu_affinity().
403fcf1ae2fSFeng Wu  */
404fcf1ae2fSFeng Wu int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
405fcf1ae2fSFeng Wu {
406fcf1ae2fSFeng Wu 	unsigned long flags;
407fcf1ae2fSFeng Wu 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
408fcf1ae2fSFeng Wu 	struct irq_data *data;
409fcf1ae2fSFeng Wu 	struct irq_chip *chip;
410fcf1ae2fSFeng Wu 	int ret = -ENOSYS;
411fcf1ae2fSFeng Wu 
412fcf1ae2fSFeng Wu 	if (!desc)
413fcf1ae2fSFeng Wu 		return -EINVAL;
414fcf1ae2fSFeng Wu 
415fcf1ae2fSFeng Wu 	data = irq_desc_get_irq_data(desc);
416fcf1ae2fSFeng Wu 	chip = irq_data_get_irq_chip(data);
417fcf1ae2fSFeng Wu 	if (chip && chip->irq_set_vcpu_affinity)
418fcf1ae2fSFeng Wu 		ret = chip->irq_set_vcpu_affinity(data, vcpu_info);
419fcf1ae2fSFeng Wu 	irq_put_desc_unlock(desc, flags);
420fcf1ae2fSFeng Wu 
421fcf1ae2fSFeng Wu 	return ret;
422fcf1ae2fSFeng Wu }
423fcf1ae2fSFeng Wu EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
424fcf1ae2fSFeng Wu 
42579ff1cdaSJiang Liu void __disable_irq(struct irq_desc *desc)
4260a0c5168SRafael J. Wysocki {
4273aae994fSThomas Gleixner 	if (!desc->depth++)
42887923470SThomas Gleixner 		irq_disable(desc);
4290a0c5168SRafael J. Wysocki }
4300a0c5168SRafael J. Wysocki 
43102725e74SThomas Gleixner static int __disable_irq_nosync(unsigned int irq)
43202725e74SThomas Gleixner {
43302725e74SThomas Gleixner 	unsigned long flags;
43431d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
43502725e74SThomas Gleixner 
43602725e74SThomas Gleixner 	if (!desc)
43702725e74SThomas Gleixner 		return -EINVAL;
43879ff1cdaSJiang Liu 	__disable_irq(desc);
43902725e74SThomas Gleixner 	irq_put_desc_busunlock(desc, flags);
44002725e74SThomas Gleixner 	return 0;
44102725e74SThomas Gleixner }
44202725e74SThomas Gleixner 
4431da177e4SLinus Torvalds /**
4441da177e4SLinus Torvalds  *	disable_irq_nosync - disable an irq without waiting
4451da177e4SLinus Torvalds  *	@irq: Interrupt to disable
4461da177e4SLinus Torvalds  *
4471da177e4SLinus Torvalds  *	Disable the selected interrupt line.  Disables and Enables are
4481da177e4SLinus Torvalds  *	nested.
4491da177e4SLinus Torvalds  *	Unlike disable_irq(), this function does not ensure existing
4501da177e4SLinus Torvalds  *	instances of the IRQ handler have completed before returning.
4511da177e4SLinus Torvalds  *
4521da177e4SLinus Torvalds  *	This function may be called from IRQ context.
4531da177e4SLinus Torvalds  */
4541da177e4SLinus Torvalds void disable_irq_nosync(unsigned int irq)
4551da177e4SLinus Torvalds {
45602725e74SThomas Gleixner 	__disable_irq_nosync(irq);
4571da177e4SLinus Torvalds }
4581da177e4SLinus Torvalds EXPORT_SYMBOL(disable_irq_nosync);
4591da177e4SLinus Torvalds 
4601da177e4SLinus Torvalds /**
4611da177e4SLinus Torvalds  *	disable_irq - disable an irq and wait for completion
4621da177e4SLinus Torvalds  *	@irq: Interrupt to disable
4631da177e4SLinus Torvalds  *
4641da177e4SLinus Torvalds  *	Disable the selected interrupt line.  Enables and Disables are
4651da177e4SLinus Torvalds  *	nested.
4661da177e4SLinus Torvalds  *	This function waits for any pending IRQ handlers for this interrupt
4671da177e4SLinus Torvalds  *	to complete before returning. If you use this function while
4681da177e4SLinus Torvalds  *	holding a resource the IRQ handler may need you will deadlock.
4691da177e4SLinus Torvalds  *
4701da177e4SLinus Torvalds  *	This function may be called - with care - from IRQ context.
4711da177e4SLinus Torvalds  */
4721da177e4SLinus Torvalds void disable_irq(unsigned int irq)
4731da177e4SLinus Torvalds {
47402725e74SThomas Gleixner 	if (!__disable_irq_nosync(irq))
4751da177e4SLinus Torvalds 		synchronize_irq(irq);
4761da177e4SLinus Torvalds }
4771da177e4SLinus Torvalds EXPORT_SYMBOL(disable_irq);
4781da177e4SLinus Torvalds 
47902cea395SPeter Zijlstra /**
48002cea395SPeter Zijlstra  *	disable_hardirq - disables an irq and waits for hardirq completion
48102cea395SPeter Zijlstra  *	@irq: Interrupt to disable
48202cea395SPeter Zijlstra  *
48302cea395SPeter Zijlstra  *	Disable the selected interrupt line.  Enables and Disables are
48402cea395SPeter Zijlstra  *	nested.
48502cea395SPeter Zijlstra  *	This function waits for any pending hard IRQ handlers for this
48602cea395SPeter Zijlstra  *	interrupt to complete before returning. If you use this function while
48702cea395SPeter Zijlstra  *	holding a resource the hard IRQ handler may need you will deadlock.
48802cea395SPeter Zijlstra  *
48902cea395SPeter Zijlstra  *	When used to optimistically disable an interrupt from atomic context
49002cea395SPeter Zijlstra  *	the return value must be checked.
49102cea395SPeter Zijlstra  *
49202cea395SPeter Zijlstra  *	Returns: false if a threaded handler is active.
49302cea395SPeter Zijlstra  *
49402cea395SPeter Zijlstra  *	This function may be called - with care - from IRQ context.
49502cea395SPeter Zijlstra  */
49602cea395SPeter Zijlstra bool disable_hardirq(unsigned int irq)
49702cea395SPeter Zijlstra {
49802cea395SPeter Zijlstra 	if (!__disable_irq_nosync(irq))
49902cea395SPeter Zijlstra 		return synchronize_hardirq(irq);
50002cea395SPeter Zijlstra 
50102cea395SPeter Zijlstra 	return false;
50202cea395SPeter Zijlstra }
50302cea395SPeter Zijlstra EXPORT_SYMBOL_GPL(disable_hardirq);
50402cea395SPeter Zijlstra 
50579ff1cdaSJiang Liu void __enable_irq(struct irq_desc *desc)
5061adb0850SThomas Gleixner {
5071adb0850SThomas Gleixner 	switch (desc->depth) {
5081adb0850SThomas Gleixner 	case 0:
5090a0c5168SRafael J. Wysocki  err_out:
51079ff1cdaSJiang Liu 		WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
51179ff1cdaSJiang Liu 		     irq_desc_get_irq(desc));
5121adb0850SThomas Gleixner 		break;
5131adb0850SThomas Gleixner 	case 1: {
514c531e836SThomas Gleixner 		if (desc->istate & IRQS_SUSPENDED)
5150a0c5168SRafael J. Wysocki 			goto err_out;
5161adb0850SThomas Gleixner 		/* Prevent probing on this irq: */
5171ccb4e61SThomas Gleixner 		irq_settings_set_noprobe(desc);
5183aae994fSThomas Gleixner 		irq_enable(desc);
5190798abebSJiang Liu 		check_irq_resend(desc);
5201adb0850SThomas Gleixner 		/* fall-through */
5211adb0850SThomas Gleixner 	}
5221adb0850SThomas Gleixner 	default:
5231adb0850SThomas Gleixner 		desc->depth--;
5241adb0850SThomas Gleixner 	}
5251adb0850SThomas Gleixner }
5261adb0850SThomas Gleixner 
5271da177e4SLinus Torvalds /**
5281da177e4SLinus Torvalds  *	enable_irq - enable handling of an irq
5291da177e4SLinus Torvalds  *	@irq: Interrupt to enable
5301da177e4SLinus Torvalds  *
5311da177e4SLinus Torvalds  *	Undoes the effect of one call to disable_irq().  If this
5321da177e4SLinus Torvalds  *	matches the last disable, processing of interrupts on this
5331da177e4SLinus Torvalds  *	IRQ line is re-enabled.
5341da177e4SLinus Torvalds  *
53570aedd24SThomas Gleixner  *	This function may be called from IRQ context only when
5366b8ff312SThomas Gleixner  *	desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
5371da177e4SLinus Torvalds  */
5381da177e4SLinus Torvalds void enable_irq(unsigned int irq)
5391da177e4SLinus Torvalds {
5401da177e4SLinus Torvalds 	unsigned long flags;
54131d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
5421da177e4SLinus Torvalds 
5437d94f7caSYinghai Lu 	if (!desc)
544c2b5a251SMatthew Wilcox 		return;
54550f7c032SThomas Gleixner 	if (WARN(!desc->irq_data.chip,
5462656c366SThomas Gleixner 		 KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
54702725e74SThomas Gleixner 		goto out;
5482656c366SThomas Gleixner 
54979ff1cdaSJiang Liu 	__enable_irq(desc);
55002725e74SThomas Gleixner out:
55102725e74SThomas Gleixner 	irq_put_desc_busunlock(desc, flags);
5521da177e4SLinus Torvalds }
5531da177e4SLinus Torvalds EXPORT_SYMBOL(enable_irq);
5541da177e4SLinus Torvalds 
5550c5d1eb7SDavid Brownell static int set_irq_wake_real(unsigned int irq, unsigned int on)
5562db87321SUwe Kleine-König {
55708678b08SYinghai Lu 	struct irq_desc *desc = irq_to_desc(irq);
5582db87321SUwe Kleine-König 	int ret = -ENXIO;
5592db87321SUwe Kleine-König 
56060f96b41SSantosh Shilimkar 	if (irq_desc_get_chip(desc)->flags &  IRQCHIP_SKIP_SET_WAKE)
56160f96b41SSantosh Shilimkar 		return 0;
56260f96b41SSantosh Shilimkar 
5632f7e99bbSThomas Gleixner 	if (desc->irq_data.chip->irq_set_wake)
5642f7e99bbSThomas Gleixner 		ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
5652db87321SUwe Kleine-König 
5662db87321SUwe Kleine-König 	return ret;
5672db87321SUwe Kleine-König }
5682db87321SUwe Kleine-König 
569ba9a2331SThomas Gleixner /**
570a0cd9ca2SThomas Gleixner  *	irq_set_irq_wake - control irq power management wakeup
571ba9a2331SThomas Gleixner  *	@irq:	interrupt to control
572ba9a2331SThomas Gleixner  *	@on:	enable/disable power management wakeup
573ba9a2331SThomas Gleixner  *
57415a647ebSDavid Brownell  *	Enable/disable power management wakeup mode, which is
57515a647ebSDavid Brownell  *	disabled by default.  Enables and disables must match,
57615a647ebSDavid Brownell  *	just as they match for non-wakeup mode support.
57715a647ebSDavid Brownell  *
57815a647ebSDavid Brownell  *	Wakeup mode lets this IRQ wake the system from sleep
57915a647ebSDavid Brownell  *	states like "suspend to RAM".
580ba9a2331SThomas Gleixner  */
581a0cd9ca2SThomas Gleixner int irq_set_irq_wake(unsigned int irq, unsigned int on)
582ba9a2331SThomas Gleixner {
583ba9a2331SThomas Gleixner 	unsigned long flags;
58431d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
5852db87321SUwe Kleine-König 	int ret = 0;
586ba9a2331SThomas Gleixner 
58713863a66SJesper Juhl 	if (!desc)
58813863a66SJesper Juhl 		return -EINVAL;
58913863a66SJesper Juhl 
59015a647ebSDavid Brownell 	/* wakeup-capable irqs can be shared between drivers that
59115a647ebSDavid Brownell 	 * don't need to have the same sleep mode behaviors.
59215a647ebSDavid Brownell 	 */
59315a647ebSDavid Brownell 	if (on) {
5942db87321SUwe Kleine-König 		if (desc->wake_depth++ == 0) {
5952db87321SUwe Kleine-König 			ret = set_irq_wake_real(irq, on);
5962db87321SUwe Kleine-König 			if (ret)
5972db87321SUwe Kleine-König 				desc->wake_depth = 0;
59815a647ebSDavid Brownell 			else
5997f94226fSThomas Gleixner 				irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
6002db87321SUwe Kleine-König 		}
60115a647ebSDavid Brownell 	} else {
60215a647ebSDavid Brownell 		if (desc->wake_depth == 0) {
6037a2c4770SArjan van de Ven 			WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
6042db87321SUwe Kleine-König 		} else if (--desc->wake_depth == 0) {
6052db87321SUwe Kleine-König 			ret = set_irq_wake_real(irq, on);
6062db87321SUwe Kleine-König 			if (ret)
6072db87321SUwe Kleine-König 				desc->wake_depth = 1;
60815a647ebSDavid Brownell 			else
6097f94226fSThomas Gleixner 				irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
61015a647ebSDavid Brownell 		}
6112db87321SUwe Kleine-König 	}
61202725e74SThomas Gleixner 	irq_put_desc_busunlock(desc, flags);
613ba9a2331SThomas Gleixner 	return ret;
614ba9a2331SThomas Gleixner }
615a0cd9ca2SThomas Gleixner EXPORT_SYMBOL(irq_set_irq_wake);
616ba9a2331SThomas Gleixner 
6171da177e4SLinus Torvalds /*
6181da177e4SLinus Torvalds  * Internal function that tells the architecture code whether a
6191da177e4SLinus Torvalds  * particular irq has been exclusively allocated or is available
6201da177e4SLinus Torvalds  * for driver use.
6211da177e4SLinus Torvalds  */
6221da177e4SLinus Torvalds int can_request_irq(unsigned int irq, unsigned long irqflags)
6231da177e4SLinus Torvalds {
624cc8c3b78SThomas Gleixner 	unsigned long flags;
62531d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
62602725e74SThomas Gleixner 	int canrequest = 0;
6271da177e4SLinus Torvalds 
6287d94f7caSYinghai Lu 	if (!desc)
6297d94f7caSYinghai Lu 		return 0;
6307d94f7caSYinghai Lu 
63102725e74SThomas Gleixner 	if (irq_settings_can_request(desc)) {
6322779db8dSBen Hutchings 		if (!desc->action ||
6332779db8dSBen Hutchings 		    irqflags & desc->action->flags & IRQF_SHARED)
63402725e74SThomas Gleixner 			canrequest = 1;
63502725e74SThomas Gleixner 	}
63602725e74SThomas Gleixner 	irq_put_desc_unlock(desc, flags);
63702725e74SThomas Gleixner 	return canrequest;
6381da177e4SLinus Torvalds }
6391da177e4SLinus Torvalds 
640a1ff541aSJiang Liu int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
64182736f4dSUwe Kleine-König {
6426b8ff312SThomas Gleixner 	struct irq_chip *chip = desc->irq_data.chip;
643d4d5e089SThomas Gleixner 	int ret, unmask = 0;
64482736f4dSUwe Kleine-König 
645b2ba2c30SThomas Gleixner 	if (!chip || !chip->irq_set_type) {
64682736f4dSUwe Kleine-König 		/*
64782736f4dSUwe Kleine-König 		 * IRQF_TRIGGER_* but the PIC does not support multiple
64882736f4dSUwe Kleine-König 		 * flow-types?
64982736f4dSUwe Kleine-König 		 */
650a1ff541aSJiang Liu 		pr_debug("No set_type function for IRQ %d (%s)\n",
651a1ff541aSJiang Liu 			 irq_desc_get_irq(desc),
65282736f4dSUwe Kleine-König 			 chip ? (chip->name ? : "unknown") : "unknown");
65382736f4dSUwe Kleine-König 		return 0;
65482736f4dSUwe Kleine-König 	}
65582736f4dSUwe Kleine-König 
656876dbd4cSThomas Gleixner 	flags &= IRQ_TYPE_SENSE_MASK;
657d4d5e089SThomas Gleixner 
658d4d5e089SThomas Gleixner 	if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
65932f4125eSThomas Gleixner 		if (!irqd_irq_masked(&desc->irq_data))
660d4d5e089SThomas Gleixner 			mask_irq(desc);
66132f4125eSThomas Gleixner 		if (!irqd_irq_disabled(&desc->irq_data))
662d4d5e089SThomas Gleixner 			unmask = 1;
663d4d5e089SThomas Gleixner 	}
664d4d5e089SThomas Gleixner 
665f2b662daSDavid Brownell 	/* caller masked out all except trigger mode flags */
666b2ba2c30SThomas Gleixner 	ret = chip->irq_set_type(&desc->irq_data, flags);
66782736f4dSUwe Kleine-König 
668876dbd4cSThomas Gleixner 	switch (ret) {
669876dbd4cSThomas Gleixner 	case IRQ_SET_MASK_OK:
6702cb62547SJiang Liu 	case IRQ_SET_MASK_OK_DONE:
671876dbd4cSThomas Gleixner 		irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
672876dbd4cSThomas Gleixner 		irqd_set(&desc->irq_data, flags);
673876dbd4cSThomas Gleixner 
674876dbd4cSThomas Gleixner 	case IRQ_SET_MASK_OK_NOCOPY:
675876dbd4cSThomas Gleixner 		flags = irqd_get_trigger_type(&desc->irq_data);
676876dbd4cSThomas Gleixner 		irq_settings_set_trigger_mask(desc, flags);
677876dbd4cSThomas Gleixner 		irqd_clear(&desc->irq_data, IRQD_LEVEL);
678876dbd4cSThomas Gleixner 		irq_settings_clr_level(desc);
679876dbd4cSThomas Gleixner 		if (flags & IRQ_TYPE_LEVEL_MASK) {
680876dbd4cSThomas Gleixner 			irq_settings_set_level(desc);
681876dbd4cSThomas Gleixner 			irqd_set(&desc->irq_data, IRQD_LEVEL);
682876dbd4cSThomas Gleixner 		}
68346732475SThomas Gleixner 
684d4d5e089SThomas Gleixner 		ret = 0;
6858fff39e0SThomas Gleixner 		break;
686876dbd4cSThomas Gleixner 	default:
68797fd75b7SAndrew Morton 		pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
688a1ff541aSJiang Liu 		       flags, irq_desc_get_irq(desc), chip->irq_set_type);
6890c5d1eb7SDavid Brownell 	}
690d4d5e089SThomas Gleixner 	if (unmask)
691d4d5e089SThomas Gleixner 		unmask_irq(desc);
69282736f4dSUwe Kleine-König 	return ret;
69382736f4dSUwe Kleine-König }
69482736f4dSUwe Kleine-König 
695293a7a0aSThomas Gleixner #ifdef CONFIG_HARDIRQS_SW_RESEND
696293a7a0aSThomas Gleixner int irq_set_parent(int irq, int parent_irq)
697293a7a0aSThomas Gleixner {
698293a7a0aSThomas Gleixner 	unsigned long flags;
699293a7a0aSThomas Gleixner 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
700293a7a0aSThomas Gleixner 
701293a7a0aSThomas Gleixner 	if (!desc)
702293a7a0aSThomas Gleixner 		return -EINVAL;
703293a7a0aSThomas Gleixner 
704293a7a0aSThomas Gleixner 	desc->parent_irq = parent_irq;
705293a7a0aSThomas Gleixner 
706293a7a0aSThomas Gleixner 	irq_put_desc_unlock(desc, flags);
707293a7a0aSThomas Gleixner 	return 0;
708293a7a0aSThomas Gleixner }
709293a7a0aSThomas Gleixner #endif
710293a7a0aSThomas Gleixner 
711b25c340cSThomas Gleixner /*
712b25c340cSThomas Gleixner  * Default primary interrupt handler for threaded interrupts. Is
713b25c340cSThomas Gleixner  * assigned as primary handler when request_threaded_irq is called
714b25c340cSThomas Gleixner  * with handler == NULL. Useful for oneshot interrupts.
715b25c340cSThomas Gleixner  */
716b25c340cSThomas Gleixner static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
717b25c340cSThomas Gleixner {
718b25c340cSThomas Gleixner 	return IRQ_WAKE_THREAD;
719b25c340cSThomas Gleixner }
720b25c340cSThomas Gleixner 
721399b5da2SThomas Gleixner /*
722399b5da2SThomas Gleixner  * Primary handler for nested threaded interrupts. Should never be
723399b5da2SThomas Gleixner  * called.
724399b5da2SThomas Gleixner  */
725399b5da2SThomas Gleixner static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
726399b5da2SThomas Gleixner {
727399b5da2SThomas Gleixner 	WARN(1, "Primary handler called for nested irq %d\n", irq);
728399b5da2SThomas Gleixner 	return IRQ_NONE;
729399b5da2SThomas Gleixner }
730399b5da2SThomas Gleixner 
7312a1d3ab8SThomas Gleixner static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
7322a1d3ab8SThomas Gleixner {
7332a1d3ab8SThomas Gleixner 	WARN(1, "Secondary action handler called for irq %d\n", irq);
7342a1d3ab8SThomas Gleixner 	return IRQ_NONE;
7352a1d3ab8SThomas Gleixner }
7362a1d3ab8SThomas Gleixner 
7373aa551c9SThomas Gleixner static int irq_wait_for_interrupt(struct irqaction *action)
7383aa551c9SThomas Gleixner {
7393aa551c9SThomas Gleixner 	set_current_state(TASK_INTERRUPTIBLE);
740f48fe81eSThomas Gleixner 
741550acb19SIdo Yariv 	while (!kthread_should_stop()) {
742550acb19SIdo Yariv 
743f48fe81eSThomas Gleixner 		if (test_and_clear_bit(IRQTF_RUNTHREAD,
744f48fe81eSThomas Gleixner 				       &action->thread_flags)) {
7453aa551c9SThomas Gleixner 			__set_current_state(TASK_RUNNING);
7463aa551c9SThomas Gleixner 			return 0;
747f48fe81eSThomas Gleixner 		}
7483aa551c9SThomas Gleixner 		schedule();
749550acb19SIdo Yariv 		set_current_state(TASK_INTERRUPTIBLE);
7503aa551c9SThomas Gleixner 	}
751550acb19SIdo Yariv 	__set_current_state(TASK_RUNNING);
7523aa551c9SThomas Gleixner 	return -1;
7533aa551c9SThomas Gleixner }
7543aa551c9SThomas Gleixner 
755b25c340cSThomas Gleixner /*
756b25c340cSThomas Gleixner  * Oneshot interrupts keep the irq line masked until the threaded
757b25c340cSThomas Gleixner  * handler finished. unmask if the interrupt has not been disabled and
758b25c340cSThomas Gleixner  * is marked MASKED.
759b25c340cSThomas Gleixner  */
760b5faba21SThomas Gleixner static void irq_finalize_oneshot(struct irq_desc *desc,
761f3f79e38SAlexander Gordeev 				 struct irqaction *action)
762b25c340cSThomas Gleixner {
7632a1d3ab8SThomas Gleixner 	if (!(desc->istate & IRQS_ONESHOT) ||
7642a1d3ab8SThomas Gleixner 	    action->handler == irq_forced_secondary_handler)
765b5faba21SThomas Gleixner 		return;
7660b1adaa0SThomas Gleixner again:
7673876ec9eSThomas Gleixner 	chip_bus_lock(desc);
768239007b8SThomas Gleixner 	raw_spin_lock_irq(&desc->lock);
7690b1adaa0SThomas Gleixner 
7700b1adaa0SThomas Gleixner 	/*
7710b1adaa0SThomas Gleixner 	 * Implausible though it may be we need to protect us against
7720b1adaa0SThomas Gleixner 	 * the following scenario:
7730b1adaa0SThomas Gleixner 	 *
7740b1adaa0SThomas Gleixner 	 * The thread is faster done than the hard interrupt handler
7750b1adaa0SThomas Gleixner 	 * on the other CPU. If we unmask the irq line then the
7760b1adaa0SThomas Gleixner 	 * interrupt can come in again and masks the line, leaves due
777009b4c3bSThomas Gleixner 	 * to IRQS_INPROGRESS and the irq line is masked forever.
778b5faba21SThomas Gleixner 	 *
779b5faba21SThomas Gleixner 	 * This also serializes the state of shared oneshot handlers
780b5faba21SThomas Gleixner 	 * versus "desc->threads_onehsot |= action->thread_mask;" in
781b5faba21SThomas Gleixner 	 * irq_wake_thread(). See the comment there which explains the
782b5faba21SThomas Gleixner 	 * serialization.
7830b1adaa0SThomas Gleixner 	 */
78432f4125eSThomas Gleixner 	if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
7850b1adaa0SThomas Gleixner 		raw_spin_unlock_irq(&desc->lock);
7863876ec9eSThomas Gleixner 		chip_bus_sync_unlock(desc);
7870b1adaa0SThomas Gleixner 		cpu_relax();
7880b1adaa0SThomas Gleixner 		goto again;
7890b1adaa0SThomas Gleixner 	}
7900b1adaa0SThomas Gleixner 
791b5faba21SThomas Gleixner 	/*
792b5faba21SThomas Gleixner 	 * Now check again, whether the thread should run. Otherwise
793b5faba21SThomas Gleixner 	 * we would clear the threads_oneshot bit of this thread which
794b5faba21SThomas Gleixner 	 * was just set.
795b5faba21SThomas Gleixner 	 */
796f3f79e38SAlexander Gordeev 	if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
797b5faba21SThomas Gleixner 		goto out_unlock;
798b5faba21SThomas Gleixner 
799b5faba21SThomas Gleixner 	desc->threads_oneshot &= ~action->thread_mask;
800b5faba21SThomas Gleixner 
80132f4125eSThomas Gleixner 	if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
80232f4125eSThomas Gleixner 	    irqd_irq_masked(&desc->irq_data))
803328a4978SThomas Gleixner 		unmask_threaded_irq(desc);
80432f4125eSThomas Gleixner 
805b5faba21SThomas Gleixner out_unlock:
806239007b8SThomas Gleixner 	raw_spin_unlock_irq(&desc->lock);
8073876ec9eSThomas Gleixner 	chip_bus_sync_unlock(desc);
808b25c340cSThomas Gleixner }
809b25c340cSThomas Gleixner 
81061f38261SBruno Premont #ifdef CONFIG_SMP
8113aa551c9SThomas Gleixner /*
812b04c644eSChuansheng Liu  * Check whether we need to change the affinity of the interrupt thread.
813591d2fb0SThomas Gleixner  */
814591d2fb0SThomas Gleixner static void
815591d2fb0SThomas Gleixner irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
816591d2fb0SThomas Gleixner {
817591d2fb0SThomas Gleixner 	cpumask_var_t mask;
81804aa530eSThomas Gleixner 	bool valid = true;
819591d2fb0SThomas Gleixner 
820591d2fb0SThomas Gleixner 	if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
821591d2fb0SThomas Gleixner 		return;
822591d2fb0SThomas Gleixner 
823591d2fb0SThomas Gleixner 	/*
824591d2fb0SThomas Gleixner 	 * In case we are out of memory we set IRQTF_AFFINITY again and
825591d2fb0SThomas Gleixner 	 * try again next time
826591d2fb0SThomas Gleixner 	 */
827591d2fb0SThomas Gleixner 	if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
828591d2fb0SThomas Gleixner 		set_bit(IRQTF_AFFINITY, &action->thread_flags);
829591d2fb0SThomas Gleixner 		return;
830591d2fb0SThomas Gleixner 	}
831591d2fb0SThomas Gleixner 
832239007b8SThomas Gleixner 	raw_spin_lock_irq(&desc->lock);
83304aa530eSThomas Gleixner 	/*
83404aa530eSThomas Gleixner 	 * This code is triggered unconditionally. Check the affinity
83504aa530eSThomas Gleixner 	 * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
83604aa530eSThomas Gleixner 	 */
8379df872faSJiang Liu 	if (desc->irq_common_data.affinity)
8389df872faSJiang Liu 		cpumask_copy(mask, desc->irq_common_data.affinity);
83904aa530eSThomas Gleixner 	else
84004aa530eSThomas Gleixner 		valid = false;
841239007b8SThomas Gleixner 	raw_spin_unlock_irq(&desc->lock);
842591d2fb0SThomas Gleixner 
84304aa530eSThomas Gleixner 	if (valid)
844591d2fb0SThomas Gleixner 		set_cpus_allowed_ptr(current, mask);
845591d2fb0SThomas Gleixner 	free_cpumask_var(mask);
846591d2fb0SThomas Gleixner }
84761f38261SBruno Premont #else
84861f38261SBruno Premont static inline void
84961f38261SBruno Premont irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
85061f38261SBruno Premont #endif
851591d2fb0SThomas Gleixner 
852591d2fb0SThomas Gleixner /*
8538d32a307SThomas Gleixner  * Interrupts which are not explicitely requested as threaded
8548d32a307SThomas Gleixner  * interrupts rely on the implicit bh/preempt disable of the hard irq
8558d32a307SThomas Gleixner  * context. So we need to disable bh here to avoid deadlocks and other
8568d32a307SThomas Gleixner  * side effects.
8578d32a307SThomas Gleixner  */
8583a43e05fSSebastian Andrzej Siewior static irqreturn_t
8598d32a307SThomas Gleixner irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
8608d32a307SThomas Gleixner {
8613a43e05fSSebastian Andrzej Siewior 	irqreturn_t ret;
8623a43e05fSSebastian Andrzej Siewior 
8638d32a307SThomas Gleixner 	local_bh_disable();
8643a43e05fSSebastian Andrzej Siewior 	ret = action->thread_fn(action->irq, action->dev_id);
865f3f79e38SAlexander Gordeev 	irq_finalize_oneshot(desc, action);
8668d32a307SThomas Gleixner 	local_bh_enable();
8673a43e05fSSebastian Andrzej Siewior 	return ret;
8688d32a307SThomas Gleixner }
8698d32a307SThomas Gleixner 
8708d32a307SThomas Gleixner /*
871f788e7bfSXie XiuQi  * Interrupts explicitly requested as threaded interrupts want to be
8728d32a307SThomas Gleixner  * preemtible - many of them need to sleep and wait for slow busses to
8738d32a307SThomas Gleixner  * complete.
8748d32a307SThomas Gleixner  */
8753a43e05fSSebastian Andrzej Siewior static irqreturn_t irq_thread_fn(struct irq_desc *desc,
8763a43e05fSSebastian Andrzej Siewior 		struct irqaction *action)
8778d32a307SThomas Gleixner {
8783a43e05fSSebastian Andrzej Siewior 	irqreturn_t ret;
8793a43e05fSSebastian Andrzej Siewior 
8803a43e05fSSebastian Andrzej Siewior 	ret = action->thread_fn(action->irq, action->dev_id);
881f3f79e38SAlexander Gordeev 	irq_finalize_oneshot(desc, action);
8823a43e05fSSebastian Andrzej Siewior 	return ret;
8838d32a307SThomas Gleixner }
8848d32a307SThomas Gleixner 
8857140ea19SIdo Yariv static void wake_threads_waitq(struct irq_desc *desc)
8867140ea19SIdo Yariv {
887c685689fSChuansheng Liu 	if (atomic_dec_and_test(&desc->threads_active))
8887140ea19SIdo Yariv 		wake_up(&desc->wait_for_threads);
8897140ea19SIdo Yariv }
8907140ea19SIdo Yariv 
89167d12145SAl Viro static void irq_thread_dtor(struct callback_head *unused)
8924d1d61a6SOleg Nesterov {
8934d1d61a6SOleg Nesterov 	struct task_struct *tsk = current;
8944d1d61a6SOleg Nesterov 	struct irq_desc *desc;
8954d1d61a6SOleg Nesterov 	struct irqaction *action;
8964d1d61a6SOleg Nesterov 
8974d1d61a6SOleg Nesterov 	if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
8984d1d61a6SOleg Nesterov 		return;
8994d1d61a6SOleg Nesterov 
9004d1d61a6SOleg Nesterov 	action = kthread_data(tsk);
9014d1d61a6SOleg Nesterov 
902fb21affaSLinus Torvalds 	pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
90319af395dSAlan Cox 	       tsk->comm, tsk->pid, action->irq);
9044d1d61a6SOleg Nesterov 
9054d1d61a6SOleg Nesterov 
9064d1d61a6SOleg Nesterov 	desc = irq_to_desc(action->irq);
9074d1d61a6SOleg Nesterov 	/*
9084d1d61a6SOleg Nesterov 	 * If IRQTF_RUNTHREAD is set, we need to decrement
9094d1d61a6SOleg Nesterov 	 * desc->threads_active and wake possible waiters.
9104d1d61a6SOleg Nesterov 	 */
9114d1d61a6SOleg Nesterov 	if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
9124d1d61a6SOleg Nesterov 		wake_threads_waitq(desc);
9134d1d61a6SOleg Nesterov 
9144d1d61a6SOleg Nesterov 	/* Prevent a stale desc->threads_oneshot */
9154d1d61a6SOleg Nesterov 	irq_finalize_oneshot(desc, action);
9164d1d61a6SOleg Nesterov }
9174d1d61a6SOleg Nesterov 
9182a1d3ab8SThomas Gleixner static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
9192a1d3ab8SThomas Gleixner {
9202a1d3ab8SThomas Gleixner 	struct irqaction *secondary = action->secondary;
9212a1d3ab8SThomas Gleixner 
9222a1d3ab8SThomas Gleixner 	if (WARN_ON_ONCE(!secondary))
9232a1d3ab8SThomas Gleixner 		return;
9242a1d3ab8SThomas Gleixner 
9252a1d3ab8SThomas Gleixner 	raw_spin_lock_irq(&desc->lock);
9262a1d3ab8SThomas Gleixner 	__irq_wake_thread(desc, secondary);
9272a1d3ab8SThomas Gleixner 	raw_spin_unlock_irq(&desc->lock);
9282a1d3ab8SThomas Gleixner }
9292a1d3ab8SThomas Gleixner 
9308d32a307SThomas Gleixner /*
9313aa551c9SThomas Gleixner  * Interrupt handler thread
9323aa551c9SThomas Gleixner  */
9333aa551c9SThomas Gleixner static int irq_thread(void *data)
9343aa551c9SThomas Gleixner {
93567d12145SAl Viro 	struct callback_head on_exit_work;
9363aa551c9SThomas Gleixner 	struct irqaction *action = data;
9373aa551c9SThomas Gleixner 	struct irq_desc *desc = irq_to_desc(action->irq);
9383a43e05fSSebastian Andrzej Siewior 	irqreturn_t (*handler_fn)(struct irq_desc *desc,
9393a43e05fSSebastian Andrzej Siewior 			struct irqaction *action);
9403aa551c9SThomas Gleixner 
941540b60e2SAlexander Gordeev 	if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
9428d32a307SThomas Gleixner 					&action->thread_flags))
9438d32a307SThomas Gleixner 		handler_fn = irq_forced_thread_fn;
9448d32a307SThomas Gleixner 	else
9458d32a307SThomas Gleixner 		handler_fn = irq_thread_fn;
9468d32a307SThomas Gleixner 
94741f9d29fSAl Viro 	init_task_work(&on_exit_work, irq_thread_dtor);
9484d1d61a6SOleg Nesterov 	task_work_add(current, &on_exit_work, false);
9493aa551c9SThomas Gleixner 
950f3de44edSSankara Muthukrishnan 	irq_thread_check_affinity(desc, action);
951f3de44edSSankara Muthukrishnan 
9523aa551c9SThomas Gleixner 	while (!irq_wait_for_interrupt(action)) {
9537140ea19SIdo Yariv 		irqreturn_t action_ret;
9543aa551c9SThomas Gleixner 
955591d2fb0SThomas Gleixner 		irq_thread_check_affinity(desc, action);
956591d2fb0SThomas Gleixner 
9573a43e05fSSebastian Andrzej Siewior 		action_ret = handler_fn(desc, action);
9581e77d0a1SThomas Gleixner 		if (action_ret == IRQ_HANDLED)
9591e77d0a1SThomas Gleixner 			atomic_inc(&desc->threads_handled);
9602a1d3ab8SThomas Gleixner 		if (action_ret == IRQ_WAKE_THREAD)
9612a1d3ab8SThomas Gleixner 			irq_wake_secondary(desc, action);
9627140ea19SIdo Yariv 
9637140ea19SIdo Yariv 		wake_threads_waitq(desc);
9643aa551c9SThomas Gleixner 	}
9653aa551c9SThomas Gleixner 
9667140ea19SIdo Yariv 	/*
9677140ea19SIdo Yariv 	 * This is the regular exit path. __free_irq() is stopping the
9687140ea19SIdo Yariv 	 * thread via kthread_stop() after calling
9697140ea19SIdo Yariv 	 * synchronize_irq(). So neither IRQTF_RUNTHREAD nor the
970e04268b0SThomas Gleixner 	 * oneshot mask bit can be set. We cannot verify that as we
971e04268b0SThomas Gleixner 	 * cannot touch the oneshot mask at this point anymore as
972e04268b0SThomas Gleixner 	 * __setup_irq() might have given out currents thread_mask
973e04268b0SThomas Gleixner 	 * again.
9743aa551c9SThomas Gleixner 	 */
9754d1d61a6SOleg Nesterov 	task_work_cancel(current, irq_thread_dtor);
9763aa551c9SThomas Gleixner 	return 0;
9773aa551c9SThomas Gleixner }
9783aa551c9SThomas Gleixner 
979a92444c6SThomas Gleixner /**
980a92444c6SThomas Gleixner  *	irq_wake_thread - wake the irq thread for the action identified by dev_id
981a92444c6SThomas Gleixner  *	@irq:		Interrupt line
982a92444c6SThomas Gleixner  *	@dev_id:	Device identity for which the thread should be woken
983a92444c6SThomas Gleixner  *
984a92444c6SThomas Gleixner  */
985a92444c6SThomas Gleixner void irq_wake_thread(unsigned int irq, void *dev_id)
986a92444c6SThomas Gleixner {
987a92444c6SThomas Gleixner 	struct irq_desc *desc = irq_to_desc(irq);
988a92444c6SThomas Gleixner 	struct irqaction *action;
989a92444c6SThomas Gleixner 	unsigned long flags;
990a92444c6SThomas Gleixner 
991a92444c6SThomas Gleixner 	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
992a92444c6SThomas Gleixner 		return;
993a92444c6SThomas Gleixner 
994a92444c6SThomas Gleixner 	raw_spin_lock_irqsave(&desc->lock, flags);
995f944b5a7SDaniel Lezcano 	for_each_action_of_desc(desc, action) {
996a92444c6SThomas Gleixner 		if (action->dev_id == dev_id) {
997a92444c6SThomas Gleixner 			if (action->thread)
998a92444c6SThomas Gleixner 				__irq_wake_thread(desc, action);
999a92444c6SThomas Gleixner 			break;
1000a92444c6SThomas Gleixner 		}
1001a92444c6SThomas Gleixner 	}
1002a92444c6SThomas Gleixner 	raw_spin_unlock_irqrestore(&desc->lock, flags);
1003a92444c6SThomas Gleixner }
1004a92444c6SThomas Gleixner EXPORT_SYMBOL_GPL(irq_wake_thread);
1005a92444c6SThomas Gleixner 
10062a1d3ab8SThomas Gleixner static int irq_setup_forced_threading(struct irqaction *new)
10078d32a307SThomas Gleixner {
10088d32a307SThomas Gleixner 	if (!force_irqthreads)
10092a1d3ab8SThomas Gleixner 		return 0;
10108d32a307SThomas Gleixner 	if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
10112a1d3ab8SThomas Gleixner 		return 0;
10128d32a307SThomas Gleixner 
10138d32a307SThomas Gleixner 	new->flags |= IRQF_ONESHOT;
10148d32a307SThomas Gleixner 
10152a1d3ab8SThomas Gleixner 	/*
10162a1d3ab8SThomas Gleixner 	 * Handle the case where we have a real primary handler and a
10172a1d3ab8SThomas Gleixner 	 * thread handler. We force thread them as well by creating a
10182a1d3ab8SThomas Gleixner 	 * secondary action.
10192a1d3ab8SThomas Gleixner 	 */
10202a1d3ab8SThomas Gleixner 	if (new->handler != irq_default_primary_handler && new->thread_fn) {
10212a1d3ab8SThomas Gleixner 		/* Allocate the secondary action */
10222a1d3ab8SThomas Gleixner 		new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
10232a1d3ab8SThomas Gleixner 		if (!new->secondary)
10242a1d3ab8SThomas Gleixner 			return -ENOMEM;
10252a1d3ab8SThomas Gleixner 		new->secondary->handler = irq_forced_secondary_handler;
10262a1d3ab8SThomas Gleixner 		new->secondary->thread_fn = new->thread_fn;
10272a1d3ab8SThomas Gleixner 		new->secondary->dev_id = new->dev_id;
10282a1d3ab8SThomas Gleixner 		new->secondary->irq = new->irq;
10292a1d3ab8SThomas Gleixner 		new->secondary->name = new->name;
10302a1d3ab8SThomas Gleixner 	}
10312a1d3ab8SThomas Gleixner 	/* Deal with the primary handler */
10328d32a307SThomas Gleixner 	set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
10338d32a307SThomas Gleixner 	new->thread_fn = new->handler;
10348d32a307SThomas Gleixner 	new->handler = irq_default_primary_handler;
10352a1d3ab8SThomas Gleixner 	return 0;
10368d32a307SThomas Gleixner }
10378d32a307SThomas Gleixner 
1038c1bacbaeSThomas Gleixner static int irq_request_resources(struct irq_desc *desc)
1039c1bacbaeSThomas Gleixner {
1040c1bacbaeSThomas Gleixner 	struct irq_data *d = &desc->irq_data;
1041c1bacbaeSThomas Gleixner 	struct irq_chip *c = d->chip;
1042c1bacbaeSThomas Gleixner 
1043c1bacbaeSThomas Gleixner 	return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1044c1bacbaeSThomas Gleixner }
1045c1bacbaeSThomas Gleixner 
1046c1bacbaeSThomas Gleixner static void irq_release_resources(struct irq_desc *desc)
1047c1bacbaeSThomas Gleixner {
1048c1bacbaeSThomas Gleixner 	struct irq_data *d = &desc->irq_data;
1049c1bacbaeSThomas Gleixner 	struct irq_chip *c = d->chip;
1050c1bacbaeSThomas Gleixner 
1051c1bacbaeSThomas Gleixner 	if (c->irq_release_resources)
1052c1bacbaeSThomas Gleixner 		c->irq_release_resources(d);
1053c1bacbaeSThomas Gleixner }
1054c1bacbaeSThomas Gleixner 
10552a1d3ab8SThomas Gleixner static int
10562a1d3ab8SThomas Gleixner setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
10572a1d3ab8SThomas Gleixner {
10582a1d3ab8SThomas Gleixner 	struct task_struct *t;
10592a1d3ab8SThomas Gleixner 	struct sched_param param = {
10602a1d3ab8SThomas Gleixner 		.sched_priority = MAX_USER_RT_PRIO/2,
10612a1d3ab8SThomas Gleixner 	};
10622a1d3ab8SThomas Gleixner 
10632a1d3ab8SThomas Gleixner 	if (!secondary) {
10642a1d3ab8SThomas Gleixner 		t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
10652a1d3ab8SThomas Gleixner 				   new->name);
10662a1d3ab8SThomas Gleixner 	} else {
10672a1d3ab8SThomas Gleixner 		t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
10682a1d3ab8SThomas Gleixner 				   new->name);
10692a1d3ab8SThomas Gleixner 		param.sched_priority -= 1;
10702a1d3ab8SThomas Gleixner 	}
10712a1d3ab8SThomas Gleixner 
10722a1d3ab8SThomas Gleixner 	if (IS_ERR(t))
10732a1d3ab8SThomas Gleixner 		return PTR_ERR(t);
10742a1d3ab8SThomas Gleixner 
10752a1d3ab8SThomas Gleixner 	sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
10762a1d3ab8SThomas Gleixner 
10772a1d3ab8SThomas Gleixner 	/*
10782a1d3ab8SThomas Gleixner 	 * We keep the reference to the task struct even if
10792a1d3ab8SThomas Gleixner 	 * the thread dies to avoid that the interrupt code
10802a1d3ab8SThomas Gleixner 	 * references an already freed task_struct.
10812a1d3ab8SThomas Gleixner 	 */
10822a1d3ab8SThomas Gleixner 	get_task_struct(t);
10832a1d3ab8SThomas Gleixner 	new->thread = t;
10842a1d3ab8SThomas Gleixner 	/*
10852a1d3ab8SThomas Gleixner 	 * Tell the thread to set its affinity. This is
10862a1d3ab8SThomas Gleixner 	 * important for shared interrupt handlers as we do
10872a1d3ab8SThomas Gleixner 	 * not invoke setup_affinity() for the secondary
10882a1d3ab8SThomas Gleixner 	 * handlers as everything is already set up. Even for
10892a1d3ab8SThomas Gleixner 	 * interrupts marked with IRQF_NO_BALANCE this is
10902a1d3ab8SThomas Gleixner 	 * correct as we want the thread to move to the cpu(s)
10912a1d3ab8SThomas Gleixner 	 * on which the requesting code placed the interrupt.
10922a1d3ab8SThomas Gleixner 	 */
10932a1d3ab8SThomas Gleixner 	set_bit(IRQTF_AFFINITY, &new->thread_flags);
10942a1d3ab8SThomas Gleixner 	return 0;
10952a1d3ab8SThomas Gleixner }
10962a1d3ab8SThomas Gleixner 
10971da177e4SLinus Torvalds /*
10981da177e4SLinus Torvalds  * Internal function to register an irqaction - typically used to
10991da177e4SLinus Torvalds  * allocate special interrupts that are part of the architecture.
11001da177e4SLinus Torvalds  */
1101d3c60047SThomas Gleixner static int
1102d3c60047SThomas Gleixner __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
11031da177e4SLinus Torvalds {
1104f17c7545SIngo Molnar 	struct irqaction *old, **old_ptr;
1105b5faba21SThomas Gleixner 	unsigned long flags, thread_mask = 0;
11063b8249e7SThomas Gleixner 	int ret, nested, shared = 0;
11073b8249e7SThomas Gleixner 	cpumask_var_t mask;
11081da177e4SLinus Torvalds 
11097d94f7caSYinghai Lu 	if (!desc)
1110c2b5a251SMatthew Wilcox 		return -EINVAL;
1111c2b5a251SMatthew Wilcox 
11126b8ff312SThomas Gleixner 	if (desc->irq_data.chip == &no_irq_chip)
11131da177e4SLinus Torvalds 		return -ENOSYS;
1114b6873807SSebastian Andrzej Siewior 	if (!try_module_get(desc->owner))
1115b6873807SSebastian Andrzej Siewior 		return -ENODEV;
11161da177e4SLinus Torvalds 
11172a1d3ab8SThomas Gleixner 	new->irq = irq;
11182a1d3ab8SThomas Gleixner 
11191da177e4SLinus Torvalds 	/*
1120*4b357daeSJon Hunter 	 * If the trigger type is not specified by the caller,
1121*4b357daeSJon Hunter 	 * then use the default for this interrupt.
1122*4b357daeSJon Hunter 	 */
1123*4b357daeSJon Hunter 	if (!(new->flags & IRQF_TRIGGER_MASK))
1124*4b357daeSJon Hunter 		new->flags |= irqd_get_trigger_type(&desc->irq_data);
1125*4b357daeSJon Hunter 
1126*4b357daeSJon Hunter 	/*
1127399b5da2SThomas Gleixner 	 * Check whether the interrupt nests into another interrupt
1128399b5da2SThomas Gleixner 	 * thread.
11293aa551c9SThomas Gleixner 	 */
11301ccb4e61SThomas Gleixner 	nested = irq_settings_is_nested_thread(desc);
1131399b5da2SThomas Gleixner 	if (nested) {
1132b6873807SSebastian Andrzej Siewior 		if (!new->thread_fn) {
1133b6873807SSebastian Andrzej Siewior 			ret = -EINVAL;
1134b6873807SSebastian Andrzej Siewior 			goto out_mput;
1135b6873807SSebastian Andrzej Siewior 		}
1136399b5da2SThomas Gleixner 		/*
1137399b5da2SThomas Gleixner 		 * Replace the primary handler which was provided from
1138399b5da2SThomas Gleixner 		 * the driver for non nested interrupt handling by the
1139399b5da2SThomas Gleixner 		 * dummy function which warns when called.
1140399b5da2SThomas Gleixner 		 */
1141399b5da2SThomas Gleixner 		new->handler = irq_nested_primary_handler;
11428d32a307SThomas Gleixner 	} else {
11432a1d3ab8SThomas Gleixner 		if (irq_settings_can_thread(desc)) {
11442a1d3ab8SThomas Gleixner 			ret = irq_setup_forced_threading(new);
11452a1d3ab8SThomas Gleixner 			if (ret)
11462a1d3ab8SThomas Gleixner 				goto out_mput;
11472a1d3ab8SThomas Gleixner 		}
1148399b5da2SThomas Gleixner 	}
1149399b5da2SThomas Gleixner 
1150399b5da2SThomas Gleixner 	/*
1151399b5da2SThomas Gleixner 	 * Create a handler thread when a thread function is supplied
1152399b5da2SThomas Gleixner 	 * and the interrupt does not nest into another interrupt
1153399b5da2SThomas Gleixner 	 * thread.
1154399b5da2SThomas Gleixner 	 */
1155399b5da2SThomas Gleixner 	if (new->thread_fn && !nested) {
11562a1d3ab8SThomas Gleixner 		ret = setup_irq_thread(new, irq, false);
11572a1d3ab8SThomas Gleixner 		if (ret)
1158b6873807SSebastian Andrzej Siewior 			goto out_mput;
11592a1d3ab8SThomas Gleixner 		if (new->secondary) {
11602a1d3ab8SThomas Gleixner 			ret = setup_irq_thread(new->secondary, irq, true);
11612a1d3ab8SThomas Gleixner 			if (ret)
11622a1d3ab8SThomas Gleixner 				goto out_thread;
1163b6873807SSebastian Andrzej Siewior 		}
11643aa551c9SThomas Gleixner 	}
11653aa551c9SThomas Gleixner 
11663b8249e7SThomas Gleixner 	if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
11673b8249e7SThomas Gleixner 		ret = -ENOMEM;
11683b8249e7SThomas Gleixner 		goto out_thread;
11693b8249e7SThomas Gleixner 	}
11703b8249e7SThomas Gleixner 
11713aa551c9SThomas Gleixner 	/*
1172dc9b229aSThomas Gleixner 	 * Drivers are often written to work w/o knowledge about the
1173dc9b229aSThomas Gleixner 	 * underlying irq chip implementation, so a request for a
1174dc9b229aSThomas Gleixner 	 * threaded irq without a primary hard irq context handler
1175dc9b229aSThomas Gleixner 	 * requires the ONESHOT flag to be set. Some irq chips like
1176dc9b229aSThomas Gleixner 	 * MSI based interrupts are per se one shot safe. Check the
1177dc9b229aSThomas Gleixner 	 * chip flags, so we can avoid the unmask dance at the end of
1178dc9b229aSThomas Gleixner 	 * the threaded handler for those.
1179dc9b229aSThomas Gleixner 	 */
1180dc9b229aSThomas Gleixner 	if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1181dc9b229aSThomas Gleixner 		new->flags &= ~IRQF_ONESHOT;
1182dc9b229aSThomas Gleixner 
1183dc9b229aSThomas Gleixner 	/*
11841da177e4SLinus Torvalds 	 * The following block of code has to be executed atomically
11851da177e4SLinus Torvalds 	 */
1186239007b8SThomas Gleixner 	raw_spin_lock_irqsave(&desc->lock, flags);
1187f17c7545SIngo Molnar 	old_ptr = &desc->action;
1188f17c7545SIngo Molnar 	old = *old_ptr;
118906fcb0c6SIngo Molnar 	if (old) {
1190e76de9f8SThomas Gleixner 		/*
1191e76de9f8SThomas Gleixner 		 * Can't share interrupts unless both agree to and are
1192e76de9f8SThomas Gleixner 		 * the same type (level, edge, polarity). So both flag
11933cca53b0SThomas Gleixner 		 * fields must have IRQF_SHARED set and the bits which
11949d591eddSThomas Gleixner 		 * set the trigger type must match. Also all must
11959d591eddSThomas Gleixner 		 * agree on ONESHOT.
1196e76de9f8SThomas Gleixner 		 */
11973cca53b0SThomas Gleixner 		if (!((old->flags & new->flags) & IRQF_SHARED) ||
11989d591eddSThomas Gleixner 		    ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK) ||
1199f5d89470SThomas Gleixner 		    ((old->flags ^ new->flags) & IRQF_ONESHOT))
1200f5163427SDimitri Sivanich 			goto mismatch;
1201f5163427SDimitri Sivanich 
1202f5163427SDimitri Sivanich 		/* All handlers must agree on per-cpuness */
12033cca53b0SThomas Gleixner 		if ((old->flags & IRQF_PERCPU) !=
12043cca53b0SThomas Gleixner 		    (new->flags & IRQF_PERCPU))
1205f5163427SDimitri Sivanich 			goto mismatch;
12061da177e4SLinus Torvalds 
12071da177e4SLinus Torvalds 		/* add new interrupt at end of irq queue */
12081da177e4SLinus Torvalds 		do {
120952abb700SThomas Gleixner 			/*
121052abb700SThomas Gleixner 			 * Or all existing action->thread_mask bits,
121152abb700SThomas Gleixner 			 * so we can find the next zero bit for this
121252abb700SThomas Gleixner 			 * new action.
121352abb700SThomas Gleixner 			 */
1214b5faba21SThomas Gleixner 			thread_mask |= old->thread_mask;
1215f17c7545SIngo Molnar 			old_ptr = &old->next;
1216f17c7545SIngo Molnar 			old = *old_ptr;
12171da177e4SLinus Torvalds 		} while (old);
12181da177e4SLinus Torvalds 		shared = 1;
12191da177e4SLinus Torvalds 	}
12201da177e4SLinus Torvalds 
1221b5faba21SThomas Gleixner 	/*
122252abb700SThomas Gleixner 	 * Setup the thread mask for this irqaction for ONESHOT. For
122352abb700SThomas Gleixner 	 * !ONESHOT irqs the thread mask is 0 so we can avoid a
122452abb700SThomas Gleixner 	 * conditional in irq_wake_thread().
1225b5faba21SThomas Gleixner 	 */
122652abb700SThomas Gleixner 	if (new->flags & IRQF_ONESHOT) {
122752abb700SThomas Gleixner 		/*
122852abb700SThomas Gleixner 		 * Unlikely to have 32 resp 64 irqs sharing one line,
122952abb700SThomas Gleixner 		 * but who knows.
123052abb700SThomas Gleixner 		 */
123152abb700SThomas Gleixner 		if (thread_mask == ~0UL) {
1232b5faba21SThomas Gleixner 			ret = -EBUSY;
1233b5faba21SThomas Gleixner 			goto out_mask;
1234b5faba21SThomas Gleixner 		}
123552abb700SThomas Gleixner 		/*
123652abb700SThomas Gleixner 		 * The thread_mask for the action is or'ed to
123752abb700SThomas Gleixner 		 * desc->thread_active to indicate that the
123852abb700SThomas Gleixner 		 * IRQF_ONESHOT thread handler has been woken, but not
123952abb700SThomas Gleixner 		 * yet finished. The bit is cleared when a thread
124052abb700SThomas Gleixner 		 * completes. When all threads of a shared interrupt
124152abb700SThomas Gleixner 		 * line have completed desc->threads_active becomes
124252abb700SThomas Gleixner 		 * zero and the interrupt line is unmasked. See
124352abb700SThomas Gleixner 		 * handle.c:irq_wake_thread() for further information.
124452abb700SThomas Gleixner 		 *
124552abb700SThomas Gleixner 		 * If no thread is woken by primary (hard irq context)
124652abb700SThomas Gleixner 		 * interrupt handlers, then desc->threads_active is
124752abb700SThomas Gleixner 		 * also checked for zero to unmask the irq line in the
124852abb700SThomas Gleixner 		 * affected hard irq flow handlers
124952abb700SThomas Gleixner 		 * (handle_[fasteoi|level]_irq).
125052abb700SThomas Gleixner 		 *
125152abb700SThomas Gleixner 		 * The new action gets the first zero bit of
125252abb700SThomas Gleixner 		 * thread_mask assigned. See the loop above which or's
125352abb700SThomas Gleixner 		 * all existing action->thread_mask bits.
125452abb700SThomas Gleixner 		 */
1255b5faba21SThomas Gleixner 		new->thread_mask = 1 << ffz(thread_mask);
12561c6c6952SThomas Gleixner 
1257dc9b229aSThomas Gleixner 	} else if (new->handler == irq_default_primary_handler &&
1258dc9b229aSThomas Gleixner 		   !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
12591c6c6952SThomas Gleixner 		/*
12601c6c6952SThomas Gleixner 		 * The interrupt was requested with handler = NULL, so
12611c6c6952SThomas Gleixner 		 * we use the default primary handler for it. But it
12621c6c6952SThomas Gleixner 		 * does not have the oneshot flag set. In combination
12631c6c6952SThomas Gleixner 		 * with level interrupts this is deadly, because the
12641c6c6952SThomas Gleixner 		 * default primary handler just wakes the thread, then
12651c6c6952SThomas Gleixner 		 * the irq lines is reenabled, but the device still
12661c6c6952SThomas Gleixner 		 * has the level irq asserted. Rinse and repeat....
12671c6c6952SThomas Gleixner 		 *
12681c6c6952SThomas Gleixner 		 * While this works for edge type interrupts, we play
12691c6c6952SThomas Gleixner 		 * it safe and reject unconditionally because we can't
12701c6c6952SThomas Gleixner 		 * say for sure which type this interrupt really
12711c6c6952SThomas Gleixner 		 * has. The type flags are unreliable as the
12721c6c6952SThomas Gleixner 		 * underlying chip implementation can override them.
12731c6c6952SThomas Gleixner 		 */
127497fd75b7SAndrew Morton 		pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
12751c6c6952SThomas Gleixner 		       irq);
12761c6c6952SThomas Gleixner 		ret = -EINVAL;
12771c6c6952SThomas Gleixner 		goto out_mask;
127852abb700SThomas Gleixner 	}
1279b5faba21SThomas Gleixner 
12801da177e4SLinus Torvalds 	if (!shared) {
1281c1bacbaeSThomas Gleixner 		ret = irq_request_resources(desc);
1282c1bacbaeSThomas Gleixner 		if (ret) {
1283c1bacbaeSThomas Gleixner 			pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1284c1bacbaeSThomas Gleixner 			       new->name, irq, desc->irq_data.chip->name);
1285c1bacbaeSThomas Gleixner 			goto out_mask;
1286c1bacbaeSThomas Gleixner 		}
1287c1bacbaeSThomas Gleixner 
12883aa551c9SThomas Gleixner 		init_waitqueue_head(&desc->wait_for_threads);
12893aa551c9SThomas Gleixner 
129082736f4dSUwe Kleine-König 		/* Setup the type (level, edge polarity) if configured: */
129182736f4dSUwe Kleine-König 		if (new->flags & IRQF_TRIGGER_MASK) {
1292a1ff541aSJiang Liu 			ret = __irq_set_trigger(desc,
1293f2b662daSDavid Brownell 						new->flags & IRQF_TRIGGER_MASK);
129482736f4dSUwe Kleine-König 
12953aa551c9SThomas Gleixner 			if (ret)
12963b8249e7SThomas Gleixner 				goto out_mask;
1297091738a2SThomas Gleixner 		}
1298f75d222bSAhmed S. Darwish 
1299009b4c3bSThomas Gleixner 		desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
130032f4125eSThomas Gleixner 				  IRQS_ONESHOT | IRQS_WAITING);
130132f4125eSThomas Gleixner 		irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
130294d39e1fSThomas Gleixner 
1303a005677bSThomas Gleixner 		if (new->flags & IRQF_PERCPU) {
1304a005677bSThomas Gleixner 			irqd_set(&desc->irq_data, IRQD_PER_CPU);
1305a005677bSThomas Gleixner 			irq_settings_set_per_cpu(desc);
1306a005677bSThomas Gleixner 		}
13076a58fb3bSThomas Gleixner 
1308b25c340cSThomas Gleixner 		if (new->flags & IRQF_ONESHOT)
13093d67baecSThomas Gleixner 			desc->istate |= IRQS_ONESHOT;
1310b25c340cSThomas Gleixner 
13111ccb4e61SThomas Gleixner 		if (irq_settings_can_autoenable(desc))
1312b4bc724eSThomas Gleixner 			irq_startup(desc, true);
131346999238SThomas Gleixner 		else
1314e76de9f8SThomas Gleixner 			/* Undo nested disables: */
1315e76de9f8SThomas Gleixner 			desc->depth = 1;
131618404756SMax Krasnyansky 
1317612e3684SThomas Gleixner 		/* Exclude IRQ from balancing if requested */
1318a005677bSThomas Gleixner 		if (new->flags & IRQF_NOBALANCING) {
1319a005677bSThomas Gleixner 			irq_settings_set_no_balancing(desc);
1320a005677bSThomas Gleixner 			irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1321a005677bSThomas Gleixner 		}
1322612e3684SThomas Gleixner 
132318404756SMax Krasnyansky 		/* Set default affinity mask once everything is setup */
1324a8a98eacSJiang Liu 		setup_affinity(desc, mask);
13250c5d1eb7SDavid Brownell 
1326876dbd4cSThomas Gleixner 	} else if (new->flags & IRQF_TRIGGER_MASK) {
1327876dbd4cSThomas Gleixner 		unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1328876dbd4cSThomas Gleixner 		unsigned int omsk = irq_settings_get_trigger_mask(desc);
1329876dbd4cSThomas Gleixner 
1330876dbd4cSThomas Gleixner 		if (nmsk != omsk)
1331876dbd4cSThomas Gleixner 			/* hope the handler works with current  trigger mode */
1332a395d6a7SJoe Perches 			pr_warn("irq %d uses trigger mode %u; requested %u\n",
1333876dbd4cSThomas Gleixner 				irq, nmsk, omsk);
133494d39e1fSThomas Gleixner 	}
133582736f4dSUwe Kleine-König 
1336f17c7545SIngo Molnar 	*old_ptr = new;
133782736f4dSUwe Kleine-König 
1338cab303beSThomas Gleixner 	irq_pm_install_action(desc, new);
1339cab303beSThomas Gleixner 
13408528b0f1SLinus Torvalds 	/* Reset broken irq detection when installing new handler */
13418528b0f1SLinus Torvalds 	desc->irq_count = 0;
13428528b0f1SLinus Torvalds 	desc->irqs_unhandled = 0;
13431adb0850SThomas Gleixner 
13441adb0850SThomas Gleixner 	/*
13451adb0850SThomas Gleixner 	 * Check whether we disabled the irq via the spurious handler
13461adb0850SThomas Gleixner 	 * before. Reenable it and give it another chance.
13471adb0850SThomas Gleixner 	 */
13487acdd53eSThomas Gleixner 	if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
13497acdd53eSThomas Gleixner 		desc->istate &= ~IRQS_SPURIOUS_DISABLED;
135079ff1cdaSJiang Liu 		__enable_irq(desc);
13511adb0850SThomas Gleixner 	}
13521adb0850SThomas Gleixner 
1353239007b8SThomas Gleixner 	raw_spin_unlock_irqrestore(&desc->lock, flags);
13541da177e4SLinus Torvalds 
135569ab8494SThomas Gleixner 	/*
135669ab8494SThomas Gleixner 	 * Strictly no need to wake it up, but hung_task complains
135769ab8494SThomas Gleixner 	 * when no hard interrupt wakes the thread up.
135869ab8494SThomas Gleixner 	 */
135969ab8494SThomas Gleixner 	if (new->thread)
136069ab8494SThomas Gleixner 		wake_up_process(new->thread);
13612a1d3ab8SThomas Gleixner 	if (new->secondary)
13622a1d3ab8SThomas Gleixner 		wake_up_process(new->secondary->thread);
136369ab8494SThomas Gleixner 
13642c6927a3SYinghai Lu 	register_irq_proc(irq, desc);
13651da177e4SLinus Torvalds 	new->dir = NULL;
13661da177e4SLinus Torvalds 	register_handler_proc(irq, new);
13674f5058c3SXiaotian Feng 	free_cpumask_var(mask);
13681da177e4SLinus Torvalds 
13691da177e4SLinus Torvalds 	return 0;
1370f5163427SDimitri Sivanich 
1371f5163427SDimitri Sivanich mismatch:
13723cca53b0SThomas Gleixner 	if (!(new->flags & IRQF_PROBE_SHARED)) {
137397fd75b7SAndrew Morton 		pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1374f5d89470SThomas Gleixner 		       irq, new->flags, new->name, old->flags, old->name);
1375f5d89470SThomas Gleixner #ifdef CONFIG_DEBUG_SHIRQ
1376f5163427SDimitri Sivanich 		dump_stack();
13773f050447SAlan Cox #endif
1378f5d89470SThomas Gleixner 	}
13793aa551c9SThomas Gleixner 	ret = -EBUSY;
13803aa551c9SThomas Gleixner 
13813b8249e7SThomas Gleixner out_mask:
13821c389795SDan Carpenter 	raw_spin_unlock_irqrestore(&desc->lock, flags);
13833b8249e7SThomas Gleixner 	free_cpumask_var(mask);
13843b8249e7SThomas Gleixner 
13853aa551c9SThomas Gleixner out_thread:
13863aa551c9SThomas Gleixner 	if (new->thread) {
13873aa551c9SThomas Gleixner 		struct task_struct *t = new->thread;
13883aa551c9SThomas Gleixner 
13893aa551c9SThomas Gleixner 		new->thread = NULL;
13903aa551c9SThomas Gleixner 		kthread_stop(t);
13913aa551c9SThomas Gleixner 		put_task_struct(t);
13923aa551c9SThomas Gleixner 	}
13932a1d3ab8SThomas Gleixner 	if (new->secondary && new->secondary->thread) {
13942a1d3ab8SThomas Gleixner 		struct task_struct *t = new->secondary->thread;
13952a1d3ab8SThomas Gleixner 
13962a1d3ab8SThomas Gleixner 		new->secondary->thread = NULL;
13972a1d3ab8SThomas Gleixner 		kthread_stop(t);
13982a1d3ab8SThomas Gleixner 		put_task_struct(t);
13992a1d3ab8SThomas Gleixner 	}
1400b6873807SSebastian Andrzej Siewior out_mput:
1401b6873807SSebastian Andrzej Siewior 	module_put(desc->owner);
14023aa551c9SThomas Gleixner 	return ret;
14031da177e4SLinus Torvalds }
14041da177e4SLinus Torvalds 
14051da177e4SLinus Torvalds /**
1406d3c60047SThomas Gleixner  *	setup_irq - setup an interrupt
1407d3c60047SThomas Gleixner  *	@irq: Interrupt line to setup
1408d3c60047SThomas Gleixner  *	@act: irqaction for the interrupt
1409d3c60047SThomas Gleixner  *
1410d3c60047SThomas Gleixner  * Used to statically setup interrupts in the early boot process.
1411d3c60047SThomas Gleixner  */
1412d3c60047SThomas Gleixner int setup_irq(unsigned int irq, struct irqaction *act)
1413d3c60047SThomas Gleixner {
1414986c011dSDavid Daney 	int retval;
1415d3c60047SThomas Gleixner 	struct irq_desc *desc = irq_to_desc(irq);
1416d3c60047SThomas Gleixner 
14179b5d585dSJon Hunter 	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
141831d9d9b6SMarc Zyngier 		return -EINVAL;
1419986c011dSDavid Daney 	chip_bus_lock(desc);
1420986c011dSDavid Daney 	retval = __setup_irq(irq, desc, act);
1421986c011dSDavid Daney 	chip_bus_sync_unlock(desc);
1422986c011dSDavid Daney 
1423986c011dSDavid Daney 	return retval;
1424d3c60047SThomas Gleixner }
1425eb53b4e8SMagnus Damm EXPORT_SYMBOL_GPL(setup_irq);
1426d3c60047SThomas Gleixner 
1427cbf94f06SMagnus Damm /*
1428cbf94f06SMagnus Damm  * Internal function to unregister an irqaction - used to free
1429cbf94f06SMagnus Damm  * regular and special interrupts that are part of the architecture.
14301da177e4SLinus Torvalds  */
1431cbf94f06SMagnus Damm static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
14321da177e4SLinus Torvalds {
1433d3c60047SThomas Gleixner 	struct irq_desc *desc = irq_to_desc(irq);
1434f17c7545SIngo Molnar 	struct irqaction *action, **action_ptr;
14351da177e4SLinus Torvalds 	unsigned long flags;
14361da177e4SLinus Torvalds 
1437ae88a23bSIngo Molnar 	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
14387d94f7caSYinghai Lu 
14397d94f7caSYinghai Lu 	if (!desc)
1440f21cfb25SMagnus Damm 		return NULL;
14411da177e4SLinus Torvalds 
1442abc7e40cSThomas Gleixner 	chip_bus_lock(desc);
1443239007b8SThomas Gleixner 	raw_spin_lock_irqsave(&desc->lock, flags);
1444ae88a23bSIngo Molnar 
1445ae88a23bSIngo Molnar 	/*
1446ae88a23bSIngo Molnar 	 * There can be multiple actions per IRQ descriptor, find the right
1447ae88a23bSIngo Molnar 	 * one based on the dev_id:
1448ae88a23bSIngo Molnar 	 */
1449f17c7545SIngo Molnar 	action_ptr = &desc->action;
14501da177e4SLinus Torvalds 	for (;;) {
1451f17c7545SIngo Molnar 		action = *action_ptr;
14521da177e4SLinus Torvalds 
1453ae88a23bSIngo Molnar 		if (!action) {
1454ae88a23bSIngo Molnar 			WARN(1, "Trying to free already-free IRQ %d\n", irq);
1455239007b8SThomas Gleixner 			raw_spin_unlock_irqrestore(&desc->lock, flags);
1456abc7e40cSThomas Gleixner 			chip_bus_sync_unlock(desc);
1457f21cfb25SMagnus Damm 			return NULL;
1458ae88a23bSIngo Molnar 		}
14591da177e4SLinus Torvalds 
14608316e381SIngo Molnar 		if (action->dev_id == dev_id)
1461ae88a23bSIngo Molnar 			break;
1462f17c7545SIngo Molnar 		action_ptr = &action->next;
1463ae88a23bSIngo Molnar 	}
1464ae88a23bSIngo Molnar 
1465ae88a23bSIngo Molnar 	/* Found it - now remove it from the list of entries: */
1466f17c7545SIngo Molnar 	*action_ptr = action->next;
1467dbce706eSPaolo 'Blaisorblade' Giarrusso 
1468cab303beSThomas Gleixner 	irq_pm_remove_action(desc, action);
1469cab303beSThomas Gleixner 
1470ae88a23bSIngo Molnar 	/* If this was the last handler, shut down the IRQ line: */
1471c1bacbaeSThomas Gleixner 	if (!desc->action) {
1472e9849777SThomas Gleixner 		irq_settings_clr_disable_unlazy(desc);
147346999238SThomas Gleixner 		irq_shutdown(desc);
1474c1bacbaeSThomas Gleixner 		irq_release_resources(desc);
1475c1bacbaeSThomas Gleixner 	}
14763aa551c9SThomas Gleixner 
1477e7a297b0SPeter P Waskiewicz Jr #ifdef CONFIG_SMP
1478e7a297b0SPeter P Waskiewicz Jr 	/* make sure affinity_hint is cleaned up */
1479e7a297b0SPeter P Waskiewicz Jr 	if (WARN_ON_ONCE(desc->affinity_hint))
1480e7a297b0SPeter P Waskiewicz Jr 		desc->affinity_hint = NULL;
1481e7a297b0SPeter P Waskiewicz Jr #endif
1482e7a297b0SPeter P Waskiewicz Jr 
1483239007b8SThomas Gleixner 	raw_spin_unlock_irqrestore(&desc->lock, flags);
1484abc7e40cSThomas Gleixner 	chip_bus_sync_unlock(desc);
1485ae88a23bSIngo Molnar 
14861da177e4SLinus Torvalds 	unregister_handler_proc(irq, action);
14871da177e4SLinus Torvalds 
1488ae88a23bSIngo Molnar 	/* Make sure it's not being used on another CPU: */
14891da177e4SLinus Torvalds 	synchronize_irq(irq);
1490ae88a23bSIngo Molnar 
14911d99493bSDavid Woodhouse #ifdef CONFIG_DEBUG_SHIRQ
14921d99493bSDavid Woodhouse 	/*
1493ae88a23bSIngo Molnar 	 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1494ae88a23bSIngo Molnar 	 * event to happen even now it's being freed, so let's make sure that
1495ae88a23bSIngo Molnar 	 * is so by doing an extra call to the handler ....
1496ae88a23bSIngo Molnar 	 *
1497ae88a23bSIngo Molnar 	 * ( We do this after actually deregistering it, to make sure that a
1498ae88a23bSIngo Molnar 	 *   'real' IRQ doesn't run in * parallel with our fake. )
14991d99493bSDavid Woodhouse 	 */
15001d99493bSDavid Woodhouse 	if (action->flags & IRQF_SHARED) {
15011d99493bSDavid Woodhouse 		local_irq_save(flags);
15021d99493bSDavid Woodhouse 		action->handler(irq, dev_id);
15031d99493bSDavid Woodhouse 		local_irq_restore(flags);
15041d99493bSDavid Woodhouse 	}
15051d99493bSDavid Woodhouse #endif
15062d860ad7SLinus Torvalds 
15072d860ad7SLinus Torvalds 	if (action->thread) {
15082d860ad7SLinus Torvalds 		kthread_stop(action->thread);
15092d860ad7SLinus Torvalds 		put_task_struct(action->thread);
15102a1d3ab8SThomas Gleixner 		if (action->secondary && action->secondary->thread) {
15112a1d3ab8SThomas Gleixner 			kthread_stop(action->secondary->thread);
15122a1d3ab8SThomas Gleixner 			put_task_struct(action->secondary->thread);
15132a1d3ab8SThomas Gleixner 		}
15142d860ad7SLinus Torvalds 	}
15152d860ad7SLinus Torvalds 
1516b6873807SSebastian Andrzej Siewior 	module_put(desc->owner);
15172a1d3ab8SThomas Gleixner 	kfree(action->secondary);
1518f21cfb25SMagnus Damm 	return action;
1519f21cfb25SMagnus Damm }
15201da177e4SLinus Torvalds 
15211da177e4SLinus Torvalds /**
1522cbf94f06SMagnus Damm  *	remove_irq - free an interrupt
1523cbf94f06SMagnus Damm  *	@irq: Interrupt line to free
1524cbf94f06SMagnus Damm  *	@act: irqaction for the interrupt
1525cbf94f06SMagnus Damm  *
1526cbf94f06SMagnus Damm  * Used to remove interrupts statically setup by the early boot process.
1527cbf94f06SMagnus Damm  */
1528cbf94f06SMagnus Damm void remove_irq(unsigned int irq, struct irqaction *act)
1529cbf94f06SMagnus Damm {
153031d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_to_desc(irq);
153131d9d9b6SMarc Zyngier 
153231d9d9b6SMarc Zyngier 	if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1533cbf94f06SMagnus Damm 	    __free_irq(irq, act->dev_id);
1534cbf94f06SMagnus Damm }
1535eb53b4e8SMagnus Damm EXPORT_SYMBOL_GPL(remove_irq);
1536cbf94f06SMagnus Damm 
1537cbf94f06SMagnus Damm /**
1538f21cfb25SMagnus Damm  *	free_irq - free an interrupt allocated with request_irq
15391da177e4SLinus Torvalds  *	@irq: Interrupt line to free
15401da177e4SLinus Torvalds  *	@dev_id: Device identity to free
15411da177e4SLinus Torvalds  *
15421da177e4SLinus Torvalds  *	Remove an interrupt handler. The handler is removed and if the
15431da177e4SLinus Torvalds  *	interrupt line is no longer in use by any driver it is disabled.
15441da177e4SLinus Torvalds  *	On a shared IRQ the caller must ensure the interrupt is disabled
15451da177e4SLinus Torvalds  *	on the card it drives before calling this function. The function
15461da177e4SLinus Torvalds  *	does not return until any executing interrupts for this IRQ
15471da177e4SLinus Torvalds  *	have completed.
15481da177e4SLinus Torvalds  *
15491da177e4SLinus Torvalds  *	This function must not be called from interrupt context.
15501da177e4SLinus Torvalds  */
15511da177e4SLinus Torvalds void free_irq(unsigned int irq, void *dev_id)
15521da177e4SLinus Torvalds {
155370aedd24SThomas Gleixner 	struct irq_desc *desc = irq_to_desc(irq);
155470aedd24SThomas Gleixner 
155531d9d9b6SMarc Zyngier 	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
155670aedd24SThomas Gleixner 		return;
155770aedd24SThomas Gleixner 
1558cd7eab44SBen Hutchings #ifdef CONFIG_SMP
1559cd7eab44SBen Hutchings 	if (WARN_ON(desc->affinity_notify))
1560cd7eab44SBen Hutchings 		desc->affinity_notify = NULL;
1561cd7eab44SBen Hutchings #endif
1562cd7eab44SBen Hutchings 
1563cbf94f06SMagnus Damm 	kfree(__free_irq(irq, dev_id));
15641da177e4SLinus Torvalds }
15651da177e4SLinus Torvalds EXPORT_SYMBOL(free_irq);
15661da177e4SLinus Torvalds 
15671da177e4SLinus Torvalds /**
15683aa551c9SThomas Gleixner  *	request_threaded_irq - allocate an interrupt line
15691da177e4SLinus Torvalds  *	@irq: Interrupt line to allocate
15703aa551c9SThomas Gleixner  *	@handler: Function to be called when the IRQ occurs.
15713aa551c9SThomas Gleixner  *		  Primary handler for threaded interrupts
1572b25c340cSThomas Gleixner  *		  If NULL and thread_fn != NULL the default
1573b25c340cSThomas Gleixner  *		  primary handler is installed
15743aa551c9SThomas Gleixner  *	@thread_fn: Function called from the irq handler thread
15753aa551c9SThomas Gleixner  *		    If NULL, no irq thread is created
15761da177e4SLinus Torvalds  *	@irqflags: Interrupt type flags
15771da177e4SLinus Torvalds  *	@devname: An ascii name for the claiming device
15781da177e4SLinus Torvalds  *	@dev_id: A cookie passed back to the handler function
15791da177e4SLinus Torvalds  *
15801da177e4SLinus Torvalds  *	This call allocates interrupt resources and enables the
15811da177e4SLinus Torvalds  *	interrupt line and IRQ handling. From the point this
15821da177e4SLinus Torvalds  *	call is made your handler function may be invoked. Since
15831da177e4SLinus Torvalds  *	your handler function must clear any interrupt the board
15841da177e4SLinus Torvalds  *	raises, you must take care both to initialise your hardware
15851da177e4SLinus Torvalds  *	and to set up the interrupt handler in the right order.
15861da177e4SLinus Torvalds  *
15873aa551c9SThomas Gleixner  *	If you want to set up a threaded irq handler for your device
15886d21af4fSJavi Merino  *	then you need to supply @handler and @thread_fn. @handler is
15893aa551c9SThomas Gleixner  *	still called in hard interrupt context and has to check
15903aa551c9SThomas Gleixner  *	whether the interrupt originates from the device. If yes it
15913aa551c9SThomas Gleixner  *	needs to disable the interrupt on the device and return
159239a2eddbSSteven Rostedt  *	IRQ_WAKE_THREAD which will wake up the handler thread and run
15933aa551c9SThomas Gleixner  *	@thread_fn. This split handler design is necessary to support
15943aa551c9SThomas Gleixner  *	shared interrupts.
15953aa551c9SThomas Gleixner  *
15961da177e4SLinus Torvalds  *	Dev_id must be globally unique. Normally the address of the
15971da177e4SLinus Torvalds  *	device data structure is used as the cookie. Since the handler
15981da177e4SLinus Torvalds  *	receives this value it makes sense to use it.
15991da177e4SLinus Torvalds  *
16001da177e4SLinus Torvalds  *	If your interrupt is shared you must pass a non NULL dev_id
16011da177e4SLinus Torvalds  *	as this is required when freeing the interrupt.
16021da177e4SLinus Torvalds  *
16031da177e4SLinus Torvalds  *	Flags:
16041da177e4SLinus Torvalds  *
16053cca53b0SThomas Gleixner  *	IRQF_SHARED		Interrupt is shared
16060c5d1eb7SDavid Brownell  *	IRQF_TRIGGER_*		Specify active edge(s) or level
16071da177e4SLinus Torvalds  *
16081da177e4SLinus Torvalds  */
16093aa551c9SThomas Gleixner int request_threaded_irq(unsigned int irq, irq_handler_t handler,
16103aa551c9SThomas Gleixner 			 irq_handler_t thread_fn, unsigned long irqflags,
16113aa551c9SThomas Gleixner 			 const char *devname, void *dev_id)
16121da177e4SLinus Torvalds {
16131da177e4SLinus Torvalds 	struct irqaction *action;
161408678b08SYinghai Lu 	struct irq_desc *desc;
1615d3c60047SThomas Gleixner 	int retval;
16161da177e4SLinus Torvalds 
1617e237a551SChen Fan 	if (irq == IRQ_NOTCONNECTED)
1618e237a551SChen Fan 		return -ENOTCONN;
1619e237a551SChen Fan 
1620470c6623SDavid Brownell 	/*
16211da177e4SLinus Torvalds 	 * Sanity-check: shared interrupts must pass in a real dev-ID,
16221da177e4SLinus Torvalds 	 * otherwise we'll have trouble later trying to figure out
16231da177e4SLinus Torvalds 	 * which interrupt is which (messes up the interrupt freeing
16241da177e4SLinus Torvalds 	 * logic etc).
162517f48034SRafael J. Wysocki 	 *
162617f48034SRafael J. Wysocki 	 * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
162717f48034SRafael J. Wysocki 	 * it cannot be set along with IRQF_NO_SUSPEND.
16281da177e4SLinus Torvalds 	 */
162917f48034SRafael J. Wysocki 	if (((irqflags & IRQF_SHARED) && !dev_id) ||
163017f48034SRafael J. Wysocki 	    (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
163117f48034SRafael J. Wysocki 	    ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
16321da177e4SLinus Torvalds 		return -EINVAL;
16337d94f7caSYinghai Lu 
1634cb5bc832SYinghai Lu 	desc = irq_to_desc(irq);
16357d94f7caSYinghai Lu 	if (!desc)
16361da177e4SLinus Torvalds 		return -EINVAL;
16377d94f7caSYinghai Lu 
163831d9d9b6SMarc Zyngier 	if (!irq_settings_can_request(desc) ||
163931d9d9b6SMarc Zyngier 	    WARN_ON(irq_settings_is_per_cpu_devid(desc)))
16406550c775SThomas Gleixner 		return -EINVAL;
1641b25c340cSThomas Gleixner 
1642b25c340cSThomas Gleixner 	if (!handler) {
1643b25c340cSThomas Gleixner 		if (!thread_fn)
16441da177e4SLinus Torvalds 			return -EINVAL;
1645b25c340cSThomas Gleixner 		handler = irq_default_primary_handler;
1646b25c340cSThomas Gleixner 	}
16471da177e4SLinus Torvalds 
164845535732SThomas Gleixner 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
16491da177e4SLinus Torvalds 	if (!action)
16501da177e4SLinus Torvalds 		return -ENOMEM;
16511da177e4SLinus Torvalds 
16521da177e4SLinus Torvalds 	action->handler = handler;
16533aa551c9SThomas Gleixner 	action->thread_fn = thread_fn;
16541da177e4SLinus Torvalds 	action->flags = irqflags;
16551da177e4SLinus Torvalds 	action->name = devname;
16561da177e4SLinus Torvalds 	action->dev_id = dev_id;
16571da177e4SLinus Torvalds 
16583876ec9eSThomas Gleixner 	chip_bus_lock(desc);
1659d3c60047SThomas Gleixner 	retval = __setup_irq(irq, desc, action);
16603876ec9eSThomas Gleixner 	chip_bus_sync_unlock(desc);
166170aedd24SThomas Gleixner 
16622a1d3ab8SThomas Gleixner 	if (retval) {
16632a1d3ab8SThomas Gleixner 		kfree(action->secondary);
1664377bf1e4SAnton Vorontsov 		kfree(action);
16652a1d3ab8SThomas Gleixner 	}
1666377bf1e4SAnton Vorontsov 
16676d83f94dSThomas Gleixner #ifdef CONFIG_DEBUG_SHIRQ_FIXME
16686ce51c43SLuis Henriques 	if (!retval && (irqflags & IRQF_SHARED)) {
1669a304e1b8SDavid Woodhouse 		/*
1670a304e1b8SDavid Woodhouse 		 * It's a shared IRQ -- the driver ought to be prepared for it
1671a304e1b8SDavid Woodhouse 		 * to happen immediately, so let's make sure....
1672377bf1e4SAnton Vorontsov 		 * We disable the irq to make sure that a 'real' IRQ doesn't
1673377bf1e4SAnton Vorontsov 		 * run in parallel with our fake.
1674a304e1b8SDavid Woodhouse 		 */
1675a304e1b8SDavid Woodhouse 		unsigned long flags;
1676a304e1b8SDavid Woodhouse 
1677377bf1e4SAnton Vorontsov 		disable_irq(irq);
1678a304e1b8SDavid Woodhouse 		local_irq_save(flags);
1679377bf1e4SAnton Vorontsov 
1680a304e1b8SDavid Woodhouse 		handler(irq, dev_id);
1681377bf1e4SAnton Vorontsov 
1682a304e1b8SDavid Woodhouse 		local_irq_restore(flags);
1683377bf1e4SAnton Vorontsov 		enable_irq(irq);
1684a304e1b8SDavid Woodhouse 	}
1685a304e1b8SDavid Woodhouse #endif
16861da177e4SLinus Torvalds 	return retval;
16871da177e4SLinus Torvalds }
16883aa551c9SThomas Gleixner EXPORT_SYMBOL(request_threaded_irq);
1689ae731f8dSMarc Zyngier 
1690ae731f8dSMarc Zyngier /**
1691ae731f8dSMarc Zyngier  *	request_any_context_irq - allocate an interrupt line
1692ae731f8dSMarc Zyngier  *	@irq: Interrupt line to allocate
1693ae731f8dSMarc Zyngier  *	@handler: Function to be called when the IRQ occurs.
1694ae731f8dSMarc Zyngier  *		  Threaded handler for threaded interrupts.
1695ae731f8dSMarc Zyngier  *	@flags: Interrupt type flags
1696ae731f8dSMarc Zyngier  *	@name: An ascii name for the claiming device
1697ae731f8dSMarc Zyngier  *	@dev_id: A cookie passed back to the handler function
1698ae731f8dSMarc Zyngier  *
1699ae731f8dSMarc Zyngier  *	This call allocates interrupt resources and enables the
1700ae731f8dSMarc Zyngier  *	interrupt line and IRQ handling. It selects either a
1701ae731f8dSMarc Zyngier  *	hardirq or threaded handling method depending on the
1702ae731f8dSMarc Zyngier  *	context.
1703ae731f8dSMarc Zyngier  *
1704ae731f8dSMarc Zyngier  *	On failure, it returns a negative value. On success,
1705ae731f8dSMarc Zyngier  *	it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
1706ae731f8dSMarc Zyngier  */
1707ae731f8dSMarc Zyngier int request_any_context_irq(unsigned int irq, irq_handler_t handler,
1708ae731f8dSMarc Zyngier 			    unsigned long flags, const char *name, void *dev_id)
1709ae731f8dSMarc Zyngier {
1710e237a551SChen Fan 	struct irq_desc *desc;
1711ae731f8dSMarc Zyngier 	int ret;
1712ae731f8dSMarc Zyngier 
1713e237a551SChen Fan 	if (irq == IRQ_NOTCONNECTED)
1714e237a551SChen Fan 		return -ENOTCONN;
1715e237a551SChen Fan 
1716e237a551SChen Fan 	desc = irq_to_desc(irq);
1717ae731f8dSMarc Zyngier 	if (!desc)
1718ae731f8dSMarc Zyngier 		return -EINVAL;
1719ae731f8dSMarc Zyngier 
17201ccb4e61SThomas Gleixner 	if (irq_settings_is_nested_thread(desc)) {
1721ae731f8dSMarc Zyngier 		ret = request_threaded_irq(irq, NULL, handler,
1722ae731f8dSMarc Zyngier 					   flags, name, dev_id);
1723ae731f8dSMarc Zyngier 		return !ret ? IRQC_IS_NESTED : ret;
1724ae731f8dSMarc Zyngier 	}
1725ae731f8dSMarc Zyngier 
1726ae731f8dSMarc Zyngier 	ret = request_irq(irq, handler, flags, name, dev_id);
1727ae731f8dSMarc Zyngier 	return !ret ? IRQC_IS_HARDIRQ : ret;
1728ae731f8dSMarc Zyngier }
1729ae731f8dSMarc Zyngier EXPORT_SYMBOL_GPL(request_any_context_irq);
173031d9d9b6SMarc Zyngier 
17311e7c5fd2SMarc Zyngier void enable_percpu_irq(unsigned int irq, unsigned int type)
173231d9d9b6SMarc Zyngier {
173331d9d9b6SMarc Zyngier 	unsigned int cpu = smp_processor_id();
173431d9d9b6SMarc Zyngier 	unsigned long flags;
173531d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
173631d9d9b6SMarc Zyngier 
173731d9d9b6SMarc Zyngier 	if (!desc)
173831d9d9b6SMarc Zyngier 		return;
173931d9d9b6SMarc Zyngier 
17401e7c5fd2SMarc Zyngier 	type &= IRQ_TYPE_SENSE_MASK;
17411e7c5fd2SMarc Zyngier 	if (type != IRQ_TYPE_NONE) {
17421e7c5fd2SMarc Zyngier 		int ret;
17431e7c5fd2SMarc Zyngier 
1744a1ff541aSJiang Liu 		ret = __irq_set_trigger(desc, type);
17451e7c5fd2SMarc Zyngier 
17461e7c5fd2SMarc Zyngier 		if (ret) {
174732cffddeSThomas Gleixner 			WARN(1, "failed to set type for IRQ%d\n", irq);
17481e7c5fd2SMarc Zyngier 			goto out;
17491e7c5fd2SMarc Zyngier 		}
17501e7c5fd2SMarc Zyngier 	}
17511e7c5fd2SMarc Zyngier 
175231d9d9b6SMarc Zyngier 	irq_percpu_enable(desc, cpu);
17531e7c5fd2SMarc Zyngier out:
175431d9d9b6SMarc Zyngier 	irq_put_desc_unlock(desc, flags);
175531d9d9b6SMarc Zyngier }
175636a5df85SChris Metcalf EXPORT_SYMBOL_GPL(enable_percpu_irq);
175731d9d9b6SMarc Zyngier 
1758f0cb3220SThomas Petazzoni /**
1759f0cb3220SThomas Petazzoni  * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
1760f0cb3220SThomas Petazzoni  * @irq:	Linux irq number to check for
1761f0cb3220SThomas Petazzoni  *
1762f0cb3220SThomas Petazzoni  * Must be called from a non migratable context. Returns the enable
1763f0cb3220SThomas Petazzoni  * state of a per cpu interrupt on the current cpu.
1764f0cb3220SThomas Petazzoni  */
1765f0cb3220SThomas Petazzoni bool irq_percpu_is_enabled(unsigned int irq)
1766f0cb3220SThomas Petazzoni {
1767f0cb3220SThomas Petazzoni 	unsigned int cpu = smp_processor_id();
1768f0cb3220SThomas Petazzoni 	struct irq_desc *desc;
1769f0cb3220SThomas Petazzoni 	unsigned long flags;
1770f0cb3220SThomas Petazzoni 	bool is_enabled;
1771f0cb3220SThomas Petazzoni 
1772f0cb3220SThomas Petazzoni 	desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1773f0cb3220SThomas Petazzoni 	if (!desc)
1774f0cb3220SThomas Petazzoni 		return false;
1775f0cb3220SThomas Petazzoni 
1776f0cb3220SThomas Petazzoni 	is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
1777f0cb3220SThomas Petazzoni 	irq_put_desc_unlock(desc, flags);
1778f0cb3220SThomas Petazzoni 
1779f0cb3220SThomas Petazzoni 	return is_enabled;
1780f0cb3220SThomas Petazzoni }
1781f0cb3220SThomas Petazzoni EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
1782f0cb3220SThomas Petazzoni 
178331d9d9b6SMarc Zyngier void disable_percpu_irq(unsigned int irq)
178431d9d9b6SMarc Zyngier {
178531d9d9b6SMarc Zyngier 	unsigned int cpu = smp_processor_id();
178631d9d9b6SMarc Zyngier 	unsigned long flags;
178731d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
178831d9d9b6SMarc Zyngier 
178931d9d9b6SMarc Zyngier 	if (!desc)
179031d9d9b6SMarc Zyngier 		return;
179131d9d9b6SMarc Zyngier 
179231d9d9b6SMarc Zyngier 	irq_percpu_disable(desc, cpu);
179331d9d9b6SMarc Zyngier 	irq_put_desc_unlock(desc, flags);
179431d9d9b6SMarc Zyngier }
179536a5df85SChris Metcalf EXPORT_SYMBOL_GPL(disable_percpu_irq);
179631d9d9b6SMarc Zyngier 
179731d9d9b6SMarc Zyngier /*
179831d9d9b6SMarc Zyngier  * Internal function to unregister a percpu irqaction.
179931d9d9b6SMarc Zyngier  */
180031d9d9b6SMarc Zyngier static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
180131d9d9b6SMarc Zyngier {
180231d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_to_desc(irq);
180331d9d9b6SMarc Zyngier 	struct irqaction *action;
180431d9d9b6SMarc Zyngier 	unsigned long flags;
180531d9d9b6SMarc Zyngier 
180631d9d9b6SMarc Zyngier 	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
180731d9d9b6SMarc Zyngier 
180831d9d9b6SMarc Zyngier 	if (!desc)
180931d9d9b6SMarc Zyngier 		return NULL;
181031d9d9b6SMarc Zyngier 
181131d9d9b6SMarc Zyngier 	raw_spin_lock_irqsave(&desc->lock, flags);
181231d9d9b6SMarc Zyngier 
181331d9d9b6SMarc Zyngier 	action = desc->action;
181431d9d9b6SMarc Zyngier 	if (!action || action->percpu_dev_id != dev_id) {
181531d9d9b6SMarc Zyngier 		WARN(1, "Trying to free already-free IRQ %d\n", irq);
181631d9d9b6SMarc Zyngier 		goto bad;
181731d9d9b6SMarc Zyngier 	}
181831d9d9b6SMarc Zyngier 
181931d9d9b6SMarc Zyngier 	if (!cpumask_empty(desc->percpu_enabled)) {
182031d9d9b6SMarc Zyngier 		WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
182131d9d9b6SMarc Zyngier 		     irq, cpumask_first(desc->percpu_enabled));
182231d9d9b6SMarc Zyngier 		goto bad;
182331d9d9b6SMarc Zyngier 	}
182431d9d9b6SMarc Zyngier 
182531d9d9b6SMarc Zyngier 	/* Found it - now remove it from the list of entries: */
182631d9d9b6SMarc Zyngier 	desc->action = NULL;
182731d9d9b6SMarc Zyngier 
182831d9d9b6SMarc Zyngier 	raw_spin_unlock_irqrestore(&desc->lock, flags);
182931d9d9b6SMarc Zyngier 
183031d9d9b6SMarc Zyngier 	unregister_handler_proc(irq, action);
183131d9d9b6SMarc Zyngier 
183231d9d9b6SMarc Zyngier 	module_put(desc->owner);
183331d9d9b6SMarc Zyngier 	return action;
183431d9d9b6SMarc Zyngier 
183531d9d9b6SMarc Zyngier bad:
183631d9d9b6SMarc Zyngier 	raw_spin_unlock_irqrestore(&desc->lock, flags);
183731d9d9b6SMarc Zyngier 	return NULL;
183831d9d9b6SMarc Zyngier }
183931d9d9b6SMarc Zyngier 
184031d9d9b6SMarc Zyngier /**
184131d9d9b6SMarc Zyngier  *	remove_percpu_irq - free a per-cpu interrupt
184231d9d9b6SMarc Zyngier  *	@irq: Interrupt line to free
184331d9d9b6SMarc Zyngier  *	@act: irqaction for the interrupt
184431d9d9b6SMarc Zyngier  *
184531d9d9b6SMarc Zyngier  * Used to remove interrupts statically setup by the early boot process.
184631d9d9b6SMarc Zyngier  */
184731d9d9b6SMarc Zyngier void remove_percpu_irq(unsigned int irq, struct irqaction *act)
184831d9d9b6SMarc Zyngier {
184931d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_to_desc(irq);
185031d9d9b6SMarc Zyngier 
185131d9d9b6SMarc Zyngier 	if (desc && irq_settings_is_per_cpu_devid(desc))
185231d9d9b6SMarc Zyngier 	    __free_percpu_irq(irq, act->percpu_dev_id);
185331d9d9b6SMarc Zyngier }
185431d9d9b6SMarc Zyngier 
185531d9d9b6SMarc Zyngier /**
185631d9d9b6SMarc Zyngier  *	free_percpu_irq - free an interrupt allocated with request_percpu_irq
185731d9d9b6SMarc Zyngier  *	@irq: Interrupt line to free
185831d9d9b6SMarc Zyngier  *	@dev_id: Device identity to free
185931d9d9b6SMarc Zyngier  *
186031d9d9b6SMarc Zyngier  *	Remove a percpu interrupt handler. The handler is removed, but
186131d9d9b6SMarc Zyngier  *	the interrupt line is not disabled. This must be done on each
186231d9d9b6SMarc Zyngier  *	CPU before calling this function. The function does not return
186331d9d9b6SMarc Zyngier  *	until any executing interrupts for this IRQ have completed.
186431d9d9b6SMarc Zyngier  *
186531d9d9b6SMarc Zyngier  *	This function must not be called from interrupt context.
186631d9d9b6SMarc Zyngier  */
186731d9d9b6SMarc Zyngier void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
186831d9d9b6SMarc Zyngier {
186931d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_to_desc(irq);
187031d9d9b6SMarc Zyngier 
187131d9d9b6SMarc Zyngier 	if (!desc || !irq_settings_is_per_cpu_devid(desc))
187231d9d9b6SMarc Zyngier 		return;
187331d9d9b6SMarc Zyngier 
187431d9d9b6SMarc Zyngier 	chip_bus_lock(desc);
187531d9d9b6SMarc Zyngier 	kfree(__free_percpu_irq(irq, dev_id));
187631d9d9b6SMarc Zyngier 	chip_bus_sync_unlock(desc);
187731d9d9b6SMarc Zyngier }
1878aec2e2adSMaxime Ripard EXPORT_SYMBOL_GPL(free_percpu_irq);
187931d9d9b6SMarc Zyngier 
188031d9d9b6SMarc Zyngier /**
188131d9d9b6SMarc Zyngier  *	setup_percpu_irq - setup a per-cpu interrupt
188231d9d9b6SMarc Zyngier  *	@irq: Interrupt line to setup
188331d9d9b6SMarc Zyngier  *	@act: irqaction for the interrupt
188431d9d9b6SMarc Zyngier  *
188531d9d9b6SMarc Zyngier  * Used to statically setup per-cpu interrupts in the early boot process.
188631d9d9b6SMarc Zyngier  */
188731d9d9b6SMarc Zyngier int setup_percpu_irq(unsigned int irq, struct irqaction *act)
188831d9d9b6SMarc Zyngier {
188931d9d9b6SMarc Zyngier 	struct irq_desc *desc = irq_to_desc(irq);
189031d9d9b6SMarc Zyngier 	int retval;
189131d9d9b6SMarc Zyngier 
189231d9d9b6SMarc Zyngier 	if (!desc || !irq_settings_is_per_cpu_devid(desc))
189331d9d9b6SMarc Zyngier 		return -EINVAL;
189431d9d9b6SMarc Zyngier 	chip_bus_lock(desc);
189531d9d9b6SMarc Zyngier 	retval = __setup_irq(irq, desc, act);
189631d9d9b6SMarc Zyngier 	chip_bus_sync_unlock(desc);
189731d9d9b6SMarc Zyngier 
189831d9d9b6SMarc Zyngier 	return retval;
189931d9d9b6SMarc Zyngier }
190031d9d9b6SMarc Zyngier 
190131d9d9b6SMarc Zyngier /**
190231d9d9b6SMarc Zyngier  *	request_percpu_irq - allocate a percpu interrupt line
190331d9d9b6SMarc Zyngier  *	@irq: Interrupt line to allocate
190431d9d9b6SMarc Zyngier  *	@handler: Function to be called when the IRQ occurs.
190531d9d9b6SMarc Zyngier  *	@devname: An ascii name for the claiming device
190631d9d9b6SMarc Zyngier  *	@dev_id: A percpu cookie passed back to the handler function
190731d9d9b6SMarc Zyngier  *
1908a1b7febdSMaxime Ripard  *	This call allocates interrupt resources and enables the
1909a1b7febdSMaxime Ripard  *	interrupt on the local CPU. If the interrupt is supposed to be
1910a1b7febdSMaxime Ripard  *	enabled on other CPUs, it has to be done on each CPU using
1911a1b7febdSMaxime Ripard  *	enable_percpu_irq().
191231d9d9b6SMarc Zyngier  *
191331d9d9b6SMarc Zyngier  *	Dev_id must be globally unique. It is a per-cpu variable, and
191431d9d9b6SMarc Zyngier  *	the handler gets called with the interrupted CPU's instance of
191531d9d9b6SMarc Zyngier  *	that variable.
191631d9d9b6SMarc Zyngier  */
191731d9d9b6SMarc Zyngier int request_percpu_irq(unsigned int irq, irq_handler_t handler,
191831d9d9b6SMarc Zyngier 		       const char *devname, void __percpu *dev_id)
191931d9d9b6SMarc Zyngier {
192031d9d9b6SMarc Zyngier 	struct irqaction *action;
192131d9d9b6SMarc Zyngier 	struct irq_desc *desc;
192231d9d9b6SMarc Zyngier 	int retval;
192331d9d9b6SMarc Zyngier 
192431d9d9b6SMarc Zyngier 	if (!dev_id)
192531d9d9b6SMarc Zyngier 		return -EINVAL;
192631d9d9b6SMarc Zyngier 
192731d9d9b6SMarc Zyngier 	desc = irq_to_desc(irq);
192831d9d9b6SMarc Zyngier 	if (!desc || !irq_settings_can_request(desc) ||
192931d9d9b6SMarc Zyngier 	    !irq_settings_is_per_cpu_devid(desc))
193031d9d9b6SMarc Zyngier 		return -EINVAL;
193131d9d9b6SMarc Zyngier 
193231d9d9b6SMarc Zyngier 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
193331d9d9b6SMarc Zyngier 	if (!action)
193431d9d9b6SMarc Zyngier 		return -ENOMEM;
193531d9d9b6SMarc Zyngier 
193631d9d9b6SMarc Zyngier 	action->handler = handler;
19372ed0e645SMarc Zyngier 	action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND;
193831d9d9b6SMarc Zyngier 	action->name = devname;
193931d9d9b6SMarc Zyngier 	action->percpu_dev_id = dev_id;
194031d9d9b6SMarc Zyngier 
194131d9d9b6SMarc Zyngier 	chip_bus_lock(desc);
194231d9d9b6SMarc Zyngier 	retval = __setup_irq(irq, desc, action);
194331d9d9b6SMarc Zyngier 	chip_bus_sync_unlock(desc);
194431d9d9b6SMarc Zyngier 
194531d9d9b6SMarc Zyngier 	if (retval)
194631d9d9b6SMarc Zyngier 		kfree(action);
194731d9d9b6SMarc Zyngier 
194831d9d9b6SMarc Zyngier 	return retval;
194931d9d9b6SMarc Zyngier }
1950aec2e2adSMaxime Ripard EXPORT_SYMBOL_GPL(request_percpu_irq);
19511b7047edSMarc Zyngier 
19521b7047edSMarc Zyngier /**
19531b7047edSMarc Zyngier  *	irq_get_irqchip_state - returns the irqchip state of a interrupt.
19541b7047edSMarc Zyngier  *	@irq: Interrupt line that is forwarded to a VM
19551b7047edSMarc Zyngier  *	@which: One of IRQCHIP_STATE_* the caller wants to know about
19561b7047edSMarc Zyngier  *	@state: a pointer to a boolean where the state is to be storeed
19571b7047edSMarc Zyngier  *
19581b7047edSMarc Zyngier  *	This call snapshots the internal irqchip state of an
19591b7047edSMarc Zyngier  *	interrupt, returning into @state the bit corresponding to
19601b7047edSMarc Zyngier  *	stage @which
19611b7047edSMarc Zyngier  *
19621b7047edSMarc Zyngier  *	This function should be called with preemption disabled if the
19631b7047edSMarc Zyngier  *	interrupt controller has per-cpu registers.
19641b7047edSMarc Zyngier  */
19651b7047edSMarc Zyngier int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
19661b7047edSMarc Zyngier 			  bool *state)
19671b7047edSMarc Zyngier {
19681b7047edSMarc Zyngier 	struct irq_desc *desc;
19691b7047edSMarc Zyngier 	struct irq_data *data;
19701b7047edSMarc Zyngier 	struct irq_chip *chip;
19711b7047edSMarc Zyngier 	unsigned long flags;
19721b7047edSMarc Zyngier 	int err = -EINVAL;
19731b7047edSMarc Zyngier 
19741b7047edSMarc Zyngier 	desc = irq_get_desc_buslock(irq, &flags, 0);
19751b7047edSMarc Zyngier 	if (!desc)
19761b7047edSMarc Zyngier 		return err;
19771b7047edSMarc Zyngier 
19781b7047edSMarc Zyngier 	data = irq_desc_get_irq_data(desc);
19791b7047edSMarc Zyngier 
19801b7047edSMarc Zyngier 	do {
19811b7047edSMarc Zyngier 		chip = irq_data_get_irq_chip(data);
19821b7047edSMarc Zyngier 		if (chip->irq_get_irqchip_state)
19831b7047edSMarc Zyngier 			break;
19841b7047edSMarc Zyngier #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
19851b7047edSMarc Zyngier 		data = data->parent_data;
19861b7047edSMarc Zyngier #else
19871b7047edSMarc Zyngier 		data = NULL;
19881b7047edSMarc Zyngier #endif
19891b7047edSMarc Zyngier 	} while (data);
19901b7047edSMarc Zyngier 
19911b7047edSMarc Zyngier 	if (data)
19921b7047edSMarc Zyngier 		err = chip->irq_get_irqchip_state(data, which, state);
19931b7047edSMarc Zyngier 
19941b7047edSMarc Zyngier 	irq_put_desc_busunlock(desc, flags);
19951b7047edSMarc Zyngier 	return err;
19961b7047edSMarc Zyngier }
19971ee4fb3eSBjorn Andersson EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
19981b7047edSMarc Zyngier 
19991b7047edSMarc Zyngier /**
20001b7047edSMarc Zyngier  *	irq_set_irqchip_state - set the state of a forwarded interrupt.
20011b7047edSMarc Zyngier  *	@irq: Interrupt line that is forwarded to a VM
20021b7047edSMarc Zyngier  *	@which: State to be restored (one of IRQCHIP_STATE_*)
20031b7047edSMarc Zyngier  *	@val: Value corresponding to @which
20041b7047edSMarc Zyngier  *
20051b7047edSMarc Zyngier  *	This call sets the internal irqchip state of an interrupt,
20061b7047edSMarc Zyngier  *	depending on the value of @which.
20071b7047edSMarc Zyngier  *
20081b7047edSMarc Zyngier  *	This function should be called with preemption disabled if the
20091b7047edSMarc Zyngier  *	interrupt controller has per-cpu registers.
20101b7047edSMarc Zyngier  */
20111b7047edSMarc Zyngier int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
20121b7047edSMarc Zyngier 			  bool val)
20131b7047edSMarc Zyngier {
20141b7047edSMarc Zyngier 	struct irq_desc *desc;
20151b7047edSMarc Zyngier 	struct irq_data *data;
20161b7047edSMarc Zyngier 	struct irq_chip *chip;
20171b7047edSMarc Zyngier 	unsigned long flags;
20181b7047edSMarc Zyngier 	int err = -EINVAL;
20191b7047edSMarc Zyngier 
20201b7047edSMarc Zyngier 	desc = irq_get_desc_buslock(irq, &flags, 0);
20211b7047edSMarc Zyngier 	if (!desc)
20221b7047edSMarc Zyngier 		return err;
20231b7047edSMarc Zyngier 
20241b7047edSMarc Zyngier 	data = irq_desc_get_irq_data(desc);
20251b7047edSMarc Zyngier 
20261b7047edSMarc Zyngier 	do {
20271b7047edSMarc Zyngier 		chip = irq_data_get_irq_chip(data);
20281b7047edSMarc Zyngier 		if (chip->irq_set_irqchip_state)
20291b7047edSMarc Zyngier 			break;
20301b7047edSMarc Zyngier #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
20311b7047edSMarc Zyngier 		data = data->parent_data;
20321b7047edSMarc Zyngier #else
20331b7047edSMarc Zyngier 		data = NULL;
20341b7047edSMarc Zyngier #endif
20351b7047edSMarc Zyngier 	} while (data);
20361b7047edSMarc Zyngier 
20371b7047edSMarc Zyngier 	if (data)
20381b7047edSMarc Zyngier 		err = chip->irq_set_irqchip_state(data, which, val);
20391b7047edSMarc Zyngier 
20401b7047edSMarc Zyngier 	irq_put_desc_busunlock(desc, flags);
20411b7047edSMarc Zyngier 	return err;
20421b7047edSMarc Zyngier }
20431ee4fb3eSBjorn Andersson EXPORT_SYMBOL_GPL(irq_set_irqchip_state);
2044