1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright The Asahi Linux Contributors
4  *
5  * Based on irq-lpc32xx:
6  *   Copyright 2015-2016 Vladimir Zapolskiy <vz@mleia.com>
7  * Based on irq-bcm2836:
8  *   Copyright 2015 Broadcom
9  */
10 
11 /*
12  * AIC is a fairly simple interrupt controller with the following features:
13  *
14  * - 896 level-triggered hardware IRQs
15  *   - Single mask bit per IRQ
16  *   - Per-IRQ affinity setting
17  *   - Automatic masking on event delivery (auto-ack)
18  *   - Software triggering (ORed with hw line)
19  * - 2 per-CPU IPIs (meant as "self" and "other", but they are
20  *   interchangeable if not symmetric)
21  * - Automatic prioritization (single event/ack register per CPU, lower IRQs =
22  *   higher priority)
23  * - Automatic masking on ack
24  * - Default "this CPU" register view and explicit per-CPU views
25  *
26  * In addition, this driver also handles FIQs, as these are routed to the same
27  * IRQ vector. These are used for Fast IPIs, the ARMv8 timer IRQs, and
28  * performance counters (TODO).
29  *
30  * Implementation notes:
31  *
32  * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs,
33  *   and one for IPIs.
34  * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller
35  *   and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused).
36  * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu.
37  * - DT bindings use 3-cell form (like GIC):
38  *   - <0 nr flags> - hwirq #nr
39  *   - <1 nr flags> - FIQ #nr
40  *     - nr=0  Physical HV timer
41  *     - nr=1  Virtual HV timer
42  *     - nr=2  Physical guest timer
43  *     - nr=3  Virtual guest timer
44  */
45 
46 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
47 
48 #include <linux/bits.h>
49 #include <linux/bitfield.h>
50 #include <linux/cpuhotplug.h>
51 #include <linux/io.h>
52 #include <linux/irqchip.h>
53 #include <linux/irqchip/arm-vgic-info.h>
54 #include <linux/irqdomain.h>
55 #include <linux/jump_label.h>
56 #include <linux/limits.h>
57 #include <linux/of_address.h>
58 #include <linux/slab.h>
59 #include <asm/apple_m1_pmu.h>
60 #include <asm/cputype.h>
61 #include <asm/exception.h>
62 #include <asm/sysreg.h>
63 #include <asm/virt.h>
64 
65 #include <dt-bindings/interrupt-controller/apple-aic.h>
66 
67 /*
68  * AIC v1 registers (MMIO)
69  */
70 
71 #define AIC_INFO		0x0004
72 #define AIC_INFO_NR_IRQ		GENMASK(15, 0)
73 
74 #define AIC_CONFIG		0x0010
75 
76 #define AIC_WHOAMI		0x2000
77 #define AIC_EVENT		0x2004
78 #define AIC_EVENT_DIE		GENMASK(31, 24)
79 #define AIC_EVENT_TYPE		GENMASK(23, 16)
80 #define AIC_EVENT_NUM		GENMASK(15, 0)
81 
82 #define AIC_EVENT_TYPE_FIQ	0 /* Software use */
83 #define AIC_EVENT_TYPE_IRQ	1
84 #define AIC_EVENT_TYPE_IPI	4
85 #define AIC_EVENT_IPI_OTHER	1
86 #define AIC_EVENT_IPI_SELF	2
87 
88 #define AIC_IPI_SEND		0x2008
89 #define AIC_IPI_ACK		0x200c
90 #define AIC_IPI_MASK_SET	0x2024
91 #define AIC_IPI_MASK_CLR	0x2028
92 
93 #define AIC_IPI_SEND_CPU(cpu)	BIT(cpu)
94 
95 #define AIC_IPI_OTHER		BIT(0)
96 #define AIC_IPI_SELF		BIT(31)
97 
98 #define AIC_TARGET_CPU		0x3000
99 
100 #define AIC_CPU_IPI_SET(cpu)	(0x5008 + ((cpu) << 7))
101 #define AIC_CPU_IPI_CLR(cpu)	(0x500c + ((cpu) << 7))
102 #define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7))
103 #define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7))
104 
105 #define AIC_MAX_IRQ		0x400
106 
107 /*
108  * AIC v2 registers (MMIO)
109  */
110 
111 #define AIC2_VERSION		0x0000
112 #define AIC2_VERSION_VER	GENMASK(7, 0)
113 
114 #define AIC2_INFO1		0x0004
115 #define AIC2_INFO1_NR_IRQ	GENMASK(15, 0)
116 #define AIC2_INFO1_LAST_DIE	GENMASK(27, 24)
117 
118 #define AIC2_INFO2		0x0008
119 
120 #define AIC2_INFO3		0x000c
121 #define AIC2_INFO3_MAX_IRQ	GENMASK(15, 0)
122 #define AIC2_INFO3_MAX_DIE	GENMASK(27, 24)
123 
124 #define AIC2_RESET		0x0010
125 #define AIC2_RESET_RESET	BIT(0)
126 
127 #define AIC2_CONFIG		0x0014
128 #define AIC2_CONFIG_ENABLE	BIT(0)
129 #define AIC2_CONFIG_PREFER_PCPU	BIT(28)
130 
131 #define AIC2_TIMEOUT		0x0028
132 #define AIC2_CLUSTER_PRIO	0x0030
133 #define AIC2_DELAY_GROUPS	0x0100
134 
135 #define AIC2_IRQ_CFG		0x2000
136 
137 /*
138  * AIC2 registers are laid out like this, starting at AIC2_IRQ_CFG:
139  *
140  * Repeat for each die:
141  *   IRQ_CFG: u32 * MAX_IRQS
142  *   SW_SET: u32 * (MAX_IRQS / 32)
143  *   SW_CLR: u32 * (MAX_IRQS / 32)
144  *   MASK_SET: u32 * (MAX_IRQS / 32)
145  *   MASK_CLR: u32 * (MAX_IRQS / 32)
146  *   HW_STATE: u32 * (MAX_IRQS / 32)
147  *
148  * This is followed by a set of event registers, each 16K page aligned.
149  * The first one is the AP event register we will use. Unfortunately,
150  * the actual implemented die count is not specified anywhere in the
151  * capability registers, so we have to explicitly specify the event
152  * register as a second reg entry in the device tree to remain
153  * forward-compatible.
154  */
155 
156 #define AIC2_IRQ_CFG_TARGET	GENMASK(3, 0)
157 #define AIC2_IRQ_CFG_DELAY_IDX	GENMASK(7, 5)
158 
159 #define MASK_REG(x)		(4 * ((x) >> 5))
160 #define MASK_BIT(x)		BIT((x) & GENMASK(4, 0))
161 
162 /*
163  * IMP-DEF sysregs that control FIQ sources
164  */
165 
166 /* IPI request registers */
167 #define SYS_IMP_APL_IPI_RR_LOCAL_EL1	sys_reg(3, 5, 15, 0, 0)
168 #define SYS_IMP_APL_IPI_RR_GLOBAL_EL1	sys_reg(3, 5, 15, 0, 1)
169 #define IPI_RR_CPU			GENMASK(7, 0)
170 /* Cluster only used for the GLOBAL register */
171 #define IPI_RR_CLUSTER			GENMASK(23, 16)
172 #define IPI_RR_TYPE			GENMASK(29, 28)
173 #define IPI_RR_IMMEDIATE		0
174 #define IPI_RR_RETRACT			1
175 #define IPI_RR_DEFERRED			2
176 #define IPI_RR_NOWAKE			3
177 
178 /* IPI status register */
179 #define SYS_IMP_APL_IPI_SR_EL1		sys_reg(3, 5, 15, 1, 1)
180 #define IPI_SR_PENDING			BIT(0)
181 
182 /* Guest timer FIQ enable register */
183 #define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2	sys_reg(3, 5, 15, 1, 3)
184 #define VM_TMR_FIQ_ENABLE_V		BIT(0)
185 #define VM_TMR_FIQ_ENABLE_P		BIT(1)
186 
187 /* Deferred IPI countdown register */
188 #define SYS_IMP_APL_IPI_CR_EL1		sys_reg(3, 5, 15, 3, 1)
189 
190 /* Uncore PMC control register */
191 #define SYS_IMP_APL_UPMCR0_EL1		sys_reg(3, 7, 15, 0, 4)
192 #define UPMCR0_IMODE			GENMASK(18, 16)
193 #define UPMCR0_IMODE_OFF		0
194 #define UPMCR0_IMODE_AIC		2
195 #define UPMCR0_IMODE_HALT		3
196 #define UPMCR0_IMODE_FIQ		4
197 
198 /* Uncore PMC status register */
199 #define SYS_IMP_APL_UPMSR_EL1		sys_reg(3, 7, 15, 6, 4)
200 #define UPMSR_IACT			BIT(0)
201 
202 /* MPIDR fields */
203 #define MPIDR_CPU(x)			MPIDR_AFFINITY_LEVEL(x, 0)
204 #define MPIDR_CLUSTER(x)		MPIDR_AFFINITY_LEVEL(x, 1)
205 
206 #define AIC_IRQ_HWIRQ(die, irq)	(FIELD_PREP(AIC_EVENT_DIE, die) | \
207 				 FIELD_PREP(AIC_EVENT_TYPE, AIC_EVENT_TYPE_IRQ) | \
208 				 FIELD_PREP(AIC_EVENT_NUM, irq))
209 #define AIC_FIQ_HWIRQ(x)	(FIELD_PREP(AIC_EVENT_TYPE, AIC_EVENT_TYPE_FIQ) | \
210 				 FIELD_PREP(AIC_EVENT_NUM, x))
211 #define AIC_HWIRQ_IRQ(x)	FIELD_GET(AIC_EVENT_NUM, x)
212 #define AIC_HWIRQ_DIE(x)	FIELD_GET(AIC_EVENT_DIE, x)
213 #define AIC_NR_FIQ		6
214 #define AIC_NR_SWIPI		32
215 
216 /*
217  * FIQ hwirq index definitions: FIQ sources use the DT binding defines
218  * directly, except that timers are special. At the irqchip level, the
219  * two timer types are represented by their access method: _EL0 registers
220  * or _EL02 registers. In the DT binding, the timers are represented
221  * by their purpose (HV or guest). This mapping is for when the kernel is
222  * running at EL2 (with VHE). When the kernel is running at EL1, the
223  * mapping differs and aic_irq_domain_translate() performs the remapping.
224  */
225 
226 #define AIC_TMR_EL0_PHYS	AIC_TMR_HV_PHYS
227 #define AIC_TMR_EL0_VIRT	AIC_TMR_HV_VIRT
228 #define AIC_TMR_EL02_PHYS	AIC_TMR_GUEST_PHYS
229 #define AIC_TMR_EL02_VIRT	AIC_TMR_GUEST_VIRT
230 
231 DEFINE_STATIC_KEY_TRUE(use_fast_ipi);
232 
233 struct aic_info {
234 	int version;
235 
236 	/* Register offsets */
237 	u32 event;
238 	u32 target_cpu;
239 	u32 irq_cfg;
240 	u32 sw_set;
241 	u32 sw_clr;
242 	u32 mask_set;
243 	u32 mask_clr;
244 
245 	u32 die_stride;
246 
247 	/* Features */
248 	bool fast_ipi;
249 };
250 
251 static const struct aic_info aic1_info = {
252 	.version	= 1,
253 
254 	.event		= AIC_EVENT,
255 	.target_cpu	= AIC_TARGET_CPU,
256 };
257 
258 static const struct aic_info aic1_fipi_info = {
259 	.version	= 1,
260 
261 	.event		= AIC_EVENT,
262 	.target_cpu	= AIC_TARGET_CPU,
263 
264 	.fast_ipi	= true,
265 };
266 
267 static const struct aic_info aic2_info = {
268 	.version	= 2,
269 
270 	.irq_cfg	= AIC2_IRQ_CFG,
271 
272 	.fast_ipi	= true,
273 };
274 
275 static const struct of_device_id aic_info_match[] = {
276 	{
277 		.compatible = "apple,t8103-aic",
278 		.data = &aic1_fipi_info,
279 	},
280 	{
281 		.compatible = "apple,aic",
282 		.data = &aic1_info,
283 	},
284 	{
285 		.compatible = "apple,aic2",
286 		.data = &aic2_info,
287 	},
288 	{}
289 };
290 
291 struct aic_irq_chip {
292 	void __iomem *base;
293 	void __iomem *event;
294 	struct irq_domain *hw_domain;
295 	struct irq_domain *ipi_domain;
296 	struct {
297 		cpumask_t aff;
298 	} *fiq_aff[AIC_NR_FIQ];
299 
300 	int nr_irq;
301 	int max_irq;
302 	int nr_die;
303 	int max_die;
304 
305 	struct aic_info info;
306 };
307 
308 static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked);
309 
310 static DEFINE_PER_CPU(atomic_t, aic_vipi_flag);
311 static DEFINE_PER_CPU(atomic_t, aic_vipi_enable);
312 
313 static struct aic_irq_chip *aic_irqc;
314 
315 static void aic_handle_ipi(struct pt_regs *regs);
316 
317 static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg)
318 {
319 	return readl_relaxed(ic->base + reg);
320 }
321 
322 static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val)
323 {
324 	writel_relaxed(val, ic->base + reg);
325 }
326 
327 /*
328  * IRQ irqchip
329  */
330 
331 static void aic_irq_mask(struct irq_data *d)
332 {
333 	irq_hw_number_t hwirq = irqd_to_hwirq(d);
334 	struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d);
335 
336 	u32 off = AIC_HWIRQ_DIE(hwirq) * ic->info.die_stride;
337 	u32 irq = AIC_HWIRQ_IRQ(hwirq);
338 
339 	aic_ic_write(ic, ic->info.mask_set + off + MASK_REG(irq), MASK_BIT(irq));
340 }
341 
342 static void aic_irq_unmask(struct irq_data *d)
343 {
344 	irq_hw_number_t hwirq = irqd_to_hwirq(d);
345 	struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d);
346 
347 	u32 off = AIC_HWIRQ_DIE(hwirq) * ic->info.die_stride;
348 	u32 irq = AIC_HWIRQ_IRQ(hwirq);
349 
350 	aic_ic_write(ic, ic->info.mask_clr + off + MASK_REG(irq), MASK_BIT(irq));
351 }
352 
353 static void aic_irq_eoi(struct irq_data *d)
354 {
355 	/*
356 	 * Reading the interrupt reason automatically acknowledges and masks
357 	 * the IRQ, so we just unmask it here if needed.
358 	 */
359 	if (!irqd_irq_masked(d))
360 		aic_irq_unmask(d);
361 }
362 
363 static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs)
364 {
365 	struct aic_irq_chip *ic = aic_irqc;
366 	u32 event, type, irq;
367 
368 	do {
369 		/*
370 		 * We cannot use a relaxed read here, as reads from DMA buffers
371 		 * need to be ordered after the IRQ fires.
372 		 */
373 		event = readl(ic->event + ic->info.event);
374 		type = FIELD_GET(AIC_EVENT_TYPE, event);
375 		irq = FIELD_GET(AIC_EVENT_NUM, event);
376 
377 		if (type == AIC_EVENT_TYPE_IRQ)
378 			generic_handle_domain_irq(aic_irqc->hw_domain, event);
379 		else if (type == AIC_EVENT_TYPE_IPI && irq == 1)
380 			aic_handle_ipi(regs);
381 		else if (event != 0)
382 			pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq);
383 	} while (event);
384 
385 	/*
386 	 * vGIC maintenance interrupts end up here too, so we need to check
387 	 * for them separately. This should never trigger if KVM is working
388 	 * properly, because it will have already taken care of clearing it
389 	 * on guest exit before this handler runs.
390 	 */
391 	if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) &&
392 		read_sysreg_s(SYS_ICH_MISR_EL2) != 0) {
393 		pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n");
394 		sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0);
395 	}
396 }
397 
398 static int aic_irq_set_affinity(struct irq_data *d,
399 				const struct cpumask *mask_val, bool force)
400 {
401 	irq_hw_number_t hwirq = irqd_to_hwirq(d);
402 	struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d);
403 	int cpu;
404 
405 	BUG_ON(!ic->info.target_cpu);
406 
407 	if (force)
408 		cpu = cpumask_first(mask_val);
409 	else
410 		cpu = cpumask_any_and(mask_val, cpu_online_mask);
411 
412 	aic_ic_write(ic, ic->info.target_cpu + AIC_HWIRQ_IRQ(hwirq) * 4, BIT(cpu));
413 	irq_data_update_effective_affinity(d, cpumask_of(cpu));
414 
415 	return IRQ_SET_MASK_OK;
416 }
417 
418 static int aic_irq_set_type(struct irq_data *d, unsigned int type)
419 {
420 	/*
421 	 * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't
422 	 * have a way to find out the type of any given IRQ, so just allow both.
423 	 */
424 	return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL;
425 }
426 
427 static struct irq_chip aic_chip = {
428 	.name = "AIC",
429 	.irq_mask = aic_irq_mask,
430 	.irq_unmask = aic_irq_unmask,
431 	.irq_eoi = aic_irq_eoi,
432 	.irq_set_affinity = aic_irq_set_affinity,
433 	.irq_set_type = aic_irq_set_type,
434 };
435 
436 static struct irq_chip aic2_chip = {
437 	.name = "AIC2",
438 	.irq_mask = aic_irq_mask,
439 	.irq_unmask = aic_irq_unmask,
440 	.irq_eoi = aic_irq_eoi,
441 	.irq_set_type = aic_irq_set_type,
442 };
443 
444 /*
445  * FIQ irqchip
446  */
447 
448 static unsigned long aic_fiq_get_idx(struct irq_data *d)
449 {
450 	return AIC_HWIRQ_IRQ(irqd_to_hwirq(d));
451 }
452 
453 static void aic_fiq_set_mask(struct irq_data *d)
454 {
455 	/* Only the guest timers have real mask bits, unfortunately. */
456 	switch (aic_fiq_get_idx(d)) {
457 	case AIC_TMR_EL02_PHYS:
458 		sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0);
459 		isb();
460 		break;
461 	case AIC_TMR_EL02_VIRT:
462 		sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0);
463 		isb();
464 		break;
465 	default:
466 		break;
467 	}
468 }
469 
470 static void aic_fiq_clear_mask(struct irq_data *d)
471 {
472 	switch (aic_fiq_get_idx(d)) {
473 	case AIC_TMR_EL02_PHYS:
474 		sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P);
475 		isb();
476 		break;
477 	case AIC_TMR_EL02_VIRT:
478 		sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V);
479 		isb();
480 		break;
481 	default:
482 		break;
483 	}
484 }
485 
486 static void aic_fiq_mask(struct irq_data *d)
487 {
488 	aic_fiq_set_mask(d);
489 	__this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d)));
490 }
491 
492 static void aic_fiq_unmask(struct irq_data *d)
493 {
494 	aic_fiq_clear_mask(d);
495 	__this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d)));
496 }
497 
498 static void aic_fiq_eoi(struct irq_data *d)
499 {
500 	/* We mask to ack (where we can), so we need to unmask at EOI. */
501 	if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d)))
502 		aic_fiq_clear_mask(d);
503 }
504 
505 #define TIMER_FIRING(x)                                                        \
506 	(((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK |            \
507 		 ARCH_TIMER_CTRL_IT_STAT)) ==                                  \
508 	 (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT))
509 
510 static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs)
511 {
512 	/*
513 	 * It would be really nice if we had a system register that lets us get
514 	 * the FIQ source state without having to peek down into sources...
515 	 * but such a register does not seem to exist.
516 	 *
517 	 * So, we have these potential sources to test for:
518 	 *  - Fast IPIs (not yet used)
519 	 *  - The 4 timers (CNTP, CNTV for each of HV and guest)
520 	 *  - Per-core PMCs (not yet supported)
521 	 *  - Per-cluster uncore PMCs (not yet supported)
522 	 *
523 	 * Since not dealing with any of these results in a FIQ storm,
524 	 * we check for everything here, even things we don't support yet.
525 	 */
526 
527 	if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) {
528 		if (static_branch_likely(&use_fast_ipi)) {
529 			aic_handle_ipi(regs);
530 		} else {
531 			pr_err_ratelimited("Fast IPI fired. Acking.\n");
532 			write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1);
533 		}
534 	}
535 
536 	if (TIMER_FIRING(read_sysreg(cntp_ctl_el0)))
537 		generic_handle_domain_irq(aic_irqc->hw_domain,
538 					  AIC_FIQ_HWIRQ(AIC_TMR_EL0_PHYS));
539 
540 	if (TIMER_FIRING(read_sysreg(cntv_ctl_el0)))
541 		generic_handle_domain_irq(aic_irqc->hw_domain,
542 					  AIC_FIQ_HWIRQ(AIC_TMR_EL0_VIRT));
543 
544 	if (is_kernel_in_hyp_mode()) {
545 		uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2);
546 
547 		if ((enabled & VM_TMR_FIQ_ENABLE_P) &&
548 		    TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02)))
549 			generic_handle_domain_irq(aic_irqc->hw_domain,
550 						  AIC_FIQ_HWIRQ(AIC_TMR_EL02_PHYS));
551 
552 		if ((enabled & VM_TMR_FIQ_ENABLE_V) &&
553 		    TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02)))
554 			generic_handle_domain_irq(aic_irqc->hw_domain,
555 						  AIC_FIQ_HWIRQ(AIC_TMR_EL02_VIRT));
556 	}
557 
558 	if (read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & PMCR0_IACT) {
559 		int irq;
560 		if (cpumask_test_cpu(smp_processor_id(),
561 				     &aic_irqc->fiq_aff[AIC_CPU_PMU_P]->aff))
562 			irq = AIC_CPU_PMU_P;
563 		else
564 			irq = AIC_CPU_PMU_E;
565 		generic_handle_domain_irq(aic_irqc->hw_domain,
566 					  AIC_FIQ_HWIRQ(irq));
567 	}
568 
569 	if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ &&
570 			(read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) {
571 		/* Same story with uncore PMCs */
572 		pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n");
573 		sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE,
574 				   FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF));
575 	}
576 }
577 
578 static int aic_fiq_set_type(struct irq_data *d, unsigned int type)
579 {
580 	return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL;
581 }
582 
583 static struct irq_chip fiq_chip = {
584 	.name = "AIC-FIQ",
585 	.irq_mask = aic_fiq_mask,
586 	.irq_unmask = aic_fiq_unmask,
587 	.irq_ack = aic_fiq_set_mask,
588 	.irq_eoi = aic_fiq_eoi,
589 	.irq_set_type = aic_fiq_set_type,
590 };
591 
592 /*
593  * Main IRQ domain
594  */
595 
596 static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq,
597 			      irq_hw_number_t hw)
598 {
599 	struct aic_irq_chip *ic = id->host_data;
600 	u32 type = FIELD_GET(AIC_EVENT_TYPE, hw);
601 	struct irq_chip *chip = &aic_chip;
602 
603 	if (ic->info.version == 2)
604 		chip = &aic2_chip;
605 
606 	if (type == AIC_EVENT_TYPE_IRQ) {
607 		irq_domain_set_info(id, irq, hw, chip, id->host_data,
608 				    handle_fasteoi_irq, NULL, NULL);
609 		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
610 	} else {
611 		int fiq = FIELD_GET(AIC_EVENT_NUM, hw);
612 
613 		switch (fiq) {
614 		case AIC_CPU_PMU_P:
615 		case AIC_CPU_PMU_E:
616 			irq_set_percpu_devid_partition(irq, &ic->fiq_aff[fiq]->aff);
617 			break;
618 		default:
619 			irq_set_percpu_devid(irq);
620 			break;
621 		}
622 
623 		irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data,
624 				    handle_percpu_devid_irq, NULL, NULL);
625 	}
626 
627 	return 0;
628 }
629 
630 static int aic_irq_domain_translate(struct irq_domain *id,
631 				    struct irq_fwspec *fwspec,
632 				    unsigned long *hwirq,
633 				    unsigned int *type)
634 {
635 	struct aic_irq_chip *ic = id->host_data;
636 	u32 *args;
637 	u32 die = 0;
638 
639 	if (fwspec->param_count < 3 || fwspec->param_count > 4 ||
640 	    !is_of_node(fwspec->fwnode))
641 		return -EINVAL;
642 
643 	args = &fwspec->param[1];
644 
645 	if (fwspec->param_count == 4) {
646 		die = args[0];
647 		args++;
648 	}
649 
650 	switch (fwspec->param[0]) {
651 	case AIC_IRQ:
652 		if (die >= ic->nr_die)
653 			return -EINVAL;
654 		if (args[0] >= ic->nr_irq)
655 			return -EINVAL;
656 		*hwirq = AIC_IRQ_HWIRQ(die, args[0]);
657 		break;
658 	case AIC_FIQ:
659 		if (die != 0)
660 			return -EINVAL;
661 		if (args[0] >= AIC_NR_FIQ)
662 			return -EINVAL;
663 		*hwirq = AIC_FIQ_HWIRQ(args[0]);
664 
665 		/*
666 		 * In EL1 the non-redirected registers are the guest's,
667 		 * not EL2's, so remap the hwirqs to match.
668 		 */
669 		if (!is_kernel_in_hyp_mode()) {
670 			switch (args[0]) {
671 			case AIC_TMR_GUEST_PHYS:
672 				*hwirq = AIC_FIQ_HWIRQ(AIC_TMR_EL0_PHYS);
673 				break;
674 			case AIC_TMR_GUEST_VIRT:
675 				*hwirq = AIC_FIQ_HWIRQ(AIC_TMR_EL0_VIRT);
676 				break;
677 			case AIC_TMR_HV_PHYS:
678 			case AIC_TMR_HV_VIRT:
679 				return -ENOENT;
680 			default:
681 				break;
682 			}
683 		}
684 		break;
685 	default:
686 		return -EINVAL;
687 	}
688 
689 	*type = args[1] & IRQ_TYPE_SENSE_MASK;
690 
691 	return 0;
692 }
693 
694 static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
695 				unsigned int nr_irqs, void *arg)
696 {
697 	unsigned int type = IRQ_TYPE_NONE;
698 	struct irq_fwspec *fwspec = arg;
699 	irq_hw_number_t hwirq;
700 	int i, ret;
701 
702 	ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type);
703 	if (ret)
704 		return ret;
705 
706 	for (i = 0; i < nr_irqs; i++) {
707 		ret = aic_irq_domain_map(domain, virq + i, hwirq + i);
708 		if (ret)
709 			return ret;
710 	}
711 
712 	return 0;
713 }
714 
715 static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
716 				unsigned int nr_irqs)
717 {
718 	int i;
719 
720 	for (i = 0; i < nr_irqs; i++) {
721 		struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
722 
723 		irq_set_handler(virq + i, NULL);
724 		irq_domain_reset_irq_data(d);
725 	}
726 }
727 
728 static const struct irq_domain_ops aic_irq_domain_ops = {
729 	.translate	= aic_irq_domain_translate,
730 	.alloc		= aic_irq_domain_alloc,
731 	.free		= aic_irq_domain_free,
732 };
733 
734 /*
735  * IPI irqchip
736  */
737 
738 static void aic_ipi_send_fast(int cpu)
739 {
740 	u64 mpidr = cpu_logical_map(cpu);
741 	u64 my_mpidr = read_cpuid_mpidr();
742 	u64 cluster = MPIDR_CLUSTER(mpidr);
743 	u64 idx = MPIDR_CPU(mpidr);
744 
745 	if (MPIDR_CLUSTER(my_mpidr) == cluster)
746 		write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx),
747 			       SYS_IMP_APL_IPI_RR_LOCAL_EL1);
748 	else
749 		write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx) | FIELD_PREP(IPI_RR_CLUSTER, cluster),
750 			       SYS_IMP_APL_IPI_RR_GLOBAL_EL1);
751 	isb();
752 }
753 
754 static void aic_ipi_mask(struct irq_data *d)
755 {
756 	u32 irq_bit = BIT(irqd_to_hwirq(d));
757 
758 	/* No specific ordering requirements needed here. */
759 	atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable));
760 }
761 
762 static void aic_ipi_unmask(struct irq_data *d)
763 {
764 	struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d);
765 	u32 irq_bit = BIT(irqd_to_hwirq(d));
766 
767 	atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable));
768 
769 	/*
770 	 * The atomic_or() above must complete before the atomic_read()
771 	 * below to avoid racing aic_ipi_send_mask().
772 	 */
773 	smp_mb__after_atomic();
774 
775 	/*
776 	 * If a pending vIPI was unmasked, raise a HW IPI to ourselves.
777 	 * No barriers needed here since this is a self-IPI.
778 	 */
779 	if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) {
780 		if (static_branch_likely(&use_fast_ipi))
781 			aic_ipi_send_fast(smp_processor_id());
782 		else
783 			aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id()));
784 	}
785 }
786 
787 static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
788 {
789 	struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d);
790 	u32 irq_bit = BIT(irqd_to_hwirq(d));
791 	u32 send = 0;
792 	int cpu;
793 	unsigned long pending;
794 
795 	for_each_cpu(cpu, mask) {
796 		/*
797 		 * This sequence is the mirror of the one in aic_ipi_unmask();
798 		 * see the comment there. Additionally, release semantics
799 		 * ensure that the vIPI flag set is ordered after any shared
800 		 * memory accesses that precede it. This therefore also pairs
801 		 * with the atomic_fetch_andnot in aic_handle_ipi().
802 		 */
803 		pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu));
804 
805 		/*
806 		 * The atomic_fetch_or_release() above must complete before the
807 		 * atomic_read() below to avoid racing aic_ipi_unmask().
808 		 */
809 		smp_mb__after_atomic();
810 
811 		if (!(pending & irq_bit) &&
812 		    (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) {
813 			if (static_branch_likely(&use_fast_ipi))
814 				aic_ipi_send_fast(cpu);
815 			else
816 				send |= AIC_IPI_SEND_CPU(cpu);
817 		}
818 	}
819 
820 	/*
821 	 * The flag writes must complete before the physical IPI is issued
822 	 * to another CPU. This is implied by the control dependency on
823 	 * the result of atomic_read_acquire() above, which is itself
824 	 * already ordered after the vIPI flag write.
825 	 */
826 	if (send)
827 		aic_ic_write(ic, AIC_IPI_SEND, send);
828 }
829 
830 static struct irq_chip ipi_chip = {
831 	.name = "AIC-IPI",
832 	.irq_mask = aic_ipi_mask,
833 	.irq_unmask = aic_ipi_unmask,
834 	.ipi_send_mask = aic_ipi_send_mask,
835 };
836 
837 /*
838  * IPI IRQ domain
839  */
840 
841 static void aic_handle_ipi(struct pt_regs *regs)
842 {
843 	int i;
844 	unsigned long enabled, firing;
845 
846 	/*
847 	 * Ack the IPI. We need to order this after the AIC event read, but
848 	 * that is enforced by normal MMIO ordering guarantees.
849 	 *
850 	 * For the Fast IPI case, this needs to be ordered before the vIPI
851 	 * handling below, so we need to isb();
852 	 */
853 	if (static_branch_likely(&use_fast_ipi)) {
854 		write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1);
855 		isb();
856 	} else {
857 		aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER);
858 	}
859 
860 	/*
861 	 * The mask read does not need to be ordered. Only we can change
862 	 * our own mask anyway, so no races are possible here, as long as
863 	 * we are properly in the interrupt handler (which is covered by
864 	 * the barrier that is part of the top-level AIC handler's readl()).
865 	 */
866 	enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable));
867 
868 	/*
869 	 * Clear the IPIs we are about to handle. This pairs with the
870 	 * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be
871 	 * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and
872 	 * before IPI handling code (to avoid races handling vIPIs before they
873 	 * are signaled). The former is taken care of by the release semantics
874 	 * of the write portion, while the latter is taken care of by the
875 	 * acquire semantics of the read portion.
876 	 */
877 	firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled;
878 
879 	for_each_set_bit(i, &firing, AIC_NR_SWIPI)
880 		generic_handle_domain_irq(aic_irqc->ipi_domain, i);
881 
882 	/*
883 	 * No ordering needed here; at worst this just changes the timing of
884 	 * when the next IPI will be delivered.
885 	 */
886 	if (!static_branch_likely(&use_fast_ipi))
887 		aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER);
888 }
889 
890 static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq,
891 			 unsigned int nr_irqs, void *args)
892 {
893 	int i;
894 
895 	for (i = 0; i < nr_irqs; i++) {
896 		irq_set_percpu_devid(virq + i);
897 		irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data,
898 				    handle_percpu_devid_irq, NULL, NULL);
899 	}
900 
901 	return 0;
902 }
903 
904 static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
905 {
906 	/* Not freeing IPIs */
907 }
908 
909 static const struct irq_domain_ops aic_ipi_domain_ops = {
910 	.alloc = aic_ipi_alloc,
911 	.free = aic_ipi_free,
912 };
913 
914 static int __init aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node)
915 {
916 	struct irq_domain *ipi_domain;
917 	int base_ipi;
918 
919 	ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI,
920 					      &aic_ipi_domain_ops, irqc);
921 	if (WARN_ON(!ipi_domain))
922 		return -ENODEV;
923 
924 	ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE;
925 	irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI);
926 
927 	base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI,
928 					   NUMA_NO_NODE, NULL, false, NULL);
929 
930 	if (WARN_ON(!base_ipi)) {
931 		irq_domain_remove(ipi_domain);
932 		return -ENODEV;
933 	}
934 
935 	set_smp_ipi_range(base_ipi, AIC_NR_SWIPI);
936 
937 	irqc->ipi_domain = ipi_domain;
938 
939 	return 0;
940 }
941 
942 static int aic_init_cpu(unsigned int cpu)
943 {
944 	/* Mask all hard-wired per-CPU IRQ/FIQ sources */
945 
946 	/* Pending Fast IPI FIQs */
947 	write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1);
948 
949 	/* Timer FIQs */
950 	sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK);
951 	sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK);
952 
953 	/* EL2-only (VHE mode) IRQ sources */
954 	if (is_kernel_in_hyp_mode()) {
955 		/* Guest timers */
956 		sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2,
957 				   VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0);
958 
959 		/* vGIC maintenance IRQ */
960 		sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0);
961 	}
962 
963 	/* PMC FIQ */
964 	sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT,
965 			   FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF));
966 
967 	/* Uncore PMC FIQ */
968 	sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE,
969 			   FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF));
970 
971 	/* Commit all of the above */
972 	isb();
973 
974 	if (aic_irqc->info.version == 1) {
975 		/*
976 		 * Make sure the kernel's idea of logical CPU order is the same as AIC's
977 		 * If we ever end up with a mismatch here, we will have to introduce
978 		 * a mapping table similar to what other irqchip drivers do.
979 		 */
980 		WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id());
981 
982 		/*
983 		 * Always keep IPIs unmasked at the hardware level (except auto-masking
984 		 * by AIC during processing). We manage masks at the vIPI level.
985 		 * These registers only exist on AICv1, AICv2 always uses fast IPIs.
986 		 */
987 		aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER);
988 		if (static_branch_likely(&use_fast_ipi)) {
989 			aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF | AIC_IPI_OTHER);
990 		} else {
991 			aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF);
992 			aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER);
993 		}
994 	}
995 
996 	/* Initialize the local mask state */
997 	__this_cpu_write(aic_fiq_unmasked, 0);
998 
999 	return 0;
1000 }
1001 
1002 static struct gic_kvm_info vgic_info __initdata = {
1003 	.type			= GIC_V3,
1004 	.no_maint_irq_mask	= true,
1005 	.no_hw_deactivation	= true,
1006 };
1007 
1008 static void build_fiq_affinity(struct aic_irq_chip *ic, struct device_node *aff)
1009 {
1010 	int i, n;
1011 	u32 fiq;
1012 
1013 	if (of_property_read_u32(aff, "apple,fiq-index", &fiq) ||
1014 	    WARN_ON(fiq >= AIC_NR_FIQ) || ic->fiq_aff[fiq])
1015 		return;
1016 
1017 	n = of_property_count_elems_of_size(aff, "cpus", sizeof(u32));
1018 	if (WARN_ON(n < 0))
1019 		return;
1020 
1021 	ic->fiq_aff[fiq] = kzalloc(sizeof(*ic->fiq_aff[fiq]), GFP_KERNEL);
1022 	if (!ic->fiq_aff[fiq])
1023 		return;
1024 
1025 	for (i = 0; i < n; i++) {
1026 		struct device_node *cpu_node;
1027 		u32 cpu_phandle;
1028 		int cpu;
1029 
1030 		if (of_property_read_u32_index(aff, "cpus", i, &cpu_phandle))
1031 			continue;
1032 
1033 		cpu_node = of_find_node_by_phandle(cpu_phandle);
1034 		if (WARN_ON(!cpu_node))
1035 			continue;
1036 
1037 		cpu = of_cpu_node_to_id(cpu_node);
1038 		of_node_put(cpu_node);
1039 		if (WARN_ON(cpu < 0))
1040 			continue;
1041 
1042 		cpumask_set_cpu(cpu, &ic->fiq_aff[fiq]->aff);
1043 	}
1044 }
1045 
1046 static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent)
1047 {
1048 	int i, die;
1049 	u32 off, start_off;
1050 	void __iomem *regs;
1051 	struct aic_irq_chip *irqc;
1052 	struct device_node *affs;
1053 	const struct of_device_id *match;
1054 
1055 	regs = of_iomap(node, 0);
1056 	if (WARN_ON(!regs))
1057 		return -EIO;
1058 
1059 	irqc = kzalloc(sizeof(*irqc), GFP_KERNEL);
1060 	if (!irqc) {
1061 		iounmap(regs);
1062 		return -ENOMEM;
1063 	}
1064 
1065 	irqc->base = regs;
1066 
1067 	match = of_match_node(aic_info_match, node);
1068 	if (!match)
1069 		goto err_unmap;
1070 
1071 	irqc->info = *(struct aic_info *)match->data;
1072 
1073 	aic_irqc = irqc;
1074 
1075 	switch (irqc->info.version) {
1076 	case 1: {
1077 		u32 info;
1078 
1079 		info = aic_ic_read(irqc, AIC_INFO);
1080 		irqc->nr_irq = FIELD_GET(AIC_INFO_NR_IRQ, info);
1081 		irqc->max_irq = AIC_MAX_IRQ;
1082 		irqc->nr_die = irqc->max_die = 1;
1083 
1084 		off = start_off = irqc->info.target_cpu;
1085 		off += sizeof(u32) * irqc->max_irq; /* TARGET_CPU */
1086 
1087 		irqc->event = irqc->base;
1088 
1089 		break;
1090 	}
1091 	case 2: {
1092 		u32 info1, info3;
1093 
1094 		info1 = aic_ic_read(irqc, AIC2_INFO1);
1095 		info3 = aic_ic_read(irqc, AIC2_INFO3);
1096 
1097 		irqc->nr_irq = FIELD_GET(AIC2_INFO1_NR_IRQ, info1);
1098 		irqc->max_irq = FIELD_GET(AIC2_INFO3_MAX_IRQ, info3);
1099 		irqc->nr_die = FIELD_GET(AIC2_INFO1_LAST_DIE, info1) + 1;
1100 		irqc->max_die = FIELD_GET(AIC2_INFO3_MAX_DIE, info3);
1101 
1102 		off = start_off = irqc->info.irq_cfg;
1103 		off += sizeof(u32) * irqc->max_irq; /* IRQ_CFG */
1104 
1105 		irqc->event = of_iomap(node, 1);
1106 		if (WARN_ON(!irqc->event))
1107 			goto err_unmap;
1108 
1109 		break;
1110 	}
1111 	}
1112 
1113 	irqc->info.sw_set = off;
1114 	off += sizeof(u32) * (irqc->max_irq >> 5); /* SW_SET */
1115 	irqc->info.sw_clr = off;
1116 	off += sizeof(u32) * (irqc->max_irq >> 5); /* SW_CLR */
1117 	irqc->info.mask_set = off;
1118 	off += sizeof(u32) * (irqc->max_irq >> 5); /* MASK_SET */
1119 	irqc->info.mask_clr = off;
1120 	off += sizeof(u32) * (irqc->max_irq >> 5); /* MASK_CLR */
1121 	off += sizeof(u32) * (irqc->max_irq >> 5); /* HW_STATE */
1122 
1123 	if (irqc->info.fast_ipi)
1124 		static_branch_enable(&use_fast_ipi);
1125 	else
1126 		static_branch_disable(&use_fast_ipi);
1127 
1128 	irqc->info.die_stride = off - start_off;
1129 
1130 	irqc->hw_domain = irq_domain_create_tree(of_node_to_fwnode(node),
1131 						 &aic_irq_domain_ops, irqc);
1132 	if (WARN_ON(!irqc->hw_domain))
1133 		goto err_unmap;
1134 
1135 	irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED);
1136 
1137 	if (aic_init_smp(irqc, node))
1138 		goto err_remove_domain;
1139 
1140 	affs = of_get_child_by_name(node, "affinities");
1141 	if (affs) {
1142 		struct device_node *chld;
1143 
1144 		for_each_child_of_node(affs, chld)
1145 			build_fiq_affinity(irqc, chld);
1146 	}
1147 	of_node_put(affs);
1148 
1149 	set_handle_irq(aic_handle_irq);
1150 	set_handle_fiq(aic_handle_fiq);
1151 
1152 	off = 0;
1153 	for (die = 0; die < irqc->nr_die; die++) {
1154 		for (i = 0; i < BITS_TO_U32(irqc->nr_irq); i++)
1155 			aic_ic_write(irqc, irqc->info.mask_set + off + i * 4, U32_MAX);
1156 		for (i = 0; i < BITS_TO_U32(irqc->nr_irq); i++)
1157 			aic_ic_write(irqc, irqc->info.sw_clr + off + i * 4, U32_MAX);
1158 		if (irqc->info.target_cpu)
1159 			for (i = 0; i < irqc->nr_irq; i++)
1160 				aic_ic_write(irqc, irqc->info.target_cpu + off + i * 4, 1);
1161 		off += irqc->info.die_stride;
1162 	}
1163 
1164 	if (irqc->info.version == 2) {
1165 		u32 config = aic_ic_read(irqc, AIC2_CONFIG);
1166 
1167 		config |= AIC2_CONFIG_ENABLE;
1168 		aic_ic_write(irqc, AIC2_CONFIG, config);
1169 	}
1170 
1171 	if (!is_kernel_in_hyp_mode())
1172 		pr_info("Kernel running in EL1, mapping interrupts");
1173 
1174 	if (static_branch_likely(&use_fast_ipi))
1175 		pr_info("Using Fast IPIs");
1176 
1177 	cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING,
1178 			  "irqchip/apple-aic/ipi:starting",
1179 			  aic_init_cpu, NULL);
1180 
1181 	vgic_set_kvm_info(&vgic_info);
1182 
1183 	pr_info("Initialized with %d/%d IRQs * %d/%d die(s), %d FIQs, %d vIPIs",
1184 		irqc->nr_irq, irqc->max_irq, irqc->nr_die, irqc->max_die, AIC_NR_FIQ, AIC_NR_SWIPI);
1185 
1186 	return 0;
1187 
1188 err_remove_domain:
1189 	irq_domain_remove(irqc->hw_domain);
1190 err_unmap:
1191 	if (irqc->event && irqc->event != irqc->base)
1192 		iounmap(irqc->event);
1193 	iounmap(irqc->base);
1194 	kfree(irqc);
1195 	return -ENODEV;
1196 }
1197 
1198 IRQCHIP_DECLARE(apple_aic, "apple,aic", aic_of_ic_init);
1199 IRQCHIP_DECLARE(apple_aic2, "apple,aic2", aic_of_ic_init);
1200