1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2021f6537SMarc Zyngier /*
30edc23eaSMarc Zyngier * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4021f6537SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com>
5021f6537SMarc Zyngier */
6021f6537SMarc Zyngier
768628bb8SJulien Grall #define pr_fmt(fmt) "GICv3: " fmt
868628bb8SJulien Grall
9ffa7d616STomasz Nowicki #include <linux/acpi.h>
10021f6537SMarc Zyngier #include <linux/cpu.h>
113708d52fSSudeep Holla #include <linux/cpu_pm.h>
12021f6537SMarc Zyngier #include <linux/delay.h>
13021f6537SMarc Zyngier #include <linux/interrupt.h>
14ffa7d616STomasz Nowicki #include <linux/irqdomain.h>
155e279739SChristophe JAILLET #include <linux/kstrtox.h>
16021f6537SMarc Zyngier #include <linux/of.h>
17021f6537SMarc Zyngier #include <linux/of_address.h>
18021f6537SMarc Zyngier #include <linux/of_irq.h>
19021f6537SMarc Zyngier #include <linux/percpu.h>
20101b35f7SJulien Thierry #include <linux/refcount.h>
21021f6537SMarc Zyngier #include <linux/slab.h>
22021f6537SMarc Zyngier
2341a83e06SJoel Porquet #include <linux/irqchip.h>
241839e576SJulien Grall #include <linux/irqchip/arm-gic-common.h>
25021f6537SMarc Zyngier #include <linux/irqchip/arm-gic-v3.h>
26e3825ba1SMarc Zyngier #include <linux/irqchip/irq-partition-percpu.h>
2735727af2SShanker Donthineni #include <linux/bitfield.h>
2835727af2SShanker Donthineni #include <linux/bits.h>
2935727af2SShanker Donthineni #include <linux/arm-smccc.h>
30021f6537SMarc Zyngier
31021f6537SMarc Zyngier #include <asm/cputype.h>
32021f6537SMarc Zyngier #include <asm/exception.h>
33021f6537SMarc Zyngier #include <asm/smp_plat.h>
340b6a3da9SMarc Zyngier #include <asm/virt.h>
35021f6537SMarc Zyngier
36021f6537SMarc Zyngier #include "irq-gic-common.h"
37021f6537SMarc Zyngier
38f32c9266SJulien Thierry #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80)
39f32c9266SJulien Thierry
409c8114c2SSrinivas Kandagatla #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
41d01fd161SMarc Zyngier #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1)
4244bd78ddSDouglas Anderson #define FLAGS_WORKAROUND_MTK_GICR_SAVE (1ULL << 2)
43b4d81fabSzhengyan #define FLAGS_WORKAROUND_ASR_ERRATUM_8601001 (1ULL << 3)
449c8114c2SSrinivas Kandagatla
4564b499d8SMarc Zyngier #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1)
4664b499d8SMarc Zyngier
47f5c1434cSMarc Zyngier struct redist_region {
48f5c1434cSMarc Zyngier void __iomem *redist_base;
49f5c1434cSMarc Zyngier phys_addr_t phys_base;
50b70fb7afSTomasz Nowicki bool single_redist;
51f5c1434cSMarc Zyngier };
52f5c1434cSMarc Zyngier
53021f6537SMarc Zyngier struct gic_chip_data {
54e3825ba1SMarc Zyngier struct fwnode_handle *fwnode;
5535727af2SShanker Donthineni phys_addr_t dist_phys_base;
56021f6537SMarc Zyngier void __iomem *dist_base;
57f5c1434cSMarc Zyngier struct redist_region *redist_regions;
58f5c1434cSMarc Zyngier struct rdists rdists;
59021f6537SMarc Zyngier struct irq_domain *domain;
60021f6537SMarc Zyngier u64 redist_stride;
61f5c1434cSMarc Zyngier u32 nr_redist_regions;
629c8114c2SSrinivas Kandagatla u64 flags;
63eda0d04aSShanker Donthineni bool has_rss;
641a60e1e6SMarc Zyngier unsigned int ppi_nr;
6552085d3fSMarc Zyngier struct partition_desc **ppi_descs;
66021f6537SMarc Zyngier };
67021f6537SMarc Zyngier
6835727af2SShanker Donthineni #define T241_CHIPS_MAX 4
6935727af2SShanker Donthineni static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly;
7035727af2SShanker Donthineni static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum);
7135727af2SShanker Donthineni
726fe5c68eSLorenzo Pieralisi static DEFINE_STATIC_KEY_FALSE(gic_arm64_2941627_erratum);
736fe5c68eSLorenzo Pieralisi
74021f6537SMarc Zyngier static struct gic_chip_data gic_data __read_mostly;
75d01d3274SDavidlohr Bueso static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
76021f6537SMarc Zyngier
77211bddd2SMarc Zyngier #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
78c107d613SZenghui Yu #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
79211bddd2SMarc Zyngier #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
80211bddd2SMarc Zyngier
81d98d0a99SJulien Thierry /*
82d98d0a99SJulien Thierry * The behaviours of RPR and PMR registers differ depending on the value of
83d98d0a99SJulien Thierry * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
84d98d0a99SJulien Thierry * distributor and redistributors depends on whether security is enabled in the
85d98d0a99SJulien Thierry * GIC.
86d98d0a99SJulien Thierry *
87d98d0a99SJulien Thierry * When security is enabled, non-secure priority values from the (re)distributor
88d98d0a99SJulien Thierry * are presented to the GIC CPUIF as follow:
89d98d0a99SJulien Thierry * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
90d98d0a99SJulien Thierry *
91d4034114SLorenzo Pieralisi * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure
92d98d0a99SJulien Thierry * EL1 are subject to a similar operation thus matching the priorities presented
9333678059SAlexandru Elisei * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0,
94d4034114SLorenzo Pieralisi * these values are unchanged by the GIC.
95d98d0a99SJulien Thierry *
96d98d0a99SJulien Thierry * see GICv3/GICv4 Architecture Specification (IHI0069D):
97d98d0a99SJulien Thierry * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
98d98d0a99SJulien Thierry * priorities.
99d98d0a99SJulien Thierry * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
100d98d0a99SJulien Thierry * interrupt.
101d98d0a99SJulien Thierry */
102d98d0a99SJulien Thierry static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
103d98d0a99SJulien Thierry
10433678059SAlexandru Elisei DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities);
10533678059SAlexandru Elisei EXPORT_SYMBOL(gic_nonsecure_priorities);
10633678059SAlexandru Elisei
1078d474deaSChen-Yu Tsai /*
1088d474deaSChen-Yu Tsai * When the Non-secure world has access to group 0 interrupts (as a
1098d474deaSChen-Yu Tsai * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will
1108d474deaSChen-Yu Tsai * return the Distributor's view of the interrupt priority.
1118d474deaSChen-Yu Tsai *
1128d474deaSChen-Yu Tsai * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority
1138d474deaSChen-Yu Tsai * written by software is moved to the Non-secure range by the Distributor.
1148d474deaSChen-Yu Tsai *
1158d474deaSChen-Yu Tsai * If both are true (which is when gic_nonsecure_priorities gets enabled),
1168d474deaSChen-Yu Tsai * we need to shift down the priority programmed by software to match it
1178d474deaSChen-Yu Tsai * against the value returned by ICC_RPR_EL1.
1188d474deaSChen-Yu Tsai */
1198d474deaSChen-Yu Tsai #define GICD_INT_RPR_PRI(priority) \
1208d474deaSChen-Yu Tsai ({ \
1218d474deaSChen-Yu Tsai u32 __priority = (priority); \
1228d474deaSChen-Yu Tsai if (static_branch_unlikely(&gic_nonsecure_priorities)) \
1238d474deaSChen-Yu Tsai __priority = 0x80 | (__priority >> 1); \
1248d474deaSChen-Yu Tsai \
1258d474deaSChen-Yu Tsai __priority; \
1268d474deaSChen-Yu Tsai })
1278d474deaSChen-Yu Tsai
128101b35f7SJulien Thierry /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
12981a43273SMarc Zyngier static refcount_t *ppi_nmi_refs;
130101b35f7SJulien Thierry
1310e5cb777SMarc Zyngier static struct gic_kvm_info gic_v3_kvm_info __initdata;
132eda0d04aSShanker Donthineni static DEFINE_PER_CPU(bool, has_rss);
1331839e576SJulien Grall
134eda0d04aSShanker Donthineni #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4)
135f5c1434cSMarc Zyngier #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist))
136f5c1434cSMarc Zyngier #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
137021f6537SMarc Zyngier #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K)
138021f6537SMarc Zyngier
139021f6537SMarc Zyngier /* Our default, arbitrary priority value. Linux only uses one anyway. */
140021f6537SMarc Zyngier #define DEFAULT_PMR_VALUE 0xf0
141021f6537SMarc Zyngier
142e91b036eSMarc Zyngier enum gic_intid_range {
14370a29c32SMarc Zyngier SGI_RANGE,
144e91b036eSMarc Zyngier PPI_RANGE,
145e91b036eSMarc Zyngier SPI_RANGE,
1465f51f803SMarc Zyngier EPPI_RANGE,
147211bddd2SMarc Zyngier ESPI_RANGE,
148e91b036eSMarc Zyngier LPI_RANGE,
149e91b036eSMarc Zyngier __INVALID_RANGE__
150e91b036eSMarc Zyngier };
151e91b036eSMarc Zyngier
__get_intid_range(irq_hw_number_t hwirq)152e91b036eSMarc Zyngier static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
153e91b036eSMarc Zyngier {
154e91b036eSMarc Zyngier switch (hwirq) {
15570a29c32SMarc Zyngier case 0 ... 15:
15670a29c32SMarc Zyngier return SGI_RANGE;
157e91b036eSMarc Zyngier case 16 ... 31:
158e91b036eSMarc Zyngier return PPI_RANGE;
159e91b036eSMarc Zyngier case 32 ... 1019:
160e91b036eSMarc Zyngier return SPI_RANGE;
1615f51f803SMarc Zyngier case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
1625f51f803SMarc Zyngier return EPPI_RANGE;
163211bddd2SMarc Zyngier case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
164211bddd2SMarc Zyngier return ESPI_RANGE;
165e91b036eSMarc Zyngier case 8192 ... GENMASK(23, 0):
166e91b036eSMarc Zyngier return LPI_RANGE;
167e91b036eSMarc Zyngier default:
168e91b036eSMarc Zyngier return __INVALID_RANGE__;
169e91b036eSMarc Zyngier }
170e91b036eSMarc Zyngier }
171e91b036eSMarc Zyngier
get_intid_range(struct irq_data * d)172e91b036eSMarc Zyngier static enum gic_intid_range get_intid_range(struct irq_data *d)
173e91b036eSMarc Zyngier {
174e91b036eSMarc Zyngier return __get_intid_range(d->hwirq);
175e91b036eSMarc Zyngier }
176e91b036eSMarc Zyngier
gic_irq(struct irq_data * d)177021f6537SMarc Zyngier static inline unsigned int gic_irq(struct irq_data *d)
178021f6537SMarc Zyngier {
179021f6537SMarc Zyngier return d->hwirq;
180021f6537SMarc Zyngier }
181021f6537SMarc Zyngier
gic_irq_in_rdist(struct irq_data * d)18270a29c32SMarc Zyngier static inline bool gic_irq_in_rdist(struct irq_data *d)
183021f6537SMarc Zyngier {
18470a29c32SMarc Zyngier switch (get_intid_range(d)) {
18570a29c32SMarc Zyngier case SGI_RANGE:
18670a29c32SMarc Zyngier case PPI_RANGE:
18770a29c32SMarc Zyngier case EPPI_RANGE:
18870a29c32SMarc Zyngier return true;
18970a29c32SMarc Zyngier default:
19070a29c32SMarc Zyngier return false;
19170a29c32SMarc Zyngier }
192021f6537SMarc Zyngier }
193021f6537SMarc Zyngier
gic_dist_base_alias(struct irq_data * d)19435727af2SShanker Donthineni static inline void __iomem *gic_dist_base_alias(struct irq_data *d)
19535727af2SShanker Donthineni {
19635727af2SShanker Donthineni if (static_branch_unlikely(&gic_nvidia_t241_erratum)) {
19735727af2SShanker Donthineni irq_hw_number_t hwirq = irqd_to_hwirq(d);
19835727af2SShanker Donthineni u32 chip;
19935727af2SShanker Donthineni
20035727af2SShanker Donthineni /*
20135727af2SShanker Donthineni * For the erratum T241-FABRIC-4, read accesses to GICD_In{E}
20235727af2SShanker Donthineni * registers are directed to the chip that owns the SPI. The
20335727af2SShanker Donthineni * the alias region can also be used for writes to the
20435727af2SShanker Donthineni * GICD_In{E} except GICD_ICENABLERn. Each chip has support
20535727af2SShanker Donthineni * for 320 {E}SPIs. Mappings for all 4 chips:
20635727af2SShanker Donthineni * Chip0 = 32-351
20735727af2SShanker Donthineni * Chip1 = 352-671
20835727af2SShanker Donthineni * Chip2 = 672-991
20935727af2SShanker Donthineni * Chip3 = 4096-4415
21035727af2SShanker Donthineni */
21135727af2SShanker Donthineni switch (__get_intid_range(hwirq)) {
21235727af2SShanker Donthineni case SPI_RANGE:
21335727af2SShanker Donthineni chip = (hwirq - 32) / 320;
21435727af2SShanker Donthineni break;
21535727af2SShanker Donthineni case ESPI_RANGE:
21635727af2SShanker Donthineni chip = 3;
21735727af2SShanker Donthineni break;
21835727af2SShanker Donthineni default:
21935727af2SShanker Donthineni unreachable();
22035727af2SShanker Donthineni }
22135727af2SShanker Donthineni return t241_dist_base_alias[chip];
22235727af2SShanker Donthineni }
22335727af2SShanker Donthineni
22435727af2SShanker Donthineni return gic_data.dist_base;
22535727af2SShanker Donthineni }
22635727af2SShanker Donthineni
gic_dist_base(struct irq_data * d)227021f6537SMarc Zyngier static inline void __iomem *gic_dist_base(struct irq_data *d)
228021f6537SMarc Zyngier {
229e91b036eSMarc Zyngier switch (get_intid_range(d)) {
23070a29c32SMarc Zyngier case SGI_RANGE:
231e91b036eSMarc Zyngier case PPI_RANGE:
2325f51f803SMarc Zyngier case EPPI_RANGE:
233e91b036eSMarc Zyngier /* SGI+PPI -> SGI_base for this CPU */
234021f6537SMarc Zyngier return gic_data_rdist_sgi_base();
235021f6537SMarc Zyngier
236e91b036eSMarc Zyngier case SPI_RANGE:
237211bddd2SMarc Zyngier case ESPI_RANGE:
238e91b036eSMarc Zyngier /* SPI -> dist_base */
239021f6537SMarc Zyngier return gic_data.dist_base;
240021f6537SMarc Zyngier
241e91b036eSMarc Zyngier default:
242021f6537SMarc Zyngier return NULL;
243021f6537SMarc Zyngier }
244e91b036eSMarc Zyngier }
245021f6537SMarc Zyngier
gic_do_wait_for_rwp(void __iomem * base,u32 bit)2460df66645SMarc Zyngier static void gic_do_wait_for_rwp(void __iomem *base, u32 bit)
247021f6537SMarc Zyngier {
248021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */
249021f6537SMarc Zyngier
2500df66645SMarc Zyngier while (readl_relaxed(base + GICD_CTLR) & bit) {
251021f6537SMarc Zyngier count--;
252021f6537SMarc Zyngier if (!count) {
253021f6537SMarc Zyngier pr_err_ratelimited("RWP timeout, gone fishing\n");
254021f6537SMarc Zyngier return;
255021f6537SMarc Zyngier }
256021f6537SMarc Zyngier cpu_relax();
257021f6537SMarc Zyngier udelay(1);
2582c542426SDaode Huang }
259021f6537SMarc Zyngier }
260021f6537SMarc Zyngier
261021f6537SMarc Zyngier /* Wait for completion of a distributor change */
gic_dist_wait_for_rwp(void)262021f6537SMarc Zyngier static void gic_dist_wait_for_rwp(void)
263021f6537SMarc Zyngier {
2640df66645SMarc Zyngier gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP);
265021f6537SMarc Zyngier }
266021f6537SMarc Zyngier
267021f6537SMarc Zyngier /* Wait for completion of a redistributor change */
gic_redist_wait_for_rwp(void)268021f6537SMarc Zyngier static void gic_redist_wait_for_rwp(void)
269021f6537SMarc Zyngier {
2700df66645SMarc Zyngier gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP);
271021f6537SMarc Zyngier }
272021f6537SMarc Zyngier
2737936e914SJean-Philippe Brucker #ifdef CONFIG_ARM64
2746d4e11c5SRobert Richter
gic_read_iar(void)2756d4e11c5SRobert Richter static u64 __maybe_unused gic_read_iar(void)
2766d4e11c5SRobert Richter {
277a4023f68SSuzuki K Poulose if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154))
2786d4e11c5SRobert Richter return gic_read_iar_cavium_thunderx();
2796d4e11c5SRobert Richter else
2806d4e11c5SRobert Richter return gic_read_iar_common();
2816d4e11c5SRobert Richter }
2827936e914SJean-Philippe Brucker #endif
283021f6537SMarc Zyngier
gic_enable_redist(bool enable)284a2c22510SSudeep Holla static void gic_enable_redist(bool enable)
285021f6537SMarc Zyngier {
286021f6537SMarc Zyngier void __iomem *rbase;
287021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */
288021f6537SMarc Zyngier u32 val;
289021f6537SMarc Zyngier
2909c8114c2SSrinivas Kandagatla if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
2919c8114c2SSrinivas Kandagatla return;
2929c8114c2SSrinivas Kandagatla
293021f6537SMarc Zyngier rbase = gic_data_rdist_rd_base();
294021f6537SMarc Zyngier
295021f6537SMarc Zyngier val = readl_relaxed(rbase + GICR_WAKER);
296a2c22510SSudeep Holla if (enable)
297a2c22510SSudeep Holla /* Wake up this CPU redistributor */
298021f6537SMarc Zyngier val &= ~GICR_WAKER_ProcessorSleep;
299a2c22510SSudeep Holla else
300a2c22510SSudeep Holla val |= GICR_WAKER_ProcessorSleep;
301021f6537SMarc Zyngier writel_relaxed(val, rbase + GICR_WAKER);
302021f6537SMarc Zyngier
303a2c22510SSudeep Holla if (!enable) { /* Check that GICR_WAKER is writeable */
304a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER);
305a2c22510SSudeep Holla if (!(val & GICR_WAKER_ProcessorSleep))
306a2c22510SSudeep Holla return; /* No PM support in this redistributor */
307021f6537SMarc Zyngier }
308a2c22510SSudeep Holla
309d102eb5cSDan Carpenter while (--count) {
310a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER);
311cf1d9d11SAndrew Jones if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep))
312a2c22510SSudeep Holla break;
313021f6537SMarc Zyngier cpu_relax();
314021f6537SMarc Zyngier udelay(1);
3152c542426SDaode Huang }
316a2c22510SSudeep Holla if (!count)
317a2c22510SSudeep Holla pr_err_ratelimited("redistributor failed to %s...\n",
318a2c22510SSudeep Holla enable ? "wakeup" : "sleep");
319021f6537SMarc Zyngier }
320021f6537SMarc Zyngier
321021f6537SMarc Zyngier /*
322021f6537SMarc Zyngier * Routines to disable, enable, EOI and route interrupts
323021f6537SMarc Zyngier */
convert_offset_index(struct irq_data * d,u32 offset,u32 * index)324e91b036eSMarc Zyngier static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
325e91b036eSMarc Zyngier {
326e91b036eSMarc Zyngier switch (get_intid_range(d)) {
32770a29c32SMarc Zyngier case SGI_RANGE:
328e91b036eSMarc Zyngier case PPI_RANGE:
329e91b036eSMarc Zyngier case SPI_RANGE:
330e91b036eSMarc Zyngier *index = d->hwirq;
331e91b036eSMarc Zyngier return offset;
3325f51f803SMarc Zyngier case EPPI_RANGE:
3335f51f803SMarc Zyngier /*
3345f51f803SMarc Zyngier * Contrary to the ESPI range, the EPPI range is contiguous
3355f51f803SMarc Zyngier * to the PPI range in the registers, so let's adjust the
3365f51f803SMarc Zyngier * displacement accordingly. Consistency is overrated.
3375f51f803SMarc Zyngier */
3385f51f803SMarc Zyngier *index = d->hwirq - EPPI_BASE_INTID + 32;
3395f51f803SMarc Zyngier return offset;
340211bddd2SMarc Zyngier case ESPI_RANGE:
341211bddd2SMarc Zyngier *index = d->hwirq - ESPI_BASE_INTID;
342211bddd2SMarc Zyngier switch (offset) {
343211bddd2SMarc Zyngier case GICD_ISENABLER:
344211bddd2SMarc Zyngier return GICD_ISENABLERnE;
345211bddd2SMarc Zyngier case GICD_ICENABLER:
346211bddd2SMarc Zyngier return GICD_ICENABLERnE;
347211bddd2SMarc Zyngier case GICD_ISPENDR:
348211bddd2SMarc Zyngier return GICD_ISPENDRnE;
349211bddd2SMarc Zyngier case GICD_ICPENDR:
350211bddd2SMarc Zyngier return GICD_ICPENDRnE;
351211bddd2SMarc Zyngier case GICD_ISACTIVER:
352211bddd2SMarc Zyngier return GICD_ISACTIVERnE;
353211bddd2SMarc Zyngier case GICD_ICACTIVER:
354211bddd2SMarc Zyngier return GICD_ICACTIVERnE;
355211bddd2SMarc Zyngier case GICD_IPRIORITYR:
356211bddd2SMarc Zyngier return GICD_IPRIORITYRnE;
357211bddd2SMarc Zyngier case GICD_ICFGR:
358211bddd2SMarc Zyngier return GICD_ICFGRnE;
359211bddd2SMarc Zyngier case GICD_IROUTER:
360211bddd2SMarc Zyngier return GICD_IROUTERnE;
361211bddd2SMarc Zyngier default:
362211bddd2SMarc Zyngier break;
363211bddd2SMarc Zyngier }
364211bddd2SMarc Zyngier break;
365e91b036eSMarc Zyngier default:
366e91b036eSMarc Zyngier break;
367e91b036eSMarc Zyngier }
368e91b036eSMarc Zyngier
369e91b036eSMarc Zyngier WARN_ON(1);
370e91b036eSMarc Zyngier *index = d->hwirq;
371e91b036eSMarc Zyngier return offset;
372e91b036eSMarc Zyngier }
373e91b036eSMarc Zyngier
gic_peek_irq(struct irq_data * d,u32 offset)374b594c6e2SMarc Zyngier static int gic_peek_irq(struct irq_data *d, u32 offset)
375b594c6e2SMarc Zyngier {
376b594c6e2SMarc Zyngier void __iomem *base;
377e91b036eSMarc Zyngier u32 index, mask;
378e91b036eSMarc Zyngier
379e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index);
380e91b036eSMarc Zyngier mask = 1 << (index % 32);
381b594c6e2SMarc Zyngier
382b594c6e2SMarc Zyngier if (gic_irq_in_rdist(d))
383b594c6e2SMarc Zyngier base = gic_data_rdist_sgi_base();
384b594c6e2SMarc Zyngier else
38535727af2SShanker Donthineni base = gic_dist_base_alias(d);
386b594c6e2SMarc Zyngier
387e91b036eSMarc Zyngier return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
388b594c6e2SMarc Zyngier }
389b594c6e2SMarc Zyngier
gic_poke_irq(struct irq_data * d,u32 offset)390021f6537SMarc Zyngier static void gic_poke_irq(struct irq_data *d, u32 offset)
391021f6537SMarc Zyngier {
392021f6537SMarc Zyngier void __iomem *base;
393e91b036eSMarc Zyngier u32 index, mask;
394e91b036eSMarc Zyngier
395e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index);
396e91b036eSMarc Zyngier mask = 1 << (index % 32);
397021f6537SMarc Zyngier
39863f13483SMarc Zyngier if (gic_irq_in_rdist(d))
399021f6537SMarc Zyngier base = gic_data_rdist_sgi_base();
40063f13483SMarc Zyngier else
401021f6537SMarc Zyngier base = gic_data.dist_base;
402021f6537SMarc Zyngier
403e91b036eSMarc Zyngier writel_relaxed(mask, base + offset + (index / 32) * 4);
404021f6537SMarc Zyngier }
405021f6537SMarc Zyngier
gic_mask_irq(struct irq_data * d)406021f6537SMarc Zyngier static void gic_mask_irq(struct irq_data *d)
407021f6537SMarc Zyngier {
408021f6537SMarc Zyngier gic_poke_irq(d, GICD_ICENABLER);
40963f13483SMarc Zyngier if (gic_irq_in_rdist(d))
41063f13483SMarc Zyngier gic_redist_wait_for_rwp();
41163f13483SMarc Zyngier else
41263f13483SMarc Zyngier gic_dist_wait_for_rwp();
413021f6537SMarc Zyngier }
414021f6537SMarc Zyngier
gic_eoimode1_mask_irq(struct irq_data * d)4150b6a3da9SMarc Zyngier static void gic_eoimode1_mask_irq(struct irq_data *d)
4160b6a3da9SMarc Zyngier {
4170b6a3da9SMarc Zyngier gic_mask_irq(d);
418530bf353SMarc Zyngier /*
419530bf353SMarc Zyngier * When masking a forwarded interrupt, make sure it is
420530bf353SMarc Zyngier * deactivated as well.
421530bf353SMarc Zyngier *
422530bf353SMarc Zyngier * This ensures that an interrupt that is getting
423530bf353SMarc Zyngier * disabled/masked will not get "stuck", because there is
424530bf353SMarc Zyngier * noone to deactivate it (guest is being terminated).
425530bf353SMarc Zyngier */
4264df7f54dSThomas Gleixner if (irqd_is_forwarded_to_vcpu(d))
427530bf353SMarc Zyngier gic_poke_irq(d, GICD_ICACTIVER);
4280b6a3da9SMarc Zyngier }
4290b6a3da9SMarc Zyngier
gic_unmask_irq(struct irq_data * d)430021f6537SMarc Zyngier static void gic_unmask_irq(struct irq_data *d)
431021f6537SMarc Zyngier {
432021f6537SMarc Zyngier gic_poke_irq(d, GICD_ISENABLER);
433021f6537SMarc Zyngier }
434021f6537SMarc Zyngier
gic_supports_nmi(void)435d98d0a99SJulien Thierry static inline bool gic_supports_nmi(void)
436d98d0a99SJulien Thierry {
437d98d0a99SJulien Thierry return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
438d98d0a99SJulien Thierry static_branch_likely(&supports_pseudo_nmis);
439d98d0a99SJulien Thierry }
440d98d0a99SJulien Thierry
gic_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool val)441b594c6e2SMarc Zyngier static int gic_irq_set_irqchip_state(struct irq_data *d,
442b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool val)
443b594c6e2SMarc Zyngier {
444b594c6e2SMarc Zyngier u32 reg;
445b594c6e2SMarc Zyngier
44664b499d8SMarc Zyngier if (d->hwirq >= 8192) /* SGI/PPI/SPI only */
447b594c6e2SMarc Zyngier return -EINVAL;
448b594c6e2SMarc Zyngier
449b594c6e2SMarc Zyngier switch (which) {
450b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING:
451b594c6e2SMarc Zyngier reg = val ? GICD_ISPENDR : GICD_ICPENDR;
452b594c6e2SMarc Zyngier break;
453b594c6e2SMarc Zyngier
454b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE:
455b594c6e2SMarc Zyngier reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
456b594c6e2SMarc Zyngier break;
457b594c6e2SMarc Zyngier
458b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED:
45963f13483SMarc Zyngier if (val) {
46063f13483SMarc Zyngier gic_mask_irq(d);
46163f13483SMarc Zyngier return 0;
46263f13483SMarc Zyngier }
46363f13483SMarc Zyngier reg = GICD_ISENABLER;
464b594c6e2SMarc Zyngier break;
465b594c6e2SMarc Zyngier
466b594c6e2SMarc Zyngier default:
467b594c6e2SMarc Zyngier return -EINVAL;
468b594c6e2SMarc Zyngier }
469b594c6e2SMarc Zyngier
470b594c6e2SMarc Zyngier gic_poke_irq(d, reg);
471b594c6e2SMarc Zyngier return 0;
472b594c6e2SMarc Zyngier }
473b594c6e2SMarc Zyngier
gic_irq_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * val)474b594c6e2SMarc Zyngier static int gic_irq_get_irqchip_state(struct irq_data *d,
475b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool *val)
476b594c6e2SMarc Zyngier {
477211bddd2SMarc Zyngier if (d->hwirq >= 8192) /* PPI/SPI only */
478b594c6e2SMarc Zyngier return -EINVAL;
479b594c6e2SMarc Zyngier
480b594c6e2SMarc Zyngier switch (which) {
481b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING:
482b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISPENDR);
483b594c6e2SMarc Zyngier break;
484b594c6e2SMarc Zyngier
485b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE:
486b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISACTIVER);
487b594c6e2SMarc Zyngier break;
488b594c6e2SMarc Zyngier
489b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED:
490b594c6e2SMarc Zyngier *val = !gic_peek_irq(d, GICD_ISENABLER);
491b594c6e2SMarc Zyngier break;
492b594c6e2SMarc Zyngier
493b594c6e2SMarc Zyngier default:
494b594c6e2SMarc Zyngier return -EINVAL;
495b594c6e2SMarc Zyngier }
496b594c6e2SMarc Zyngier
497b594c6e2SMarc Zyngier return 0;
498b594c6e2SMarc Zyngier }
499b594c6e2SMarc Zyngier
gic_irq_set_prio(struct irq_data * d,u8 prio)500101b35f7SJulien Thierry static void gic_irq_set_prio(struct irq_data *d, u8 prio)
501101b35f7SJulien Thierry {
502101b35f7SJulien Thierry void __iomem *base = gic_dist_base(d);
503e91b036eSMarc Zyngier u32 offset, index;
504101b35f7SJulien Thierry
505e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
506e91b036eSMarc Zyngier
507e91b036eSMarc Zyngier writeb_relaxed(prio, base + offset + index);
508101b35f7SJulien Thierry }
509101b35f7SJulien Thierry
__gic_get_ppi_index(irq_hw_number_t hwirq)510bfa80ee9SJames Morse static u32 __gic_get_ppi_index(irq_hw_number_t hwirq)
51181a43273SMarc Zyngier {
512bfa80ee9SJames Morse switch (__get_intid_range(hwirq)) {
51381a43273SMarc Zyngier case PPI_RANGE:
514bfa80ee9SJames Morse return hwirq - 16;
5155f51f803SMarc Zyngier case EPPI_RANGE:
516bfa80ee9SJames Morse return hwirq - EPPI_BASE_INTID + 16;
51781a43273SMarc Zyngier default:
51881a43273SMarc Zyngier unreachable();
51981a43273SMarc Zyngier }
52081a43273SMarc Zyngier }
52181a43273SMarc Zyngier
gic_get_ppi_index(struct irq_data * d)522bfa80ee9SJames Morse static u32 gic_get_ppi_index(struct irq_data *d)
523bfa80ee9SJames Morse {
524bfa80ee9SJames Morse return __gic_get_ppi_index(d->hwirq);
525bfa80ee9SJames Morse }
526bfa80ee9SJames Morse
gic_irq_nmi_setup(struct irq_data * d)527101b35f7SJulien Thierry static int gic_irq_nmi_setup(struct irq_data *d)
528101b35f7SJulien Thierry {
529101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq);
530101b35f7SJulien Thierry
531101b35f7SJulien Thierry if (!gic_supports_nmi())
532101b35f7SJulien Thierry return -EINVAL;
533101b35f7SJulien Thierry
534101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) {
535101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
536101b35f7SJulien Thierry return -EINVAL;
537101b35f7SJulien Thierry }
538101b35f7SJulien Thierry
539101b35f7SJulien Thierry /*
540101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request,
541101b35f7SJulien Thierry * it should not be possible to get there
542101b35f7SJulien Thierry */
543101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192))
544101b35f7SJulien Thierry return -EINVAL;
545101b35f7SJulien Thierry
546101b35f7SJulien Thierry /* desc lock should already be held */
54781a43273SMarc Zyngier if (gic_irq_in_rdist(d)) {
54881a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d);
54981a43273SMarc Zyngier
550101b35f7SJulien Thierry /* Setting up PPI as NMI, only switch handler for first NMI */
55181a43273SMarc Zyngier if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) {
55281a43273SMarc Zyngier refcount_set(&ppi_nmi_refs[idx], 1);
553101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
554101b35f7SJulien Thierry }
555101b35f7SJulien Thierry } else {
556101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_nmi;
557101b35f7SJulien Thierry }
558101b35f7SJulien Thierry
559101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_NMI_PRI);
560101b35f7SJulien Thierry
561101b35f7SJulien Thierry return 0;
562101b35f7SJulien Thierry }
563101b35f7SJulien Thierry
gic_irq_nmi_teardown(struct irq_data * d)564101b35f7SJulien Thierry static void gic_irq_nmi_teardown(struct irq_data *d)
565101b35f7SJulien Thierry {
566101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq);
567101b35f7SJulien Thierry
568101b35f7SJulien Thierry if (WARN_ON(!gic_supports_nmi()))
569101b35f7SJulien Thierry return;
570101b35f7SJulien Thierry
571101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) {
572101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
573101b35f7SJulien Thierry return;
574101b35f7SJulien Thierry }
575101b35f7SJulien Thierry
576101b35f7SJulien Thierry /*
577101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request,
578101b35f7SJulien Thierry * it should not be possible to get there
579101b35f7SJulien Thierry */
580101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192))
581101b35f7SJulien Thierry return;
582101b35f7SJulien Thierry
583101b35f7SJulien Thierry /* desc lock should already be held */
58481a43273SMarc Zyngier if (gic_irq_in_rdist(d)) {
58581a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d);
58681a43273SMarc Zyngier
587101b35f7SJulien Thierry /* Tearing down NMI, only switch handler for last NMI */
58881a43273SMarc Zyngier if (refcount_dec_and_test(&ppi_nmi_refs[idx]))
589101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_irq;
590101b35f7SJulien Thierry } else {
591101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_irq;
592101b35f7SJulien Thierry }
593101b35f7SJulien Thierry
594101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_DEF_PRI);
595101b35f7SJulien Thierry }
596101b35f7SJulien Thierry
gic_arm64_erratum_2941627_needed(struct irq_data * d)5976fe5c68eSLorenzo Pieralisi static bool gic_arm64_erratum_2941627_needed(struct irq_data *d)
5986fe5c68eSLorenzo Pieralisi {
5996fe5c68eSLorenzo Pieralisi enum gic_intid_range range;
6006fe5c68eSLorenzo Pieralisi
6016fe5c68eSLorenzo Pieralisi if (!static_branch_unlikely(&gic_arm64_2941627_erratum))
6026fe5c68eSLorenzo Pieralisi return false;
6036fe5c68eSLorenzo Pieralisi
6046fe5c68eSLorenzo Pieralisi range = get_intid_range(d);
6056fe5c68eSLorenzo Pieralisi
6066fe5c68eSLorenzo Pieralisi /*
6076fe5c68eSLorenzo Pieralisi * The workaround is needed if the IRQ is an SPI and
6086fe5c68eSLorenzo Pieralisi * the target cpu is different from the one we are
6096fe5c68eSLorenzo Pieralisi * executing on.
6106fe5c68eSLorenzo Pieralisi */
6116fe5c68eSLorenzo Pieralisi return (range == SPI_RANGE || range == ESPI_RANGE) &&
6126fe5c68eSLorenzo Pieralisi !cpumask_test_cpu(raw_smp_processor_id(),
6136fe5c68eSLorenzo Pieralisi irq_data_get_effective_affinity_mask(d));
6146fe5c68eSLorenzo Pieralisi }
6156fe5c68eSLorenzo Pieralisi
gic_eoi_irq(struct irq_data * d)616021f6537SMarc Zyngier static void gic_eoi_irq(struct irq_data *d)
617021f6537SMarc Zyngier {
6186efb5092SMark Rutland write_gicreg(gic_irq(d), ICC_EOIR1_EL1);
6196efb5092SMark Rutland isb();
6206fe5c68eSLorenzo Pieralisi
6216fe5c68eSLorenzo Pieralisi if (gic_arm64_erratum_2941627_needed(d)) {
6226fe5c68eSLorenzo Pieralisi /*
6236fe5c68eSLorenzo Pieralisi * Make sure the GIC stream deactivate packet
6246fe5c68eSLorenzo Pieralisi * issued by ICC_EOIR1_EL1 has completed before
6256fe5c68eSLorenzo Pieralisi * deactivating through GICD_IACTIVER.
6266fe5c68eSLorenzo Pieralisi */
6276fe5c68eSLorenzo Pieralisi dsb(sy);
6286fe5c68eSLorenzo Pieralisi gic_poke_irq(d, GICD_ICACTIVER);
6296fe5c68eSLorenzo Pieralisi }
630021f6537SMarc Zyngier }
631021f6537SMarc Zyngier
gic_eoimode1_eoi_irq(struct irq_data * d)6320b6a3da9SMarc Zyngier static void gic_eoimode1_eoi_irq(struct irq_data *d)
6330b6a3da9SMarc Zyngier {
6340b6a3da9SMarc Zyngier /*
635530bf353SMarc Zyngier * No need to deactivate an LPI, or an interrupt that
636530bf353SMarc Zyngier * is is getting forwarded to a vcpu.
6370b6a3da9SMarc Zyngier */
6384df7f54dSThomas Gleixner if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
6390b6a3da9SMarc Zyngier return;
6406fe5c68eSLorenzo Pieralisi
6416fe5c68eSLorenzo Pieralisi if (!gic_arm64_erratum_2941627_needed(d))
6420b6a3da9SMarc Zyngier gic_write_dir(gic_irq(d));
6436fe5c68eSLorenzo Pieralisi else
6446fe5c68eSLorenzo Pieralisi gic_poke_irq(d, GICD_ICACTIVER);
6450b6a3da9SMarc Zyngier }
6460b6a3da9SMarc Zyngier
gic_set_type(struct irq_data * d,unsigned int type)647021f6537SMarc Zyngier static int gic_set_type(struct irq_data *d, unsigned int type)
648021f6537SMarc Zyngier {
6495f51f803SMarc Zyngier enum gic_intid_range range;
650021f6537SMarc Zyngier unsigned int irq = gic_irq(d);
651021f6537SMarc Zyngier void __iomem *base;
652e91b036eSMarc Zyngier u32 offset, index;
65313d22e2eSMarc Zyngier int ret;
654021f6537SMarc Zyngier
6555f51f803SMarc Zyngier range = get_intid_range(d);
6565f51f803SMarc Zyngier
65764b499d8SMarc Zyngier /* Interrupt configuration for SGIs can't be changed */
65864b499d8SMarc Zyngier if (range == SGI_RANGE)
65964b499d8SMarc Zyngier return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0;
66064b499d8SMarc Zyngier
661fb7e7debSLiviu Dudau /* SPIs have restrictions on the supported types */
6625f51f803SMarc Zyngier if ((range == SPI_RANGE || range == ESPI_RANGE) &&
6635f51f803SMarc Zyngier type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
664021f6537SMarc Zyngier return -EINVAL;
665021f6537SMarc Zyngier
66663f13483SMarc Zyngier if (gic_irq_in_rdist(d))
667021f6537SMarc Zyngier base = gic_data_rdist_sgi_base();
66863f13483SMarc Zyngier else
66935727af2SShanker Donthineni base = gic_dist_base_alias(d);
670021f6537SMarc Zyngier
671e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_ICFGR, &index);
67213d22e2eSMarc Zyngier
67363f13483SMarc Zyngier ret = gic_configure_irq(index, type, base + offset, NULL);
6745f51f803SMarc Zyngier if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
67513d22e2eSMarc Zyngier /* Misconfigured PPIs are usually not fatal */
6765f51f803SMarc Zyngier pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
67713d22e2eSMarc Zyngier ret = 0;
67813d22e2eSMarc Zyngier }
67913d22e2eSMarc Zyngier
68013d22e2eSMarc Zyngier return ret;
681021f6537SMarc Zyngier }
682021f6537SMarc Zyngier
gic_irq_set_vcpu_affinity(struct irq_data * d,void * vcpu)683530bf353SMarc Zyngier static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
684530bf353SMarc Zyngier {
68564b499d8SMarc Zyngier if (get_intid_range(d) == SGI_RANGE)
68664b499d8SMarc Zyngier return -EINVAL;
68764b499d8SMarc Zyngier
6884df7f54dSThomas Gleixner if (vcpu)
6894df7f54dSThomas Gleixner irqd_set_forwarded_to_vcpu(d);
6904df7f54dSThomas Gleixner else
6914df7f54dSThomas Gleixner irqd_clr_forwarded_to_vcpu(d);
692530bf353SMarc Zyngier return 0;
693530bf353SMarc Zyngier }
694530bf353SMarc Zyngier
gic_cpu_to_affinity(int cpu)6953c65cbb7SMarc Zyngier static u64 gic_cpu_to_affinity(int cpu)
696021f6537SMarc Zyngier {
6973c65cbb7SMarc Zyngier u64 mpidr = cpu_logical_map(cpu);
698021f6537SMarc Zyngier u64 aff;
699021f6537SMarc Zyngier
700b4d81fabSzhengyan /* ASR8601 needs to have its affinities shifted down... */
701b4d81fabSzhengyan if (unlikely(gic_data.flags & FLAGS_WORKAROUND_ASR_ERRATUM_8601001))
702b4d81fabSzhengyan mpidr = (MPIDR_AFFINITY_LEVEL(mpidr, 1) |
703b4d81fabSzhengyan (MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8));
704b4d81fabSzhengyan
705f6c86a41SJean-Philippe Brucker aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
706021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
707021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
708021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0));
709021f6537SMarc Zyngier
710021f6537SMarc Zyngier return aff;
711021f6537SMarc Zyngier }
712021f6537SMarc Zyngier
gic_deactivate_unhandled(u32 irqnr)713f32c9266SJulien Thierry static void gic_deactivate_unhandled(u32 irqnr)
714f32c9266SJulien Thierry {
715f32c9266SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) {
716f32c9266SJulien Thierry if (irqnr < 8192)
717f32c9266SJulien Thierry gic_write_dir(irqnr);
718f32c9266SJulien Thierry } else {
7196efb5092SMark Rutland write_gicreg(irqnr, ICC_EOIR1_EL1);
7206efb5092SMark Rutland isb();
721f32c9266SJulien Thierry }
722f32c9266SJulien Thierry }
723f32c9266SJulien Thierry
724f32c9266SJulien Thierry /*
7256efb5092SMark Rutland * Follow a read of the IAR with any HW maintenance that needs to happen prior
7266efb5092SMark Rutland * to invoking the relevant IRQ handler. We must do two things:
7276efb5092SMark Rutland *
7286efb5092SMark Rutland * (1) Ensure instruction ordering between a read of IAR and subsequent
7296efb5092SMark Rutland * instructions in the IRQ handler using an ISB.
7306efb5092SMark Rutland *
7316efb5092SMark Rutland * It is possible for the IAR to report an IRQ which was signalled *after*
7326efb5092SMark Rutland * the CPU took an IRQ exception as multiple interrupts can race to be
7336efb5092SMark Rutland * recognized by the GIC, earlier interrupts could be withdrawn, and/or
7346efb5092SMark Rutland * later interrupts could be prioritized by the GIC.
7356efb5092SMark Rutland *
7366efb5092SMark Rutland * For devices which are tightly coupled to the CPU, such as PMUs, a
7376efb5092SMark Rutland * context synchronization event is necessary to ensure that system
7386efb5092SMark Rutland * register state is not stale, as these may have been indirectly written
7396efb5092SMark Rutland * *after* exception entry.
7406efb5092SMark Rutland *
7416efb5092SMark Rutland * (2) Deactivate the interrupt when EOI mode 1 is in use.
742f32c9266SJulien Thierry */
gic_complete_ack(u32 irqnr)7436efb5092SMark Rutland static inline void gic_complete_ack(u32 irqnr)
7446efb5092SMark Rutland {
7456efb5092SMark Rutland if (static_branch_likely(&supports_deactivate_key))
7466efb5092SMark Rutland write_gicreg(irqnr, ICC_EOIR1_EL1);
74717ce302fSJulien Thierry
7486efb5092SMark Rutland isb();
7496efb5092SMark Rutland }
7506efb5092SMark Rutland
gic_rpr_is_nmi_prio(void)751614ab80cSMark Rutland static bool gic_rpr_is_nmi_prio(void)
752f32c9266SJulien Thierry {
753614ab80cSMark Rutland if (!gic_supports_nmi())
754614ab80cSMark Rutland return false;
755f32c9266SJulien Thierry
756614ab80cSMark Rutland return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI));
757614ab80cSMark Rutland }
758614ab80cSMark Rutland
gic_irqnr_is_special(u32 irqnr)759614ab80cSMark Rutland static bool gic_irqnr_is_special(u32 irqnr)
760614ab80cSMark Rutland {
761614ab80cSMark Rutland return irqnr >= 1020 && irqnr <= 1023;
762614ab80cSMark Rutland }
763614ab80cSMark Rutland
__gic_handle_irq(u32 irqnr,struct pt_regs * regs)764614ab80cSMark Rutland static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs)
765614ab80cSMark Rutland {
766614ab80cSMark Rutland if (gic_irqnr_is_special(irqnr))
767614ab80cSMark Rutland return;
768f32c9266SJulien Thierry
7696efb5092SMark Rutland gic_complete_ack(irqnr);
770adf14453SMark Rutland
771614ab80cSMark Rutland if (generic_handle_domain_irq(gic_data.domain, irqnr)) {
772614ab80cSMark Rutland WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr);
773f32c9266SJulien Thierry gic_deactivate_unhandled(irqnr);
774614ab80cSMark Rutland }
775614ab80cSMark Rutland }
77617ce302fSJulien Thierry
__gic_handle_nmi(u32 irqnr,struct pt_regs * regs)777614ab80cSMark Rutland static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
778614ab80cSMark Rutland {
779614ab80cSMark Rutland if (gic_irqnr_is_special(irqnr))
780614ab80cSMark Rutland return;
781614ab80cSMark Rutland
782614ab80cSMark Rutland gic_complete_ack(irqnr);
783614ab80cSMark Rutland
784614ab80cSMark Rutland if (generic_handle_domain_nmi(gic_data.domain, irqnr)) {
785614ab80cSMark Rutland WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr);
786614ab80cSMark Rutland gic_deactivate_unhandled(irqnr);
787614ab80cSMark Rutland }
788614ab80cSMark Rutland }
789614ab80cSMark Rutland
790614ab80cSMark Rutland /*
791614ab80cSMark Rutland * An exception has been taken from a context with IRQs enabled, and this could
792614ab80cSMark Rutland * be an IRQ or an NMI.
793614ab80cSMark Rutland *
794614ab80cSMark Rutland * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear
795614ab80cSMark Rutland * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning,
796614ab80cSMark Rutland * after handling any NMI but before handling any IRQ.
797614ab80cSMark Rutland *
798614ab80cSMark Rutland * The entry code has performed IRQ entry, and if an NMI is detected we must
799614ab80cSMark Rutland * perform NMI entry/exit around invoking the handler.
800614ab80cSMark Rutland */
__gic_handle_irq_from_irqson(struct pt_regs * regs)801614ab80cSMark Rutland static void __gic_handle_irq_from_irqson(struct pt_regs *regs)
802614ab80cSMark Rutland {
803614ab80cSMark Rutland bool is_nmi;
804614ab80cSMark Rutland u32 irqnr;
805614ab80cSMark Rutland
806614ab80cSMark Rutland irqnr = gic_read_iar();
807614ab80cSMark Rutland
808614ab80cSMark Rutland is_nmi = gic_rpr_is_nmi_prio();
809614ab80cSMark Rutland
810614ab80cSMark Rutland if (is_nmi) {
811614ab80cSMark Rutland nmi_enter();
812614ab80cSMark Rutland __gic_handle_nmi(irqnr, regs);
81317ce302fSJulien Thierry nmi_exit();
814f32c9266SJulien Thierry }
815f32c9266SJulien Thierry
816614ab80cSMark Rutland if (gic_prio_masking_enabled()) {
817614ab80cSMark Rutland gic_pmr_mask_irqs();
818614ab80cSMark Rutland gic_arch_enable_irqs();
819614ab80cSMark Rutland }
820382e6e17SMarc Zyngier
821614ab80cSMark Rutland if (!is_nmi)
822614ab80cSMark Rutland __gic_handle_irq(irqnr, regs);
823614ab80cSMark Rutland }
824614ab80cSMark Rutland
825614ab80cSMark Rutland /*
826614ab80cSMark Rutland * An exception has been taken from a context with IRQs disabled, which can only
827614ab80cSMark Rutland * be an NMI.
828614ab80cSMark Rutland *
829614ab80cSMark Rutland * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave
830614ab80cSMark Rutland * DAIF.IF (and ICC_PMR_EL1) unchanged.
831614ab80cSMark Rutland *
832614ab80cSMark Rutland * The entry code has performed NMI entry.
833614ab80cSMark Rutland */
__gic_handle_irq_from_irqsoff(struct pt_regs * regs)834614ab80cSMark Rutland static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs)
835614ab80cSMark Rutland {
836382e6e17SMarc Zyngier u64 pmr;
837614ab80cSMark Rutland u32 irqnr;
838382e6e17SMarc Zyngier
839382e6e17SMarc Zyngier /*
840382e6e17SMarc Zyngier * We were in a context with IRQs disabled. However, the
841382e6e17SMarc Zyngier * entry code has set PMR to a value that allows any
842382e6e17SMarc Zyngier * interrupt to be acknowledged, and not just NMIs. This can
843382e6e17SMarc Zyngier * lead to surprising effects if the NMI has been retired in
844382e6e17SMarc Zyngier * the meantime, and that there is an IRQ pending. The IRQ
845382e6e17SMarc Zyngier * would then be taken in NMI context, something that nobody
846382e6e17SMarc Zyngier * wants to debug twice.
847382e6e17SMarc Zyngier *
848382e6e17SMarc Zyngier * Until we sort this, drop PMR again to a level that will
849382e6e17SMarc Zyngier * actually only allow NMIs before reading IAR, and then
850382e6e17SMarc Zyngier * restore it to what it was.
851382e6e17SMarc Zyngier */
852382e6e17SMarc Zyngier pmr = gic_read_pmr();
853382e6e17SMarc Zyngier gic_pmr_mask_irqs();
854382e6e17SMarc Zyngier isb();
855614ab80cSMark Rutland irqnr = gic_read_iar();
856382e6e17SMarc Zyngier gic_write_pmr(pmr);
857382e6e17SMarc Zyngier
858614ab80cSMark Rutland __gic_handle_nmi(irqnr, regs);
859382e6e17SMarc Zyngier }
860382e6e17SMarc Zyngier
gic_handle_irq(struct pt_regs * regs)861021f6537SMarc Zyngier static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
862021f6537SMarc Zyngier {
863614ab80cSMark Rutland if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs)))
864614ab80cSMark Rutland __gic_handle_irq_from_irqsoff(regs);
86539a06b67SWill Deacon else
866614ab80cSMark Rutland __gic_handle_irq_from_irqson(regs);
867021f6537SMarc Zyngier }
868021f6537SMarc Zyngier
gic_get_pribits(void)869b5cf6073SJulien Thierry static u32 gic_get_pribits(void)
870b5cf6073SJulien Thierry {
871b5cf6073SJulien Thierry u32 pribits;
872b5cf6073SJulien Thierry
873b5cf6073SJulien Thierry pribits = gic_read_ctlr();
874b5cf6073SJulien Thierry pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
875b5cf6073SJulien Thierry pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
876b5cf6073SJulien Thierry pribits++;
877b5cf6073SJulien Thierry
878b5cf6073SJulien Thierry return pribits;
879b5cf6073SJulien Thierry }
880b5cf6073SJulien Thierry
gic_has_group0(void)881b5cf6073SJulien Thierry static bool gic_has_group0(void)
882b5cf6073SJulien Thierry {
883b5cf6073SJulien Thierry u32 val;
884e7932188SJulien Thierry u32 old_pmr;
885e7932188SJulien Thierry
886e7932188SJulien Thierry old_pmr = gic_read_pmr();
887b5cf6073SJulien Thierry
888b5cf6073SJulien Thierry /*
889b5cf6073SJulien Thierry * Let's find out if Group0 is under control of EL3 or not by
890b5cf6073SJulien Thierry * setting the highest possible, non-zero priority in PMR.
891b5cf6073SJulien Thierry *
892b5cf6073SJulien Thierry * If SCR_EL3.FIQ is set, the priority gets shifted down in
893b5cf6073SJulien Thierry * order for the CPU interface to set bit 7, and keep the
894b5cf6073SJulien Thierry * actual priority in the non-secure range. In the process, it
895b5cf6073SJulien Thierry * looses the least significant bit and the actual priority
896b5cf6073SJulien Thierry * becomes 0x80. Reading it back returns 0, indicating that
897b5cf6073SJulien Thierry * we're don't have access to Group0.
898b5cf6073SJulien Thierry */
899b5cf6073SJulien Thierry gic_write_pmr(BIT(8 - gic_get_pribits()));
900b5cf6073SJulien Thierry val = gic_read_pmr();
901b5cf6073SJulien Thierry
902e7932188SJulien Thierry gic_write_pmr(old_pmr);
903e7932188SJulien Thierry
904b5cf6073SJulien Thierry return val != 0;
905b5cf6073SJulien Thierry }
906b5cf6073SJulien Thierry
gic_dist_init(void)907021f6537SMarc Zyngier static void __init gic_dist_init(void)
908021f6537SMarc Zyngier {
909021f6537SMarc Zyngier unsigned int i;
910021f6537SMarc Zyngier u64 affinity;
911021f6537SMarc Zyngier void __iomem *base = gic_data.dist_base;
9120b04758bSMarc Zyngier u32 val;
913021f6537SMarc Zyngier
914021f6537SMarc Zyngier /* Disable the distributor */
915021f6537SMarc Zyngier writel_relaxed(0, base + GICD_CTLR);
916021f6537SMarc Zyngier gic_dist_wait_for_rwp();
917021f6537SMarc Zyngier
9187c9b9730SMarc Zyngier /*
9197c9b9730SMarc Zyngier * Configure SPIs as non-secure Group-1. This will only matter
9207c9b9730SMarc Zyngier * if the GIC only has a single security state. This will not
9217c9b9730SMarc Zyngier * do the right thing if the kernel is running in secure mode,
9227c9b9730SMarc Zyngier * but that's not the intended use case anyway.
9237c9b9730SMarc Zyngier */
924211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i += 32)
9257c9b9730SMarc Zyngier writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
9267c9b9730SMarc Zyngier
927211bddd2SMarc Zyngier /* Extended SPI range, not handled by the GICv2/GICv3 common code */
928211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) {
929211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
930211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
931211bddd2SMarc Zyngier }
932211bddd2SMarc Zyngier
933211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32)
934211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
935211bddd2SMarc Zyngier
936211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 16)
937211bddd2SMarc Zyngier writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
938211bddd2SMarc Zyngier
939211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 4)
940211bddd2SMarc Zyngier writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
941211bddd2SMarc Zyngier
94263f13483SMarc Zyngier /* Now do the common stuff */
94363f13483SMarc Zyngier gic_dist_config(base, GIC_LINE_NR, NULL);
944021f6537SMarc Zyngier
9450b04758bSMarc Zyngier val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1;
9460b04758bSMarc Zyngier if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) {
9470b04758bSMarc Zyngier pr_info("Enabling SGIs without active state\n");
9480b04758bSMarc Zyngier val |= GICD_CTLR_nASSGIreq;
9490b04758bSMarc Zyngier }
9500b04758bSMarc Zyngier
95163f13483SMarc Zyngier /* Enable distributor with ARE, Group1, and wait for it to drain */
9520b04758bSMarc Zyngier writel_relaxed(val, base + GICD_CTLR);
95363f13483SMarc Zyngier gic_dist_wait_for_rwp();
954021f6537SMarc Zyngier
955021f6537SMarc Zyngier /*
956021f6537SMarc Zyngier * Set all global interrupts to the boot CPU only. ARE must be
957021f6537SMarc Zyngier * enabled.
958021f6537SMarc Zyngier */
9593c65cbb7SMarc Zyngier affinity = gic_cpu_to_affinity(smp_processor_id());
960211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i++)
96172c97126SJean-Philippe Brucker gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
962211bddd2SMarc Zyngier
963211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i++)
964211bddd2SMarc Zyngier gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
965021f6537SMarc Zyngier }
966021f6537SMarc Zyngier
gic_iterate_rdists(int (* fn)(struct redist_region *,void __iomem *))9670d94ded2SMarc Zyngier static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
968021f6537SMarc Zyngier {
9690d94ded2SMarc Zyngier int ret = -ENODEV;
970021f6537SMarc Zyngier int i;
971021f6537SMarc Zyngier
972f5c1434cSMarc Zyngier for (i = 0; i < gic_data.nr_redist_regions; i++) {
973f5c1434cSMarc Zyngier void __iomem *ptr = gic_data.redist_regions[i].redist_base;
9740d94ded2SMarc Zyngier u64 typer;
975021f6537SMarc Zyngier u32 reg;
976021f6537SMarc Zyngier
977021f6537SMarc Zyngier reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
978021f6537SMarc Zyngier if (reg != GIC_PIDR2_ARCH_GICv3 &&
979021f6537SMarc Zyngier reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
980021f6537SMarc Zyngier pr_warn("No redistributor present @%p\n", ptr);
981021f6537SMarc Zyngier break;
982021f6537SMarc Zyngier }
983021f6537SMarc Zyngier
984021f6537SMarc Zyngier do {
98572c97126SJean-Philippe Brucker typer = gic_read_typer(ptr + GICR_TYPER);
9860d94ded2SMarc Zyngier ret = fn(gic_data.redist_regions + i, ptr);
9870d94ded2SMarc Zyngier if (!ret)
988021f6537SMarc Zyngier return 0;
989021f6537SMarc Zyngier
990b70fb7afSTomasz Nowicki if (gic_data.redist_regions[i].single_redist)
991b70fb7afSTomasz Nowicki break;
992b70fb7afSTomasz Nowicki
993021f6537SMarc Zyngier if (gic_data.redist_stride) {
994021f6537SMarc Zyngier ptr += gic_data.redist_stride;
995021f6537SMarc Zyngier } else {
996021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
997021f6537SMarc Zyngier if (typer & GICR_TYPER_VLPIS)
998021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
999021f6537SMarc Zyngier }
1000021f6537SMarc Zyngier } while (!(typer & GICR_TYPER_LAST));
1001021f6537SMarc Zyngier }
1002021f6537SMarc Zyngier
10030d94ded2SMarc Zyngier return ret ? -ENODEV : 0;
10040d94ded2SMarc Zyngier }
10050d94ded2SMarc Zyngier
__gic_populate_rdist(struct redist_region * region,void __iomem * ptr)10060d94ded2SMarc Zyngier static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
10070d94ded2SMarc Zyngier {
10083c65cbb7SMarc Zyngier unsigned long mpidr;
10090d94ded2SMarc Zyngier u64 typer;
10100d94ded2SMarc Zyngier u32 aff;
10110d94ded2SMarc Zyngier
10120d94ded2SMarc Zyngier /*
10130d94ded2SMarc Zyngier * Convert affinity to a 32bit value that can be matched to
10140d94ded2SMarc Zyngier * GICR_TYPER bits [63:32].
10150d94ded2SMarc Zyngier */
10163c65cbb7SMarc Zyngier mpidr = gic_cpu_to_affinity(smp_processor_id());
10173c65cbb7SMarc Zyngier
10180d94ded2SMarc Zyngier aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
10190d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
10200d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
10210d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0));
10220d94ded2SMarc Zyngier
10230d94ded2SMarc Zyngier typer = gic_read_typer(ptr + GICR_TYPER);
10240d94ded2SMarc Zyngier if ((typer >> 32) == aff) {
10250d94ded2SMarc Zyngier u64 offset = ptr - region->redist_base;
10269058a4e9SMarc Zyngier raw_spin_lock_init(&gic_data_rdist()->rd_lock);
10270d94ded2SMarc Zyngier gic_data_rdist_rd_base() = ptr;
10280d94ded2SMarc Zyngier gic_data_rdist()->phys_base = region->phys_base + offset;
10290d94ded2SMarc Zyngier
10300d94ded2SMarc Zyngier pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
10310d94ded2SMarc Zyngier smp_processor_id(), mpidr,
10320d94ded2SMarc Zyngier (int)(region - gic_data.redist_regions),
10330d94ded2SMarc Zyngier &gic_data_rdist()->phys_base);
10340d94ded2SMarc Zyngier return 0;
10350d94ded2SMarc Zyngier }
10360d94ded2SMarc Zyngier
10370d94ded2SMarc Zyngier /* Try next one */
10380d94ded2SMarc Zyngier return 1;
10390d94ded2SMarc Zyngier }
10400d94ded2SMarc Zyngier
gic_populate_rdist(void)10410d94ded2SMarc Zyngier static int gic_populate_rdist(void)
10420d94ded2SMarc Zyngier {
10430d94ded2SMarc Zyngier if (gic_iterate_rdists(__gic_populate_rdist) == 0)
10440d94ded2SMarc Zyngier return 0;
10450d94ded2SMarc Zyngier
1046021f6537SMarc Zyngier /* We couldn't even deal with ourselves... */
1047f6c86a41SJean-Philippe Brucker WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
10480d94ded2SMarc Zyngier smp_processor_id(),
10490d94ded2SMarc Zyngier (unsigned long)cpu_logical_map(smp_processor_id()));
1050021f6537SMarc Zyngier return -ENODEV;
1051021f6537SMarc Zyngier }
1052021f6537SMarc Zyngier
__gic_update_rdist_properties(struct redist_region * region,void __iomem * ptr)10531a60e1e6SMarc Zyngier static int __gic_update_rdist_properties(struct redist_region *region,
10540edc23eaSMarc Zyngier void __iomem *ptr)
10550edc23eaSMarc Zyngier {
10560edc23eaSMarc Zyngier u64 typer = gic_read_typer(ptr + GICR_TYPER);
1057a837ed36SMarc Zyngier u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
1058b25319d2SMarc Zyngier
10594d968297SZhiyuan Dai /* Boot-time cleanup */
106079a7f77bSMarc Zyngier if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
106179a7f77bSMarc Zyngier u64 val;
106279a7f77bSMarc Zyngier
106379a7f77bSMarc Zyngier /* Deactivate any present vPE */
106479a7f77bSMarc Zyngier val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER);
106579a7f77bSMarc Zyngier if (val & GICR_VPENDBASER_Valid)
106679a7f77bSMarc Zyngier gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
106779a7f77bSMarc Zyngier ptr + SZ_128K + GICR_VPENDBASER);
106879a7f77bSMarc Zyngier
106979a7f77bSMarc Zyngier /* Mark the VPE table as invalid */
107079a7f77bSMarc Zyngier val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER);
107179a7f77bSMarc Zyngier val &= ~GICR_VPROPBASER_4_1_VALID;
107279a7f77bSMarc Zyngier gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER);
107379a7f77bSMarc Zyngier }
107479a7f77bSMarc Zyngier
10750edc23eaSMarc Zyngier gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
1076b25319d2SMarc Zyngier
1077a837ed36SMarc Zyngier /*
1078a837ed36SMarc Zyngier * TYPER.RVPEID implies some form of DirectLPI, no matter what the
1079a837ed36SMarc Zyngier * doc says... :-/ And CTLR.IR implies another subset of DirectLPI
1080a837ed36SMarc Zyngier * that the ITS driver can make use of for LPIs (and not VLPIs).
1081a837ed36SMarc Zyngier *
1082a837ed36SMarc Zyngier * These are 3 different ways to express the same thing, depending
1083a837ed36SMarc Zyngier * on the revision of the architecture and its relaxations over
1084a837ed36SMarc Zyngier * time. Just group them under the 'direct_lpi' banner.
1085a837ed36SMarc Zyngier */
1086b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID);
1087b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) |
1088a837ed36SMarc Zyngier !!(ctlr & GICR_CTLR_IR) |
1089b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid);
109096806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY);
1091b25319d2SMarc Zyngier
1092b25319d2SMarc Zyngier /* Detect non-sensical configurations */
1093b25319d2SMarc Zyngier if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) {
1094b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi = false;
1095b25319d2SMarc Zyngier gic_data.rdists.has_vlpis = false;
1096b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = false;
1097b25319d2SMarc Zyngier }
1098b25319d2SMarc Zyngier
10995f51f803SMarc Zyngier gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
11000edc23eaSMarc Zyngier
11010edc23eaSMarc Zyngier return 1;
11020edc23eaSMarc Zyngier }
11030edc23eaSMarc Zyngier
gic_update_rdist_properties(void)11041a60e1e6SMarc Zyngier static void gic_update_rdist_properties(void)
11050edc23eaSMarc Zyngier {
11061a60e1e6SMarc Zyngier gic_data.ppi_nr = UINT_MAX;
11071a60e1e6SMarc Zyngier gic_iterate_rdists(__gic_update_rdist_properties);
11081a60e1e6SMarc Zyngier if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
11091a60e1e6SMarc Zyngier gic_data.ppi_nr = 0;
1110a837ed36SMarc Zyngier pr_info("GICv3 features: %d PPIs%s%s\n",
1111a837ed36SMarc Zyngier gic_data.ppi_nr,
1112a837ed36SMarc Zyngier gic_data.has_rss ? ", RSS" : "",
1113a837ed36SMarc Zyngier gic_data.rdists.has_direct_lpi ? ", DirectLPI" : "");
1114a837ed36SMarc Zyngier
111596806229SMarc Zyngier if (gic_data.rdists.has_vlpis)
111696806229SMarc Zyngier pr_info("GICv4 features: %s%s%s\n",
111796806229SMarc Zyngier gic_data.rdists.has_direct_lpi ? "DirectLPI " : "",
111896806229SMarc Zyngier gic_data.rdists.has_rvpeid ? "RVPEID " : "",
111996806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : "");
11200edc23eaSMarc Zyngier }
11210edc23eaSMarc Zyngier
1122d98d0a99SJulien Thierry /* Check whether it's single security state view */
gic_dist_security_disabled(void)1123d98d0a99SJulien Thierry static inline bool gic_dist_security_disabled(void)
1124d98d0a99SJulien Thierry {
1125d98d0a99SJulien Thierry return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
1126d98d0a99SJulien Thierry }
1127d98d0a99SJulien Thierry
gic_cpu_sys_reg_init(void)11283708d52fSSudeep Holla static void gic_cpu_sys_reg_init(void)
1129021f6537SMarc Zyngier {
1130eda0d04aSShanker Donthineni int i, cpu = smp_processor_id();
11313c65cbb7SMarc Zyngier u64 mpidr = gic_cpu_to_affinity(cpu);
1132eda0d04aSShanker Donthineni u64 need_rss = MPIDR_RS(mpidr);
113333625282SMarc Zyngier bool group0;
1134b5cf6073SJulien Thierry u32 pribits;
1135eda0d04aSShanker Donthineni
11367cabd008SMarc Zyngier /*
11377cabd008SMarc Zyngier * Need to check that the SRE bit has actually been set. If
11387cabd008SMarc Zyngier * not, it means that SRE is disabled at EL2. We're going to
11397cabd008SMarc Zyngier * die painfully, and there is nothing we can do about it.
11407cabd008SMarc Zyngier *
11417cabd008SMarc Zyngier * Kindly inform the luser.
11427cabd008SMarc Zyngier */
11437cabd008SMarc Zyngier if (!gic_enable_sre())
11447cabd008SMarc Zyngier pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
1145021f6537SMarc Zyngier
1146b5cf6073SJulien Thierry pribits = gic_get_pribits();
114733625282SMarc Zyngier
1148b5cf6073SJulien Thierry group0 = gic_has_group0();
114933625282SMarc Zyngier
1150021f6537SMarc Zyngier /* Set priority mask register */
1151d98d0a99SJulien Thierry if (!gic_prio_masking_enabled()) {
115233625282SMarc Zyngier write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
115333678059SAlexandru Elisei } else if (gic_supports_nmi()) {
1154d98d0a99SJulien Thierry /*
1155d98d0a99SJulien Thierry * Mismatch configuration with boot CPU, the system is likely
1156d98d0a99SJulien Thierry * to die as interrupt masking will not work properly on all
1157d98d0a99SJulien Thierry * CPUs
115833678059SAlexandru Elisei *
115933678059SAlexandru Elisei * The boot CPU calls this function before enabling NMI support,
116033678059SAlexandru Elisei * and as a result we'll never see this warning in the boot path
116133678059SAlexandru Elisei * for that CPU.
1162d98d0a99SJulien Thierry */
116333678059SAlexandru Elisei if (static_branch_unlikely(&gic_nonsecure_priorities))
116433678059SAlexandru Elisei WARN_ON(!group0 || gic_dist_security_disabled());
116533678059SAlexandru Elisei else
116633678059SAlexandru Elisei WARN_ON(group0 && !gic_dist_security_disabled());
1167d98d0a99SJulien Thierry }
1168021f6537SMarc Zyngier
116991ef8442SDaniel Thompson /*
117091ef8442SDaniel Thompson * Some firmwares hand over to the kernel with the BPR changed from
117191ef8442SDaniel Thompson * its reset value (and with a value large enough to prevent
117291ef8442SDaniel Thompson * any pre-emptive interrupts from working at all). Writing a zero
117391ef8442SDaniel Thompson * to BPR restores is reset value.
117491ef8442SDaniel Thompson */
117591ef8442SDaniel Thompson gic_write_bpr1(0);
117691ef8442SDaniel Thompson
1177d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) {
11780b6a3da9SMarc Zyngier /* EOI drops priority only (mode 1) */
11790b6a3da9SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
11800b6a3da9SMarc Zyngier } else {
1181021f6537SMarc Zyngier /* EOI deactivates interrupt too (mode 0) */
1182021f6537SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
11830b6a3da9SMarc Zyngier }
1184021f6537SMarc Zyngier
118533625282SMarc Zyngier /* Always whack Group0 before Group1 */
118633625282SMarc Zyngier if (group0) {
118733625282SMarc Zyngier switch(pribits) {
118833625282SMarc Zyngier case 8:
118933625282SMarc Zyngier case 7:
119033625282SMarc Zyngier write_gicreg(0, ICC_AP0R3_EL1);
119133625282SMarc Zyngier write_gicreg(0, ICC_AP0R2_EL1);
1192df561f66SGustavo A. R. Silva fallthrough;
119333625282SMarc Zyngier case 6:
119433625282SMarc Zyngier write_gicreg(0, ICC_AP0R1_EL1);
1195df561f66SGustavo A. R. Silva fallthrough;
119633625282SMarc Zyngier case 5:
119733625282SMarc Zyngier case 4:
119833625282SMarc Zyngier write_gicreg(0, ICC_AP0R0_EL1);
119933625282SMarc Zyngier }
1200d6062a6dSMarc Zyngier
120133625282SMarc Zyngier isb();
120233625282SMarc Zyngier }
120333625282SMarc Zyngier
120433625282SMarc Zyngier switch(pribits) {
1205d6062a6dSMarc Zyngier case 8:
1206d6062a6dSMarc Zyngier case 7:
1207d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R3_EL1);
1208d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R2_EL1);
1209df561f66SGustavo A. R. Silva fallthrough;
1210d6062a6dSMarc Zyngier case 6:
1211d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R1_EL1);
1212df561f66SGustavo A. R. Silva fallthrough;
1213d6062a6dSMarc Zyngier case 5:
1214d6062a6dSMarc Zyngier case 4:
1215d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R0_EL1);
1216d6062a6dSMarc Zyngier }
1217d6062a6dSMarc Zyngier
1218d6062a6dSMarc Zyngier isb();
1219d6062a6dSMarc Zyngier
1220021f6537SMarc Zyngier /* ... and let's hit the road... */
1221021f6537SMarc Zyngier gic_write_grpen1(1);
1222eda0d04aSShanker Donthineni
1223eda0d04aSShanker Donthineni /* Keep the RSS capability status in per_cpu variable */
1224eda0d04aSShanker Donthineni per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
1225eda0d04aSShanker Donthineni
1226eda0d04aSShanker Donthineni /* Check all the CPUs have capable of sending SGIs to other CPUs */
1227eda0d04aSShanker Donthineni for_each_online_cpu(i) {
1228eda0d04aSShanker Donthineni bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
1229eda0d04aSShanker Donthineni
12303c65cbb7SMarc Zyngier need_rss |= MPIDR_RS(gic_cpu_to_affinity(i));
1231eda0d04aSShanker Donthineni if (need_rss && (!have_rss))
1232eda0d04aSShanker Donthineni pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
1233eda0d04aSShanker Donthineni cpu, (unsigned long)mpidr,
12343c65cbb7SMarc Zyngier i, (unsigned long)gic_cpu_to_affinity(i));
1235eda0d04aSShanker Donthineni }
1236eda0d04aSShanker Donthineni
1237eda0d04aSShanker Donthineni /**
1238eda0d04aSShanker Donthineni * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
1239eda0d04aSShanker Donthineni * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
1240eda0d04aSShanker Donthineni * UNPREDICTABLE choice of :
1241eda0d04aSShanker Donthineni * - The write is ignored.
1242eda0d04aSShanker Donthineni * - The RS field is treated as 0.
1243eda0d04aSShanker Donthineni */
1244eda0d04aSShanker Donthineni if (need_rss && (!gic_data.has_rss))
1245eda0d04aSShanker Donthineni pr_crit_once("RSS is required but GICD doesn't support it\n");
1246021f6537SMarc Zyngier }
1247021f6537SMarc Zyngier
1248f736d65dSMarc Zyngier static bool gicv3_nolpi;
1249f736d65dSMarc Zyngier
gicv3_nolpi_cfg(char * buf)1250f736d65dSMarc Zyngier static int __init gicv3_nolpi_cfg(char *buf)
1251f736d65dSMarc Zyngier {
12525e279739SChristophe JAILLET return kstrtobool(buf, &gicv3_nolpi);
1253f736d65dSMarc Zyngier }
1254f736d65dSMarc Zyngier early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
1255f736d65dSMarc Zyngier
gic_dist_supports_lpis(void)1256da33f31dSMarc Zyngier static int gic_dist_supports_lpis(void)
1257da33f31dSMarc Zyngier {
1258d38a71c5SMarc Zyngier return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
1259d38a71c5SMarc Zyngier !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
1260d38a71c5SMarc Zyngier !gicv3_nolpi);
1261da33f31dSMarc Zyngier }
1262da33f31dSMarc Zyngier
gic_cpu_init(void)1263021f6537SMarc Zyngier static void gic_cpu_init(void)
1264021f6537SMarc Zyngier {
1265021f6537SMarc Zyngier void __iomem *rbase;
12661a60e1e6SMarc Zyngier int i;
1267021f6537SMarc Zyngier
1268021f6537SMarc Zyngier /* Register ourselves with the rest of the world */
1269021f6537SMarc Zyngier if (gic_populate_rdist())
1270021f6537SMarc Zyngier return;
1271021f6537SMarc Zyngier
1272a2c22510SSudeep Holla gic_enable_redist(true);
1273021f6537SMarc Zyngier
1274ad5a78d3SMarc Zyngier WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
1275ad5a78d3SMarc Zyngier !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
1276ad5a78d3SMarc Zyngier "Distributor has extended ranges, but CPU%d doesn't\n",
1277ad5a78d3SMarc Zyngier smp_processor_id());
1278ad5a78d3SMarc Zyngier
1279021f6537SMarc Zyngier rbase = gic_data_rdist_sgi_base();
1280021f6537SMarc Zyngier
12817c9b9730SMarc Zyngier /* Configure SGIs/PPIs as non-secure Group-1 */
12821a60e1e6SMarc Zyngier for (i = 0; i < gic_data.ppi_nr + 16; i += 32)
12831a60e1e6SMarc Zyngier writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
12847c9b9730SMarc Zyngier
12851a60e1e6SMarc Zyngier gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp);
1286021f6537SMarc Zyngier
12873708d52fSSudeep Holla /* initialise system registers */
12883708d52fSSudeep Holla gic_cpu_sys_reg_init();
1289021f6537SMarc Zyngier }
1290021f6537SMarc Zyngier
1291021f6537SMarc Zyngier #ifdef CONFIG_SMP
1292021f6537SMarc Zyngier
1293eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
1294eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL)
1295eda0d04aSShanker Donthineni
gic_starting_cpu(unsigned int cpu)12966670a6d8SRichard Cochran static int gic_starting_cpu(unsigned int cpu)
12976670a6d8SRichard Cochran {
12986670a6d8SRichard Cochran gic_cpu_init();
1299d38a71c5SMarc Zyngier
1300d38a71c5SMarc Zyngier if (gic_dist_supports_lpis())
1301d38a71c5SMarc Zyngier its_cpu_init();
1302d38a71c5SMarc Zyngier
13036670a6d8SRichard Cochran return 0;
13046670a6d8SRichard Cochran }
1305021f6537SMarc Zyngier
gic_compute_target_list(int * base_cpu,const struct cpumask * mask,unsigned long cluster_id)1306021f6537SMarc Zyngier static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
1307f6c86a41SJean-Philippe Brucker unsigned long cluster_id)
1308021f6537SMarc Zyngier {
1309727653d6SJames Morse int next_cpu, cpu = *base_cpu;
13103c65cbb7SMarc Zyngier unsigned long mpidr;
1311021f6537SMarc Zyngier u16 tlist = 0;
1312021f6537SMarc Zyngier
13133c65cbb7SMarc Zyngier mpidr = gic_cpu_to_affinity(cpu);
13143c65cbb7SMarc Zyngier
1315021f6537SMarc Zyngier while (cpu < nr_cpu_ids) {
1316021f6537SMarc Zyngier tlist |= 1 << (mpidr & 0xf);
1317021f6537SMarc Zyngier
1318727653d6SJames Morse next_cpu = cpumask_next(cpu, mask);
1319727653d6SJames Morse if (next_cpu >= nr_cpu_ids)
1320021f6537SMarc Zyngier goto out;
1321727653d6SJames Morse cpu = next_cpu;
1322021f6537SMarc Zyngier
13233c65cbb7SMarc Zyngier mpidr = gic_cpu_to_affinity(cpu);
1324021f6537SMarc Zyngier
1325eda0d04aSShanker Donthineni if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
1326021f6537SMarc Zyngier cpu--;
1327021f6537SMarc Zyngier goto out;
1328021f6537SMarc Zyngier }
1329021f6537SMarc Zyngier }
1330021f6537SMarc Zyngier out:
1331021f6537SMarc Zyngier *base_cpu = cpu;
1332021f6537SMarc Zyngier return tlist;
1333021f6537SMarc Zyngier }
1334021f6537SMarc Zyngier
13357e580278SAndre Przywara #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
13367e580278SAndre Przywara (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
13377e580278SAndre Przywara << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
13387e580278SAndre Przywara
gic_send_sgi(u64 cluster_id,u16 tlist,unsigned int irq)1339021f6537SMarc Zyngier static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
1340021f6537SMarc Zyngier {
1341021f6537SMarc Zyngier u64 val;
1342021f6537SMarc Zyngier
13437e580278SAndre Przywara val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) |
13447e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 2) |
13457e580278SAndre Przywara irq << ICC_SGI1R_SGI_ID_SHIFT |
13467e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 1) |
1347eda0d04aSShanker Donthineni MPIDR_TO_SGI_RS(cluster_id) |
13487e580278SAndre Przywara tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
1349021f6537SMarc Zyngier
1350b6dd4d83SMark Salter pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
1351021f6537SMarc Zyngier gic_write_sgi1r(val);
1352021f6537SMarc Zyngier }
1353021f6537SMarc Zyngier
gic_ipi_send_mask(struct irq_data * d,const struct cpumask * mask)135464b499d8SMarc Zyngier static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
1355021f6537SMarc Zyngier {
1356021f6537SMarc Zyngier int cpu;
1357021f6537SMarc Zyngier
135864b499d8SMarc Zyngier if (WARN_ON(d->hwirq >= 16))
1359021f6537SMarc Zyngier return;
1360021f6537SMarc Zyngier
1361021f6537SMarc Zyngier /*
1362021f6537SMarc Zyngier * Ensure that stores to Normal memory are visible to the
1363021f6537SMarc Zyngier * other CPUs before issuing the IPI.
1364021f6537SMarc Zyngier */
136580e4e1f4SBarry Song dsb(ishst);
1366021f6537SMarc Zyngier
1367f9b531feSRusty Russell for_each_cpu(cpu, mask) {
13683c65cbb7SMarc Zyngier u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(gic_cpu_to_affinity(cpu));
1369021f6537SMarc Zyngier u16 tlist;
1370021f6537SMarc Zyngier
1371021f6537SMarc Zyngier tlist = gic_compute_target_list(&cpu, mask, cluster_id);
137264b499d8SMarc Zyngier gic_send_sgi(cluster_id, tlist, d->hwirq);
1373021f6537SMarc Zyngier }
1374021f6537SMarc Zyngier
1375021f6537SMarc Zyngier /* Force the above writes to ICC_SGI1R_EL1 to be executed */
1376021f6537SMarc Zyngier isb();
1377021f6537SMarc Zyngier }
1378021f6537SMarc Zyngier
gic_smp_init(void)13798a94c1abSIngo Rohloff static void __init gic_smp_init(void)
1380021f6537SMarc Zyngier {
138164b499d8SMarc Zyngier struct irq_fwspec sgi_fwspec = {
138264b499d8SMarc Zyngier .fwnode = gic_data.fwnode,
138364b499d8SMarc Zyngier .param_count = 1,
138464b499d8SMarc Zyngier };
138564b499d8SMarc Zyngier int base_sgi;
138664b499d8SMarc Zyngier
13876896bcd1SThomas Gleixner cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
138873c1b41eSThomas Gleixner "irqchip/arm/gicv3:starting",
138973c1b41eSThomas Gleixner gic_starting_cpu, NULL);
139064b499d8SMarc Zyngier
139164b499d8SMarc Zyngier /* Register all 8 non-secure SGIs */
13920e2213feSJohan Hovold base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec);
139364b499d8SMarc Zyngier if (WARN_ON(base_sgi <= 0))
139464b499d8SMarc Zyngier return;
139564b499d8SMarc Zyngier
139664b499d8SMarc Zyngier set_smp_ipi_range(base_sgi, 8);
1397021f6537SMarc Zyngier }
1398021f6537SMarc Zyngier
gic_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)1399021f6537SMarc Zyngier static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1400021f6537SMarc Zyngier bool force)
1401021f6537SMarc Zyngier {
140265a30f8bSSuzuki K Poulose unsigned int cpu;
1403e91b036eSMarc Zyngier u32 offset, index;
1404021f6537SMarc Zyngier void __iomem *reg;
1405021f6537SMarc Zyngier int enabled;
1406021f6537SMarc Zyngier u64 val;
1407021f6537SMarc Zyngier
140865a30f8bSSuzuki K Poulose if (force)
140965a30f8bSSuzuki K Poulose cpu = cpumask_first(mask_val);
141065a30f8bSSuzuki K Poulose else
141165a30f8bSSuzuki K Poulose cpu = cpumask_any_and(mask_val, cpu_online_mask);
141265a30f8bSSuzuki K Poulose
1413866d7c1bSSuzuki K Poulose if (cpu >= nr_cpu_ids)
1414866d7c1bSSuzuki K Poulose return -EINVAL;
1415866d7c1bSSuzuki K Poulose
1416021f6537SMarc Zyngier if (gic_irq_in_rdist(d))
1417021f6537SMarc Zyngier return -EINVAL;
1418021f6537SMarc Zyngier
1419021f6537SMarc Zyngier /* If interrupt was enabled, disable it first */
1420021f6537SMarc Zyngier enabled = gic_peek_irq(d, GICD_ISENABLER);
1421021f6537SMarc Zyngier if (enabled)
1422021f6537SMarc Zyngier gic_mask_irq(d);
1423021f6537SMarc Zyngier
1424e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IROUTER, &index);
1425e91b036eSMarc Zyngier reg = gic_dist_base(d) + offset + (index * 8);
14263c65cbb7SMarc Zyngier val = gic_cpu_to_affinity(cpu);
1427021f6537SMarc Zyngier
142872c97126SJean-Philippe Brucker gic_write_irouter(val, reg);
1429021f6537SMarc Zyngier
1430021f6537SMarc Zyngier /*
1431021f6537SMarc Zyngier * If the interrupt was enabled, enabled it again. Otherwise,
1432021f6537SMarc Zyngier * just wait for the distributor to have digested our changes.
1433021f6537SMarc Zyngier */
1434021f6537SMarc Zyngier if (enabled)
1435021f6537SMarc Zyngier gic_unmask_irq(d);
1436021f6537SMarc Zyngier
1437956ae91aSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu));
1438956ae91aSMarc Zyngier
14390fc6fa29SAntoine Tenart return IRQ_SET_MASK_OK_DONE;
1440021f6537SMarc Zyngier }
1441021f6537SMarc Zyngier #else
1442021f6537SMarc Zyngier #define gic_set_affinity NULL
144364b499d8SMarc Zyngier #define gic_ipi_send_mask NULL
1444021f6537SMarc Zyngier #define gic_smp_init() do { } while(0)
1445021f6537SMarc Zyngier #endif
1446021f6537SMarc Zyngier
gic_retrigger(struct irq_data * data)144717f644e9SValentin Schneider static int gic_retrigger(struct irq_data *data)
144817f644e9SValentin Schneider {
144917f644e9SValentin Schneider return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true);
145017f644e9SValentin Schneider }
145117f644e9SValentin Schneider
14523708d52fSSudeep Holla #ifdef CONFIG_CPU_PM
gic_cpu_pm_notifier(struct notifier_block * self,unsigned long cmd,void * v)14533708d52fSSudeep Holla static int gic_cpu_pm_notifier(struct notifier_block *self,
14543708d52fSSudeep Holla unsigned long cmd, void *v)
14553708d52fSSudeep Holla {
14563708d52fSSudeep Holla if (cmd == CPU_PM_EXIT) {
1457ccd9432aSSudeep Holla if (gic_dist_security_disabled())
14583708d52fSSudeep Holla gic_enable_redist(true);
14593708d52fSSudeep Holla gic_cpu_sys_reg_init();
1460ccd9432aSSudeep Holla } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
14613708d52fSSudeep Holla gic_write_grpen1(0);
14623708d52fSSudeep Holla gic_enable_redist(false);
14633708d52fSSudeep Holla }
14643708d52fSSudeep Holla return NOTIFY_OK;
14653708d52fSSudeep Holla }
14663708d52fSSudeep Holla
14673708d52fSSudeep Holla static struct notifier_block gic_cpu_pm_notifier_block = {
14683708d52fSSudeep Holla .notifier_call = gic_cpu_pm_notifier,
14693708d52fSSudeep Holla };
14703708d52fSSudeep Holla
gic_cpu_pm_init(void)14713708d52fSSudeep Holla static void gic_cpu_pm_init(void)
14723708d52fSSudeep Holla {
14733708d52fSSudeep Holla cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
14743708d52fSSudeep Holla }
14753708d52fSSudeep Holla
14763708d52fSSudeep Holla #else
gic_cpu_pm_init(void)14773708d52fSSudeep Holla static inline void gic_cpu_pm_init(void) { }
14783708d52fSSudeep Holla #endif /* CONFIG_CPU_PM */
14793708d52fSSudeep Holla
1480021f6537SMarc Zyngier static struct irq_chip gic_chip = {
1481021f6537SMarc Zyngier .name = "GICv3",
1482021f6537SMarc Zyngier .irq_mask = gic_mask_irq,
1483021f6537SMarc Zyngier .irq_unmask = gic_unmask_irq,
1484021f6537SMarc Zyngier .irq_eoi = gic_eoi_irq,
1485021f6537SMarc Zyngier .irq_set_type = gic_set_type,
1486021f6537SMarc Zyngier .irq_set_affinity = gic_set_affinity,
148717f644e9SValentin Schneider .irq_retrigger = gic_retrigger,
1488b594c6e2SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state,
1489b594c6e2SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state,
1490101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup,
1491101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown,
149264b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask,
14934110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED |
14944110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE |
14954110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND,
1496021f6537SMarc Zyngier };
1497021f6537SMarc Zyngier
14980b6a3da9SMarc Zyngier static struct irq_chip gic_eoimode1_chip = {
14990b6a3da9SMarc Zyngier .name = "GICv3",
15000b6a3da9SMarc Zyngier .irq_mask = gic_eoimode1_mask_irq,
15010b6a3da9SMarc Zyngier .irq_unmask = gic_unmask_irq,
15020b6a3da9SMarc Zyngier .irq_eoi = gic_eoimode1_eoi_irq,
15030b6a3da9SMarc Zyngier .irq_set_type = gic_set_type,
15040b6a3da9SMarc Zyngier .irq_set_affinity = gic_set_affinity,
150517f644e9SValentin Schneider .irq_retrigger = gic_retrigger,
15060b6a3da9SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state,
15070b6a3da9SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state,
1508530bf353SMarc Zyngier .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
1509101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup,
1510101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown,
151164b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask,
15124110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED |
15134110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE |
15144110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND,
15150b6a3da9SMarc Zyngier };
15160b6a3da9SMarc Zyngier
gic_irq_domain_map(struct irq_domain * d,unsigned int irq,irq_hw_number_t hw)1517021f6537SMarc Zyngier static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
1518021f6537SMarc Zyngier irq_hw_number_t hw)
1519021f6537SMarc Zyngier {
15200b6a3da9SMarc Zyngier struct irq_chip *chip = &gic_chip;
15211b57d91bSValentin Schneider struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq));
15220b6a3da9SMarc Zyngier
1523d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key))
15240b6a3da9SMarc Zyngier chip = &gic_eoimode1_chip;
15250b6a3da9SMarc Zyngier
1526e91b036eSMarc Zyngier switch (__get_intid_range(hw)) {
152770a29c32SMarc Zyngier case SGI_RANGE:
1528e91b036eSMarc Zyngier case PPI_RANGE:
15295f51f803SMarc Zyngier case EPPI_RANGE:
1530021f6537SMarc Zyngier irq_set_percpu_devid(irq);
15310b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data,
1532443acc4fSMarc Zyngier handle_percpu_devid_irq, NULL, NULL);
1533e91b036eSMarc Zyngier break;
1534e91b036eSMarc Zyngier
1535e91b036eSMarc Zyngier case SPI_RANGE:
1536211bddd2SMarc Zyngier case ESPI_RANGE:
15370b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data,
1538443acc4fSMarc Zyngier handle_fasteoi_irq, NULL, NULL);
1539d17cab44SRob Herring irq_set_probe(irq);
15401b57d91bSValentin Schneider irqd_set_single_target(irqd);
1541e91b036eSMarc Zyngier break;
1542e91b036eSMarc Zyngier
1543e91b036eSMarc Zyngier case LPI_RANGE:
1544da33f31dSMarc Zyngier if (!gic_dist_supports_lpis())
1545da33f31dSMarc Zyngier return -EPERM;
15460b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data,
1547da33f31dSMarc Zyngier handle_fasteoi_irq, NULL, NULL);
1548e91b036eSMarc Zyngier break;
1549e91b036eSMarc Zyngier
1550e91b036eSMarc Zyngier default:
1551e91b036eSMarc Zyngier return -EPERM;
1552da33f31dSMarc Zyngier }
1553da33f31dSMarc Zyngier
15541b57d91bSValentin Schneider /* Prevents SW retriggers which mess up the ACK/EOI ordering */
15551b57d91bSValentin Schneider irqd_set_handle_enforce_irqctx(irqd);
1556021f6537SMarc Zyngier return 0;
1557021f6537SMarc Zyngier }
1558021f6537SMarc Zyngier
gic_irq_domain_translate(struct irq_domain * d,struct irq_fwspec * fwspec,unsigned long * hwirq,unsigned int * type)1559f833f57fSMarc Zyngier static int gic_irq_domain_translate(struct irq_domain *d,
1560f833f57fSMarc Zyngier struct irq_fwspec *fwspec,
1561f833f57fSMarc Zyngier unsigned long *hwirq,
1562f833f57fSMarc Zyngier unsigned int *type)
1563021f6537SMarc Zyngier {
156464b499d8SMarc Zyngier if (fwspec->param_count == 1 && fwspec->param[0] < 16) {
156564b499d8SMarc Zyngier *hwirq = fwspec->param[0];
156664b499d8SMarc Zyngier *type = IRQ_TYPE_EDGE_RISING;
156764b499d8SMarc Zyngier return 0;
156864b499d8SMarc Zyngier }
156964b499d8SMarc Zyngier
1570f833f57fSMarc Zyngier if (is_of_node(fwspec->fwnode)) {
1571f833f57fSMarc Zyngier if (fwspec->param_count < 3)
1572021f6537SMarc Zyngier return -EINVAL;
1573021f6537SMarc Zyngier
1574db8c70ecSMarc Zyngier switch (fwspec->param[0]) {
1575db8c70ecSMarc Zyngier case 0: /* SPI */
1576db8c70ecSMarc Zyngier *hwirq = fwspec->param[1] + 32;
1577db8c70ecSMarc Zyngier break;
1578db8c70ecSMarc Zyngier case 1: /* PPI */
1579f833f57fSMarc Zyngier *hwirq = fwspec->param[1] + 16;
1580db8c70ecSMarc Zyngier break;
1581211bddd2SMarc Zyngier case 2: /* ESPI */
1582211bddd2SMarc Zyngier *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
1583211bddd2SMarc Zyngier break;
15845f51f803SMarc Zyngier case 3: /* EPPI */
15855f51f803SMarc Zyngier *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
15865f51f803SMarc Zyngier break;
1587db8c70ecSMarc Zyngier case GIC_IRQ_TYPE_LPI: /* LPI */
1588db8c70ecSMarc Zyngier *hwirq = fwspec->param[1];
1589db8c70ecSMarc Zyngier break;
15905f51f803SMarc Zyngier case GIC_IRQ_TYPE_PARTITION:
15915f51f803SMarc Zyngier *hwirq = fwspec->param[1];
15925f51f803SMarc Zyngier if (fwspec->param[1] >= 16)
15935f51f803SMarc Zyngier *hwirq += EPPI_BASE_INTID - 16;
15945f51f803SMarc Zyngier else
15955f51f803SMarc Zyngier *hwirq += 16;
15965f51f803SMarc Zyngier break;
1597db8c70ecSMarc Zyngier default:
1598db8c70ecSMarc Zyngier return -EINVAL;
1599db8c70ecSMarc Zyngier }
1600f833f57fSMarc Zyngier
1601f833f57fSMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
16026ef6386eSMarc Zyngier
160365da7d19SMarc Zyngier /*
160465da7d19SMarc Zyngier * Make it clear that broken DTs are... broken.
1605a359f757SIngo Molnar * Partitioned PPIs are an unfortunate exception.
160665da7d19SMarc Zyngier */
160765da7d19SMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE &&
160865da7d19SMarc Zyngier fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
1609f833f57fSMarc Zyngier return 0;
1610021f6537SMarc Zyngier }
1611021f6537SMarc Zyngier
1612ffa7d616STomasz Nowicki if (is_fwnode_irqchip(fwspec->fwnode)) {
1613ffa7d616STomasz Nowicki if(fwspec->param_count != 2)
1614ffa7d616STomasz Nowicki return -EINVAL;
1615ffa7d616STomasz Nowicki
1616544808f7SAndre Przywara if (fwspec->param[0] < 16) {
1617544808f7SAndre Przywara pr_err(FW_BUG "Illegal GSI%d translation request\n",
1618544808f7SAndre Przywara fwspec->param[0]);
1619544808f7SAndre Przywara return -EINVAL;
1620544808f7SAndre Przywara }
1621544808f7SAndre Przywara
1622ffa7d616STomasz Nowicki *hwirq = fwspec->param[0];
1623ffa7d616STomasz Nowicki *type = fwspec->param[1];
16246ef6386eSMarc Zyngier
16256ef6386eSMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE);
1626ffa7d616STomasz Nowicki return 0;
1627ffa7d616STomasz Nowicki }
1628ffa7d616STomasz Nowicki
1629f833f57fSMarc Zyngier return -EINVAL;
1630021f6537SMarc Zyngier }
1631021f6537SMarc Zyngier
gic_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)1632443acc4fSMarc Zyngier static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1633443acc4fSMarc Zyngier unsigned int nr_irqs, void *arg)
1634443acc4fSMarc Zyngier {
1635443acc4fSMarc Zyngier int i, ret;
1636443acc4fSMarc Zyngier irq_hw_number_t hwirq;
1637443acc4fSMarc Zyngier unsigned int type = IRQ_TYPE_NONE;
1638f833f57fSMarc Zyngier struct irq_fwspec *fwspec = arg;
1639443acc4fSMarc Zyngier
1640f833f57fSMarc Zyngier ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1641443acc4fSMarc Zyngier if (ret)
1642443acc4fSMarc Zyngier return ret;
1643443acc4fSMarc Zyngier
164463c16c6eSSuzuki K Poulose for (i = 0; i < nr_irqs; i++) {
164563c16c6eSSuzuki K Poulose ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
164663c16c6eSSuzuki K Poulose if (ret)
164763c16c6eSSuzuki K Poulose return ret;
164863c16c6eSSuzuki K Poulose }
1649443acc4fSMarc Zyngier
1650443acc4fSMarc Zyngier return 0;
1651443acc4fSMarc Zyngier }
1652443acc4fSMarc Zyngier
gic_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)1653443acc4fSMarc Zyngier static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1654443acc4fSMarc Zyngier unsigned int nr_irqs)
1655443acc4fSMarc Zyngier {
1656443acc4fSMarc Zyngier int i;
1657443acc4fSMarc Zyngier
1658443acc4fSMarc Zyngier for (i = 0; i < nr_irqs; i++) {
1659443acc4fSMarc Zyngier struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
1660443acc4fSMarc Zyngier irq_set_handler(virq + i, NULL);
1661443acc4fSMarc Zyngier irq_domain_reset_irq_data(d);
1662443acc4fSMarc Zyngier }
1663443acc4fSMarc Zyngier }
1664443acc4fSMarc Zyngier
fwspec_is_partitioned_ppi(struct irq_fwspec * fwspec,irq_hw_number_t hwirq)1665d753f849SJames Morse static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec,
1666d753f849SJames Morse irq_hw_number_t hwirq)
1667d753f849SJames Morse {
1668d753f849SJames Morse enum gic_intid_range range;
1669d753f849SJames Morse
1670d753f849SJames Morse if (!gic_data.ppi_descs)
1671d753f849SJames Morse return false;
1672d753f849SJames Morse
1673d753f849SJames Morse if (!is_of_node(fwspec->fwnode))
1674d753f849SJames Morse return false;
1675d753f849SJames Morse
1676d753f849SJames Morse if (fwspec->param_count < 4 || !fwspec->param[3])
1677d753f849SJames Morse return false;
1678d753f849SJames Morse
1679d753f849SJames Morse range = __get_intid_range(hwirq);
1680d753f849SJames Morse if (range != PPI_RANGE && range != EPPI_RANGE)
1681d753f849SJames Morse return false;
1682d753f849SJames Morse
1683d753f849SJames Morse return true;
1684d753f849SJames Morse }
1685d753f849SJames Morse
gic_irq_domain_select(struct irq_domain * d,struct irq_fwspec * fwspec,enum irq_domain_bus_token bus_token)1686e3825ba1SMarc Zyngier static int gic_irq_domain_select(struct irq_domain *d,
1687e3825ba1SMarc Zyngier struct irq_fwspec *fwspec,
1688e3825ba1SMarc Zyngier enum irq_domain_bus_token bus_token)
1689e3825ba1SMarc Zyngier {
1690d753f849SJames Morse unsigned int type, ret, ppi_idx;
1691d753f849SJames Morse irq_hw_number_t hwirq;
1692d753f849SJames Morse
1693e3825ba1SMarc Zyngier /* Not for us */
1694e3825ba1SMarc Zyngier if (fwspec->fwnode != d->fwnode)
1695e3825ba1SMarc Zyngier return 0;
1696e3825ba1SMarc Zyngier
1697e3825ba1SMarc Zyngier /* If this is not DT, then we have a single domain */
1698e3825ba1SMarc Zyngier if (!is_of_node(fwspec->fwnode))
1699e3825ba1SMarc Zyngier return 1;
1700e3825ba1SMarc Zyngier
1701d753f849SJames Morse ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type);
1702d753f849SJames Morse if (WARN_ON_ONCE(ret))
1703d753f849SJames Morse return 0;
1704d753f849SJames Morse
1705d753f849SJames Morse if (!fwspec_is_partitioned_ppi(fwspec, hwirq))
1706d753f849SJames Morse return d == gic_data.domain;
1707d753f849SJames Morse
1708e3825ba1SMarc Zyngier /*
1709e3825ba1SMarc Zyngier * If this is a PPI and we have a 4th (non-null) parameter,
1710e3825ba1SMarc Zyngier * then we need to match the partition domain.
1711e3825ba1SMarc Zyngier */
1712d753f849SJames Morse ppi_idx = __gic_get_ppi_index(hwirq);
1713d753f849SJames Morse return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]);
1714e3825ba1SMarc Zyngier }
1715e3825ba1SMarc Zyngier
1716021f6537SMarc Zyngier static const struct irq_domain_ops gic_irq_domain_ops = {
1717f833f57fSMarc Zyngier .translate = gic_irq_domain_translate,
1718443acc4fSMarc Zyngier .alloc = gic_irq_domain_alloc,
1719443acc4fSMarc Zyngier .free = gic_irq_domain_free,
1720e3825ba1SMarc Zyngier .select = gic_irq_domain_select,
1721e3825ba1SMarc Zyngier };
1722e3825ba1SMarc Zyngier
partition_domain_translate(struct irq_domain * d,struct irq_fwspec * fwspec,unsigned long * hwirq,unsigned int * type)1723e3825ba1SMarc Zyngier static int partition_domain_translate(struct irq_domain *d,
1724e3825ba1SMarc Zyngier struct irq_fwspec *fwspec,
1725e3825ba1SMarc Zyngier unsigned long *hwirq,
1726e3825ba1SMarc Zyngier unsigned int *type)
1727e3825ba1SMarc Zyngier {
1728d753f849SJames Morse unsigned long ppi_intid;
1729e3825ba1SMarc Zyngier struct device_node *np;
1730d753f849SJames Morse unsigned int ppi_idx;
1731e3825ba1SMarc Zyngier int ret;
1732e3825ba1SMarc Zyngier
173352085d3fSMarc Zyngier if (!gic_data.ppi_descs)
173452085d3fSMarc Zyngier return -ENOMEM;
173552085d3fSMarc Zyngier
1736e3825ba1SMarc Zyngier np = of_find_node_by_phandle(fwspec->param[3]);
1737e3825ba1SMarc Zyngier if (WARN_ON(!np))
1738e3825ba1SMarc Zyngier return -EINVAL;
1739e3825ba1SMarc Zyngier
1740d753f849SJames Morse ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type);
1741d753f849SJames Morse if (WARN_ON_ONCE(ret))
1742d753f849SJames Morse return 0;
1743d753f849SJames Morse
1744d753f849SJames Morse ppi_idx = __gic_get_ppi_index(ppi_intid);
1745d753f849SJames Morse ret = partition_translate_id(gic_data.ppi_descs[ppi_idx],
1746e3825ba1SMarc Zyngier of_node_to_fwnode(np));
1747e3825ba1SMarc Zyngier if (ret < 0)
1748e3825ba1SMarc Zyngier return ret;
1749e3825ba1SMarc Zyngier
1750e3825ba1SMarc Zyngier *hwirq = ret;
1751e3825ba1SMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1752e3825ba1SMarc Zyngier
1753e3825ba1SMarc Zyngier return 0;
1754e3825ba1SMarc Zyngier }
1755e3825ba1SMarc Zyngier
1756e3825ba1SMarc Zyngier static const struct irq_domain_ops partition_domain_ops = {
1757e3825ba1SMarc Zyngier .translate = partition_domain_translate,
1758e3825ba1SMarc Zyngier .select = gic_irq_domain_select,
1759021f6537SMarc Zyngier };
1760021f6537SMarc Zyngier
gic_enable_quirk_msm8996(void * data)17619c8114c2SSrinivas Kandagatla static bool gic_enable_quirk_msm8996(void *data)
17629c8114c2SSrinivas Kandagatla {
17639c8114c2SSrinivas Kandagatla struct gic_chip_data *d = data;
17649c8114c2SSrinivas Kandagatla
17659c8114c2SSrinivas Kandagatla d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
17669c8114c2SSrinivas Kandagatla
17679c8114c2SSrinivas Kandagatla return true;
17689c8114c2SSrinivas Kandagatla }
17699c8114c2SSrinivas Kandagatla
gic_enable_quirk_mtk_gicr(void * data)177044bd78ddSDouglas Anderson static bool gic_enable_quirk_mtk_gicr(void *data)
177144bd78ddSDouglas Anderson {
177244bd78ddSDouglas Anderson struct gic_chip_data *d = data;
177344bd78ddSDouglas Anderson
177444bd78ddSDouglas Anderson d->flags |= FLAGS_WORKAROUND_MTK_GICR_SAVE;
177544bd78ddSDouglas Anderson
177644bd78ddSDouglas Anderson return true;
177744bd78ddSDouglas Anderson }
177844bd78ddSDouglas Anderson
gic_enable_quirk_cavium_38539(void * data)1779d01fd161SMarc Zyngier static bool gic_enable_quirk_cavium_38539(void *data)
1780d01fd161SMarc Zyngier {
1781d01fd161SMarc Zyngier struct gic_chip_data *d = data;
1782d01fd161SMarc Zyngier
1783d01fd161SMarc Zyngier d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539;
1784d01fd161SMarc Zyngier
1785d01fd161SMarc Zyngier return true;
1786d01fd161SMarc Zyngier }
1787d01fd161SMarc Zyngier
gic_enable_quirk_hip06_07(void * data)17887f2481b3SMarc Zyngier static bool gic_enable_quirk_hip06_07(void *data)
17897f2481b3SMarc Zyngier {
17907f2481b3SMarc Zyngier struct gic_chip_data *d = data;
17917f2481b3SMarc Zyngier
17927f2481b3SMarc Zyngier /*
17937f2481b3SMarc Zyngier * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
17947f2481b3SMarc Zyngier * not being an actual ARM implementation). The saving grace is
17957f2481b3SMarc Zyngier * that GIC-600 doesn't have ESPI, so nothing to do in that case.
17967f2481b3SMarc Zyngier * HIP07 doesn't even have a proper IIDR, and still pretends to
17977f2481b3SMarc Zyngier * have ESPI. In both cases, put them right.
17987f2481b3SMarc Zyngier */
17997f2481b3SMarc Zyngier if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
18007f2481b3SMarc Zyngier /* Zero both ESPI and the RES0 field next to it... */
18017f2481b3SMarc Zyngier d->rdists.gicd_typer &= ~GENMASK(9, 8);
18027f2481b3SMarc Zyngier return true;
18037f2481b3SMarc Zyngier }
18047f2481b3SMarc Zyngier
18057f2481b3SMarc Zyngier return false;
18067f2481b3SMarc Zyngier }
18077f2481b3SMarc Zyngier
180835727af2SShanker Donthineni #define T241_CHIPN_MASK GENMASK_ULL(45, 44)
180935727af2SShanker Donthineni #define T241_CHIP_GICDA_OFFSET 0x1580000
181035727af2SShanker Donthineni #define SMCCC_SOC_ID_T241 0x036b0241
181135727af2SShanker Donthineni
gic_enable_quirk_nvidia_t241(void * data)181235727af2SShanker Donthineni static bool gic_enable_quirk_nvidia_t241(void *data)
181335727af2SShanker Donthineni {
181435727af2SShanker Donthineni s32 soc_id = arm_smccc_get_soc_id_version();
181535727af2SShanker Donthineni unsigned long chip_bmask = 0;
181635727af2SShanker Donthineni phys_addr_t phys;
181735727af2SShanker Donthineni u32 i;
181835727af2SShanker Donthineni
181935727af2SShanker Donthineni /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */
182035727af2SShanker Donthineni if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241))
182135727af2SShanker Donthineni return false;
182235727af2SShanker Donthineni
182335727af2SShanker Donthineni /* Find the chips based on GICR regions PHYS addr */
182435727af2SShanker Donthineni for (i = 0; i < gic_data.nr_redist_regions; i++) {
182535727af2SShanker Donthineni chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK,
182635727af2SShanker Donthineni (u64)gic_data.redist_regions[i].phys_base));
182735727af2SShanker Donthineni }
182835727af2SShanker Donthineni
182935727af2SShanker Donthineni if (hweight32(chip_bmask) < 3)
183035727af2SShanker Donthineni return false;
183135727af2SShanker Donthineni
183235727af2SShanker Donthineni /* Setup GICD alias regions */
183335727af2SShanker Donthineni for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) {
183435727af2SShanker Donthineni if (chip_bmask & BIT(i)) {
183535727af2SShanker Donthineni phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET;
183635727af2SShanker Donthineni phys |= FIELD_PREP(T241_CHIPN_MASK, i);
183735727af2SShanker Donthineni t241_dist_base_alias[i] = ioremap(phys, SZ_64K);
183835727af2SShanker Donthineni WARN_ON_ONCE(!t241_dist_base_alias[i]);
183935727af2SShanker Donthineni }
184035727af2SShanker Donthineni }
184135727af2SShanker Donthineni static_branch_enable(&gic_nvidia_t241_erratum);
184235727af2SShanker Donthineni return true;
184335727af2SShanker Donthineni }
184435727af2SShanker Donthineni
gic_enable_quirk_asr8601(void * data)1845b4d81fabSzhengyan static bool gic_enable_quirk_asr8601(void *data)
1846b4d81fabSzhengyan {
1847b4d81fabSzhengyan struct gic_chip_data *d = data;
1848b4d81fabSzhengyan
1849b4d81fabSzhengyan d->flags |= FLAGS_WORKAROUND_ASR_ERRATUM_8601001;
1850b4d81fabSzhengyan
1851b4d81fabSzhengyan return true;
1852b4d81fabSzhengyan }
1853b4d81fabSzhengyan
gic_enable_quirk_arm64_2941627(void * data)18546fe5c68eSLorenzo Pieralisi static bool gic_enable_quirk_arm64_2941627(void *data)
18556fe5c68eSLorenzo Pieralisi {
18566fe5c68eSLorenzo Pieralisi static_branch_enable(&gic_arm64_2941627_erratum);
18576fe5c68eSLorenzo Pieralisi return true;
18586fe5c68eSLorenzo Pieralisi }
18596fe5c68eSLorenzo Pieralisi
rd_set_non_coherent(void * data)1860*3a0fff0fSLorenzo Pieralisi static bool rd_set_non_coherent(void *data)
1861*3a0fff0fSLorenzo Pieralisi {
1862*3a0fff0fSLorenzo Pieralisi struct gic_chip_data *d = data;
1863*3a0fff0fSLorenzo Pieralisi
1864*3a0fff0fSLorenzo Pieralisi d->rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
1865*3a0fff0fSLorenzo Pieralisi return true;
1866*3a0fff0fSLorenzo Pieralisi }
1867*3a0fff0fSLorenzo Pieralisi
18687f2481b3SMarc Zyngier static const struct gic_quirk gic_quirks[] = {
18697f2481b3SMarc Zyngier {
18707f2481b3SMarc Zyngier .desc = "GICv3: Qualcomm MSM8996 broken firmware",
18717f2481b3SMarc Zyngier .compatible = "qcom,msm8996-gic-v3",
18727f2481b3SMarc Zyngier .init = gic_enable_quirk_msm8996,
18737f2481b3SMarc Zyngier },
18747f2481b3SMarc Zyngier {
1875b4d81fabSzhengyan .desc = "GICv3: ASR erratum 8601001",
1876b4d81fabSzhengyan .compatible = "asr,asr8601-gic-v3",
1877b4d81fabSzhengyan .init = gic_enable_quirk_asr8601,
1878b4d81fabSzhengyan },
1879b4d81fabSzhengyan {
188044bd78ddSDouglas Anderson .desc = "GICv3: Mediatek Chromebook GICR save problem",
188144bd78ddSDouglas Anderson .property = "mediatek,broken-save-restore-fw",
188244bd78ddSDouglas Anderson .init = gic_enable_quirk_mtk_gicr,
188344bd78ddSDouglas Anderson },
188444bd78ddSDouglas Anderson {
18857f2481b3SMarc Zyngier .desc = "GICv3: HIP06 erratum 161010803",
18867f2481b3SMarc Zyngier .iidr = 0x0204043b,
18877f2481b3SMarc Zyngier .mask = 0xffffffff,
18887f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07,
18897f2481b3SMarc Zyngier },
18907f2481b3SMarc Zyngier {
18917f2481b3SMarc Zyngier .desc = "GICv3: HIP07 erratum 161010803",
18927f2481b3SMarc Zyngier .iidr = 0x00000000,
18937f2481b3SMarc Zyngier .mask = 0xffffffff,
18947f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07,
18957f2481b3SMarc Zyngier },
18967f2481b3SMarc Zyngier {
1897d01fd161SMarc Zyngier /*
1898d01fd161SMarc Zyngier * Reserved register accesses generate a Synchronous
1899d01fd161SMarc Zyngier * External Abort. This erratum applies to:
1900d01fd161SMarc Zyngier * - ThunderX: CN88xx
1901d01fd161SMarc Zyngier * - OCTEON TX: CN83xx, CN81xx
1902d01fd161SMarc Zyngier * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
1903d01fd161SMarc Zyngier */
1904d01fd161SMarc Zyngier .desc = "GICv3: Cavium erratum 38539",
1905d01fd161SMarc Zyngier .iidr = 0xa000034c,
1906d01fd161SMarc Zyngier .mask = 0xe8f00fff,
1907d01fd161SMarc Zyngier .init = gic_enable_quirk_cavium_38539,
1908d01fd161SMarc Zyngier },
1909d01fd161SMarc Zyngier {
191035727af2SShanker Donthineni .desc = "GICv3: NVIDIA erratum T241-FABRIC-4",
191135727af2SShanker Donthineni .iidr = 0x0402043b,
191235727af2SShanker Donthineni .mask = 0xffffffff,
191335727af2SShanker Donthineni .init = gic_enable_quirk_nvidia_t241,
191435727af2SShanker Donthineni },
191535727af2SShanker Donthineni {
19166fe5c68eSLorenzo Pieralisi /*
19176fe5c68eSLorenzo Pieralisi * GIC-700: 2941627 workaround - IP variant [0,1]
19186fe5c68eSLorenzo Pieralisi *
19196fe5c68eSLorenzo Pieralisi */
19206fe5c68eSLorenzo Pieralisi .desc = "GICv3: ARM64 erratum 2941627",
19216fe5c68eSLorenzo Pieralisi .iidr = 0x0400043b,
19226fe5c68eSLorenzo Pieralisi .mask = 0xff0e0fff,
19236fe5c68eSLorenzo Pieralisi .init = gic_enable_quirk_arm64_2941627,
19246fe5c68eSLorenzo Pieralisi },
19256fe5c68eSLorenzo Pieralisi {
19266fe5c68eSLorenzo Pieralisi /*
19276fe5c68eSLorenzo Pieralisi * GIC-700: 2941627 workaround - IP variant [2]
19286fe5c68eSLorenzo Pieralisi */
19296fe5c68eSLorenzo Pieralisi .desc = "GICv3: ARM64 erratum 2941627",
19306fe5c68eSLorenzo Pieralisi .iidr = 0x0402043b,
19316fe5c68eSLorenzo Pieralisi .mask = 0xff0f0fff,
19326fe5c68eSLorenzo Pieralisi .init = gic_enable_quirk_arm64_2941627,
19336fe5c68eSLorenzo Pieralisi },
19346fe5c68eSLorenzo Pieralisi {
1935*3a0fff0fSLorenzo Pieralisi .desc = "GICv3: non-coherent attribute",
1936*3a0fff0fSLorenzo Pieralisi .property = "dma-noncoherent",
1937*3a0fff0fSLorenzo Pieralisi .init = rd_set_non_coherent,
1938*3a0fff0fSLorenzo Pieralisi },
1939*3a0fff0fSLorenzo Pieralisi {
19407f2481b3SMarc Zyngier }
19417f2481b3SMarc Zyngier };
19427f2481b3SMarc Zyngier
gic_enable_nmi_support(void)1943d98d0a99SJulien Thierry static void gic_enable_nmi_support(void)
1944d98d0a99SJulien Thierry {
1945101b35f7SJulien Thierry int i;
1946101b35f7SJulien Thierry
194781a43273SMarc Zyngier if (!gic_prio_masking_enabled())
194881a43273SMarc Zyngier return;
194981a43273SMarc Zyngier
195044bd78ddSDouglas Anderson if (gic_data.flags & FLAGS_WORKAROUND_MTK_GICR_SAVE) {
195144bd78ddSDouglas Anderson pr_warn("Skipping NMI enable due to firmware issues\n");
195244bd78ddSDouglas Anderson return;
195344bd78ddSDouglas Anderson }
195444bd78ddSDouglas Anderson
195581a43273SMarc Zyngier ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
195681a43273SMarc Zyngier if (!ppi_nmi_refs)
195781a43273SMarc Zyngier return;
195881a43273SMarc Zyngier
195981a43273SMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++)
1960101b35f7SJulien Thierry refcount_set(&ppi_nmi_refs[i], 0);
1961101b35f7SJulien Thierry
19624e594ad1SAlexandru Elisei pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
19638bf0a804SMark Rutland gic_has_relaxed_pmr_sync() ? "relaxed" : "forced");
1964f2266504SMarc Zyngier
196533678059SAlexandru Elisei /*
196633678059SAlexandru Elisei * How priority values are used by the GIC depends on two things:
196733678059SAlexandru Elisei * the security state of the GIC (controlled by the GICD_CTRL.DS bit)
196833678059SAlexandru Elisei * and if Group 0 interrupts can be delivered to Linux in the non-secure
196933678059SAlexandru Elisei * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
197029517170SJason Wang * ICC_PMR_EL1 register and the priority that software assigns to
197133678059SAlexandru Elisei * interrupts:
197233678059SAlexandru Elisei *
197333678059SAlexandru Elisei * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority
197433678059SAlexandru Elisei * -----------------------------------------------------------
197533678059SAlexandru Elisei * 1 | - | unchanged | unchanged
197633678059SAlexandru Elisei * -----------------------------------------------------------
197733678059SAlexandru Elisei * 0 | 1 | non-secure | non-secure
197833678059SAlexandru Elisei * -----------------------------------------------------------
197933678059SAlexandru Elisei * 0 | 0 | unchanged | non-secure
198033678059SAlexandru Elisei *
198133678059SAlexandru Elisei * where non-secure means that the value is right-shifted by one and the
198233678059SAlexandru Elisei * MSB bit set, to make it fit in the non-secure priority range.
198333678059SAlexandru Elisei *
198433678059SAlexandru Elisei * In the first two cases, where ICC_PMR_EL1 and the interrupt priority
198533678059SAlexandru Elisei * are both either modified or unchanged, we can use the same set of
198633678059SAlexandru Elisei * priorities.
198733678059SAlexandru Elisei *
198833678059SAlexandru Elisei * In the last case, where only the interrupt priorities are modified to
198933678059SAlexandru Elisei * be in the non-secure range, we use a different PMR value to mask IRQs
199033678059SAlexandru Elisei * and the rest of the values that we use remain unchanged.
199133678059SAlexandru Elisei */
199233678059SAlexandru Elisei if (gic_has_group0() && !gic_dist_security_disabled())
199333678059SAlexandru Elisei static_branch_enable(&gic_nonsecure_priorities);
199433678059SAlexandru Elisei
1995d98d0a99SJulien Thierry static_branch_enable(&supports_pseudo_nmis);
1996101b35f7SJulien Thierry
1997101b35f7SJulien Thierry if (static_branch_likely(&supports_deactivate_key))
1998101b35f7SJulien Thierry gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1999101b35f7SJulien Thierry else
2000101b35f7SJulien Thierry gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
2001d98d0a99SJulien Thierry }
2002d98d0a99SJulien Thierry
gic_init_bases(phys_addr_t dist_phys_base,void __iomem * dist_base,struct redist_region * rdist_regs,u32 nr_redist_regions,u64 redist_stride,struct fwnode_handle * handle)200335727af2SShanker Donthineni static int __init gic_init_bases(phys_addr_t dist_phys_base,
200435727af2SShanker Donthineni void __iomem *dist_base,
2005db57d746STomasz Nowicki struct redist_region *rdist_regs,
2006db57d746STomasz Nowicki u32 nr_redist_regions,
2007db57d746STomasz Nowicki u64 redist_stride,
2008db57d746STomasz Nowicki struct fwnode_handle *handle)
2009db57d746STomasz Nowicki {
2010db57d746STomasz Nowicki u32 typer;
2011db57d746STomasz Nowicki int err;
2012db57d746STomasz Nowicki
2013db57d746STomasz Nowicki if (!is_hyp_mode_available())
2014d01d3274SDavidlohr Bueso static_branch_disable(&supports_deactivate_key);
2015db57d746STomasz Nowicki
2016d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key))
2017db57d746STomasz Nowicki pr_info("GIC: Using split EOI/Deactivate mode\n");
2018db57d746STomasz Nowicki
2019e3825ba1SMarc Zyngier gic_data.fwnode = handle;
202035727af2SShanker Donthineni gic_data.dist_phys_base = dist_phys_base;
2021db57d746STomasz Nowicki gic_data.dist_base = dist_base;
2022db57d746STomasz Nowicki gic_data.redist_regions = rdist_regs;
2023db57d746STomasz Nowicki gic_data.nr_redist_regions = nr_redist_regions;
2024db57d746STomasz Nowicki gic_data.redist_stride = redist_stride;
2025db57d746STomasz Nowicki
2026db57d746STomasz Nowicki /*
2027db57d746STomasz Nowicki * Find out how many interrupts are supported.
2028db57d746STomasz Nowicki */
2029db57d746STomasz Nowicki typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
2030a4f9edb2SMarc Zyngier gic_data.rdists.gicd_typer = typer;
20317f2481b3SMarc Zyngier
20327f2481b3SMarc Zyngier gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
20337f2481b3SMarc Zyngier gic_quirks, &gic_data);
20347f2481b3SMarc Zyngier
2035211bddd2SMarc Zyngier pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
2036211bddd2SMarc Zyngier pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
2037f2d83409SMarc Zyngier
2038d01fd161SMarc Zyngier /*
2039d01fd161SMarc Zyngier * ThunderX1 explodes on reading GICD_TYPER2, in violation of the
2040d01fd161SMarc Zyngier * architecture spec (which says that reserved registers are RES0).
2041d01fd161SMarc Zyngier */
2042d01fd161SMarc Zyngier if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539))
2043f2d83409SMarc Zyngier gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);
2044f2d83409SMarc Zyngier
2045db57d746STomasz Nowicki gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
2046db57d746STomasz Nowicki &gic_data);
2047db57d746STomasz Nowicki gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
204835727af2SShanker Donthineni if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) {
204935727af2SShanker Donthineni /* Disable GICv4.x features for the erratum T241-FABRIC-4 */
2050b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = true;
20510edc23eaSMarc Zyngier gic_data.rdists.has_vlpis = true;
20520edc23eaSMarc Zyngier gic_data.rdists.has_direct_lpi = true;
205396806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty = true;
205435727af2SShanker Donthineni }
2055db57d746STomasz Nowicki
2056db57d746STomasz Nowicki if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
2057db57d746STomasz Nowicki err = -ENOMEM;
2058db57d746STomasz Nowicki goto out_free;
2059db57d746STomasz Nowicki }
2060db57d746STomasz Nowicki
2061eeaa4b24Sluanshi irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
2062eeaa4b24Sluanshi
2063eda0d04aSShanker Donthineni gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
2064eda0d04aSShanker Donthineni
206550528752SMarc Zyngier if (typer & GICD_TYPER_MBIS) {
206650528752SMarc Zyngier err = mbi_init(handle, gic_data.domain);
206750528752SMarc Zyngier if (err)
206850528752SMarc Zyngier pr_err("Failed to initialize MBIs\n");
206950528752SMarc Zyngier }
207050528752SMarc Zyngier
2071db57d746STomasz Nowicki set_handle_irq(gic_handle_irq);
2072db57d746STomasz Nowicki
20731a60e1e6SMarc Zyngier gic_update_rdist_properties();
20740edc23eaSMarc Zyngier
2075db57d746STomasz Nowicki gic_dist_init();
2076db57d746STomasz Nowicki gic_cpu_init();
207764b499d8SMarc Zyngier gic_smp_init();
2078db57d746STomasz Nowicki gic_cpu_pm_init();
2079db57d746STomasz Nowicki
2080d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) {
2081d38a71c5SMarc Zyngier its_init(handle, &gic_data.rdists, gic_data.domain);
2082d38a71c5SMarc Zyngier its_cpu_init();
2083d23bc2bcSValentin Schneider its_lpi_memreserve_init();
208490b4c555SZeev Zilberman } else {
208590b4c555SZeev Zilberman if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
208690b4c555SZeev Zilberman gicv2m_init(handle, gic_data.domain);
2087d38a71c5SMarc Zyngier }
2088d38a71c5SMarc Zyngier
2089d98d0a99SJulien Thierry gic_enable_nmi_support();
2090d98d0a99SJulien Thierry
2091db57d746STomasz Nowicki return 0;
2092db57d746STomasz Nowicki
2093db57d746STomasz Nowicki out_free:
2094db57d746STomasz Nowicki if (gic_data.domain)
2095db57d746STomasz Nowicki irq_domain_remove(gic_data.domain);
2096db57d746STomasz Nowicki free_percpu(gic_data.rdists.rdist);
2097db57d746STomasz Nowicki return err;
2098db57d746STomasz Nowicki }
2099db57d746STomasz Nowicki
gic_validate_dist_version(void __iomem * dist_base)2100db57d746STomasz Nowicki static int __init gic_validate_dist_version(void __iomem *dist_base)
2101db57d746STomasz Nowicki {
2102db57d746STomasz Nowicki u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
2103db57d746STomasz Nowicki
2104db57d746STomasz Nowicki if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
2105db57d746STomasz Nowicki return -ENODEV;
2106db57d746STomasz Nowicki
2107db57d746STomasz Nowicki return 0;
2108db57d746STomasz Nowicki }
2109db57d746STomasz Nowicki
2110e3825ba1SMarc Zyngier /* Create all possible partitions at boot time */
gic_populate_ppi_partitions(struct device_node * gic_node)21117beaa24bSLinus Torvalds static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
2112e3825ba1SMarc Zyngier {
2113e3825ba1SMarc Zyngier struct device_node *parts_node, *child_part;
2114e3825ba1SMarc Zyngier int part_idx = 0, i;
2115e3825ba1SMarc Zyngier int nr_parts;
2116e3825ba1SMarc Zyngier struct partition_affinity *parts;
2117e3825ba1SMarc Zyngier
211800ee9a1cSJohan Hovold parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
2119e3825ba1SMarc Zyngier if (!parts_node)
2120e3825ba1SMarc Zyngier return;
2121e3825ba1SMarc Zyngier
212252085d3fSMarc Zyngier gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
212352085d3fSMarc Zyngier if (!gic_data.ppi_descs)
2124ec8401a4SMiaoqian Lin goto out_put_node;
212552085d3fSMarc Zyngier
2126e3825ba1SMarc Zyngier nr_parts = of_get_child_count(parts_node);
2127e3825ba1SMarc Zyngier
2128e3825ba1SMarc Zyngier if (!nr_parts)
212900ee9a1cSJohan Hovold goto out_put_node;
2130e3825ba1SMarc Zyngier
21316396bb22SKees Cook parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
2132e3825ba1SMarc Zyngier if (WARN_ON(!parts))
213300ee9a1cSJohan Hovold goto out_put_node;
2134e3825ba1SMarc Zyngier
2135e3825ba1SMarc Zyngier for_each_child_of_node(parts_node, child_part) {
2136e3825ba1SMarc Zyngier struct partition_affinity *part;
2137e3825ba1SMarc Zyngier int n;
2138e3825ba1SMarc Zyngier
2139e3825ba1SMarc Zyngier part = &parts[part_idx];
2140e3825ba1SMarc Zyngier
2141e3825ba1SMarc Zyngier part->partition_id = of_node_to_fwnode(child_part);
2142e3825ba1SMarc Zyngier
21432ef790dcSRob Herring pr_info("GIC: PPI partition %pOFn[%d] { ",
21442ef790dcSRob Herring child_part, part_idx);
2145e3825ba1SMarc Zyngier
2146e3825ba1SMarc Zyngier n = of_property_count_elems_of_size(child_part, "affinity",
2147e3825ba1SMarc Zyngier sizeof(u32));
2148e3825ba1SMarc Zyngier WARN_ON(n <= 0);
2149e3825ba1SMarc Zyngier
2150e3825ba1SMarc Zyngier for (i = 0; i < n; i++) {
2151e3825ba1SMarc Zyngier int err, cpu;
2152e3825ba1SMarc Zyngier u32 cpu_phandle;
2153e3825ba1SMarc Zyngier struct device_node *cpu_node;
2154e3825ba1SMarc Zyngier
2155e3825ba1SMarc Zyngier err = of_property_read_u32_index(child_part, "affinity",
2156e3825ba1SMarc Zyngier i, &cpu_phandle);
2157e3825ba1SMarc Zyngier if (WARN_ON(err))
2158e3825ba1SMarc Zyngier continue;
2159e3825ba1SMarc Zyngier
2160e3825ba1SMarc Zyngier cpu_node = of_find_node_by_phandle(cpu_phandle);
2161e3825ba1SMarc Zyngier if (WARN_ON(!cpu_node))
2162e3825ba1SMarc Zyngier continue;
2163e3825ba1SMarc Zyngier
2164c08ec7daSSuzuki K Poulose cpu = of_cpu_node_to_id(cpu_node);
2165fa1ad9d4SMiaoqian Lin if (WARN_ON(cpu < 0)) {
2166fa1ad9d4SMiaoqian Lin of_node_put(cpu_node);
2167e3825ba1SMarc Zyngier continue;
2168fa1ad9d4SMiaoqian Lin }
2169e3825ba1SMarc Zyngier
2170e81f54c6SRob Herring pr_cont("%pOF[%d] ", cpu_node, cpu);
2171e3825ba1SMarc Zyngier
2172e3825ba1SMarc Zyngier cpumask_set_cpu(cpu, &part->mask);
2173fa1ad9d4SMiaoqian Lin of_node_put(cpu_node);
2174e3825ba1SMarc Zyngier }
2175e3825ba1SMarc Zyngier
2176e3825ba1SMarc Zyngier pr_cont("}\n");
2177e3825ba1SMarc Zyngier part_idx++;
2178e3825ba1SMarc Zyngier }
2179e3825ba1SMarc Zyngier
218052085d3fSMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) {
2181e3825ba1SMarc Zyngier unsigned int irq;
2182e3825ba1SMarc Zyngier struct partition_desc *desc;
2183e3825ba1SMarc Zyngier struct irq_fwspec ppi_fwspec = {
2184e3825ba1SMarc Zyngier .fwnode = gic_data.fwnode,
2185e3825ba1SMarc Zyngier .param_count = 3,
2186e3825ba1SMarc Zyngier .param = {
218765da7d19SMarc Zyngier [0] = GIC_IRQ_TYPE_PARTITION,
2188e3825ba1SMarc Zyngier [1] = i,
2189e3825ba1SMarc Zyngier [2] = IRQ_TYPE_NONE,
2190e3825ba1SMarc Zyngier },
2191e3825ba1SMarc Zyngier };
2192e3825ba1SMarc Zyngier
2193e3825ba1SMarc Zyngier irq = irq_create_fwspec_mapping(&ppi_fwspec);
2194e3825ba1SMarc Zyngier if (WARN_ON(!irq))
2195e3825ba1SMarc Zyngier continue;
2196e3825ba1SMarc Zyngier desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
2197e3825ba1SMarc Zyngier irq, &partition_domain_ops);
2198e3825ba1SMarc Zyngier if (WARN_ON(!desc))
2199e3825ba1SMarc Zyngier continue;
2200e3825ba1SMarc Zyngier
2201e3825ba1SMarc Zyngier gic_data.ppi_descs[i] = desc;
2202e3825ba1SMarc Zyngier }
220300ee9a1cSJohan Hovold
220400ee9a1cSJohan Hovold out_put_node:
220500ee9a1cSJohan Hovold of_node_put(parts_node);
2206e3825ba1SMarc Zyngier }
2207e3825ba1SMarc Zyngier
gic_of_setup_kvm_info(struct device_node * node)22081839e576SJulien Grall static void __init gic_of_setup_kvm_info(struct device_node *node)
22091839e576SJulien Grall {
22101839e576SJulien Grall int ret;
22111839e576SJulien Grall struct resource r;
22121839e576SJulien Grall u32 gicv_idx;
22131839e576SJulien Grall
22141839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3;
22151839e576SJulien Grall
22161839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
22171839e576SJulien Grall if (!gic_v3_kvm_info.maint_irq)
22181839e576SJulien Grall return;
22191839e576SJulien Grall
22201839e576SJulien Grall if (of_property_read_u32(node, "#redistributor-regions",
22211839e576SJulien Grall &gicv_idx))
22221839e576SJulien Grall gicv_idx = 1;
22231839e576SJulien Grall
22241839e576SJulien Grall gicv_idx += 3; /* Also skip GICD, GICC, GICH */
22251839e576SJulien Grall ret = of_address_to_resource(node, gicv_idx, &r);
22261839e576SJulien Grall if (!ret)
22271839e576SJulien Grall gic_v3_kvm_info.vcpu = r;
22281839e576SJulien Grall
22294bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
22303c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
22310e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info);
22321839e576SJulien Grall }
22331839e576SJulien Grall
gic_request_region(resource_size_t base,resource_size_t size,const char * name)22344deb96e3SRobin Murphy static void gic_request_region(resource_size_t base, resource_size_t size,
22354deb96e3SRobin Murphy const char *name)
22364deb96e3SRobin Murphy {
22374deb96e3SRobin Murphy if (!request_mem_region(base, size, name))
22384deb96e3SRobin Murphy pr_warn_once(FW_BUG "%s region %pa has overlapping address\n",
22394deb96e3SRobin Murphy name, &base);
22404deb96e3SRobin Murphy }
22414deb96e3SRobin Murphy
gic_of_iomap(struct device_node * node,int idx,const char * name,struct resource * res)22424deb96e3SRobin Murphy static void __iomem *gic_of_iomap(struct device_node *node, int idx,
22434deb96e3SRobin Murphy const char *name, struct resource *res)
22444deb96e3SRobin Murphy {
22454deb96e3SRobin Murphy void __iomem *base;
22464deb96e3SRobin Murphy int ret;
22474deb96e3SRobin Murphy
22484deb96e3SRobin Murphy ret = of_address_to_resource(node, idx, res);
22494deb96e3SRobin Murphy if (ret)
22504deb96e3SRobin Murphy return IOMEM_ERR_PTR(ret);
22514deb96e3SRobin Murphy
22524deb96e3SRobin Murphy gic_request_region(res->start, resource_size(res), name);
22534deb96e3SRobin Murphy base = of_iomap(node, idx);
22544deb96e3SRobin Murphy
22554deb96e3SRobin Murphy return base ?: IOMEM_ERR_PTR(-ENOMEM);
22564deb96e3SRobin Murphy }
22574deb96e3SRobin Murphy
gic_of_init(struct device_node * node,struct device_node * parent)2258021f6537SMarc Zyngier static int __init gic_of_init(struct device_node *node, struct device_node *parent)
2259021f6537SMarc Zyngier {
226035727af2SShanker Donthineni phys_addr_t dist_phys_base;
2261021f6537SMarc Zyngier void __iomem *dist_base;
2262f5c1434cSMarc Zyngier struct redist_region *rdist_regs;
22634deb96e3SRobin Murphy struct resource res;
2264021f6537SMarc Zyngier u64 redist_stride;
2265f5c1434cSMarc Zyngier u32 nr_redist_regions;
2266db57d746STomasz Nowicki int err, i;
2267021f6537SMarc Zyngier
22684deb96e3SRobin Murphy dist_base = gic_of_iomap(node, 0, "GICD", &res);
22692b2cd74aSRobin Murphy if (IS_ERR(dist_base)) {
2270e81f54c6SRob Herring pr_err("%pOF: unable to map gic dist registers\n", node);
22712b2cd74aSRobin Murphy return PTR_ERR(dist_base);
2272021f6537SMarc Zyngier }
2273021f6537SMarc Zyngier
227435727af2SShanker Donthineni dist_phys_base = res.start;
227535727af2SShanker Donthineni
2276db57d746STomasz Nowicki err = gic_validate_dist_version(dist_base);
2277db57d746STomasz Nowicki if (err) {
2278e81f54c6SRob Herring pr_err("%pOF: no distributor detected, giving up\n", node);
2279021f6537SMarc Zyngier goto out_unmap_dist;
2280021f6537SMarc Zyngier }
2281021f6537SMarc Zyngier
2282f5c1434cSMarc Zyngier if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
2283f5c1434cSMarc Zyngier nr_redist_regions = 1;
2284021f6537SMarc Zyngier
22856396bb22SKees Cook rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
22866396bb22SKees Cook GFP_KERNEL);
2287f5c1434cSMarc Zyngier if (!rdist_regs) {
2288021f6537SMarc Zyngier err = -ENOMEM;
2289021f6537SMarc Zyngier goto out_unmap_dist;
2290021f6537SMarc Zyngier }
2291021f6537SMarc Zyngier
2292f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) {
22934deb96e3SRobin Murphy rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res);
22944deb96e3SRobin Murphy if (IS_ERR(rdist_regs[i].redist_base)) {
2295e81f54c6SRob Herring pr_err("%pOF: couldn't map region %d\n", node, i);
2296021f6537SMarc Zyngier err = -ENODEV;
2297021f6537SMarc Zyngier goto out_unmap_rdist;
2298021f6537SMarc Zyngier }
2299f5c1434cSMarc Zyngier rdist_regs[i].phys_base = res.start;
2300021f6537SMarc Zyngier }
2301021f6537SMarc Zyngier
2302021f6537SMarc Zyngier if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
2303021f6537SMarc Zyngier redist_stride = 0;
2304021f6537SMarc Zyngier
2305f70fdb42SSrinivas Kandagatla gic_enable_of_quirks(node, gic_quirks, &gic_data);
2306f70fdb42SSrinivas Kandagatla
230735727af2SShanker Donthineni err = gic_init_bases(dist_phys_base, dist_base, rdist_regs,
230835727af2SShanker Donthineni nr_redist_regions, redist_stride, &node->fwnode);
2309e3825ba1SMarc Zyngier if (err)
2310e3825ba1SMarc Zyngier goto out_unmap_rdist;
2311e3825ba1SMarc Zyngier
2312e3825ba1SMarc Zyngier gic_populate_ppi_partitions(node);
2313d33a3c8cSChristoffer Dall
2314d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key))
23151839e576SJulien Grall gic_of_setup_kvm_info(node);
2316021f6537SMarc Zyngier return 0;
2317021f6537SMarc Zyngier
2318021f6537SMarc Zyngier out_unmap_rdist:
2319f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++)
23202b2cd74aSRobin Murphy if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base))
2321f5c1434cSMarc Zyngier iounmap(rdist_regs[i].redist_base);
2322f5c1434cSMarc Zyngier kfree(rdist_regs);
2323021f6537SMarc Zyngier out_unmap_dist:
2324021f6537SMarc Zyngier iounmap(dist_base);
2325021f6537SMarc Zyngier return err;
2326021f6537SMarc Zyngier }
2327021f6537SMarc Zyngier
2328021f6537SMarc Zyngier IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
2329ffa7d616STomasz Nowicki
2330ffa7d616STomasz Nowicki #ifdef CONFIG_ACPI
2331611f039fSJulien Grall static struct
2332611f039fSJulien Grall {
2333611f039fSJulien Grall void __iomem *dist_base;
2334611f039fSJulien Grall struct redist_region *redist_regs;
2335611f039fSJulien Grall u32 nr_redist_regions;
2336611f039fSJulien Grall bool single_redist;
2337926b5dfaSMarc Zyngier int enabled_rdists;
23381839e576SJulien Grall u32 maint_irq;
23391839e576SJulien Grall int maint_irq_mode;
23401839e576SJulien Grall phys_addr_t vcpu_base;
2341611f039fSJulien Grall } acpi_data __initdata;
2342b70fb7afSTomasz Nowicki
2343b70fb7afSTomasz Nowicki static void __init
gic_acpi_register_redist(phys_addr_t phys_base,void __iomem * redist_base)2344b70fb7afSTomasz Nowicki gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
2345b70fb7afSTomasz Nowicki {
2346b70fb7afSTomasz Nowicki static int count = 0;
2347b70fb7afSTomasz Nowicki
2348611f039fSJulien Grall acpi_data.redist_regs[count].phys_base = phys_base;
2349611f039fSJulien Grall acpi_data.redist_regs[count].redist_base = redist_base;
2350611f039fSJulien Grall acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
2351b70fb7afSTomasz Nowicki count++;
2352b70fb7afSTomasz Nowicki }
2353ffa7d616STomasz Nowicki
2354ffa7d616STomasz Nowicki static int __init
gic_acpi_parse_madt_redist(union acpi_subtable_headers * header,const unsigned long end)235560574d1eSKeith Busch gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
2356ffa7d616STomasz Nowicki const unsigned long end)
2357ffa7d616STomasz Nowicki {
2358ffa7d616STomasz Nowicki struct acpi_madt_generic_redistributor *redist =
2359ffa7d616STomasz Nowicki (struct acpi_madt_generic_redistributor *)header;
2360ffa7d616STomasz Nowicki void __iomem *redist_base;
2361ffa7d616STomasz Nowicki
2362ffa7d616STomasz Nowicki redist_base = ioremap(redist->base_address, redist->length);
2363ffa7d616STomasz Nowicki if (!redist_base) {
2364ffa7d616STomasz Nowicki pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
2365ffa7d616STomasz Nowicki return -ENOMEM;
2366ffa7d616STomasz Nowicki }
23674deb96e3SRobin Murphy gic_request_region(redist->base_address, redist->length, "GICR");
2368ffa7d616STomasz Nowicki
2369b70fb7afSTomasz Nowicki gic_acpi_register_redist(redist->base_address, redist_base);
2370ffa7d616STomasz Nowicki return 0;
2371ffa7d616STomasz Nowicki }
2372ffa7d616STomasz Nowicki
2373b70fb7afSTomasz Nowicki static int __init
gic_acpi_parse_madt_gicc(union acpi_subtable_headers * header,const unsigned long end)237460574d1eSKeith Busch gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
2375b70fb7afSTomasz Nowicki const unsigned long end)
2376b70fb7afSTomasz Nowicki {
2377b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc =
2378b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header;
2379611f039fSJulien Grall u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
2380b70fb7afSTomasz Nowicki u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
2381b70fb7afSTomasz Nowicki void __iomem *redist_base;
2382b70fb7afSTomasz Nowicki
2383ebe2f871SShanker Donthineni /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
2384ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED))
2385ebe2f871SShanker Donthineni return 0;
2386ebe2f871SShanker Donthineni
2387b70fb7afSTomasz Nowicki redist_base = ioremap(gicc->gicr_base_address, size);
2388b70fb7afSTomasz Nowicki if (!redist_base)
2389b70fb7afSTomasz Nowicki return -ENOMEM;
23904deb96e3SRobin Murphy gic_request_region(gicc->gicr_base_address, size, "GICR");
2391b70fb7afSTomasz Nowicki
2392b70fb7afSTomasz Nowicki gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
2393b70fb7afSTomasz Nowicki return 0;
2394b70fb7afSTomasz Nowicki }
2395b70fb7afSTomasz Nowicki
gic_acpi_collect_gicr_base(void)2396b70fb7afSTomasz Nowicki static int __init gic_acpi_collect_gicr_base(void)
2397b70fb7afSTomasz Nowicki {
2398b70fb7afSTomasz Nowicki acpi_tbl_entry_handler redist_parser;
2399b70fb7afSTomasz Nowicki enum acpi_madt_type type;
2400b70fb7afSTomasz Nowicki
2401611f039fSJulien Grall if (acpi_data.single_redist) {
2402b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
2403b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_gicc;
2404b70fb7afSTomasz Nowicki } else {
2405b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
2406b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_redist;
2407b70fb7afSTomasz Nowicki }
2408b70fb7afSTomasz Nowicki
2409b70fb7afSTomasz Nowicki /* Collect redistributor base addresses in GICR entries */
2410b70fb7afSTomasz Nowicki if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
2411b70fb7afSTomasz Nowicki return 0;
2412b70fb7afSTomasz Nowicki
2413b70fb7afSTomasz Nowicki pr_info("No valid GICR entries exist\n");
2414b70fb7afSTomasz Nowicki return -ENODEV;
2415b70fb7afSTomasz Nowicki }
2416b70fb7afSTomasz Nowicki
gic_acpi_match_gicr(union acpi_subtable_headers * header,const unsigned long end)241760574d1eSKeith Busch static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
2418ffa7d616STomasz Nowicki const unsigned long end)
2419ffa7d616STomasz Nowicki {
2420ffa7d616STomasz Nowicki /* Subtable presence means that redist exists, that's it */
2421ffa7d616STomasz Nowicki return 0;
2422ffa7d616STomasz Nowicki }
2423ffa7d616STomasz Nowicki
gic_acpi_match_gicc(union acpi_subtable_headers * header,const unsigned long end)242460574d1eSKeith Busch static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
2425b70fb7afSTomasz Nowicki const unsigned long end)
2426b70fb7afSTomasz Nowicki {
2427b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc =
2428b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header;
2429b70fb7afSTomasz Nowicki
2430b70fb7afSTomasz Nowicki /*
2431b70fb7afSTomasz Nowicki * If GICC is enabled and has valid gicr base address, then it means
2432b70fb7afSTomasz Nowicki * GICR base is presented via GICC
2433b70fb7afSTomasz Nowicki */
2434926b5dfaSMarc Zyngier if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) {
2435926b5dfaSMarc Zyngier acpi_data.enabled_rdists++;
2436b70fb7afSTomasz Nowicki return 0;
2437926b5dfaSMarc Zyngier }
2438b70fb7afSTomasz Nowicki
2439ebe2f871SShanker Donthineni /*
2440ebe2f871SShanker Donthineni * It's perfectly valid firmware can pass disabled GICC entry, driver
2441ebe2f871SShanker Donthineni * should not treat as errors, skip the entry instead of probe fail.
2442ebe2f871SShanker Donthineni */
2443ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED))
2444ebe2f871SShanker Donthineni return 0;
2445ebe2f871SShanker Donthineni
2446b70fb7afSTomasz Nowicki return -ENODEV;
2447b70fb7afSTomasz Nowicki }
2448b70fb7afSTomasz Nowicki
gic_acpi_count_gicr_regions(void)2449b70fb7afSTomasz Nowicki static int __init gic_acpi_count_gicr_regions(void)
2450b70fb7afSTomasz Nowicki {
2451b70fb7afSTomasz Nowicki int count;
2452b70fb7afSTomasz Nowicki
2453b70fb7afSTomasz Nowicki /*
2454b70fb7afSTomasz Nowicki * Count how many redistributor regions we have. It is not allowed
2455b70fb7afSTomasz Nowicki * to mix redistributor description, GICR and GICC subtables have to be
2456b70fb7afSTomasz Nowicki * mutually exclusive.
2457b70fb7afSTomasz Nowicki */
2458b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
2459b70fb7afSTomasz Nowicki gic_acpi_match_gicr, 0);
2460b70fb7afSTomasz Nowicki if (count > 0) {
2461611f039fSJulien Grall acpi_data.single_redist = false;
2462b70fb7afSTomasz Nowicki return count;
2463b70fb7afSTomasz Nowicki }
2464b70fb7afSTomasz Nowicki
2465b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2466b70fb7afSTomasz Nowicki gic_acpi_match_gicc, 0);
2467926b5dfaSMarc Zyngier if (count > 0) {
2468611f039fSJulien Grall acpi_data.single_redist = true;
2469926b5dfaSMarc Zyngier count = acpi_data.enabled_rdists;
2470926b5dfaSMarc Zyngier }
2471b70fb7afSTomasz Nowicki
2472b70fb7afSTomasz Nowicki return count;
2473b70fb7afSTomasz Nowicki }
2474b70fb7afSTomasz Nowicki
acpi_validate_gic_table(struct acpi_subtable_header * header,struct acpi_probe_entry * ape)2475ffa7d616STomasz Nowicki static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
2476ffa7d616STomasz Nowicki struct acpi_probe_entry *ape)
2477ffa7d616STomasz Nowicki {
2478ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist;
2479ffa7d616STomasz Nowicki int count;
2480ffa7d616STomasz Nowicki
2481ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header;
2482ffa7d616STomasz Nowicki if (dist->version != ape->driver_data)
2483ffa7d616STomasz Nowicki return false;
2484ffa7d616STomasz Nowicki
2485ffa7d616STomasz Nowicki /* We need to do that exercise anyway, the sooner the better */
2486b70fb7afSTomasz Nowicki count = gic_acpi_count_gicr_regions();
2487ffa7d616STomasz Nowicki if (count <= 0)
2488ffa7d616STomasz Nowicki return false;
2489ffa7d616STomasz Nowicki
2490611f039fSJulien Grall acpi_data.nr_redist_regions = count;
2491ffa7d616STomasz Nowicki return true;
2492ffa7d616STomasz Nowicki }
2493ffa7d616STomasz Nowicki
gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers * header,const unsigned long end)249460574d1eSKeith Busch static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
24951839e576SJulien Grall const unsigned long end)
24961839e576SJulien Grall {
24971839e576SJulien Grall struct acpi_madt_generic_interrupt *gicc =
24981839e576SJulien Grall (struct acpi_madt_generic_interrupt *)header;
24991839e576SJulien Grall int maint_irq_mode;
25001839e576SJulien Grall static int first_madt = true;
25011839e576SJulien Grall
25021839e576SJulien Grall /* Skip unusable CPUs */
25031839e576SJulien Grall if (!(gicc->flags & ACPI_MADT_ENABLED))
25041839e576SJulien Grall return 0;
25051839e576SJulien Grall
25061839e576SJulien Grall maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
25071839e576SJulien Grall ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
25081839e576SJulien Grall
25091839e576SJulien Grall if (first_madt) {
25101839e576SJulien Grall first_madt = false;
25111839e576SJulien Grall
25121839e576SJulien Grall acpi_data.maint_irq = gicc->vgic_interrupt;
25131839e576SJulien Grall acpi_data.maint_irq_mode = maint_irq_mode;
25141839e576SJulien Grall acpi_data.vcpu_base = gicc->gicv_base_address;
25151839e576SJulien Grall
25161839e576SJulien Grall return 0;
25171839e576SJulien Grall }
25181839e576SJulien Grall
25191839e576SJulien Grall /*
25201839e576SJulien Grall * The maintenance interrupt and GICV should be the same for every CPU
25211839e576SJulien Grall */
25221839e576SJulien Grall if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
25231839e576SJulien Grall (acpi_data.maint_irq_mode != maint_irq_mode) ||
25241839e576SJulien Grall (acpi_data.vcpu_base != gicc->gicv_base_address))
25251839e576SJulien Grall return -EINVAL;
25261839e576SJulien Grall
25271839e576SJulien Grall return 0;
25281839e576SJulien Grall }
25291839e576SJulien Grall
gic_acpi_collect_virt_info(void)25301839e576SJulien Grall static bool __init gic_acpi_collect_virt_info(void)
25311839e576SJulien Grall {
25321839e576SJulien Grall int count;
25331839e576SJulien Grall
25341839e576SJulien Grall count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
25351839e576SJulien Grall gic_acpi_parse_virt_madt_gicc, 0);
25361839e576SJulien Grall
25371839e576SJulien Grall return (count > 0);
25381839e576SJulien Grall }
25391839e576SJulien Grall
2540ffa7d616STomasz Nowicki #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
25411839e576SJulien Grall #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K)
25421839e576SJulien Grall #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K)
25431839e576SJulien Grall
gic_acpi_setup_kvm_info(void)25441839e576SJulien Grall static void __init gic_acpi_setup_kvm_info(void)
25451839e576SJulien Grall {
25461839e576SJulien Grall int irq;
25471839e576SJulien Grall
25481839e576SJulien Grall if (!gic_acpi_collect_virt_info()) {
25491839e576SJulien Grall pr_warn("Unable to get hardware information used for virtualization\n");
25501839e576SJulien Grall return;
25511839e576SJulien Grall }
25521839e576SJulien Grall
25531839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3;
25541839e576SJulien Grall
25551839e576SJulien Grall irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
25561839e576SJulien Grall acpi_data.maint_irq_mode,
25571839e576SJulien Grall ACPI_ACTIVE_HIGH);
25581839e576SJulien Grall if (irq <= 0)
25591839e576SJulien Grall return;
25601839e576SJulien Grall
25611839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq;
25621839e576SJulien Grall
25631839e576SJulien Grall if (acpi_data.vcpu_base) {
25641839e576SJulien Grall struct resource *vcpu = &gic_v3_kvm_info.vcpu;
25651839e576SJulien Grall
25661839e576SJulien Grall vcpu->flags = IORESOURCE_MEM;
25671839e576SJulien Grall vcpu->start = acpi_data.vcpu_base;
25681839e576SJulien Grall vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
25691839e576SJulien Grall }
25701839e576SJulien Grall
25714bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
25723c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
25730e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info);
25741839e576SJulien Grall }
2575ffa7d616STomasz Nowicki
25767327b16fSMarc Zyngier static struct fwnode_handle *gsi_domain_handle;
25777327b16fSMarc Zyngier
gic_v3_get_gsi_domain_id(u32 gsi)25787327b16fSMarc Zyngier static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi)
25797327b16fSMarc Zyngier {
25807327b16fSMarc Zyngier return gsi_domain_handle;
25817327b16fSMarc Zyngier }
25827327b16fSMarc Zyngier
2583ffa7d616STomasz Nowicki static int __init
gic_acpi_init(union acpi_subtable_headers * header,const unsigned long end)2584aba3c7edSOscar Carter gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
2585ffa7d616STomasz Nowicki {
2586ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist;
2587611f039fSJulien Grall size_t size;
2588b70fb7afSTomasz Nowicki int i, err;
2589ffa7d616STomasz Nowicki
2590ffa7d616STomasz Nowicki /* Get distributor base address */
2591ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header;
2592611f039fSJulien Grall acpi_data.dist_base = ioremap(dist->base_address,
2593611f039fSJulien Grall ACPI_GICV3_DIST_MEM_SIZE);
2594611f039fSJulien Grall if (!acpi_data.dist_base) {
2595ffa7d616STomasz Nowicki pr_err("Unable to map GICD registers\n");
2596ffa7d616STomasz Nowicki return -ENOMEM;
2597ffa7d616STomasz Nowicki }
25984deb96e3SRobin Murphy gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD");
2599ffa7d616STomasz Nowicki
2600611f039fSJulien Grall err = gic_validate_dist_version(acpi_data.dist_base);
2601ffa7d616STomasz Nowicki if (err) {
260271192a68SArvind Yadav pr_err("No distributor detected at @%p, giving up\n",
2603611f039fSJulien Grall acpi_data.dist_base);
2604ffa7d616STomasz Nowicki goto out_dist_unmap;
2605ffa7d616STomasz Nowicki }
2606ffa7d616STomasz Nowicki
2607611f039fSJulien Grall size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
2608611f039fSJulien Grall acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
2609611f039fSJulien Grall if (!acpi_data.redist_regs) {
2610ffa7d616STomasz Nowicki err = -ENOMEM;
2611ffa7d616STomasz Nowicki goto out_dist_unmap;
2612ffa7d616STomasz Nowicki }
2613ffa7d616STomasz Nowicki
2614b70fb7afSTomasz Nowicki err = gic_acpi_collect_gicr_base();
2615b70fb7afSTomasz Nowicki if (err)
2616ffa7d616STomasz Nowicki goto out_redist_unmap;
2617ffa7d616STomasz Nowicki
26187327b16fSMarc Zyngier gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
26197327b16fSMarc Zyngier if (!gsi_domain_handle) {
2620ffa7d616STomasz Nowicki err = -ENOMEM;
2621ffa7d616STomasz Nowicki goto out_redist_unmap;
2622ffa7d616STomasz Nowicki }
2623ffa7d616STomasz Nowicki
262435727af2SShanker Donthineni err = gic_init_bases(dist->base_address, acpi_data.dist_base,
262535727af2SShanker Donthineni acpi_data.redist_regs, acpi_data.nr_redist_regions,
262635727af2SShanker Donthineni 0, gsi_domain_handle);
2627ffa7d616STomasz Nowicki if (err)
2628ffa7d616STomasz Nowicki goto out_fwhandle_free;
2629ffa7d616STomasz Nowicki
26307327b16fSMarc Zyngier acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id);
2631d33a3c8cSChristoffer Dall
2632d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key))
26331839e576SJulien Grall gic_acpi_setup_kvm_info();
26341839e576SJulien Grall
2635ffa7d616STomasz Nowicki return 0;
2636ffa7d616STomasz Nowicki
2637ffa7d616STomasz Nowicki out_fwhandle_free:
26387327b16fSMarc Zyngier irq_domain_free_fwnode(gsi_domain_handle);
2639ffa7d616STomasz Nowicki out_redist_unmap:
2640611f039fSJulien Grall for (i = 0; i < acpi_data.nr_redist_regions; i++)
2641611f039fSJulien Grall if (acpi_data.redist_regs[i].redist_base)
2642611f039fSJulien Grall iounmap(acpi_data.redist_regs[i].redist_base);
2643611f039fSJulien Grall kfree(acpi_data.redist_regs);
2644ffa7d616STomasz Nowicki out_dist_unmap:
2645611f039fSJulien Grall iounmap(acpi_data.dist_base);
2646ffa7d616STomasz Nowicki return err;
2647ffa7d616STomasz Nowicki }
2648ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2649ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
2650ffa7d616STomasz Nowicki gic_acpi_init);
2651ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2652ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
2653ffa7d616STomasz Nowicki gic_acpi_init);
2654ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2655ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
2656ffa7d616STomasz Nowicki gic_acpi_init);
2657ffa7d616STomasz Nowicki #endif
2658