1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2021f6537SMarc Zyngier /* 30edc23eaSMarc Zyngier * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4021f6537SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com> 5021f6537SMarc Zyngier */ 6021f6537SMarc Zyngier 768628bb8SJulien Grall #define pr_fmt(fmt) "GICv3: " fmt 868628bb8SJulien Grall 9ffa7d616STomasz Nowicki #include <linux/acpi.h> 10021f6537SMarc Zyngier #include <linux/cpu.h> 113708d52fSSudeep Holla #include <linux/cpu_pm.h> 12021f6537SMarc Zyngier #include <linux/delay.h> 13021f6537SMarc Zyngier #include <linux/interrupt.h> 14ffa7d616STomasz Nowicki #include <linux/irqdomain.h> 155e279739SChristophe JAILLET #include <linux/kstrtox.h> 16021f6537SMarc Zyngier #include <linux/of.h> 17021f6537SMarc Zyngier #include <linux/of_address.h> 18021f6537SMarc Zyngier #include <linux/of_irq.h> 19021f6537SMarc Zyngier #include <linux/percpu.h> 20101b35f7SJulien Thierry #include <linux/refcount.h> 21021f6537SMarc Zyngier #include <linux/slab.h> 22021f6537SMarc Zyngier 2341a83e06SJoel Porquet #include <linux/irqchip.h> 241839e576SJulien Grall #include <linux/irqchip/arm-gic-common.h> 25021f6537SMarc Zyngier #include <linux/irqchip/arm-gic-v3.h> 26e3825ba1SMarc Zyngier #include <linux/irqchip/irq-partition-percpu.h> 2735727af2SShanker Donthineni #include <linux/bitfield.h> 2835727af2SShanker Donthineni #include <linux/bits.h> 2935727af2SShanker Donthineni #include <linux/arm-smccc.h> 30021f6537SMarc Zyngier 31021f6537SMarc Zyngier #include <asm/cputype.h> 32021f6537SMarc Zyngier #include <asm/exception.h> 33021f6537SMarc Zyngier #include <asm/smp_plat.h> 340b6a3da9SMarc Zyngier #include <asm/virt.h> 35021f6537SMarc Zyngier 36021f6537SMarc Zyngier #include "irq-gic-common.h" 37021f6537SMarc Zyngier 38f32c9266SJulien Thierry #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 39f32c9266SJulien Thierry 409c8114c2SSrinivas Kandagatla #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 41d01fd161SMarc Zyngier #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 42*44bd78ddSDouglas Anderson #define FLAGS_WORKAROUND_MTK_GICR_SAVE (1ULL << 2) 439c8114c2SSrinivas Kandagatla 4464b499d8SMarc Zyngier #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 4564b499d8SMarc Zyngier 46f5c1434cSMarc Zyngier struct redist_region { 47f5c1434cSMarc Zyngier void __iomem *redist_base; 48f5c1434cSMarc Zyngier phys_addr_t phys_base; 49b70fb7afSTomasz Nowicki bool single_redist; 50f5c1434cSMarc Zyngier }; 51f5c1434cSMarc Zyngier 52021f6537SMarc Zyngier struct gic_chip_data { 53e3825ba1SMarc Zyngier struct fwnode_handle *fwnode; 5435727af2SShanker Donthineni phys_addr_t dist_phys_base; 55021f6537SMarc Zyngier void __iomem *dist_base; 56f5c1434cSMarc Zyngier struct redist_region *redist_regions; 57f5c1434cSMarc Zyngier struct rdists rdists; 58021f6537SMarc Zyngier struct irq_domain *domain; 59021f6537SMarc Zyngier u64 redist_stride; 60f5c1434cSMarc Zyngier u32 nr_redist_regions; 619c8114c2SSrinivas Kandagatla u64 flags; 62eda0d04aSShanker Donthineni bool has_rss; 631a60e1e6SMarc Zyngier unsigned int ppi_nr; 6452085d3fSMarc Zyngier struct partition_desc **ppi_descs; 65021f6537SMarc Zyngier }; 66021f6537SMarc Zyngier 6735727af2SShanker Donthineni #define T241_CHIPS_MAX 4 6835727af2SShanker Donthineni static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly; 6935727af2SShanker Donthineni static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum); 7035727af2SShanker Donthineni 71021f6537SMarc Zyngier static struct gic_chip_data gic_data __read_mostly; 72d01d3274SDavidlohr Bueso static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 73021f6537SMarc Zyngier 74211bddd2SMarc Zyngier #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 75c107d613SZenghui Yu #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 76211bddd2SMarc Zyngier #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 77211bddd2SMarc Zyngier 78d98d0a99SJulien Thierry /* 79d98d0a99SJulien Thierry * The behaviours of RPR and PMR registers differ depending on the value of 80d98d0a99SJulien Thierry * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 81d98d0a99SJulien Thierry * distributor and redistributors depends on whether security is enabled in the 82d98d0a99SJulien Thierry * GIC. 83d98d0a99SJulien Thierry * 84d98d0a99SJulien Thierry * When security is enabled, non-secure priority values from the (re)distributor 85d98d0a99SJulien Thierry * are presented to the GIC CPUIF as follow: 86d98d0a99SJulien Thierry * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 87d98d0a99SJulien Thierry * 88d4034114SLorenzo Pieralisi * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 89d98d0a99SJulien Thierry * EL1 are subject to a similar operation thus matching the priorities presented 9033678059SAlexandru Elisei * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 91d4034114SLorenzo Pieralisi * these values are unchanged by the GIC. 92d98d0a99SJulien Thierry * 93d98d0a99SJulien Thierry * see GICv3/GICv4 Architecture Specification (IHI0069D): 94d98d0a99SJulien Thierry * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 95d98d0a99SJulien Thierry * priorities. 96d98d0a99SJulien Thierry * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 97d98d0a99SJulien Thierry * interrupt. 98d98d0a99SJulien Thierry */ 99d98d0a99SJulien Thierry static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 100d98d0a99SJulien Thierry 10133678059SAlexandru Elisei DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 10233678059SAlexandru Elisei EXPORT_SYMBOL(gic_nonsecure_priorities); 10333678059SAlexandru Elisei 1048d474deaSChen-Yu Tsai /* 1058d474deaSChen-Yu Tsai * When the Non-secure world has access to group 0 interrupts (as a 1068d474deaSChen-Yu Tsai * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will 1078d474deaSChen-Yu Tsai * return the Distributor's view of the interrupt priority. 1088d474deaSChen-Yu Tsai * 1098d474deaSChen-Yu Tsai * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority 1108d474deaSChen-Yu Tsai * written by software is moved to the Non-secure range by the Distributor. 1118d474deaSChen-Yu Tsai * 1128d474deaSChen-Yu Tsai * If both are true (which is when gic_nonsecure_priorities gets enabled), 1138d474deaSChen-Yu Tsai * we need to shift down the priority programmed by software to match it 1148d474deaSChen-Yu Tsai * against the value returned by ICC_RPR_EL1. 1158d474deaSChen-Yu Tsai */ 1168d474deaSChen-Yu Tsai #define GICD_INT_RPR_PRI(priority) \ 1178d474deaSChen-Yu Tsai ({ \ 1188d474deaSChen-Yu Tsai u32 __priority = (priority); \ 1198d474deaSChen-Yu Tsai if (static_branch_unlikely(&gic_nonsecure_priorities)) \ 1208d474deaSChen-Yu Tsai __priority = 0x80 | (__priority >> 1); \ 1218d474deaSChen-Yu Tsai \ 1228d474deaSChen-Yu Tsai __priority; \ 1238d474deaSChen-Yu Tsai }) 1248d474deaSChen-Yu Tsai 125101b35f7SJulien Thierry /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 12681a43273SMarc Zyngier static refcount_t *ppi_nmi_refs; 127101b35f7SJulien Thierry 1280e5cb777SMarc Zyngier static struct gic_kvm_info gic_v3_kvm_info __initdata; 129eda0d04aSShanker Donthineni static DEFINE_PER_CPU(bool, has_rss); 1301839e576SJulien Grall 131eda0d04aSShanker Donthineni #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 132f5c1434cSMarc Zyngier #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 133f5c1434cSMarc Zyngier #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 134021f6537SMarc Zyngier #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 135021f6537SMarc Zyngier 136021f6537SMarc Zyngier /* Our default, arbitrary priority value. Linux only uses one anyway. */ 137021f6537SMarc Zyngier #define DEFAULT_PMR_VALUE 0xf0 138021f6537SMarc Zyngier 139e91b036eSMarc Zyngier enum gic_intid_range { 14070a29c32SMarc Zyngier SGI_RANGE, 141e91b036eSMarc Zyngier PPI_RANGE, 142e91b036eSMarc Zyngier SPI_RANGE, 1435f51f803SMarc Zyngier EPPI_RANGE, 144211bddd2SMarc Zyngier ESPI_RANGE, 145e91b036eSMarc Zyngier LPI_RANGE, 146e91b036eSMarc Zyngier __INVALID_RANGE__ 147e91b036eSMarc Zyngier }; 148e91b036eSMarc Zyngier 149e91b036eSMarc Zyngier static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 150e91b036eSMarc Zyngier { 151e91b036eSMarc Zyngier switch (hwirq) { 15270a29c32SMarc Zyngier case 0 ... 15: 15370a29c32SMarc Zyngier return SGI_RANGE; 154e91b036eSMarc Zyngier case 16 ... 31: 155e91b036eSMarc Zyngier return PPI_RANGE; 156e91b036eSMarc Zyngier case 32 ... 1019: 157e91b036eSMarc Zyngier return SPI_RANGE; 1585f51f803SMarc Zyngier case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 1595f51f803SMarc Zyngier return EPPI_RANGE; 160211bddd2SMarc Zyngier case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 161211bddd2SMarc Zyngier return ESPI_RANGE; 162e91b036eSMarc Zyngier case 8192 ... GENMASK(23, 0): 163e91b036eSMarc Zyngier return LPI_RANGE; 164e91b036eSMarc Zyngier default: 165e91b036eSMarc Zyngier return __INVALID_RANGE__; 166e91b036eSMarc Zyngier } 167e91b036eSMarc Zyngier } 168e91b036eSMarc Zyngier 169e91b036eSMarc Zyngier static enum gic_intid_range get_intid_range(struct irq_data *d) 170e91b036eSMarc Zyngier { 171e91b036eSMarc Zyngier return __get_intid_range(d->hwirq); 172e91b036eSMarc Zyngier } 173e91b036eSMarc Zyngier 174021f6537SMarc Zyngier static inline unsigned int gic_irq(struct irq_data *d) 175021f6537SMarc Zyngier { 176021f6537SMarc Zyngier return d->hwirq; 177021f6537SMarc Zyngier } 178021f6537SMarc Zyngier 17970a29c32SMarc Zyngier static inline bool gic_irq_in_rdist(struct irq_data *d) 180021f6537SMarc Zyngier { 18170a29c32SMarc Zyngier switch (get_intid_range(d)) { 18270a29c32SMarc Zyngier case SGI_RANGE: 18370a29c32SMarc Zyngier case PPI_RANGE: 18470a29c32SMarc Zyngier case EPPI_RANGE: 18570a29c32SMarc Zyngier return true; 18670a29c32SMarc Zyngier default: 18770a29c32SMarc Zyngier return false; 18870a29c32SMarc Zyngier } 189021f6537SMarc Zyngier } 190021f6537SMarc Zyngier 19135727af2SShanker Donthineni static inline void __iomem *gic_dist_base_alias(struct irq_data *d) 19235727af2SShanker Donthineni { 19335727af2SShanker Donthineni if (static_branch_unlikely(&gic_nvidia_t241_erratum)) { 19435727af2SShanker Donthineni irq_hw_number_t hwirq = irqd_to_hwirq(d); 19535727af2SShanker Donthineni u32 chip; 19635727af2SShanker Donthineni 19735727af2SShanker Donthineni /* 19835727af2SShanker Donthineni * For the erratum T241-FABRIC-4, read accesses to GICD_In{E} 19935727af2SShanker Donthineni * registers are directed to the chip that owns the SPI. The 20035727af2SShanker Donthineni * the alias region can also be used for writes to the 20135727af2SShanker Donthineni * GICD_In{E} except GICD_ICENABLERn. Each chip has support 20235727af2SShanker Donthineni * for 320 {E}SPIs. Mappings for all 4 chips: 20335727af2SShanker Donthineni * Chip0 = 32-351 20435727af2SShanker Donthineni * Chip1 = 352-671 20535727af2SShanker Donthineni * Chip2 = 672-991 20635727af2SShanker Donthineni * Chip3 = 4096-4415 20735727af2SShanker Donthineni */ 20835727af2SShanker Donthineni switch (__get_intid_range(hwirq)) { 20935727af2SShanker Donthineni case SPI_RANGE: 21035727af2SShanker Donthineni chip = (hwirq - 32) / 320; 21135727af2SShanker Donthineni break; 21235727af2SShanker Donthineni case ESPI_RANGE: 21335727af2SShanker Donthineni chip = 3; 21435727af2SShanker Donthineni break; 21535727af2SShanker Donthineni default: 21635727af2SShanker Donthineni unreachable(); 21735727af2SShanker Donthineni } 21835727af2SShanker Donthineni return t241_dist_base_alias[chip]; 21935727af2SShanker Donthineni } 22035727af2SShanker Donthineni 22135727af2SShanker Donthineni return gic_data.dist_base; 22235727af2SShanker Donthineni } 22335727af2SShanker Donthineni 224021f6537SMarc Zyngier static inline void __iomem *gic_dist_base(struct irq_data *d) 225021f6537SMarc Zyngier { 226e91b036eSMarc Zyngier switch (get_intid_range(d)) { 22770a29c32SMarc Zyngier case SGI_RANGE: 228e91b036eSMarc Zyngier case PPI_RANGE: 2295f51f803SMarc Zyngier case EPPI_RANGE: 230e91b036eSMarc Zyngier /* SGI+PPI -> SGI_base for this CPU */ 231021f6537SMarc Zyngier return gic_data_rdist_sgi_base(); 232021f6537SMarc Zyngier 233e91b036eSMarc Zyngier case SPI_RANGE: 234211bddd2SMarc Zyngier case ESPI_RANGE: 235e91b036eSMarc Zyngier /* SPI -> dist_base */ 236021f6537SMarc Zyngier return gic_data.dist_base; 237021f6537SMarc Zyngier 238e91b036eSMarc Zyngier default: 239021f6537SMarc Zyngier return NULL; 240021f6537SMarc Zyngier } 241e91b036eSMarc Zyngier } 242021f6537SMarc Zyngier 2430df66645SMarc Zyngier static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) 244021f6537SMarc Zyngier { 245021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 246021f6537SMarc Zyngier 2470df66645SMarc Zyngier while (readl_relaxed(base + GICD_CTLR) & bit) { 248021f6537SMarc Zyngier count--; 249021f6537SMarc Zyngier if (!count) { 250021f6537SMarc Zyngier pr_err_ratelimited("RWP timeout, gone fishing\n"); 251021f6537SMarc Zyngier return; 252021f6537SMarc Zyngier } 253021f6537SMarc Zyngier cpu_relax(); 254021f6537SMarc Zyngier udelay(1); 2552c542426SDaode Huang } 256021f6537SMarc Zyngier } 257021f6537SMarc Zyngier 258021f6537SMarc Zyngier /* Wait for completion of a distributor change */ 259021f6537SMarc Zyngier static void gic_dist_wait_for_rwp(void) 260021f6537SMarc Zyngier { 2610df66645SMarc Zyngier gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP); 262021f6537SMarc Zyngier } 263021f6537SMarc Zyngier 264021f6537SMarc Zyngier /* Wait for completion of a redistributor change */ 265021f6537SMarc Zyngier static void gic_redist_wait_for_rwp(void) 266021f6537SMarc Zyngier { 2670df66645SMarc Zyngier gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); 268021f6537SMarc Zyngier } 269021f6537SMarc Zyngier 2707936e914SJean-Philippe Brucker #ifdef CONFIG_ARM64 2716d4e11c5SRobert Richter 2726d4e11c5SRobert Richter static u64 __maybe_unused gic_read_iar(void) 2736d4e11c5SRobert Richter { 274a4023f68SSuzuki K Poulose if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 2756d4e11c5SRobert Richter return gic_read_iar_cavium_thunderx(); 2766d4e11c5SRobert Richter else 2776d4e11c5SRobert Richter return gic_read_iar_common(); 2786d4e11c5SRobert Richter } 2797936e914SJean-Philippe Brucker #endif 280021f6537SMarc Zyngier 281a2c22510SSudeep Holla static void gic_enable_redist(bool enable) 282021f6537SMarc Zyngier { 283021f6537SMarc Zyngier void __iomem *rbase; 284021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 285021f6537SMarc Zyngier u32 val; 286021f6537SMarc Zyngier 2879c8114c2SSrinivas Kandagatla if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 2889c8114c2SSrinivas Kandagatla return; 2899c8114c2SSrinivas Kandagatla 290021f6537SMarc Zyngier rbase = gic_data_rdist_rd_base(); 291021f6537SMarc Zyngier 292021f6537SMarc Zyngier val = readl_relaxed(rbase + GICR_WAKER); 293a2c22510SSudeep Holla if (enable) 294a2c22510SSudeep Holla /* Wake up this CPU redistributor */ 295021f6537SMarc Zyngier val &= ~GICR_WAKER_ProcessorSleep; 296a2c22510SSudeep Holla else 297a2c22510SSudeep Holla val |= GICR_WAKER_ProcessorSleep; 298021f6537SMarc Zyngier writel_relaxed(val, rbase + GICR_WAKER); 299021f6537SMarc Zyngier 300a2c22510SSudeep Holla if (!enable) { /* Check that GICR_WAKER is writeable */ 301a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 302a2c22510SSudeep Holla if (!(val & GICR_WAKER_ProcessorSleep)) 303a2c22510SSudeep Holla return; /* No PM support in this redistributor */ 304021f6537SMarc Zyngier } 305a2c22510SSudeep Holla 306d102eb5cSDan Carpenter while (--count) { 307a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 308cf1d9d11SAndrew Jones if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 309a2c22510SSudeep Holla break; 310021f6537SMarc Zyngier cpu_relax(); 311021f6537SMarc Zyngier udelay(1); 3122c542426SDaode Huang } 313a2c22510SSudeep Holla if (!count) 314a2c22510SSudeep Holla pr_err_ratelimited("redistributor failed to %s...\n", 315a2c22510SSudeep Holla enable ? "wakeup" : "sleep"); 316021f6537SMarc Zyngier } 317021f6537SMarc Zyngier 318021f6537SMarc Zyngier /* 319021f6537SMarc Zyngier * Routines to disable, enable, EOI and route interrupts 320021f6537SMarc Zyngier */ 321e91b036eSMarc Zyngier static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 322e91b036eSMarc Zyngier { 323e91b036eSMarc Zyngier switch (get_intid_range(d)) { 32470a29c32SMarc Zyngier case SGI_RANGE: 325e91b036eSMarc Zyngier case PPI_RANGE: 326e91b036eSMarc Zyngier case SPI_RANGE: 327e91b036eSMarc Zyngier *index = d->hwirq; 328e91b036eSMarc Zyngier return offset; 3295f51f803SMarc Zyngier case EPPI_RANGE: 3305f51f803SMarc Zyngier /* 3315f51f803SMarc Zyngier * Contrary to the ESPI range, the EPPI range is contiguous 3325f51f803SMarc Zyngier * to the PPI range in the registers, so let's adjust the 3335f51f803SMarc Zyngier * displacement accordingly. Consistency is overrated. 3345f51f803SMarc Zyngier */ 3355f51f803SMarc Zyngier *index = d->hwirq - EPPI_BASE_INTID + 32; 3365f51f803SMarc Zyngier return offset; 337211bddd2SMarc Zyngier case ESPI_RANGE: 338211bddd2SMarc Zyngier *index = d->hwirq - ESPI_BASE_INTID; 339211bddd2SMarc Zyngier switch (offset) { 340211bddd2SMarc Zyngier case GICD_ISENABLER: 341211bddd2SMarc Zyngier return GICD_ISENABLERnE; 342211bddd2SMarc Zyngier case GICD_ICENABLER: 343211bddd2SMarc Zyngier return GICD_ICENABLERnE; 344211bddd2SMarc Zyngier case GICD_ISPENDR: 345211bddd2SMarc Zyngier return GICD_ISPENDRnE; 346211bddd2SMarc Zyngier case GICD_ICPENDR: 347211bddd2SMarc Zyngier return GICD_ICPENDRnE; 348211bddd2SMarc Zyngier case GICD_ISACTIVER: 349211bddd2SMarc Zyngier return GICD_ISACTIVERnE; 350211bddd2SMarc Zyngier case GICD_ICACTIVER: 351211bddd2SMarc Zyngier return GICD_ICACTIVERnE; 352211bddd2SMarc Zyngier case GICD_IPRIORITYR: 353211bddd2SMarc Zyngier return GICD_IPRIORITYRnE; 354211bddd2SMarc Zyngier case GICD_ICFGR: 355211bddd2SMarc Zyngier return GICD_ICFGRnE; 356211bddd2SMarc Zyngier case GICD_IROUTER: 357211bddd2SMarc Zyngier return GICD_IROUTERnE; 358211bddd2SMarc Zyngier default: 359211bddd2SMarc Zyngier break; 360211bddd2SMarc Zyngier } 361211bddd2SMarc Zyngier break; 362e91b036eSMarc Zyngier default: 363e91b036eSMarc Zyngier break; 364e91b036eSMarc Zyngier } 365e91b036eSMarc Zyngier 366e91b036eSMarc Zyngier WARN_ON(1); 367e91b036eSMarc Zyngier *index = d->hwirq; 368e91b036eSMarc Zyngier return offset; 369e91b036eSMarc Zyngier } 370e91b036eSMarc Zyngier 371b594c6e2SMarc Zyngier static int gic_peek_irq(struct irq_data *d, u32 offset) 372b594c6e2SMarc Zyngier { 373b594c6e2SMarc Zyngier void __iomem *base; 374e91b036eSMarc Zyngier u32 index, mask; 375e91b036eSMarc Zyngier 376e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 377e91b036eSMarc Zyngier mask = 1 << (index % 32); 378b594c6e2SMarc Zyngier 379b594c6e2SMarc Zyngier if (gic_irq_in_rdist(d)) 380b594c6e2SMarc Zyngier base = gic_data_rdist_sgi_base(); 381b594c6e2SMarc Zyngier else 38235727af2SShanker Donthineni base = gic_dist_base_alias(d); 383b594c6e2SMarc Zyngier 384e91b036eSMarc Zyngier return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 385b594c6e2SMarc Zyngier } 386b594c6e2SMarc Zyngier 387021f6537SMarc Zyngier static void gic_poke_irq(struct irq_data *d, u32 offset) 388021f6537SMarc Zyngier { 389021f6537SMarc Zyngier void __iomem *base; 390e91b036eSMarc Zyngier u32 index, mask; 391e91b036eSMarc Zyngier 392e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 393e91b036eSMarc Zyngier mask = 1 << (index % 32); 394021f6537SMarc Zyngier 39563f13483SMarc Zyngier if (gic_irq_in_rdist(d)) 396021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 39763f13483SMarc Zyngier else 398021f6537SMarc Zyngier base = gic_data.dist_base; 399021f6537SMarc Zyngier 400e91b036eSMarc Zyngier writel_relaxed(mask, base + offset + (index / 32) * 4); 401021f6537SMarc Zyngier } 402021f6537SMarc Zyngier 403021f6537SMarc Zyngier static void gic_mask_irq(struct irq_data *d) 404021f6537SMarc Zyngier { 405021f6537SMarc Zyngier gic_poke_irq(d, GICD_ICENABLER); 40663f13483SMarc Zyngier if (gic_irq_in_rdist(d)) 40763f13483SMarc Zyngier gic_redist_wait_for_rwp(); 40863f13483SMarc Zyngier else 40963f13483SMarc Zyngier gic_dist_wait_for_rwp(); 410021f6537SMarc Zyngier } 411021f6537SMarc Zyngier 4120b6a3da9SMarc Zyngier static void gic_eoimode1_mask_irq(struct irq_data *d) 4130b6a3da9SMarc Zyngier { 4140b6a3da9SMarc Zyngier gic_mask_irq(d); 415530bf353SMarc Zyngier /* 416530bf353SMarc Zyngier * When masking a forwarded interrupt, make sure it is 417530bf353SMarc Zyngier * deactivated as well. 418530bf353SMarc Zyngier * 419530bf353SMarc Zyngier * This ensures that an interrupt that is getting 420530bf353SMarc Zyngier * disabled/masked will not get "stuck", because there is 421530bf353SMarc Zyngier * noone to deactivate it (guest is being terminated). 422530bf353SMarc Zyngier */ 4234df7f54dSThomas Gleixner if (irqd_is_forwarded_to_vcpu(d)) 424530bf353SMarc Zyngier gic_poke_irq(d, GICD_ICACTIVER); 4250b6a3da9SMarc Zyngier } 4260b6a3da9SMarc Zyngier 427021f6537SMarc Zyngier static void gic_unmask_irq(struct irq_data *d) 428021f6537SMarc Zyngier { 429021f6537SMarc Zyngier gic_poke_irq(d, GICD_ISENABLER); 430021f6537SMarc Zyngier } 431021f6537SMarc Zyngier 432d98d0a99SJulien Thierry static inline bool gic_supports_nmi(void) 433d98d0a99SJulien Thierry { 434d98d0a99SJulien Thierry return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 435d98d0a99SJulien Thierry static_branch_likely(&supports_pseudo_nmis); 436d98d0a99SJulien Thierry } 437d98d0a99SJulien Thierry 438b594c6e2SMarc Zyngier static int gic_irq_set_irqchip_state(struct irq_data *d, 439b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool val) 440b594c6e2SMarc Zyngier { 441b594c6e2SMarc Zyngier u32 reg; 442b594c6e2SMarc Zyngier 44364b499d8SMarc Zyngier if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 444b594c6e2SMarc Zyngier return -EINVAL; 445b594c6e2SMarc Zyngier 446b594c6e2SMarc Zyngier switch (which) { 447b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 448b594c6e2SMarc Zyngier reg = val ? GICD_ISPENDR : GICD_ICPENDR; 449b594c6e2SMarc Zyngier break; 450b594c6e2SMarc Zyngier 451b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 452b594c6e2SMarc Zyngier reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 453b594c6e2SMarc Zyngier break; 454b594c6e2SMarc Zyngier 455b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 45663f13483SMarc Zyngier if (val) { 45763f13483SMarc Zyngier gic_mask_irq(d); 45863f13483SMarc Zyngier return 0; 45963f13483SMarc Zyngier } 46063f13483SMarc Zyngier reg = GICD_ISENABLER; 461b594c6e2SMarc Zyngier break; 462b594c6e2SMarc Zyngier 463b594c6e2SMarc Zyngier default: 464b594c6e2SMarc Zyngier return -EINVAL; 465b594c6e2SMarc Zyngier } 466b594c6e2SMarc Zyngier 467b594c6e2SMarc Zyngier gic_poke_irq(d, reg); 468b594c6e2SMarc Zyngier return 0; 469b594c6e2SMarc Zyngier } 470b594c6e2SMarc Zyngier 471b594c6e2SMarc Zyngier static int gic_irq_get_irqchip_state(struct irq_data *d, 472b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool *val) 473b594c6e2SMarc Zyngier { 474211bddd2SMarc Zyngier if (d->hwirq >= 8192) /* PPI/SPI only */ 475b594c6e2SMarc Zyngier return -EINVAL; 476b594c6e2SMarc Zyngier 477b594c6e2SMarc Zyngier switch (which) { 478b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 479b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISPENDR); 480b594c6e2SMarc Zyngier break; 481b594c6e2SMarc Zyngier 482b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 483b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISACTIVER); 484b594c6e2SMarc Zyngier break; 485b594c6e2SMarc Zyngier 486b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 487b594c6e2SMarc Zyngier *val = !gic_peek_irq(d, GICD_ISENABLER); 488b594c6e2SMarc Zyngier break; 489b594c6e2SMarc Zyngier 490b594c6e2SMarc Zyngier default: 491b594c6e2SMarc Zyngier return -EINVAL; 492b594c6e2SMarc Zyngier } 493b594c6e2SMarc Zyngier 494b594c6e2SMarc Zyngier return 0; 495b594c6e2SMarc Zyngier } 496b594c6e2SMarc Zyngier 497101b35f7SJulien Thierry static void gic_irq_set_prio(struct irq_data *d, u8 prio) 498101b35f7SJulien Thierry { 499101b35f7SJulien Thierry void __iomem *base = gic_dist_base(d); 500e91b036eSMarc Zyngier u32 offset, index; 501101b35f7SJulien Thierry 502e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 503e91b036eSMarc Zyngier 504e91b036eSMarc Zyngier writeb_relaxed(prio, base + offset + index); 505101b35f7SJulien Thierry } 506101b35f7SJulien Thierry 507bfa80ee9SJames Morse static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 50881a43273SMarc Zyngier { 509bfa80ee9SJames Morse switch (__get_intid_range(hwirq)) { 51081a43273SMarc Zyngier case PPI_RANGE: 511bfa80ee9SJames Morse return hwirq - 16; 5125f51f803SMarc Zyngier case EPPI_RANGE: 513bfa80ee9SJames Morse return hwirq - EPPI_BASE_INTID + 16; 51481a43273SMarc Zyngier default: 51581a43273SMarc Zyngier unreachable(); 51681a43273SMarc Zyngier } 51781a43273SMarc Zyngier } 51881a43273SMarc Zyngier 519bfa80ee9SJames Morse static u32 gic_get_ppi_index(struct irq_data *d) 520bfa80ee9SJames Morse { 521bfa80ee9SJames Morse return __gic_get_ppi_index(d->hwirq); 522bfa80ee9SJames Morse } 523bfa80ee9SJames Morse 524101b35f7SJulien Thierry static int gic_irq_nmi_setup(struct irq_data *d) 525101b35f7SJulien Thierry { 526101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 527101b35f7SJulien Thierry 528101b35f7SJulien Thierry if (!gic_supports_nmi()) 529101b35f7SJulien Thierry return -EINVAL; 530101b35f7SJulien Thierry 531101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 532101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 533101b35f7SJulien Thierry return -EINVAL; 534101b35f7SJulien Thierry } 535101b35f7SJulien Thierry 536101b35f7SJulien Thierry /* 537101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 538101b35f7SJulien Thierry * it should not be possible to get there 539101b35f7SJulien Thierry */ 540101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 541101b35f7SJulien Thierry return -EINVAL; 542101b35f7SJulien Thierry 543101b35f7SJulien Thierry /* desc lock should already be held */ 54481a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 54581a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 54681a43273SMarc Zyngier 547101b35f7SJulien Thierry /* Setting up PPI as NMI, only switch handler for first NMI */ 54881a43273SMarc Zyngier if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 54981a43273SMarc Zyngier refcount_set(&ppi_nmi_refs[idx], 1); 550101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 551101b35f7SJulien Thierry } 552101b35f7SJulien Thierry } else { 553101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_nmi; 554101b35f7SJulien Thierry } 555101b35f7SJulien Thierry 556101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_NMI_PRI); 557101b35f7SJulien Thierry 558101b35f7SJulien Thierry return 0; 559101b35f7SJulien Thierry } 560101b35f7SJulien Thierry 561101b35f7SJulien Thierry static void gic_irq_nmi_teardown(struct irq_data *d) 562101b35f7SJulien Thierry { 563101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 564101b35f7SJulien Thierry 565101b35f7SJulien Thierry if (WARN_ON(!gic_supports_nmi())) 566101b35f7SJulien Thierry return; 567101b35f7SJulien Thierry 568101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 569101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 570101b35f7SJulien Thierry return; 571101b35f7SJulien Thierry } 572101b35f7SJulien Thierry 573101b35f7SJulien Thierry /* 574101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 575101b35f7SJulien Thierry * it should not be possible to get there 576101b35f7SJulien Thierry */ 577101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 578101b35f7SJulien Thierry return; 579101b35f7SJulien Thierry 580101b35f7SJulien Thierry /* desc lock should already be held */ 58181a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 58281a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 58381a43273SMarc Zyngier 584101b35f7SJulien Thierry /* Tearing down NMI, only switch handler for last NMI */ 58581a43273SMarc Zyngier if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 586101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_irq; 587101b35f7SJulien Thierry } else { 588101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_irq; 589101b35f7SJulien Thierry } 590101b35f7SJulien Thierry 591101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_DEF_PRI); 592101b35f7SJulien Thierry } 593101b35f7SJulien Thierry 594021f6537SMarc Zyngier static void gic_eoi_irq(struct irq_data *d) 595021f6537SMarc Zyngier { 5966efb5092SMark Rutland write_gicreg(gic_irq(d), ICC_EOIR1_EL1); 5976efb5092SMark Rutland isb(); 598021f6537SMarc Zyngier } 599021f6537SMarc Zyngier 6000b6a3da9SMarc Zyngier static void gic_eoimode1_eoi_irq(struct irq_data *d) 6010b6a3da9SMarc Zyngier { 6020b6a3da9SMarc Zyngier /* 603530bf353SMarc Zyngier * No need to deactivate an LPI, or an interrupt that 604530bf353SMarc Zyngier * is is getting forwarded to a vcpu. 6050b6a3da9SMarc Zyngier */ 6064df7f54dSThomas Gleixner if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 6070b6a3da9SMarc Zyngier return; 6080b6a3da9SMarc Zyngier gic_write_dir(gic_irq(d)); 6090b6a3da9SMarc Zyngier } 6100b6a3da9SMarc Zyngier 611021f6537SMarc Zyngier static int gic_set_type(struct irq_data *d, unsigned int type) 612021f6537SMarc Zyngier { 6135f51f803SMarc Zyngier enum gic_intid_range range; 614021f6537SMarc Zyngier unsigned int irq = gic_irq(d); 615021f6537SMarc Zyngier void __iomem *base; 616e91b036eSMarc Zyngier u32 offset, index; 61713d22e2eSMarc Zyngier int ret; 618021f6537SMarc Zyngier 6195f51f803SMarc Zyngier range = get_intid_range(d); 6205f51f803SMarc Zyngier 62164b499d8SMarc Zyngier /* Interrupt configuration for SGIs can't be changed */ 62264b499d8SMarc Zyngier if (range == SGI_RANGE) 62364b499d8SMarc Zyngier return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 62464b499d8SMarc Zyngier 625fb7e7debSLiviu Dudau /* SPIs have restrictions on the supported types */ 6265f51f803SMarc Zyngier if ((range == SPI_RANGE || range == ESPI_RANGE) && 6275f51f803SMarc Zyngier type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 628021f6537SMarc Zyngier return -EINVAL; 629021f6537SMarc Zyngier 63063f13483SMarc Zyngier if (gic_irq_in_rdist(d)) 631021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 63263f13483SMarc Zyngier else 63335727af2SShanker Donthineni base = gic_dist_base_alias(d); 634021f6537SMarc Zyngier 635e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_ICFGR, &index); 63613d22e2eSMarc Zyngier 63763f13483SMarc Zyngier ret = gic_configure_irq(index, type, base + offset, NULL); 6385f51f803SMarc Zyngier if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 63913d22e2eSMarc Zyngier /* Misconfigured PPIs are usually not fatal */ 6405f51f803SMarc Zyngier pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 64113d22e2eSMarc Zyngier ret = 0; 64213d22e2eSMarc Zyngier } 64313d22e2eSMarc Zyngier 64413d22e2eSMarc Zyngier return ret; 645021f6537SMarc Zyngier } 646021f6537SMarc Zyngier 647530bf353SMarc Zyngier static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 648530bf353SMarc Zyngier { 64964b499d8SMarc Zyngier if (get_intid_range(d) == SGI_RANGE) 65064b499d8SMarc Zyngier return -EINVAL; 65164b499d8SMarc Zyngier 6524df7f54dSThomas Gleixner if (vcpu) 6534df7f54dSThomas Gleixner irqd_set_forwarded_to_vcpu(d); 6544df7f54dSThomas Gleixner else 6554df7f54dSThomas Gleixner irqd_clr_forwarded_to_vcpu(d); 656530bf353SMarc Zyngier return 0; 657530bf353SMarc Zyngier } 658530bf353SMarc Zyngier 659f6c86a41SJean-Philippe Brucker static u64 gic_mpidr_to_affinity(unsigned long mpidr) 660021f6537SMarc Zyngier { 661021f6537SMarc Zyngier u64 aff; 662021f6537SMarc Zyngier 663f6c86a41SJean-Philippe Brucker aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 664021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 665021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 666021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 667021f6537SMarc Zyngier 668021f6537SMarc Zyngier return aff; 669021f6537SMarc Zyngier } 670021f6537SMarc Zyngier 671f32c9266SJulien Thierry static void gic_deactivate_unhandled(u32 irqnr) 672f32c9266SJulien Thierry { 673f32c9266SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) { 674f32c9266SJulien Thierry if (irqnr < 8192) 675f32c9266SJulien Thierry gic_write_dir(irqnr); 676f32c9266SJulien Thierry } else { 6776efb5092SMark Rutland write_gicreg(irqnr, ICC_EOIR1_EL1); 6786efb5092SMark Rutland isb(); 679f32c9266SJulien Thierry } 680f32c9266SJulien Thierry } 681f32c9266SJulien Thierry 682f32c9266SJulien Thierry /* 6836efb5092SMark Rutland * Follow a read of the IAR with any HW maintenance that needs to happen prior 6846efb5092SMark Rutland * to invoking the relevant IRQ handler. We must do two things: 6856efb5092SMark Rutland * 6866efb5092SMark Rutland * (1) Ensure instruction ordering between a read of IAR and subsequent 6876efb5092SMark Rutland * instructions in the IRQ handler using an ISB. 6886efb5092SMark Rutland * 6896efb5092SMark Rutland * It is possible for the IAR to report an IRQ which was signalled *after* 6906efb5092SMark Rutland * the CPU took an IRQ exception as multiple interrupts can race to be 6916efb5092SMark Rutland * recognized by the GIC, earlier interrupts could be withdrawn, and/or 6926efb5092SMark Rutland * later interrupts could be prioritized by the GIC. 6936efb5092SMark Rutland * 6946efb5092SMark Rutland * For devices which are tightly coupled to the CPU, such as PMUs, a 6956efb5092SMark Rutland * context synchronization event is necessary to ensure that system 6966efb5092SMark Rutland * register state is not stale, as these may have been indirectly written 6976efb5092SMark Rutland * *after* exception entry. 6986efb5092SMark Rutland * 6996efb5092SMark Rutland * (2) Deactivate the interrupt when EOI mode 1 is in use. 700f32c9266SJulien Thierry */ 7016efb5092SMark Rutland static inline void gic_complete_ack(u32 irqnr) 7026efb5092SMark Rutland { 7036efb5092SMark Rutland if (static_branch_likely(&supports_deactivate_key)) 7046efb5092SMark Rutland write_gicreg(irqnr, ICC_EOIR1_EL1); 70517ce302fSJulien Thierry 7066efb5092SMark Rutland isb(); 7076efb5092SMark Rutland } 7086efb5092SMark Rutland 709614ab80cSMark Rutland static bool gic_rpr_is_nmi_prio(void) 710f32c9266SJulien Thierry { 711614ab80cSMark Rutland if (!gic_supports_nmi()) 712614ab80cSMark Rutland return false; 713f32c9266SJulien Thierry 714614ab80cSMark Rutland return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI)); 715614ab80cSMark Rutland } 716614ab80cSMark Rutland 717614ab80cSMark Rutland static bool gic_irqnr_is_special(u32 irqnr) 718614ab80cSMark Rutland { 719614ab80cSMark Rutland return irqnr >= 1020 && irqnr <= 1023; 720614ab80cSMark Rutland } 721614ab80cSMark Rutland 722614ab80cSMark Rutland static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs) 723614ab80cSMark Rutland { 724614ab80cSMark Rutland if (gic_irqnr_is_special(irqnr)) 725614ab80cSMark Rutland return; 726f32c9266SJulien Thierry 7276efb5092SMark Rutland gic_complete_ack(irqnr); 728adf14453SMark Rutland 729614ab80cSMark Rutland if (generic_handle_domain_irq(gic_data.domain, irqnr)) { 730614ab80cSMark Rutland WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr); 731f32c9266SJulien Thierry gic_deactivate_unhandled(irqnr); 732614ab80cSMark Rutland } 733614ab80cSMark Rutland } 73417ce302fSJulien Thierry 735614ab80cSMark Rutland static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 736614ab80cSMark Rutland { 737614ab80cSMark Rutland if (gic_irqnr_is_special(irqnr)) 738614ab80cSMark Rutland return; 739614ab80cSMark Rutland 740614ab80cSMark Rutland gic_complete_ack(irqnr); 741614ab80cSMark Rutland 742614ab80cSMark Rutland if (generic_handle_domain_nmi(gic_data.domain, irqnr)) { 743614ab80cSMark Rutland WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr); 744614ab80cSMark Rutland gic_deactivate_unhandled(irqnr); 745614ab80cSMark Rutland } 746614ab80cSMark Rutland } 747614ab80cSMark Rutland 748614ab80cSMark Rutland /* 749614ab80cSMark Rutland * An exception has been taken from a context with IRQs enabled, and this could 750614ab80cSMark Rutland * be an IRQ or an NMI. 751614ab80cSMark Rutland * 752614ab80cSMark Rutland * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear 753614ab80cSMark Rutland * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning, 754614ab80cSMark Rutland * after handling any NMI but before handling any IRQ. 755614ab80cSMark Rutland * 756614ab80cSMark Rutland * The entry code has performed IRQ entry, and if an NMI is detected we must 757614ab80cSMark Rutland * perform NMI entry/exit around invoking the handler. 758614ab80cSMark Rutland */ 759614ab80cSMark Rutland static void __gic_handle_irq_from_irqson(struct pt_regs *regs) 760614ab80cSMark Rutland { 761614ab80cSMark Rutland bool is_nmi; 762614ab80cSMark Rutland u32 irqnr; 763614ab80cSMark Rutland 764614ab80cSMark Rutland irqnr = gic_read_iar(); 765614ab80cSMark Rutland 766614ab80cSMark Rutland is_nmi = gic_rpr_is_nmi_prio(); 767614ab80cSMark Rutland 768614ab80cSMark Rutland if (is_nmi) { 769614ab80cSMark Rutland nmi_enter(); 770614ab80cSMark Rutland __gic_handle_nmi(irqnr, regs); 77117ce302fSJulien Thierry nmi_exit(); 772f32c9266SJulien Thierry } 773f32c9266SJulien Thierry 774614ab80cSMark Rutland if (gic_prio_masking_enabled()) { 775614ab80cSMark Rutland gic_pmr_mask_irqs(); 776614ab80cSMark Rutland gic_arch_enable_irqs(); 777614ab80cSMark Rutland } 778382e6e17SMarc Zyngier 779614ab80cSMark Rutland if (!is_nmi) 780614ab80cSMark Rutland __gic_handle_irq(irqnr, regs); 781614ab80cSMark Rutland } 782614ab80cSMark Rutland 783614ab80cSMark Rutland /* 784614ab80cSMark Rutland * An exception has been taken from a context with IRQs disabled, which can only 785614ab80cSMark Rutland * be an NMI. 786614ab80cSMark Rutland * 787614ab80cSMark Rutland * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave 788614ab80cSMark Rutland * DAIF.IF (and ICC_PMR_EL1) unchanged. 789614ab80cSMark Rutland * 790614ab80cSMark Rutland * The entry code has performed NMI entry. 791614ab80cSMark Rutland */ 792614ab80cSMark Rutland static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) 793614ab80cSMark Rutland { 794382e6e17SMarc Zyngier u64 pmr; 795614ab80cSMark Rutland u32 irqnr; 796382e6e17SMarc Zyngier 797382e6e17SMarc Zyngier /* 798382e6e17SMarc Zyngier * We were in a context with IRQs disabled. However, the 799382e6e17SMarc Zyngier * entry code has set PMR to a value that allows any 800382e6e17SMarc Zyngier * interrupt to be acknowledged, and not just NMIs. This can 801382e6e17SMarc Zyngier * lead to surprising effects if the NMI has been retired in 802382e6e17SMarc Zyngier * the meantime, and that there is an IRQ pending. The IRQ 803382e6e17SMarc Zyngier * would then be taken in NMI context, something that nobody 804382e6e17SMarc Zyngier * wants to debug twice. 805382e6e17SMarc Zyngier * 806382e6e17SMarc Zyngier * Until we sort this, drop PMR again to a level that will 807382e6e17SMarc Zyngier * actually only allow NMIs before reading IAR, and then 808382e6e17SMarc Zyngier * restore it to what it was. 809382e6e17SMarc Zyngier */ 810382e6e17SMarc Zyngier pmr = gic_read_pmr(); 811382e6e17SMarc Zyngier gic_pmr_mask_irqs(); 812382e6e17SMarc Zyngier isb(); 813614ab80cSMark Rutland irqnr = gic_read_iar(); 814382e6e17SMarc Zyngier gic_write_pmr(pmr); 815382e6e17SMarc Zyngier 816614ab80cSMark Rutland __gic_handle_nmi(irqnr, regs); 817382e6e17SMarc Zyngier } 818382e6e17SMarc Zyngier 819021f6537SMarc Zyngier static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 820021f6537SMarc Zyngier { 821614ab80cSMark Rutland if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) 822614ab80cSMark Rutland __gic_handle_irq_from_irqsoff(regs); 82339a06b67SWill Deacon else 824614ab80cSMark Rutland __gic_handle_irq_from_irqson(regs); 825021f6537SMarc Zyngier } 826021f6537SMarc Zyngier 827b5cf6073SJulien Thierry static u32 gic_get_pribits(void) 828b5cf6073SJulien Thierry { 829b5cf6073SJulien Thierry u32 pribits; 830b5cf6073SJulien Thierry 831b5cf6073SJulien Thierry pribits = gic_read_ctlr(); 832b5cf6073SJulien Thierry pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 833b5cf6073SJulien Thierry pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 834b5cf6073SJulien Thierry pribits++; 835b5cf6073SJulien Thierry 836b5cf6073SJulien Thierry return pribits; 837b5cf6073SJulien Thierry } 838b5cf6073SJulien Thierry 839b5cf6073SJulien Thierry static bool gic_has_group0(void) 840b5cf6073SJulien Thierry { 841b5cf6073SJulien Thierry u32 val; 842e7932188SJulien Thierry u32 old_pmr; 843e7932188SJulien Thierry 844e7932188SJulien Thierry old_pmr = gic_read_pmr(); 845b5cf6073SJulien Thierry 846b5cf6073SJulien Thierry /* 847b5cf6073SJulien Thierry * Let's find out if Group0 is under control of EL3 or not by 848b5cf6073SJulien Thierry * setting the highest possible, non-zero priority in PMR. 849b5cf6073SJulien Thierry * 850b5cf6073SJulien Thierry * If SCR_EL3.FIQ is set, the priority gets shifted down in 851b5cf6073SJulien Thierry * order for the CPU interface to set bit 7, and keep the 852b5cf6073SJulien Thierry * actual priority in the non-secure range. In the process, it 853b5cf6073SJulien Thierry * looses the least significant bit and the actual priority 854b5cf6073SJulien Thierry * becomes 0x80. Reading it back returns 0, indicating that 855b5cf6073SJulien Thierry * we're don't have access to Group0. 856b5cf6073SJulien Thierry */ 857b5cf6073SJulien Thierry gic_write_pmr(BIT(8 - gic_get_pribits())); 858b5cf6073SJulien Thierry val = gic_read_pmr(); 859b5cf6073SJulien Thierry 860e7932188SJulien Thierry gic_write_pmr(old_pmr); 861e7932188SJulien Thierry 862b5cf6073SJulien Thierry return val != 0; 863b5cf6073SJulien Thierry } 864b5cf6073SJulien Thierry 865021f6537SMarc Zyngier static void __init gic_dist_init(void) 866021f6537SMarc Zyngier { 867021f6537SMarc Zyngier unsigned int i; 868021f6537SMarc Zyngier u64 affinity; 869021f6537SMarc Zyngier void __iomem *base = gic_data.dist_base; 8700b04758bSMarc Zyngier u32 val; 871021f6537SMarc Zyngier 872021f6537SMarc Zyngier /* Disable the distributor */ 873021f6537SMarc Zyngier writel_relaxed(0, base + GICD_CTLR); 874021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 875021f6537SMarc Zyngier 8767c9b9730SMarc Zyngier /* 8777c9b9730SMarc Zyngier * Configure SPIs as non-secure Group-1. This will only matter 8787c9b9730SMarc Zyngier * if the GIC only has a single security state. This will not 8797c9b9730SMarc Zyngier * do the right thing if the kernel is running in secure mode, 8807c9b9730SMarc Zyngier * but that's not the intended use case anyway. 8817c9b9730SMarc Zyngier */ 882211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i += 32) 8837c9b9730SMarc Zyngier writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 8847c9b9730SMarc Zyngier 885211bddd2SMarc Zyngier /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 886211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) { 887211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 888211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 889211bddd2SMarc Zyngier } 890211bddd2SMarc Zyngier 891211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) 892211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 893211bddd2SMarc Zyngier 894211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 16) 895211bddd2SMarc Zyngier writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 896211bddd2SMarc Zyngier 897211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 4) 898211bddd2SMarc Zyngier writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 899211bddd2SMarc Zyngier 90063f13483SMarc Zyngier /* Now do the common stuff */ 90163f13483SMarc Zyngier gic_dist_config(base, GIC_LINE_NR, NULL); 902021f6537SMarc Zyngier 9030b04758bSMarc Zyngier val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 9040b04758bSMarc Zyngier if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 9050b04758bSMarc Zyngier pr_info("Enabling SGIs without active state\n"); 9060b04758bSMarc Zyngier val |= GICD_CTLR_nASSGIreq; 9070b04758bSMarc Zyngier } 9080b04758bSMarc Zyngier 90963f13483SMarc Zyngier /* Enable distributor with ARE, Group1, and wait for it to drain */ 9100b04758bSMarc Zyngier writel_relaxed(val, base + GICD_CTLR); 91163f13483SMarc Zyngier gic_dist_wait_for_rwp(); 912021f6537SMarc Zyngier 913021f6537SMarc Zyngier /* 914021f6537SMarc Zyngier * Set all global interrupts to the boot CPU only. ARE must be 915021f6537SMarc Zyngier * enabled. 916021f6537SMarc Zyngier */ 917021f6537SMarc Zyngier affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 918211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i++) 91972c97126SJean-Philippe Brucker gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 920211bddd2SMarc Zyngier 921211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i++) 922211bddd2SMarc Zyngier gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 923021f6537SMarc Zyngier } 924021f6537SMarc Zyngier 9250d94ded2SMarc Zyngier static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 926021f6537SMarc Zyngier { 9270d94ded2SMarc Zyngier int ret = -ENODEV; 928021f6537SMarc Zyngier int i; 929021f6537SMarc Zyngier 930f5c1434cSMarc Zyngier for (i = 0; i < gic_data.nr_redist_regions; i++) { 931f5c1434cSMarc Zyngier void __iomem *ptr = gic_data.redist_regions[i].redist_base; 9320d94ded2SMarc Zyngier u64 typer; 933021f6537SMarc Zyngier u32 reg; 934021f6537SMarc Zyngier 935021f6537SMarc Zyngier reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 936021f6537SMarc Zyngier if (reg != GIC_PIDR2_ARCH_GICv3 && 937021f6537SMarc Zyngier reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 938021f6537SMarc Zyngier pr_warn("No redistributor present @%p\n", ptr); 939021f6537SMarc Zyngier break; 940021f6537SMarc Zyngier } 941021f6537SMarc Zyngier 942021f6537SMarc Zyngier do { 94372c97126SJean-Philippe Brucker typer = gic_read_typer(ptr + GICR_TYPER); 9440d94ded2SMarc Zyngier ret = fn(gic_data.redist_regions + i, ptr); 9450d94ded2SMarc Zyngier if (!ret) 946021f6537SMarc Zyngier return 0; 947021f6537SMarc Zyngier 948b70fb7afSTomasz Nowicki if (gic_data.redist_regions[i].single_redist) 949b70fb7afSTomasz Nowicki break; 950b70fb7afSTomasz Nowicki 951021f6537SMarc Zyngier if (gic_data.redist_stride) { 952021f6537SMarc Zyngier ptr += gic_data.redist_stride; 953021f6537SMarc Zyngier } else { 954021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 955021f6537SMarc Zyngier if (typer & GICR_TYPER_VLPIS) 956021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 957021f6537SMarc Zyngier } 958021f6537SMarc Zyngier } while (!(typer & GICR_TYPER_LAST)); 959021f6537SMarc Zyngier } 960021f6537SMarc Zyngier 9610d94ded2SMarc Zyngier return ret ? -ENODEV : 0; 9620d94ded2SMarc Zyngier } 9630d94ded2SMarc Zyngier 9640d94ded2SMarc Zyngier static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 9650d94ded2SMarc Zyngier { 9660d94ded2SMarc Zyngier unsigned long mpidr = cpu_logical_map(smp_processor_id()); 9670d94ded2SMarc Zyngier u64 typer; 9680d94ded2SMarc Zyngier u32 aff; 9690d94ded2SMarc Zyngier 9700d94ded2SMarc Zyngier /* 9710d94ded2SMarc Zyngier * Convert affinity to a 32bit value that can be matched to 9720d94ded2SMarc Zyngier * GICR_TYPER bits [63:32]. 9730d94ded2SMarc Zyngier */ 9740d94ded2SMarc Zyngier aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 9750d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 9760d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 9770d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 9780d94ded2SMarc Zyngier 9790d94ded2SMarc Zyngier typer = gic_read_typer(ptr + GICR_TYPER); 9800d94ded2SMarc Zyngier if ((typer >> 32) == aff) { 9810d94ded2SMarc Zyngier u64 offset = ptr - region->redist_base; 9829058a4e9SMarc Zyngier raw_spin_lock_init(&gic_data_rdist()->rd_lock); 9830d94ded2SMarc Zyngier gic_data_rdist_rd_base() = ptr; 9840d94ded2SMarc Zyngier gic_data_rdist()->phys_base = region->phys_base + offset; 9850d94ded2SMarc Zyngier 9860d94ded2SMarc Zyngier pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 9870d94ded2SMarc Zyngier smp_processor_id(), mpidr, 9880d94ded2SMarc Zyngier (int)(region - gic_data.redist_regions), 9890d94ded2SMarc Zyngier &gic_data_rdist()->phys_base); 9900d94ded2SMarc Zyngier return 0; 9910d94ded2SMarc Zyngier } 9920d94ded2SMarc Zyngier 9930d94ded2SMarc Zyngier /* Try next one */ 9940d94ded2SMarc Zyngier return 1; 9950d94ded2SMarc Zyngier } 9960d94ded2SMarc Zyngier 9970d94ded2SMarc Zyngier static int gic_populate_rdist(void) 9980d94ded2SMarc Zyngier { 9990d94ded2SMarc Zyngier if (gic_iterate_rdists(__gic_populate_rdist) == 0) 10000d94ded2SMarc Zyngier return 0; 10010d94ded2SMarc Zyngier 1002021f6537SMarc Zyngier /* We couldn't even deal with ourselves... */ 1003f6c86a41SJean-Philippe Brucker WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 10040d94ded2SMarc Zyngier smp_processor_id(), 10050d94ded2SMarc Zyngier (unsigned long)cpu_logical_map(smp_processor_id())); 1006021f6537SMarc Zyngier return -ENODEV; 1007021f6537SMarc Zyngier } 1008021f6537SMarc Zyngier 10091a60e1e6SMarc Zyngier static int __gic_update_rdist_properties(struct redist_region *region, 10100edc23eaSMarc Zyngier void __iomem *ptr) 10110edc23eaSMarc Zyngier { 10120edc23eaSMarc Zyngier u64 typer = gic_read_typer(ptr + GICR_TYPER); 1013a837ed36SMarc Zyngier u32 ctlr = readl_relaxed(ptr + GICR_CTLR); 1014b25319d2SMarc Zyngier 10154d968297SZhiyuan Dai /* Boot-time cleanup */ 101679a7f77bSMarc Zyngier if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { 101779a7f77bSMarc Zyngier u64 val; 101879a7f77bSMarc Zyngier 101979a7f77bSMarc Zyngier /* Deactivate any present vPE */ 102079a7f77bSMarc Zyngier val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); 102179a7f77bSMarc Zyngier if (val & GICR_VPENDBASER_Valid) 102279a7f77bSMarc Zyngier gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 102379a7f77bSMarc Zyngier ptr + SZ_128K + GICR_VPENDBASER); 102479a7f77bSMarc Zyngier 102579a7f77bSMarc Zyngier /* Mark the VPE table as invalid */ 102679a7f77bSMarc Zyngier val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); 102779a7f77bSMarc Zyngier val &= ~GICR_VPROPBASER_4_1_VALID; 102879a7f77bSMarc Zyngier gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); 102979a7f77bSMarc Zyngier } 103079a7f77bSMarc Zyngier 10310edc23eaSMarc Zyngier gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 1032b25319d2SMarc Zyngier 1033a837ed36SMarc Zyngier /* 1034a837ed36SMarc Zyngier * TYPER.RVPEID implies some form of DirectLPI, no matter what the 1035a837ed36SMarc Zyngier * doc says... :-/ And CTLR.IR implies another subset of DirectLPI 1036a837ed36SMarc Zyngier * that the ITS driver can make use of for LPIs (and not VLPIs). 1037a837ed36SMarc Zyngier * 1038a837ed36SMarc Zyngier * These are 3 different ways to express the same thing, depending 1039a837ed36SMarc Zyngier * on the revision of the architecture and its relaxations over 1040a837ed36SMarc Zyngier * time. Just group them under the 'direct_lpi' banner. 1041a837ed36SMarc Zyngier */ 1042b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 1043b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 1044a837ed36SMarc Zyngier !!(ctlr & GICR_CTLR_IR) | 1045b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid); 104696806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 1047b25319d2SMarc Zyngier 1048b25319d2SMarc Zyngier /* Detect non-sensical configurations */ 1049b25319d2SMarc Zyngier if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 1050b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi = false; 1051b25319d2SMarc Zyngier gic_data.rdists.has_vlpis = false; 1052b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = false; 1053b25319d2SMarc Zyngier } 1054b25319d2SMarc Zyngier 10555f51f803SMarc Zyngier gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 10560edc23eaSMarc Zyngier 10570edc23eaSMarc Zyngier return 1; 10580edc23eaSMarc Zyngier } 10590edc23eaSMarc Zyngier 10601a60e1e6SMarc Zyngier static void gic_update_rdist_properties(void) 10610edc23eaSMarc Zyngier { 10621a60e1e6SMarc Zyngier gic_data.ppi_nr = UINT_MAX; 10631a60e1e6SMarc Zyngier gic_iterate_rdists(__gic_update_rdist_properties); 10641a60e1e6SMarc Zyngier if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 10651a60e1e6SMarc Zyngier gic_data.ppi_nr = 0; 1066a837ed36SMarc Zyngier pr_info("GICv3 features: %d PPIs%s%s\n", 1067a837ed36SMarc Zyngier gic_data.ppi_nr, 1068a837ed36SMarc Zyngier gic_data.has_rss ? ", RSS" : "", 1069a837ed36SMarc Zyngier gic_data.rdists.has_direct_lpi ? ", DirectLPI" : ""); 1070a837ed36SMarc Zyngier 107196806229SMarc Zyngier if (gic_data.rdists.has_vlpis) 107296806229SMarc Zyngier pr_info("GICv4 features: %s%s%s\n", 107396806229SMarc Zyngier gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 107496806229SMarc Zyngier gic_data.rdists.has_rvpeid ? "RVPEID " : "", 107596806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 10760edc23eaSMarc Zyngier } 10770edc23eaSMarc Zyngier 1078d98d0a99SJulien Thierry /* Check whether it's single security state view */ 1079d98d0a99SJulien Thierry static inline bool gic_dist_security_disabled(void) 1080d98d0a99SJulien Thierry { 1081d98d0a99SJulien Thierry return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 1082d98d0a99SJulien Thierry } 1083d98d0a99SJulien Thierry 10843708d52fSSudeep Holla static void gic_cpu_sys_reg_init(void) 1085021f6537SMarc Zyngier { 1086eda0d04aSShanker Donthineni int i, cpu = smp_processor_id(); 1087eda0d04aSShanker Donthineni u64 mpidr = cpu_logical_map(cpu); 1088eda0d04aSShanker Donthineni u64 need_rss = MPIDR_RS(mpidr); 108933625282SMarc Zyngier bool group0; 1090b5cf6073SJulien Thierry u32 pribits; 1091eda0d04aSShanker Donthineni 10927cabd008SMarc Zyngier /* 10937cabd008SMarc Zyngier * Need to check that the SRE bit has actually been set. If 10947cabd008SMarc Zyngier * not, it means that SRE is disabled at EL2. We're going to 10957cabd008SMarc Zyngier * die painfully, and there is nothing we can do about it. 10967cabd008SMarc Zyngier * 10977cabd008SMarc Zyngier * Kindly inform the luser. 10987cabd008SMarc Zyngier */ 10997cabd008SMarc Zyngier if (!gic_enable_sre()) 11007cabd008SMarc Zyngier pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 1101021f6537SMarc Zyngier 1102b5cf6073SJulien Thierry pribits = gic_get_pribits(); 110333625282SMarc Zyngier 1104b5cf6073SJulien Thierry group0 = gic_has_group0(); 110533625282SMarc Zyngier 1106021f6537SMarc Zyngier /* Set priority mask register */ 1107d98d0a99SJulien Thierry if (!gic_prio_masking_enabled()) { 110833625282SMarc Zyngier write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 110933678059SAlexandru Elisei } else if (gic_supports_nmi()) { 1110d98d0a99SJulien Thierry /* 1111d98d0a99SJulien Thierry * Mismatch configuration with boot CPU, the system is likely 1112d98d0a99SJulien Thierry * to die as interrupt masking will not work properly on all 1113d98d0a99SJulien Thierry * CPUs 111433678059SAlexandru Elisei * 111533678059SAlexandru Elisei * The boot CPU calls this function before enabling NMI support, 111633678059SAlexandru Elisei * and as a result we'll never see this warning in the boot path 111733678059SAlexandru Elisei * for that CPU. 1118d98d0a99SJulien Thierry */ 111933678059SAlexandru Elisei if (static_branch_unlikely(&gic_nonsecure_priorities)) 112033678059SAlexandru Elisei WARN_ON(!group0 || gic_dist_security_disabled()); 112133678059SAlexandru Elisei else 112233678059SAlexandru Elisei WARN_ON(group0 && !gic_dist_security_disabled()); 1123d98d0a99SJulien Thierry } 1124021f6537SMarc Zyngier 112591ef8442SDaniel Thompson /* 112691ef8442SDaniel Thompson * Some firmwares hand over to the kernel with the BPR changed from 112791ef8442SDaniel Thompson * its reset value (and with a value large enough to prevent 112891ef8442SDaniel Thompson * any pre-emptive interrupts from working at all). Writing a zero 112991ef8442SDaniel Thompson * to BPR restores is reset value. 113091ef8442SDaniel Thompson */ 113191ef8442SDaniel Thompson gic_write_bpr1(0); 113291ef8442SDaniel Thompson 1133d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) { 11340b6a3da9SMarc Zyngier /* EOI drops priority only (mode 1) */ 11350b6a3da9SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 11360b6a3da9SMarc Zyngier } else { 1137021f6537SMarc Zyngier /* EOI deactivates interrupt too (mode 0) */ 1138021f6537SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 11390b6a3da9SMarc Zyngier } 1140021f6537SMarc Zyngier 114133625282SMarc Zyngier /* Always whack Group0 before Group1 */ 114233625282SMarc Zyngier if (group0) { 114333625282SMarc Zyngier switch(pribits) { 114433625282SMarc Zyngier case 8: 114533625282SMarc Zyngier case 7: 114633625282SMarc Zyngier write_gicreg(0, ICC_AP0R3_EL1); 114733625282SMarc Zyngier write_gicreg(0, ICC_AP0R2_EL1); 1148df561f66SGustavo A. R. Silva fallthrough; 114933625282SMarc Zyngier case 6: 115033625282SMarc Zyngier write_gicreg(0, ICC_AP0R1_EL1); 1151df561f66SGustavo A. R. Silva fallthrough; 115233625282SMarc Zyngier case 5: 115333625282SMarc Zyngier case 4: 115433625282SMarc Zyngier write_gicreg(0, ICC_AP0R0_EL1); 115533625282SMarc Zyngier } 1156d6062a6dSMarc Zyngier 115733625282SMarc Zyngier isb(); 115833625282SMarc Zyngier } 115933625282SMarc Zyngier 116033625282SMarc Zyngier switch(pribits) { 1161d6062a6dSMarc Zyngier case 8: 1162d6062a6dSMarc Zyngier case 7: 1163d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R3_EL1); 1164d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R2_EL1); 1165df561f66SGustavo A. R. Silva fallthrough; 1166d6062a6dSMarc Zyngier case 6: 1167d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R1_EL1); 1168df561f66SGustavo A. R. Silva fallthrough; 1169d6062a6dSMarc Zyngier case 5: 1170d6062a6dSMarc Zyngier case 4: 1171d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R0_EL1); 1172d6062a6dSMarc Zyngier } 1173d6062a6dSMarc Zyngier 1174d6062a6dSMarc Zyngier isb(); 1175d6062a6dSMarc Zyngier 1176021f6537SMarc Zyngier /* ... and let's hit the road... */ 1177021f6537SMarc Zyngier gic_write_grpen1(1); 1178eda0d04aSShanker Donthineni 1179eda0d04aSShanker Donthineni /* Keep the RSS capability status in per_cpu variable */ 1180eda0d04aSShanker Donthineni per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1181eda0d04aSShanker Donthineni 1182eda0d04aSShanker Donthineni /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1183eda0d04aSShanker Donthineni for_each_online_cpu(i) { 1184eda0d04aSShanker Donthineni bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1185eda0d04aSShanker Donthineni 1186eda0d04aSShanker Donthineni need_rss |= MPIDR_RS(cpu_logical_map(i)); 1187eda0d04aSShanker Donthineni if (need_rss && (!have_rss)) 1188eda0d04aSShanker Donthineni pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1189eda0d04aSShanker Donthineni cpu, (unsigned long)mpidr, 1190eda0d04aSShanker Donthineni i, (unsigned long)cpu_logical_map(i)); 1191eda0d04aSShanker Donthineni } 1192eda0d04aSShanker Donthineni 1193eda0d04aSShanker Donthineni /** 1194eda0d04aSShanker Donthineni * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1195eda0d04aSShanker Donthineni * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1196eda0d04aSShanker Donthineni * UNPREDICTABLE choice of : 1197eda0d04aSShanker Donthineni * - The write is ignored. 1198eda0d04aSShanker Donthineni * - The RS field is treated as 0. 1199eda0d04aSShanker Donthineni */ 1200eda0d04aSShanker Donthineni if (need_rss && (!gic_data.has_rss)) 1201eda0d04aSShanker Donthineni pr_crit_once("RSS is required but GICD doesn't support it\n"); 1202021f6537SMarc Zyngier } 1203021f6537SMarc Zyngier 1204f736d65dSMarc Zyngier static bool gicv3_nolpi; 1205f736d65dSMarc Zyngier 1206f736d65dSMarc Zyngier static int __init gicv3_nolpi_cfg(char *buf) 1207f736d65dSMarc Zyngier { 12085e279739SChristophe JAILLET return kstrtobool(buf, &gicv3_nolpi); 1209f736d65dSMarc Zyngier } 1210f736d65dSMarc Zyngier early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1211f736d65dSMarc Zyngier 1212da33f31dSMarc Zyngier static int gic_dist_supports_lpis(void) 1213da33f31dSMarc Zyngier { 1214d38a71c5SMarc Zyngier return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1215d38a71c5SMarc Zyngier !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1216d38a71c5SMarc Zyngier !gicv3_nolpi); 1217da33f31dSMarc Zyngier } 1218da33f31dSMarc Zyngier 1219021f6537SMarc Zyngier static void gic_cpu_init(void) 1220021f6537SMarc Zyngier { 1221021f6537SMarc Zyngier void __iomem *rbase; 12221a60e1e6SMarc Zyngier int i; 1223021f6537SMarc Zyngier 1224021f6537SMarc Zyngier /* Register ourselves with the rest of the world */ 1225021f6537SMarc Zyngier if (gic_populate_rdist()) 1226021f6537SMarc Zyngier return; 1227021f6537SMarc Zyngier 1228a2c22510SSudeep Holla gic_enable_redist(true); 1229021f6537SMarc Zyngier 1230ad5a78d3SMarc Zyngier WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1231ad5a78d3SMarc Zyngier !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1232ad5a78d3SMarc Zyngier "Distributor has extended ranges, but CPU%d doesn't\n", 1233ad5a78d3SMarc Zyngier smp_processor_id()); 1234ad5a78d3SMarc Zyngier 1235021f6537SMarc Zyngier rbase = gic_data_rdist_sgi_base(); 1236021f6537SMarc Zyngier 12377c9b9730SMarc Zyngier /* Configure SGIs/PPIs as non-secure Group-1 */ 12381a60e1e6SMarc Zyngier for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 12391a60e1e6SMarc Zyngier writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 12407c9b9730SMarc Zyngier 12411a60e1e6SMarc Zyngier gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1242021f6537SMarc Zyngier 12433708d52fSSudeep Holla /* initialise system registers */ 12443708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1245021f6537SMarc Zyngier } 1246021f6537SMarc Zyngier 1247021f6537SMarc Zyngier #ifdef CONFIG_SMP 1248021f6537SMarc Zyngier 1249eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1250eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1251eda0d04aSShanker Donthineni 12526670a6d8SRichard Cochran static int gic_starting_cpu(unsigned int cpu) 12536670a6d8SRichard Cochran { 12546670a6d8SRichard Cochran gic_cpu_init(); 1255d38a71c5SMarc Zyngier 1256d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) 1257d38a71c5SMarc Zyngier its_cpu_init(); 1258d38a71c5SMarc Zyngier 12596670a6d8SRichard Cochran return 0; 12606670a6d8SRichard Cochran } 1261021f6537SMarc Zyngier 1262021f6537SMarc Zyngier static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1263f6c86a41SJean-Philippe Brucker unsigned long cluster_id) 1264021f6537SMarc Zyngier { 1265727653d6SJames Morse int next_cpu, cpu = *base_cpu; 1266f6c86a41SJean-Philippe Brucker unsigned long mpidr = cpu_logical_map(cpu); 1267021f6537SMarc Zyngier u16 tlist = 0; 1268021f6537SMarc Zyngier 1269021f6537SMarc Zyngier while (cpu < nr_cpu_ids) { 1270021f6537SMarc Zyngier tlist |= 1 << (mpidr & 0xf); 1271021f6537SMarc Zyngier 1272727653d6SJames Morse next_cpu = cpumask_next(cpu, mask); 1273727653d6SJames Morse if (next_cpu >= nr_cpu_ids) 1274021f6537SMarc Zyngier goto out; 1275727653d6SJames Morse cpu = next_cpu; 1276021f6537SMarc Zyngier 1277021f6537SMarc Zyngier mpidr = cpu_logical_map(cpu); 1278021f6537SMarc Zyngier 1279eda0d04aSShanker Donthineni if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1280021f6537SMarc Zyngier cpu--; 1281021f6537SMarc Zyngier goto out; 1282021f6537SMarc Zyngier } 1283021f6537SMarc Zyngier } 1284021f6537SMarc Zyngier out: 1285021f6537SMarc Zyngier *base_cpu = cpu; 1286021f6537SMarc Zyngier return tlist; 1287021f6537SMarc Zyngier } 1288021f6537SMarc Zyngier 12897e580278SAndre Przywara #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 12907e580278SAndre Przywara (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 12917e580278SAndre Przywara << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 12927e580278SAndre Przywara 1293021f6537SMarc Zyngier static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1294021f6537SMarc Zyngier { 1295021f6537SMarc Zyngier u64 val; 1296021f6537SMarc Zyngier 12977e580278SAndre Przywara val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 12987e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 12997e580278SAndre Przywara irq << ICC_SGI1R_SGI_ID_SHIFT | 13007e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1301eda0d04aSShanker Donthineni MPIDR_TO_SGI_RS(cluster_id) | 13027e580278SAndre Przywara tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1303021f6537SMarc Zyngier 1304b6dd4d83SMark Salter pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1305021f6537SMarc Zyngier gic_write_sgi1r(val); 1306021f6537SMarc Zyngier } 1307021f6537SMarc Zyngier 130864b499d8SMarc Zyngier static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1309021f6537SMarc Zyngier { 1310021f6537SMarc Zyngier int cpu; 1311021f6537SMarc Zyngier 131264b499d8SMarc Zyngier if (WARN_ON(d->hwirq >= 16)) 1313021f6537SMarc Zyngier return; 1314021f6537SMarc Zyngier 1315021f6537SMarc Zyngier /* 1316021f6537SMarc Zyngier * Ensure that stores to Normal memory are visible to the 1317021f6537SMarc Zyngier * other CPUs before issuing the IPI. 1318021f6537SMarc Zyngier */ 131980e4e1f4SBarry Song dsb(ishst); 1320021f6537SMarc Zyngier 1321f9b531feSRusty Russell for_each_cpu(cpu, mask) { 1322eda0d04aSShanker Donthineni u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1323021f6537SMarc Zyngier u16 tlist; 1324021f6537SMarc Zyngier 1325021f6537SMarc Zyngier tlist = gic_compute_target_list(&cpu, mask, cluster_id); 132664b499d8SMarc Zyngier gic_send_sgi(cluster_id, tlist, d->hwirq); 1327021f6537SMarc Zyngier } 1328021f6537SMarc Zyngier 1329021f6537SMarc Zyngier /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1330021f6537SMarc Zyngier isb(); 1331021f6537SMarc Zyngier } 1332021f6537SMarc Zyngier 13338a94c1abSIngo Rohloff static void __init gic_smp_init(void) 1334021f6537SMarc Zyngier { 133564b499d8SMarc Zyngier struct irq_fwspec sgi_fwspec = { 133664b499d8SMarc Zyngier .fwnode = gic_data.fwnode, 133764b499d8SMarc Zyngier .param_count = 1, 133864b499d8SMarc Zyngier }; 133964b499d8SMarc Zyngier int base_sgi; 134064b499d8SMarc Zyngier 13416896bcd1SThomas Gleixner cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 134273c1b41eSThomas Gleixner "irqchip/arm/gicv3:starting", 134373c1b41eSThomas Gleixner gic_starting_cpu, NULL); 134464b499d8SMarc Zyngier 134564b499d8SMarc Zyngier /* Register all 8 non-secure SGIs */ 13460e2213feSJohan Hovold base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec); 134764b499d8SMarc Zyngier if (WARN_ON(base_sgi <= 0)) 134864b499d8SMarc Zyngier return; 134964b499d8SMarc Zyngier 135064b499d8SMarc Zyngier set_smp_ipi_range(base_sgi, 8); 1351021f6537SMarc Zyngier } 1352021f6537SMarc Zyngier 1353021f6537SMarc Zyngier static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1354021f6537SMarc Zyngier bool force) 1355021f6537SMarc Zyngier { 135665a30f8bSSuzuki K Poulose unsigned int cpu; 1357e91b036eSMarc Zyngier u32 offset, index; 1358021f6537SMarc Zyngier void __iomem *reg; 1359021f6537SMarc Zyngier int enabled; 1360021f6537SMarc Zyngier u64 val; 1361021f6537SMarc Zyngier 136265a30f8bSSuzuki K Poulose if (force) 136365a30f8bSSuzuki K Poulose cpu = cpumask_first(mask_val); 136465a30f8bSSuzuki K Poulose else 136565a30f8bSSuzuki K Poulose cpu = cpumask_any_and(mask_val, cpu_online_mask); 136665a30f8bSSuzuki K Poulose 1367866d7c1bSSuzuki K Poulose if (cpu >= nr_cpu_ids) 1368866d7c1bSSuzuki K Poulose return -EINVAL; 1369866d7c1bSSuzuki K Poulose 1370021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) 1371021f6537SMarc Zyngier return -EINVAL; 1372021f6537SMarc Zyngier 1373021f6537SMarc Zyngier /* If interrupt was enabled, disable it first */ 1374021f6537SMarc Zyngier enabled = gic_peek_irq(d, GICD_ISENABLER); 1375021f6537SMarc Zyngier if (enabled) 1376021f6537SMarc Zyngier gic_mask_irq(d); 1377021f6537SMarc Zyngier 1378e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IROUTER, &index); 1379e91b036eSMarc Zyngier reg = gic_dist_base(d) + offset + (index * 8); 1380021f6537SMarc Zyngier val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 1381021f6537SMarc Zyngier 138272c97126SJean-Philippe Brucker gic_write_irouter(val, reg); 1383021f6537SMarc Zyngier 1384021f6537SMarc Zyngier /* 1385021f6537SMarc Zyngier * If the interrupt was enabled, enabled it again. Otherwise, 1386021f6537SMarc Zyngier * just wait for the distributor to have digested our changes. 1387021f6537SMarc Zyngier */ 1388021f6537SMarc Zyngier if (enabled) 1389021f6537SMarc Zyngier gic_unmask_irq(d); 1390021f6537SMarc Zyngier 1391956ae91aSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1392956ae91aSMarc Zyngier 13930fc6fa29SAntoine Tenart return IRQ_SET_MASK_OK_DONE; 1394021f6537SMarc Zyngier } 1395021f6537SMarc Zyngier #else 1396021f6537SMarc Zyngier #define gic_set_affinity NULL 139764b499d8SMarc Zyngier #define gic_ipi_send_mask NULL 1398021f6537SMarc Zyngier #define gic_smp_init() do { } while(0) 1399021f6537SMarc Zyngier #endif 1400021f6537SMarc Zyngier 140117f644e9SValentin Schneider static int gic_retrigger(struct irq_data *data) 140217f644e9SValentin Schneider { 140317f644e9SValentin Schneider return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 140417f644e9SValentin Schneider } 140517f644e9SValentin Schneider 14063708d52fSSudeep Holla #ifdef CONFIG_CPU_PM 14073708d52fSSudeep Holla static int gic_cpu_pm_notifier(struct notifier_block *self, 14083708d52fSSudeep Holla unsigned long cmd, void *v) 14093708d52fSSudeep Holla { 14103708d52fSSudeep Holla if (cmd == CPU_PM_EXIT) { 1411ccd9432aSSudeep Holla if (gic_dist_security_disabled()) 14123708d52fSSudeep Holla gic_enable_redist(true); 14133708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1414ccd9432aSSudeep Holla } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 14153708d52fSSudeep Holla gic_write_grpen1(0); 14163708d52fSSudeep Holla gic_enable_redist(false); 14173708d52fSSudeep Holla } 14183708d52fSSudeep Holla return NOTIFY_OK; 14193708d52fSSudeep Holla } 14203708d52fSSudeep Holla 14213708d52fSSudeep Holla static struct notifier_block gic_cpu_pm_notifier_block = { 14223708d52fSSudeep Holla .notifier_call = gic_cpu_pm_notifier, 14233708d52fSSudeep Holla }; 14243708d52fSSudeep Holla 14253708d52fSSudeep Holla static void gic_cpu_pm_init(void) 14263708d52fSSudeep Holla { 14273708d52fSSudeep Holla cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 14283708d52fSSudeep Holla } 14293708d52fSSudeep Holla 14303708d52fSSudeep Holla #else 14313708d52fSSudeep Holla static inline void gic_cpu_pm_init(void) { } 14323708d52fSSudeep Holla #endif /* CONFIG_CPU_PM */ 14333708d52fSSudeep Holla 1434021f6537SMarc Zyngier static struct irq_chip gic_chip = { 1435021f6537SMarc Zyngier .name = "GICv3", 1436021f6537SMarc Zyngier .irq_mask = gic_mask_irq, 1437021f6537SMarc Zyngier .irq_unmask = gic_unmask_irq, 1438021f6537SMarc Zyngier .irq_eoi = gic_eoi_irq, 1439021f6537SMarc Zyngier .irq_set_type = gic_set_type, 1440021f6537SMarc Zyngier .irq_set_affinity = gic_set_affinity, 144117f644e9SValentin Schneider .irq_retrigger = gic_retrigger, 1442b594c6e2SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1443b594c6e2SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1444101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1445101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 144664b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask, 14474110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 14484110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 14494110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 1450021f6537SMarc Zyngier }; 1451021f6537SMarc Zyngier 14520b6a3da9SMarc Zyngier static struct irq_chip gic_eoimode1_chip = { 14530b6a3da9SMarc Zyngier .name = "GICv3", 14540b6a3da9SMarc Zyngier .irq_mask = gic_eoimode1_mask_irq, 14550b6a3da9SMarc Zyngier .irq_unmask = gic_unmask_irq, 14560b6a3da9SMarc Zyngier .irq_eoi = gic_eoimode1_eoi_irq, 14570b6a3da9SMarc Zyngier .irq_set_type = gic_set_type, 14580b6a3da9SMarc Zyngier .irq_set_affinity = gic_set_affinity, 145917f644e9SValentin Schneider .irq_retrigger = gic_retrigger, 14600b6a3da9SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 14610b6a3da9SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1462530bf353SMarc Zyngier .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1463101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1464101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 146564b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask, 14664110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 14674110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 14684110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 14690b6a3da9SMarc Zyngier }; 14700b6a3da9SMarc Zyngier 1471021f6537SMarc Zyngier static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1472021f6537SMarc Zyngier irq_hw_number_t hw) 1473021f6537SMarc Zyngier { 14740b6a3da9SMarc Zyngier struct irq_chip *chip = &gic_chip; 14751b57d91bSValentin Schneider struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 14760b6a3da9SMarc Zyngier 1477d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 14780b6a3da9SMarc Zyngier chip = &gic_eoimode1_chip; 14790b6a3da9SMarc Zyngier 1480e91b036eSMarc Zyngier switch (__get_intid_range(hw)) { 148170a29c32SMarc Zyngier case SGI_RANGE: 1482e91b036eSMarc Zyngier case PPI_RANGE: 14835f51f803SMarc Zyngier case EPPI_RANGE: 1484021f6537SMarc Zyngier irq_set_percpu_devid(irq); 14850b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1486443acc4fSMarc Zyngier handle_percpu_devid_irq, NULL, NULL); 1487e91b036eSMarc Zyngier break; 1488e91b036eSMarc Zyngier 1489e91b036eSMarc Zyngier case SPI_RANGE: 1490211bddd2SMarc Zyngier case ESPI_RANGE: 14910b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1492443acc4fSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1493d17cab44SRob Herring irq_set_probe(irq); 14941b57d91bSValentin Schneider irqd_set_single_target(irqd); 1495e91b036eSMarc Zyngier break; 1496e91b036eSMarc Zyngier 1497e91b036eSMarc Zyngier case LPI_RANGE: 1498da33f31dSMarc Zyngier if (!gic_dist_supports_lpis()) 1499da33f31dSMarc Zyngier return -EPERM; 15000b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1501da33f31dSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1502e91b036eSMarc Zyngier break; 1503e91b036eSMarc Zyngier 1504e91b036eSMarc Zyngier default: 1505e91b036eSMarc Zyngier return -EPERM; 1506da33f31dSMarc Zyngier } 1507da33f31dSMarc Zyngier 15081b57d91bSValentin Schneider /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 15091b57d91bSValentin Schneider irqd_set_handle_enforce_irqctx(irqd); 1510021f6537SMarc Zyngier return 0; 1511021f6537SMarc Zyngier } 1512021f6537SMarc Zyngier 1513f833f57fSMarc Zyngier static int gic_irq_domain_translate(struct irq_domain *d, 1514f833f57fSMarc Zyngier struct irq_fwspec *fwspec, 1515f833f57fSMarc Zyngier unsigned long *hwirq, 1516f833f57fSMarc Zyngier unsigned int *type) 1517021f6537SMarc Zyngier { 151864b499d8SMarc Zyngier if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 151964b499d8SMarc Zyngier *hwirq = fwspec->param[0]; 152064b499d8SMarc Zyngier *type = IRQ_TYPE_EDGE_RISING; 152164b499d8SMarc Zyngier return 0; 152264b499d8SMarc Zyngier } 152364b499d8SMarc Zyngier 1524f833f57fSMarc Zyngier if (is_of_node(fwspec->fwnode)) { 1525f833f57fSMarc Zyngier if (fwspec->param_count < 3) 1526021f6537SMarc Zyngier return -EINVAL; 1527021f6537SMarc Zyngier 1528db8c70ecSMarc Zyngier switch (fwspec->param[0]) { 1529db8c70ecSMarc Zyngier case 0: /* SPI */ 1530db8c70ecSMarc Zyngier *hwirq = fwspec->param[1] + 32; 1531db8c70ecSMarc Zyngier break; 1532db8c70ecSMarc Zyngier case 1: /* PPI */ 1533f833f57fSMarc Zyngier *hwirq = fwspec->param[1] + 16; 1534db8c70ecSMarc Zyngier break; 1535211bddd2SMarc Zyngier case 2: /* ESPI */ 1536211bddd2SMarc Zyngier *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1537211bddd2SMarc Zyngier break; 15385f51f803SMarc Zyngier case 3: /* EPPI */ 15395f51f803SMarc Zyngier *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 15405f51f803SMarc Zyngier break; 1541db8c70ecSMarc Zyngier case GIC_IRQ_TYPE_LPI: /* LPI */ 1542db8c70ecSMarc Zyngier *hwirq = fwspec->param[1]; 1543db8c70ecSMarc Zyngier break; 15445f51f803SMarc Zyngier case GIC_IRQ_TYPE_PARTITION: 15455f51f803SMarc Zyngier *hwirq = fwspec->param[1]; 15465f51f803SMarc Zyngier if (fwspec->param[1] >= 16) 15475f51f803SMarc Zyngier *hwirq += EPPI_BASE_INTID - 16; 15485f51f803SMarc Zyngier else 15495f51f803SMarc Zyngier *hwirq += 16; 15505f51f803SMarc Zyngier break; 1551db8c70ecSMarc Zyngier default: 1552db8c70ecSMarc Zyngier return -EINVAL; 1553db8c70ecSMarc Zyngier } 1554f833f57fSMarc Zyngier 1555f833f57fSMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 15566ef6386eSMarc Zyngier 155765da7d19SMarc Zyngier /* 155865da7d19SMarc Zyngier * Make it clear that broken DTs are... broken. 1559a359f757SIngo Molnar * Partitioned PPIs are an unfortunate exception. 156065da7d19SMarc Zyngier */ 156165da7d19SMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE && 156265da7d19SMarc Zyngier fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1563f833f57fSMarc Zyngier return 0; 1564021f6537SMarc Zyngier } 1565021f6537SMarc Zyngier 1566ffa7d616STomasz Nowicki if (is_fwnode_irqchip(fwspec->fwnode)) { 1567ffa7d616STomasz Nowicki if(fwspec->param_count != 2) 1568ffa7d616STomasz Nowicki return -EINVAL; 1569ffa7d616STomasz Nowicki 1570544808f7SAndre Przywara if (fwspec->param[0] < 16) { 1571544808f7SAndre Przywara pr_err(FW_BUG "Illegal GSI%d translation request\n", 1572544808f7SAndre Przywara fwspec->param[0]); 1573544808f7SAndre Przywara return -EINVAL; 1574544808f7SAndre Przywara } 1575544808f7SAndre Przywara 1576ffa7d616STomasz Nowicki *hwirq = fwspec->param[0]; 1577ffa7d616STomasz Nowicki *type = fwspec->param[1]; 15786ef6386eSMarc Zyngier 15796ef6386eSMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE); 1580ffa7d616STomasz Nowicki return 0; 1581ffa7d616STomasz Nowicki } 1582ffa7d616STomasz Nowicki 1583f833f57fSMarc Zyngier return -EINVAL; 1584021f6537SMarc Zyngier } 1585021f6537SMarc Zyngier 1586443acc4fSMarc Zyngier static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1587443acc4fSMarc Zyngier unsigned int nr_irqs, void *arg) 1588443acc4fSMarc Zyngier { 1589443acc4fSMarc Zyngier int i, ret; 1590443acc4fSMarc Zyngier irq_hw_number_t hwirq; 1591443acc4fSMarc Zyngier unsigned int type = IRQ_TYPE_NONE; 1592f833f57fSMarc Zyngier struct irq_fwspec *fwspec = arg; 1593443acc4fSMarc Zyngier 1594f833f57fSMarc Zyngier ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1595443acc4fSMarc Zyngier if (ret) 1596443acc4fSMarc Zyngier return ret; 1597443acc4fSMarc Zyngier 159863c16c6eSSuzuki K Poulose for (i = 0; i < nr_irqs; i++) { 159963c16c6eSSuzuki K Poulose ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 160063c16c6eSSuzuki K Poulose if (ret) 160163c16c6eSSuzuki K Poulose return ret; 160263c16c6eSSuzuki K Poulose } 1603443acc4fSMarc Zyngier 1604443acc4fSMarc Zyngier return 0; 1605443acc4fSMarc Zyngier } 1606443acc4fSMarc Zyngier 1607443acc4fSMarc Zyngier static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1608443acc4fSMarc Zyngier unsigned int nr_irqs) 1609443acc4fSMarc Zyngier { 1610443acc4fSMarc Zyngier int i; 1611443acc4fSMarc Zyngier 1612443acc4fSMarc Zyngier for (i = 0; i < nr_irqs; i++) { 1613443acc4fSMarc Zyngier struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1614443acc4fSMarc Zyngier irq_set_handler(virq + i, NULL); 1615443acc4fSMarc Zyngier irq_domain_reset_irq_data(d); 1616443acc4fSMarc Zyngier } 1617443acc4fSMarc Zyngier } 1618443acc4fSMarc Zyngier 1619d753f849SJames Morse static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, 1620d753f849SJames Morse irq_hw_number_t hwirq) 1621d753f849SJames Morse { 1622d753f849SJames Morse enum gic_intid_range range; 1623d753f849SJames Morse 1624d753f849SJames Morse if (!gic_data.ppi_descs) 1625d753f849SJames Morse return false; 1626d753f849SJames Morse 1627d753f849SJames Morse if (!is_of_node(fwspec->fwnode)) 1628d753f849SJames Morse return false; 1629d753f849SJames Morse 1630d753f849SJames Morse if (fwspec->param_count < 4 || !fwspec->param[3]) 1631d753f849SJames Morse return false; 1632d753f849SJames Morse 1633d753f849SJames Morse range = __get_intid_range(hwirq); 1634d753f849SJames Morse if (range != PPI_RANGE && range != EPPI_RANGE) 1635d753f849SJames Morse return false; 1636d753f849SJames Morse 1637d753f849SJames Morse return true; 1638d753f849SJames Morse } 1639d753f849SJames Morse 1640e3825ba1SMarc Zyngier static int gic_irq_domain_select(struct irq_domain *d, 1641e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1642e3825ba1SMarc Zyngier enum irq_domain_bus_token bus_token) 1643e3825ba1SMarc Zyngier { 1644d753f849SJames Morse unsigned int type, ret, ppi_idx; 1645d753f849SJames Morse irq_hw_number_t hwirq; 1646d753f849SJames Morse 1647e3825ba1SMarc Zyngier /* Not for us */ 1648e3825ba1SMarc Zyngier if (fwspec->fwnode != d->fwnode) 1649e3825ba1SMarc Zyngier return 0; 1650e3825ba1SMarc Zyngier 1651e3825ba1SMarc Zyngier /* If this is not DT, then we have a single domain */ 1652e3825ba1SMarc Zyngier if (!is_of_node(fwspec->fwnode)) 1653e3825ba1SMarc Zyngier return 1; 1654e3825ba1SMarc Zyngier 1655d753f849SJames Morse ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type); 1656d753f849SJames Morse if (WARN_ON_ONCE(ret)) 1657d753f849SJames Morse return 0; 1658d753f849SJames Morse 1659d753f849SJames Morse if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) 1660d753f849SJames Morse return d == gic_data.domain; 1661d753f849SJames Morse 1662e3825ba1SMarc Zyngier /* 1663e3825ba1SMarc Zyngier * If this is a PPI and we have a 4th (non-null) parameter, 1664e3825ba1SMarc Zyngier * then we need to match the partition domain. 1665e3825ba1SMarc Zyngier */ 1666d753f849SJames Morse ppi_idx = __gic_get_ppi_index(hwirq); 1667d753f849SJames Morse return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); 1668e3825ba1SMarc Zyngier } 1669e3825ba1SMarc Zyngier 1670021f6537SMarc Zyngier static const struct irq_domain_ops gic_irq_domain_ops = { 1671f833f57fSMarc Zyngier .translate = gic_irq_domain_translate, 1672443acc4fSMarc Zyngier .alloc = gic_irq_domain_alloc, 1673443acc4fSMarc Zyngier .free = gic_irq_domain_free, 1674e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1675e3825ba1SMarc Zyngier }; 1676e3825ba1SMarc Zyngier 1677e3825ba1SMarc Zyngier static int partition_domain_translate(struct irq_domain *d, 1678e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1679e3825ba1SMarc Zyngier unsigned long *hwirq, 1680e3825ba1SMarc Zyngier unsigned int *type) 1681e3825ba1SMarc Zyngier { 1682d753f849SJames Morse unsigned long ppi_intid; 1683e3825ba1SMarc Zyngier struct device_node *np; 1684d753f849SJames Morse unsigned int ppi_idx; 1685e3825ba1SMarc Zyngier int ret; 1686e3825ba1SMarc Zyngier 168752085d3fSMarc Zyngier if (!gic_data.ppi_descs) 168852085d3fSMarc Zyngier return -ENOMEM; 168952085d3fSMarc Zyngier 1690e3825ba1SMarc Zyngier np = of_find_node_by_phandle(fwspec->param[3]); 1691e3825ba1SMarc Zyngier if (WARN_ON(!np)) 1692e3825ba1SMarc Zyngier return -EINVAL; 1693e3825ba1SMarc Zyngier 1694d753f849SJames Morse ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); 1695d753f849SJames Morse if (WARN_ON_ONCE(ret)) 1696d753f849SJames Morse return 0; 1697d753f849SJames Morse 1698d753f849SJames Morse ppi_idx = __gic_get_ppi_index(ppi_intid); 1699d753f849SJames Morse ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], 1700e3825ba1SMarc Zyngier of_node_to_fwnode(np)); 1701e3825ba1SMarc Zyngier if (ret < 0) 1702e3825ba1SMarc Zyngier return ret; 1703e3825ba1SMarc Zyngier 1704e3825ba1SMarc Zyngier *hwirq = ret; 1705e3825ba1SMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1706e3825ba1SMarc Zyngier 1707e3825ba1SMarc Zyngier return 0; 1708e3825ba1SMarc Zyngier } 1709e3825ba1SMarc Zyngier 1710e3825ba1SMarc Zyngier static const struct irq_domain_ops partition_domain_ops = { 1711e3825ba1SMarc Zyngier .translate = partition_domain_translate, 1712e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1713021f6537SMarc Zyngier }; 1714021f6537SMarc Zyngier 17159c8114c2SSrinivas Kandagatla static bool gic_enable_quirk_msm8996(void *data) 17169c8114c2SSrinivas Kandagatla { 17179c8114c2SSrinivas Kandagatla struct gic_chip_data *d = data; 17189c8114c2SSrinivas Kandagatla 17199c8114c2SSrinivas Kandagatla d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 17209c8114c2SSrinivas Kandagatla 17219c8114c2SSrinivas Kandagatla return true; 17229c8114c2SSrinivas Kandagatla } 17239c8114c2SSrinivas Kandagatla 1724*44bd78ddSDouglas Anderson static bool gic_enable_quirk_mtk_gicr(void *data) 1725*44bd78ddSDouglas Anderson { 1726*44bd78ddSDouglas Anderson struct gic_chip_data *d = data; 1727*44bd78ddSDouglas Anderson 1728*44bd78ddSDouglas Anderson d->flags |= FLAGS_WORKAROUND_MTK_GICR_SAVE; 1729*44bd78ddSDouglas Anderson 1730*44bd78ddSDouglas Anderson return true; 1731*44bd78ddSDouglas Anderson } 1732*44bd78ddSDouglas Anderson 1733d01fd161SMarc Zyngier static bool gic_enable_quirk_cavium_38539(void *data) 1734d01fd161SMarc Zyngier { 1735d01fd161SMarc Zyngier struct gic_chip_data *d = data; 1736d01fd161SMarc Zyngier 1737d01fd161SMarc Zyngier d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1738d01fd161SMarc Zyngier 1739d01fd161SMarc Zyngier return true; 1740d01fd161SMarc Zyngier } 1741d01fd161SMarc Zyngier 17427f2481b3SMarc Zyngier static bool gic_enable_quirk_hip06_07(void *data) 17437f2481b3SMarc Zyngier { 17447f2481b3SMarc Zyngier struct gic_chip_data *d = data; 17457f2481b3SMarc Zyngier 17467f2481b3SMarc Zyngier /* 17477f2481b3SMarc Zyngier * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 17487f2481b3SMarc Zyngier * not being an actual ARM implementation). The saving grace is 17497f2481b3SMarc Zyngier * that GIC-600 doesn't have ESPI, so nothing to do in that case. 17507f2481b3SMarc Zyngier * HIP07 doesn't even have a proper IIDR, and still pretends to 17517f2481b3SMarc Zyngier * have ESPI. In both cases, put them right. 17527f2481b3SMarc Zyngier */ 17537f2481b3SMarc Zyngier if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 17547f2481b3SMarc Zyngier /* Zero both ESPI and the RES0 field next to it... */ 17557f2481b3SMarc Zyngier d->rdists.gicd_typer &= ~GENMASK(9, 8); 17567f2481b3SMarc Zyngier return true; 17577f2481b3SMarc Zyngier } 17587f2481b3SMarc Zyngier 17597f2481b3SMarc Zyngier return false; 17607f2481b3SMarc Zyngier } 17617f2481b3SMarc Zyngier 176235727af2SShanker Donthineni #define T241_CHIPN_MASK GENMASK_ULL(45, 44) 176335727af2SShanker Donthineni #define T241_CHIP_GICDA_OFFSET 0x1580000 176435727af2SShanker Donthineni #define SMCCC_SOC_ID_T241 0x036b0241 176535727af2SShanker Donthineni 176635727af2SShanker Donthineni static bool gic_enable_quirk_nvidia_t241(void *data) 176735727af2SShanker Donthineni { 176835727af2SShanker Donthineni s32 soc_id = arm_smccc_get_soc_id_version(); 176935727af2SShanker Donthineni unsigned long chip_bmask = 0; 177035727af2SShanker Donthineni phys_addr_t phys; 177135727af2SShanker Donthineni u32 i; 177235727af2SShanker Donthineni 177335727af2SShanker Donthineni /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */ 177435727af2SShanker Donthineni if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241)) 177535727af2SShanker Donthineni return false; 177635727af2SShanker Donthineni 177735727af2SShanker Donthineni /* Find the chips based on GICR regions PHYS addr */ 177835727af2SShanker Donthineni for (i = 0; i < gic_data.nr_redist_regions; i++) { 177935727af2SShanker Donthineni chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK, 178035727af2SShanker Donthineni (u64)gic_data.redist_regions[i].phys_base)); 178135727af2SShanker Donthineni } 178235727af2SShanker Donthineni 178335727af2SShanker Donthineni if (hweight32(chip_bmask) < 3) 178435727af2SShanker Donthineni return false; 178535727af2SShanker Donthineni 178635727af2SShanker Donthineni /* Setup GICD alias regions */ 178735727af2SShanker Donthineni for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) { 178835727af2SShanker Donthineni if (chip_bmask & BIT(i)) { 178935727af2SShanker Donthineni phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET; 179035727af2SShanker Donthineni phys |= FIELD_PREP(T241_CHIPN_MASK, i); 179135727af2SShanker Donthineni t241_dist_base_alias[i] = ioremap(phys, SZ_64K); 179235727af2SShanker Donthineni WARN_ON_ONCE(!t241_dist_base_alias[i]); 179335727af2SShanker Donthineni } 179435727af2SShanker Donthineni } 179535727af2SShanker Donthineni static_branch_enable(&gic_nvidia_t241_erratum); 179635727af2SShanker Donthineni return true; 179735727af2SShanker Donthineni } 179835727af2SShanker Donthineni 17997f2481b3SMarc Zyngier static const struct gic_quirk gic_quirks[] = { 18007f2481b3SMarc Zyngier { 18017f2481b3SMarc Zyngier .desc = "GICv3: Qualcomm MSM8996 broken firmware", 18027f2481b3SMarc Zyngier .compatible = "qcom,msm8996-gic-v3", 18037f2481b3SMarc Zyngier .init = gic_enable_quirk_msm8996, 18047f2481b3SMarc Zyngier }, 18057f2481b3SMarc Zyngier { 1806*44bd78ddSDouglas Anderson .desc = "GICv3: Mediatek Chromebook GICR save problem", 1807*44bd78ddSDouglas Anderson .property = "mediatek,broken-save-restore-fw", 1808*44bd78ddSDouglas Anderson .init = gic_enable_quirk_mtk_gicr, 1809*44bd78ddSDouglas Anderson }, 1810*44bd78ddSDouglas Anderson { 18117f2481b3SMarc Zyngier .desc = "GICv3: HIP06 erratum 161010803", 18127f2481b3SMarc Zyngier .iidr = 0x0204043b, 18137f2481b3SMarc Zyngier .mask = 0xffffffff, 18147f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 18157f2481b3SMarc Zyngier }, 18167f2481b3SMarc Zyngier { 18177f2481b3SMarc Zyngier .desc = "GICv3: HIP07 erratum 161010803", 18187f2481b3SMarc Zyngier .iidr = 0x00000000, 18197f2481b3SMarc Zyngier .mask = 0xffffffff, 18207f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 18217f2481b3SMarc Zyngier }, 18227f2481b3SMarc Zyngier { 1823d01fd161SMarc Zyngier /* 1824d01fd161SMarc Zyngier * Reserved register accesses generate a Synchronous 1825d01fd161SMarc Zyngier * External Abort. This erratum applies to: 1826d01fd161SMarc Zyngier * - ThunderX: CN88xx 1827d01fd161SMarc Zyngier * - OCTEON TX: CN83xx, CN81xx 1828d01fd161SMarc Zyngier * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1829d01fd161SMarc Zyngier */ 1830d01fd161SMarc Zyngier .desc = "GICv3: Cavium erratum 38539", 1831d01fd161SMarc Zyngier .iidr = 0xa000034c, 1832d01fd161SMarc Zyngier .mask = 0xe8f00fff, 1833d01fd161SMarc Zyngier .init = gic_enable_quirk_cavium_38539, 1834d01fd161SMarc Zyngier }, 1835d01fd161SMarc Zyngier { 183635727af2SShanker Donthineni .desc = "GICv3: NVIDIA erratum T241-FABRIC-4", 183735727af2SShanker Donthineni .iidr = 0x0402043b, 183835727af2SShanker Donthineni .mask = 0xffffffff, 183935727af2SShanker Donthineni .init = gic_enable_quirk_nvidia_t241, 184035727af2SShanker Donthineni }, 184135727af2SShanker Donthineni { 18427f2481b3SMarc Zyngier } 18437f2481b3SMarc Zyngier }; 18447f2481b3SMarc Zyngier 1845d98d0a99SJulien Thierry static void gic_enable_nmi_support(void) 1846d98d0a99SJulien Thierry { 1847101b35f7SJulien Thierry int i; 1848101b35f7SJulien Thierry 184981a43273SMarc Zyngier if (!gic_prio_masking_enabled()) 185081a43273SMarc Zyngier return; 185181a43273SMarc Zyngier 1852*44bd78ddSDouglas Anderson if (gic_data.flags & FLAGS_WORKAROUND_MTK_GICR_SAVE) { 1853*44bd78ddSDouglas Anderson pr_warn("Skipping NMI enable due to firmware issues\n"); 1854*44bd78ddSDouglas Anderson return; 1855*44bd78ddSDouglas Anderson } 1856*44bd78ddSDouglas Anderson 185781a43273SMarc Zyngier ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 185881a43273SMarc Zyngier if (!ppi_nmi_refs) 185981a43273SMarc Zyngier return; 186081a43273SMarc Zyngier 186181a43273SMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) 1862101b35f7SJulien Thierry refcount_set(&ppi_nmi_refs[i], 0); 1863101b35f7SJulien Thierry 18644e594ad1SAlexandru Elisei pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 18658bf0a804SMark Rutland gic_has_relaxed_pmr_sync() ? "relaxed" : "forced"); 1866f2266504SMarc Zyngier 186733678059SAlexandru Elisei /* 186833678059SAlexandru Elisei * How priority values are used by the GIC depends on two things: 186933678059SAlexandru Elisei * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 187033678059SAlexandru Elisei * and if Group 0 interrupts can be delivered to Linux in the non-secure 187133678059SAlexandru Elisei * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 187229517170SJason Wang * ICC_PMR_EL1 register and the priority that software assigns to 187333678059SAlexandru Elisei * interrupts: 187433678059SAlexandru Elisei * 187533678059SAlexandru Elisei * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 187633678059SAlexandru Elisei * ----------------------------------------------------------- 187733678059SAlexandru Elisei * 1 | - | unchanged | unchanged 187833678059SAlexandru Elisei * ----------------------------------------------------------- 187933678059SAlexandru Elisei * 0 | 1 | non-secure | non-secure 188033678059SAlexandru Elisei * ----------------------------------------------------------- 188133678059SAlexandru Elisei * 0 | 0 | unchanged | non-secure 188233678059SAlexandru Elisei * 188333678059SAlexandru Elisei * where non-secure means that the value is right-shifted by one and the 188433678059SAlexandru Elisei * MSB bit set, to make it fit in the non-secure priority range. 188533678059SAlexandru Elisei * 188633678059SAlexandru Elisei * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 188733678059SAlexandru Elisei * are both either modified or unchanged, we can use the same set of 188833678059SAlexandru Elisei * priorities. 188933678059SAlexandru Elisei * 189033678059SAlexandru Elisei * In the last case, where only the interrupt priorities are modified to 189133678059SAlexandru Elisei * be in the non-secure range, we use a different PMR value to mask IRQs 189233678059SAlexandru Elisei * and the rest of the values that we use remain unchanged. 189333678059SAlexandru Elisei */ 189433678059SAlexandru Elisei if (gic_has_group0() && !gic_dist_security_disabled()) 189533678059SAlexandru Elisei static_branch_enable(&gic_nonsecure_priorities); 189633678059SAlexandru Elisei 1897d98d0a99SJulien Thierry static_branch_enable(&supports_pseudo_nmis); 1898101b35f7SJulien Thierry 1899101b35f7SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) 1900101b35f7SJulien Thierry gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1901101b35f7SJulien Thierry else 1902101b35f7SJulien Thierry gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1903d98d0a99SJulien Thierry } 1904d98d0a99SJulien Thierry 190535727af2SShanker Donthineni static int __init gic_init_bases(phys_addr_t dist_phys_base, 190635727af2SShanker Donthineni void __iomem *dist_base, 1907db57d746STomasz Nowicki struct redist_region *rdist_regs, 1908db57d746STomasz Nowicki u32 nr_redist_regions, 1909db57d746STomasz Nowicki u64 redist_stride, 1910db57d746STomasz Nowicki struct fwnode_handle *handle) 1911db57d746STomasz Nowicki { 1912db57d746STomasz Nowicki u32 typer; 1913db57d746STomasz Nowicki int err; 1914db57d746STomasz Nowicki 1915db57d746STomasz Nowicki if (!is_hyp_mode_available()) 1916d01d3274SDavidlohr Bueso static_branch_disable(&supports_deactivate_key); 1917db57d746STomasz Nowicki 1918d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 1919db57d746STomasz Nowicki pr_info("GIC: Using split EOI/Deactivate mode\n"); 1920db57d746STomasz Nowicki 1921e3825ba1SMarc Zyngier gic_data.fwnode = handle; 192235727af2SShanker Donthineni gic_data.dist_phys_base = dist_phys_base; 1923db57d746STomasz Nowicki gic_data.dist_base = dist_base; 1924db57d746STomasz Nowicki gic_data.redist_regions = rdist_regs; 1925db57d746STomasz Nowicki gic_data.nr_redist_regions = nr_redist_regions; 1926db57d746STomasz Nowicki gic_data.redist_stride = redist_stride; 1927db57d746STomasz Nowicki 1928db57d746STomasz Nowicki /* 1929db57d746STomasz Nowicki * Find out how many interrupts are supported. 1930db57d746STomasz Nowicki */ 1931db57d746STomasz Nowicki typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1932a4f9edb2SMarc Zyngier gic_data.rdists.gicd_typer = typer; 19337f2481b3SMarc Zyngier 19347f2481b3SMarc Zyngier gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 19357f2481b3SMarc Zyngier gic_quirks, &gic_data); 19367f2481b3SMarc Zyngier 1937211bddd2SMarc Zyngier pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 1938211bddd2SMarc Zyngier pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 1939f2d83409SMarc Zyngier 1940d01fd161SMarc Zyngier /* 1941d01fd161SMarc Zyngier * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 1942d01fd161SMarc Zyngier * architecture spec (which says that reserved registers are RES0). 1943d01fd161SMarc Zyngier */ 1944d01fd161SMarc Zyngier if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 1945f2d83409SMarc Zyngier gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 1946f2d83409SMarc Zyngier 1947db57d746STomasz Nowicki gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1948db57d746STomasz Nowicki &gic_data); 1949db57d746STomasz Nowicki gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 195035727af2SShanker Donthineni if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) { 195135727af2SShanker Donthineni /* Disable GICv4.x features for the erratum T241-FABRIC-4 */ 1952b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = true; 19530edc23eaSMarc Zyngier gic_data.rdists.has_vlpis = true; 19540edc23eaSMarc Zyngier gic_data.rdists.has_direct_lpi = true; 195596806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty = true; 195635727af2SShanker Donthineni } 1957db57d746STomasz Nowicki 1958db57d746STomasz Nowicki if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1959db57d746STomasz Nowicki err = -ENOMEM; 1960db57d746STomasz Nowicki goto out_free; 1961db57d746STomasz Nowicki } 1962db57d746STomasz Nowicki 1963eeaa4b24Sluanshi irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1964eeaa4b24Sluanshi 1965eda0d04aSShanker Donthineni gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1966eda0d04aSShanker Donthineni 196750528752SMarc Zyngier if (typer & GICD_TYPER_MBIS) { 196850528752SMarc Zyngier err = mbi_init(handle, gic_data.domain); 196950528752SMarc Zyngier if (err) 197050528752SMarc Zyngier pr_err("Failed to initialize MBIs\n"); 197150528752SMarc Zyngier } 197250528752SMarc Zyngier 1973db57d746STomasz Nowicki set_handle_irq(gic_handle_irq); 1974db57d746STomasz Nowicki 19751a60e1e6SMarc Zyngier gic_update_rdist_properties(); 19760edc23eaSMarc Zyngier 1977db57d746STomasz Nowicki gic_dist_init(); 1978db57d746STomasz Nowicki gic_cpu_init(); 197964b499d8SMarc Zyngier gic_smp_init(); 1980db57d746STomasz Nowicki gic_cpu_pm_init(); 1981db57d746STomasz Nowicki 1982d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) { 1983d38a71c5SMarc Zyngier its_init(handle, &gic_data.rdists, gic_data.domain); 1984d38a71c5SMarc Zyngier its_cpu_init(); 1985d23bc2bcSValentin Schneider its_lpi_memreserve_init(); 198690b4c555SZeev Zilberman } else { 198790b4c555SZeev Zilberman if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 198890b4c555SZeev Zilberman gicv2m_init(handle, gic_data.domain); 1989d38a71c5SMarc Zyngier } 1990d38a71c5SMarc Zyngier 1991d98d0a99SJulien Thierry gic_enable_nmi_support(); 1992d98d0a99SJulien Thierry 1993db57d746STomasz Nowicki return 0; 1994db57d746STomasz Nowicki 1995db57d746STomasz Nowicki out_free: 1996db57d746STomasz Nowicki if (gic_data.domain) 1997db57d746STomasz Nowicki irq_domain_remove(gic_data.domain); 1998db57d746STomasz Nowicki free_percpu(gic_data.rdists.rdist); 1999db57d746STomasz Nowicki return err; 2000db57d746STomasz Nowicki } 2001db57d746STomasz Nowicki 2002db57d746STomasz Nowicki static int __init gic_validate_dist_version(void __iomem *dist_base) 2003db57d746STomasz Nowicki { 2004db57d746STomasz Nowicki u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2005db57d746STomasz Nowicki 2006db57d746STomasz Nowicki if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 2007db57d746STomasz Nowicki return -ENODEV; 2008db57d746STomasz Nowicki 2009db57d746STomasz Nowicki return 0; 2010db57d746STomasz Nowicki } 2011db57d746STomasz Nowicki 2012e3825ba1SMarc Zyngier /* Create all possible partitions at boot time */ 20137beaa24bSLinus Torvalds static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 2014e3825ba1SMarc Zyngier { 2015e3825ba1SMarc Zyngier struct device_node *parts_node, *child_part; 2016e3825ba1SMarc Zyngier int part_idx = 0, i; 2017e3825ba1SMarc Zyngier int nr_parts; 2018e3825ba1SMarc Zyngier struct partition_affinity *parts; 2019e3825ba1SMarc Zyngier 202000ee9a1cSJohan Hovold parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 2021e3825ba1SMarc Zyngier if (!parts_node) 2022e3825ba1SMarc Zyngier return; 2023e3825ba1SMarc Zyngier 202452085d3fSMarc Zyngier gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 202552085d3fSMarc Zyngier if (!gic_data.ppi_descs) 2026ec8401a4SMiaoqian Lin goto out_put_node; 202752085d3fSMarc Zyngier 2028e3825ba1SMarc Zyngier nr_parts = of_get_child_count(parts_node); 2029e3825ba1SMarc Zyngier 2030e3825ba1SMarc Zyngier if (!nr_parts) 203100ee9a1cSJohan Hovold goto out_put_node; 2032e3825ba1SMarc Zyngier 20336396bb22SKees Cook parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 2034e3825ba1SMarc Zyngier if (WARN_ON(!parts)) 203500ee9a1cSJohan Hovold goto out_put_node; 2036e3825ba1SMarc Zyngier 2037e3825ba1SMarc Zyngier for_each_child_of_node(parts_node, child_part) { 2038e3825ba1SMarc Zyngier struct partition_affinity *part; 2039e3825ba1SMarc Zyngier int n; 2040e3825ba1SMarc Zyngier 2041e3825ba1SMarc Zyngier part = &parts[part_idx]; 2042e3825ba1SMarc Zyngier 2043e3825ba1SMarc Zyngier part->partition_id = of_node_to_fwnode(child_part); 2044e3825ba1SMarc Zyngier 20452ef790dcSRob Herring pr_info("GIC: PPI partition %pOFn[%d] { ", 20462ef790dcSRob Herring child_part, part_idx); 2047e3825ba1SMarc Zyngier 2048e3825ba1SMarc Zyngier n = of_property_count_elems_of_size(child_part, "affinity", 2049e3825ba1SMarc Zyngier sizeof(u32)); 2050e3825ba1SMarc Zyngier WARN_ON(n <= 0); 2051e3825ba1SMarc Zyngier 2052e3825ba1SMarc Zyngier for (i = 0; i < n; i++) { 2053e3825ba1SMarc Zyngier int err, cpu; 2054e3825ba1SMarc Zyngier u32 cpu_phandle; 2055e3825ba1SMarc Zyngier struct device_node *cpu_node; 2056e3825ba1SMarc Zyngier 2057e3825ba1SMarc Zyngier err = of_property_read_u32_index(child_part, "affinity", 2058e3825ba1SMarc Zyngier i, &cpu_phandle); 2059e3825ba1SMarc Zyngier if (WARN_ON(err)) 2060e3825ba1SMarc Zyngier continue; 2061e3825ba1SMarc Zyngier 2062e3825ba1SMarc Zyngier cpu_node = of_find_node_by_phandle(cpu_phandle); 2063e3825ba1SMarc Zyngier if (WARN_ON(!cpu_node)) 2064e3825ba1SMarc Zyngier continue; 2065e3825ba1SMarc Zyngier 2066c08ec7daSSuzuki K Poulose cpu = of_cpu_node_to_id(cpu_node); 2067fa1ad9d4SMiaoqian Lin if (WARN_ON(cpu < 0)) { 2068fa1ad9d4SMiaoqian Lin of_node_put(cpu_node); 2069e3825ba1SMarc Zyngier continue; 2070fa1ad9d4SMiaoqian Lin } 2071e3825ba1SMarc Zyngier 2072e81f54c6SRob Herring pr_cont("%pOF[%d] ", cpu_node, cpu); 2073e3825ba1SMarc Zyngier 2074e3825ba1SMarc Zyngier cpumask_set_cpu(cpu, &part->mask); 2075fa1ad9d4SMiaoqian Lin of_node_put(cpu_node); 2076e3825ba1SMarc Zyngier } 2077e3825ba1SMarc Zyngier 2078e3825ba1SMarc Zyngier pr_cont("}\n"); 2079e3825ba1SMarc Zyngier part_idx++; 2080e3825ba1SMarc Zyngier } 2081e3825ba1SMarc Zyngier 208252085d3fSMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) { 2083e3825ba1SMarc Zyngier unsigned int irq; 2084e3825ba1SMarc Zyngier struct partition_desc *desc; 2085e3825ba1SMarc Zyngier struct irq_fwspec ppi_fwspec = { 2086e3825ba1SMarc Zyngier .fwnode = gic_data.fwnode, 2087e3825ba1SMarc Zyngier .param_count = 3, 2088e3825ba1SMarc Zyngier .param = { 208965da7d19SMarc Zyngier [0] = GIC_IRQ_TYPE_PARTITION, 2090e3825ba1SMarc Zyngier [1] = i, 2091e3825ba1SMarc Zyngier [2] = IRQ_TYPE_NONE, 2092e3825ba1SMarc Zyngier }, 2093e3825ba1SMarc Zyngier }; 2094e3825ba1SMarc Zyngier 2095e3825ba1SMarc Zyngier irq = irq_create_fwspec_mapping(&ppi_fwspec); 2096e3825ba1SMarc Zyngier if (WARN_ON(!irq)) 2097e3825ba1SMarc Zyngier continue; 2098e3825ba1SMarc Zyngier desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 2099e3825ba1SMarc Zyngier irq, &partition_domain_ops); 2100e3825ba1SMarc Zyngier if (WARN_ON(!desc)) 2101e3825ba1SMarc Zyngier continue; 2102e3825ba1SMarc Zyngier 2103e3825ba1SMarc Zyngier gic_data.ppi_descs[i] = desc; 2104e3825ba1SMarc Zyngier } 210500ee9a1cSJohan Hovold 210600ee9a1cSJohan Hovold out_put_node: 210700ee9a1cSJohan Hovold of_node_put(parts_node); 2108e3825ba1SMarc Zyngier } 2109e3825ba1SMarc Zyngier 21101839e576SJulien Grall static void __init gic_of_setup_kvm_info(struct device_node *node) 21111839e576SJulien Grall { 21121839e576SJulien Grall int ret; 21131839e576SJulien Grall struct resource r; 21141839e576SJulien Grall u32 gicv_idx; 21151839e576SJulien Grall 21161839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 21171839e576SJulien Grall 21181839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 21191839e576SJulien Grall if (!gic_v3_kvm_info.maint_irq) 21201839e576SJulien Grall return; 21211839e576SJulien Grall 21221839e576SJulien Grall if (of_property_read_u32(node, "#redistributor-regions", 21231839e576SJulien Grall &gicv_idx)) 21241839e576SJulien Grall gicv_idx = 1; 21251839e576SJulien Grall 21261839e576SJulien Grall gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 21271839e576SJulien Grall ret = of_address_to_resource(node, gicv_idx, &r); 21281839e576SJulien Grall if (!ret) 21291839e576SJulien Grall gic_v3_kvm_info.vcpu = r; 21301839e576SJulien Grall 21314bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 21323c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 21330e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info); 21341839e576SJulien Grall } 21351839e576SJulien Grall 21364deb96e3SRobin Murphy static void gic_request_region(resource_size_t base, resource_size_t size, 21374deb96e3SRobin Murphy const char *name) 21384deb96e3SRobin Murphy { 21394deb96e3SRobin Murphy if (!request_mem_region(base, size, name)) 21404deb96e3SRobin Murphy pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", 21414deb96e3SRobin Murphy name, &base); 21424deb96e3SRobin Murphy } 21434deb96e3SRobin Murphy 21444deb96e3SRobin Murphy static void __iomem *gic_of_iomap(struct device_node *node, int idx, 21454deb96e3SRobin Murphy const char *name, struct resource *res) 21464deb96e3SRobin Murphy { 21474deb96e3SRobin Murphy void __iomem *base; 21484deb96e3SRobin Murphy int ret; 21494deb96e3SRobin Murphy 21504deb96e3SRobin Murphy ret = of_address_to_resource(node, idx, res); 21514deb96e3SRobin Murphy if (ret) 21524deb96e3SRobin Murphy return IOMEM_ERR_PTR(ret); 21534deb96e3SRobin Murphy 21544deb96e3SRobin Murphy gic_request_region(res->start, resource_size(res), name); 21554deb96e3SRobin Murphy base = of_iomap(node, idx); 21564deb96e3SRobin Murphy 21574deb96e3SRobin Murphy return base ?: IOMEM_ERR_PTR(-ENOMEM); 21584deb96e3SRobin Murphy } 21594deb96e3SRobin Murphy 2160021f6537SMarc Zyngier static int __init gic_of_init(struct device_node *node, struct device_node *parent) 2161021f6537SMarc Zyngier { 216235727af2SShanker Donthineni phys_addr_t dist_phys_base; 2163021f6537SMarc Zyngier void __iomem *dist_base; 2164f5c1434cSMarc Zyngier struct redist_region *rdist_regs; 21654deb96e3SRobin Murphy struct resource res; 2166021f6537SMarc Zyngier u64 redist_stride; 2167f5c1434cSMarc Zyngier u32 nr_redist_regions; 2168db57d746STomasz Nowicki int err, i; 2169021f6537SMarc Zyngier 21704deb96e3SRobin Murphy dist_base = gic_of_iomap(node, 0, "GICD", &res); 21712b2cd74aSRobin Murphy if (IS_ERR(dist_base)) { 2172e81f54c6SRob Herring pr_err("%pOF: unable to map gic dist registers\n", node); 21732b2cd74aSRobin Murphy return PTR_ERR(dist_base); 2174021f6537SMarc Zyngier } 2175021f6537SMarc Zyngier 217635727af2SShanker Donthineni dist_phys_base = res.start; 217735727af2SShanker Donthineni 2178db57d746STomasz Nowicki err = gic_validate_dist_version(dist_base); 2179db57d746STomasz Nowicki if (err) { 2180e81f54c6SRob Herring pr_err("%pOF: no distributor detected, giving up\n", node); 2181021f6537SMarc Zyngier goto out_unmap_dist; 2182021f6537SMarc Zyngier } 2183021f6537SMarc Zyngier 2184f5c1434cSMarc Zyngier if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 2185f5c1434cSMarc Zyngier nr_redist_regions = 1; 2186021f6537SMarc Zyngier 21876396bb22SKees Cook rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 21886396bb22SKees Cook GFP_KERNEL); 2189f5c1434cSMarc Zyngier if (!rdist_regs) { 2190021f6537SMarc Zyngier err = -ENOMEM; 2191021f6537SMarc Zyngier goto out_unmap_dist; 2192021f6537SMarc Zyngier } 2193021f6537SMarc Zyngier 2194f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) { 21954deb96e3SRobin Murphy rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res); 21964deb96e3SRobin Murphy if (IS_ERR(rdist_regs[i].redist_base)) { 2197e81f54c6SRob Herring pr_err("%pOF: couldn't map region %d\n", node, i); 2198021f6537SMarc Zyngier err = -ENODEV; 2199021f6537SMarc Zyngier goto out_unmap_rdist; 2200021f6537SMarc Zyngier } 2201f5c1434cSMarc Zyngier rdist_regs[i].phys_base = res.start; 2202021f6537SMarc Zyngier } 2203021f6537SMarc Zyngier 2204021f6537SMarc Zyngier if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 2205021f6537SMarc Zyngier redist_stride = 0; 2206021f6537SMarc Zyngier 2207f70fdb42SSrinivas Kandagatla gic_enable_of_quirks(node, gic_quirks, &gic_data); 2208f70fdb42SSrinivas Kandagatla 220935727af2SShanker Donthineni err = gic_init_bases(dist_phys_base, dist_base, rdist_regs, 221035727af2SShanker Donthineni nr_redist_regions, redist_stride, &node->fwnode); 2211e3825ba1SMarc Zyngier if (err) 2212e3825ba1SMarc Zyngier goto out_unmap_rdist; 2213e3825ba1SMarc Zyngier 2214e3825ba1SMarc Zyngier gic_populate_ppi_partitions(node); 2215d33a3c8cSChristoffer Dall 2216d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 22171839e576SJulien Grall gic_of_setup_kvm_info(node); 2218021f6537SMarc Zyngier return 0; 2219021f6537SMarc Zyngier 2220021f6537SMarc Zyngier out_unmap_rdist: 2221f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) 22222b2cd74aSRobin Murphy if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base)) 2223f5c1434cSMarc Zyngier iounmap(rdist_regs[i].redist_base); 2224f5c1434cSMarc Zyngier kfree(rdist_regs); 2225021f6537SMarc Zyngier out_unmap_dist: 2226021f6537SMarc Zyngier iounmap(dist_base); 2227021f6537SMarc Zyngier return err; 2228021f6537SMarc Zyngier } 2229021f6537SMarc Zyngier 2230021f6537SMarc Zyngier IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 2231ffa7d616STomasz Nowicki 2232ffa7d616STomasz Nowicki #ifdef CONFIG_ACPI 2233611f039fSJulien Grall static struct 2234611f039fSJulien Grall { 2235611f039fSJulien Grall void __iomem *dist_base; 2236611f039fSJulien Grall struct redist_region *redist_regs; 2237611f039fSJulien Grall u32 nr_redist_regions; 2238611f039fSJulien Grall bool single_redist; 2239926b5dfaSMarc Zyngier int enabled_rdists; 22401839e576SJulien Grall u32 maint_irq; 22411839e576SJulien Grall int maint_irq_mode; 22421839e576SJulien Grall phys_addr_t vcpu_base; 2243611f039fSJulien Grall } acpi_data __initdata; 2244b70fb7afSTomasz Nowicki 2245b70fb7afSTomasz Nowicki static void __init 2246b70fb7afSTomasz Nowicki gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 2247b70fb7afSTomasz Nowicki { 2248b70fb7afSTomasz Nowicki static int count = 0; 2249b70fb7afSTomasz Nowicki 2250611f039fSJulien Grall acpi_data.redist_regs[count].phys_base = phys_base; 2251611f039fSJulien Grall acpi_data.redist_regs[count].redist_base = redist_base; 2252611f039fSJulien Grall acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 2253b70fb7afSTomasz Nowicki count++; 2254b70fb7afSTomasz Nowicki } 2255ffa7d616STomasz Nowicki 2256ffa7d616STomasz Nowicki static int __init 225760574d1eSKeith Busch gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 2258ffa7d616STomasz Nowicki const unsigned long end) 2259ffa7d616STomasz Nowicki { 2260ffa7d616STomasz Nowicki struct acpi_madt_generic_redistributor *redist = 2261ffa7d616STomasz Nowicki (struct acpi_madt_generic_redistributor *)header; 2262ffa7d616STomasz Nowicki void __iomem *redist_base; 2263ffa7d616STomasz Nowicki 2264ffa7d616STomasz Nowicki redist_base = ioremap(redist->base_address, redist->length); 2265ffa7d616STomasz Nowicki if (!redist_base) { 2266ffa7d616STomasz Nowicki pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2267ffa7d616STomasz Nowicki return -ENOMEM; 2268ffa7d616STomasz Nowicki } 22694deb96e3SRobin Murphy gic_request_region(redist->base_address, redist->length, "GICR"); 2270ffa7d616STomasz Nowicki 2271b70fb7afSTomasz Nowicki gic_acpi_register_redist(redist->base_address, redist_base); 2272ffa7d616STomasz Nowicki return 0; 2273ffa7d616STomasz Nowicki } 2274ffa7d616STomasz Nowicki 2275b70fb7afSTomasz Nowicki static int __init 227660574d1eSKeith Busch gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2277b70fb7afSTomasz Nowicki const unsigned long end) 2278b70fb7afSTomasz Nowicki { 2279b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 2280b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 2281611f039fSJulien Grall u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2282b70fb7afSTomasz Nowicki u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2283b70fb7afSTomasz Nowicki void __iomem *redist_base; 2284b70fb7afSTomasz Nowicki 2285ebe2f871SShanker Donthineni /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 2286ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 2287ebe2f871SShanker Donthineni return 0; 2288ebe2f871SShanker Donthineni 2289b70fb7afSTomasz Nowicki redist_base = ioremap(gicc->gicr_base_address, size); 2290b70fb7afSTomasz Nowicki if (!redist_base) 2291b70fb7afSTomasz Nowicki return -ENOMEM; 22924deb96e3SRobin Murphy gic_request_region(gicc->gicr_base_address, size, "GICR"); 2293b70fb7afSTomasz Nowicki 2294b70fb7afSTomasz Nowicki gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2295b70fb7afSTomasz Nowicki return 0; 2296b70fb7afSTomasz Nowicki } 2297b70fb7afSTomasz Nowicki 2298b70fb7afSTomasz Nowicki static int __init gic_acpi_collect_gicr_base(void) 2299b70fb7afSTomasz Nowicki { 2300b70fb7afSTomasz Nowicki acpi_tbl_entry_handler redist_parser; 2301b70fb7afSTomasz Nowicki enum acpi_madt_type type; 2302b70fb7afSTomasz Nowicki 2303611f039fSJulien Grall if (acpi_data.single_redist) { 2304b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2305b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_gicc; 2306b70fb7afSTomasz Nowicki } else { 2307b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2308b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_redist; 2309b70fb7afSTomasz Nowicki } 2310b70fb7afSTomasz Nowicki 2311b70fb7afSTomasz Nowicki /* Collect redistributor base addresses in GICR entries */ 2312b70fb7afSTomasz Nowicki if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2313b70fb7afSTomasz Nowicki return 0; 2314b70fb7afSTomasz Nowicki 2315b70fb7afSTomasz Nowicki pr_info("No valid GICR entries exist\n"); 2316b70fb7afSTomasz Nowicki return -ENODEV; 2317b70fb7afSTomasz Nowicki } 2318b70fb7afSTomasz Nowicki 231960574d1eSKeith Busch static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2320ffa7d616STomasz Nowicki const unsigned long end) 2321ffa7d616STomasz Nowicki { 2322ffa7d616STomasz Nowicki /* Subtable presence means that redist exists, that's it */ 2323ffa7d616STomasz Nowicki return 0; 2324ffa7d616STomasz Nowicki } 2325ffa7d616STomasz Nowicki 232660574d1eSKeith Busch static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2327b70fb7afSTomasz Nowicki const unsigned long end) 2328b70fb7afSTomasz Nowicki { 2329b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 2330b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 2331b70fb7afSTomasz Nowicki 2332b70fb7afSTomasz Nowicki /* 2333b70fb7afSTomasz Nowicki * If GICC is enabled and has valid gicr base address, then it means 2334b70fb7afSTomasz Nowicki * GICR base is presented via GICC 2335b70fb7afSTomasz Nowicki */ 2336926b5dfaSMarc Zyngier if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2337926b5dfaSMarc Zyngier acpi_data.enabled_rdists++; 2338b70fb7afSTomasz Nowicki return 0; 2339926b5dfaSMarc Zyngier } 2340b70fb7afSTomasz Nowicki 2341ebe2f871SShanker Donthineni /* 2342ebe2f871SShanker Donthineni * It's perfectly valid firmware can pass disabled GICC entry, driver 2343ebe2f871SShanker Donthineni * should not treat as errors, skip the entry instead of probe fail. 2344ebe2f871SShanker Donthineni */ 2345ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 2346ebe2f871SShanker Donthineni return 0; 2347ebe2f871SShanker Donthineni 2348b70fb7afSTomasz Nowicki return -ENODEV; 2349b70fb7afSTomasz Nowicki } 2350b70fb7afSTomasz Nowicki 2351b70fb7afSTomasz Nowicki static int __init gic_acpi_count_gicr_regions(void) 2352b70fb7afSTomasz Nowicki { 2353b70fb7afSTomasz Nowicki int count; 2354b70fb7afSTomasz Nowicki 2355b70fb7afSTomasz Nowicki /* 2356b70fb7afSTomasz Nowicki * Count how many redistributor regions we have. It is not allowed 2357b70fb7afSTomasz Nowicki * to mix redistributor description, GICR and GICC subtables have to be 2358b70fb7afSTomasz Nowicki * mutually exclusive. 2359b70fb7afSTomasz Nowicki */ 2360b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2361b70fb7afSTomasz Nowicki gic_acpi_match_gicr, 0); 2362b70fb7afSTomasz Nowicki if (count > 0) { 2363611f039fSJulien Grall acpi_data.single_redist = false; 2364b70fb7afSTomasz Nowicki return count; 2365b70fb7afSTomasz Nowicki } 2366b70fb7afSTomasz Nowicki 2367b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2368b70fb7afSTomasz Nowicki gic_acpi_match_gicc, 0); 2369926b5dfaSMarc Zyngier if (count > 0) { 2370611f039fSJulien Grall acpi_data.single_redist = true; 2371926b5dfaSMarc Zyngier count = acpi_data.enabled_rdists; 2372926b5dfaSMarc Zyngier } 2373b70fb7afSTomasz Nowicki 2374b70fb7afSTomasz Nowicki return count; 2375b70fb7afSTomasz Nowicki } 2376b70fb7afSTomasz Nowicki 2377ffa7d616STomasz Nowicki static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2378ffa7d616STomasz Nowicki struct acpi_probe_entry *ape) 2379ffa7d616STomasz Nowicki { 2380ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2381ffa7d616STomasz Nowicki int count; 2382ffa7d616STomasz Nowicki 2383ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2384ffa7d616STomasz Nowicki if (dist->version != ape->driver_data) 2385ffa7d616STomasz Nowicki return false; 2386ffa7d616STomasz Nowicki 2387ffa7d616STomasz Nowicki /* We need to do that exercise anyway, the sooner the better */ 2388b70fb7afSTomasz Nowicki count = gic_acpi_count_gicr_regions(); 2389ffa7d616STomasz Nowicki if (count <= 0) 2390ffa7d616STomasz Nowicki return false; 2391ffa7d616STomasz Nowicki 2392611f039fSJulien Grall acpi_data.nr_redist_regions = count; 2393ffa7d616STomasz Nowicki return true; 2394ffa7d616STomasz Nowicki } 2395ffa7d616STomasz Nowicki 239660574d1eSKeith Busch static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 23971839e576SJulien Grall const unsigned long end) 23981839e576SJulien Grall { 23991839e576SJulien Grall struct acpi_madt_generic_interrupt *gicc = 24001839e576SJulien Grall (struct acpi_madt_generic_interrupt *)header; 24011839e576SJulien Grall int maint_irq_mode; 24021839e576SJulien Grall static int first_madt = true; 24031839e576SJulien Grall 24041839e576SJulien Grall /* Skip unusable CPUs */ 24051839e576SJulien Grall if (!(gicc->flags & ACPI_MADT_ENABLED)) 24061839e576SJulien Grall return 0; 24071839e576SJulien Grall 24081839e576SJulien Grall maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 24091839e576SJulien Grall ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 24101839e576SJulien Grall 24111839e576SJulien Grall if (first_madt) { 24121839e576SJulien Grall first_madt = false; 24131839e576SJulien Grall 24141839e576SJulien Grall acpi_data.maint_irq = gicc->vgic_interrupt; 24151839e576SJulien Grall acpi_data.maint_irq_mode = maint_irq_mode; 24161839e576SJulien Grall acpi_data.vcpu_base = gicc->gicv_base_address; 24171839e576SJulien Grall 24181839e576SJulien Grall return 0; 24191839e576SJulien Grall } 24201839e576SJulien Grall 24211839e576SJulien Grall /* 24221839e576SJulien Grall * The maintenance interrupt and GICV should be the same for every CPU 24231839e576SJulien Grall */ 24241839e576SJulien Grall if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 24251839e576SJulien Grall (acpi_data.maint_irq_mode != maint_irq_mode) || 24261839e576SJulien Grall (acpi_data.vcpu_base != gicc->gicv_base_address)) 24271839e576SJulien Grall return -EINVAL; 24281839e576SJulien Grall 24291839e576SJulien Grall return 0; 24301839e576SJulien Grall } 24311839e576SJulien Grall 24321839e576SJulien Grall static bool __init gic_acpi_collect_virt_info(void) 24331839e576SJulien Grall { 24341839e576SJulien Grall int count; 24351839e576SJulien Grall 24361839e576SJulien Grall count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 24371839e576SJulien Grall gic_acpi_parse_virt_madt_gicc, 0); 24381839e576SJulien Grall 24391839e576SJulien Grall return (count > 0); 24401839e576SJulien Grall } 24411839e576SJulien Grall 2442ffa7d616STomasz Nowicki #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 24431839e576SJulien Grall #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 24441839e576SJulien Grall #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 24451839e576SJulien Grall 24461839e576SJulien Grall static void __init gic_acpi_setup_kvm_info(void) 24471839e576SJulien Grall { 24481839e576SJulien Grall int irq; 24491839e576SJulien Grall 24501839e576SJulien Grall if (!gic_acpi_collect_virt_info()) { 24511839e576SJulien Grall pr_warn("Unable to get hardware information used for virtualization\n"); 24521839e576SJulien Grall return; 24531839e576SJulien Grall } 24541839e576SJulien Grall 24551839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 24561839e576SJulien Grall 24571839e576SJulien Grall irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 24581839e576SJulien Grall acpi_data.maint_irq_mode, 24591839e576SJulien Grall ACPI_ACTIVE_HIGH); 24601839e576SJulien Grall if (irq <= 0) 24611839e576SJulien Grall return; 24621839e576SJulien Grall 24631839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq; 24641839e576SJulien Grall 24651839e576SJulien Grall if (acpi_data.vcpu_base) { 24661839e576SJulien Grall struct resource *vcpu = &gic_v3_kvm_info.vcpu; 24671839e576SJulien Grall 24681839e576SJulien Grall vcpu->flags = IORESOURCE_MEM; 24691839e576SJulien Grall vcpu->start = acpi_data.vcpu_base; 24701839e576SJulien Grall vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 24711839e576SJulien Grall } 24721839e576SJulien Grall 24734bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 24743c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 24750e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info); 24761839e576SJulien Grall } 2477ffa7d616STomasz Nowicki 24787327b16fSMarc Zyngier static struct fwnode_handle *gsi_domain_handle; 24797327b16fSMarc Zyngier 24807327b16fSMarc Zyngier static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi) 24817327b16fSMarc Zyngier { 24827327b16fSMarc Zyngier return gsi_domain_handle; 24837327b16fSMarc Zyngier } 24847327b16fSMarc Zyngier 2485ffa7d616STomasz Nowicki static int __init 2486aba3c7edSOscar Carter gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2487ffa7d616STomasz Nowicki { 2488ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2489611f039fSJulien Grall size_t size; 2490b70fb7afSTomasz Nowicki int i, err; 2491ffa7d616STomasz Nowicki 2492ffa7d616STomasz Nowicki /* Get distributor base address */ 2493ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2494611f039fSJulien Grall acpi_data.dist_base = ioremap(dist->base_address, 2495611f039fSJulien Grall ACPI_GICV3_DIST_MEM_SIZE); 2496611f039fSJulien Grall if (!acpi_data.dist_base) { 2497ffa7d616STomasz Nowicki pr_err("Unable to map GICD registers\n"); 2498ffa7d616STomasz Nowicki return -ENOMEM; 2499ffa7d616STomasz Nowicki } 25004deb96e3SRobin Murphy gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD"); 2501ffa7d616STomasz Nowicki 2502611f039fSJulien Grall err = gic_validate_dist_version(acpi_data.dist_base); 2503ffa7d616STomasz Nowicki if (err) { 250471192a68SArvind Yadav pr_err("No distributor detected at @%p, giving up\n", 2505611f039fSJulien Grall acpi_data.dist_base); 2506ffa7d616STomasz Nowicki goto out_dist_unmap; 2507ffa7d616STomasz Nowicki } 2508ffa7d616STomasz Nowicki 2509611f039fSJulien Grall size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2510611f039fSJulien Grall acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2511611f039fSJulien Grall if (!acpi_data.redist_regs) { 2512ffa7d616STomasz Nowicki err = -ENOMEM; 2513ffa7d616STomasz Nowicki goto out_dist_unmap; 2514ffa7d616STomasz Nowicki } 2515ffa7d616STomasz Nowicki 2516b70fb7afSTomasz Nowicki err = gic_acpi_collect_gicr_base(); 2517b70fb7afSTomasz Nowicki if (err) 2518ffa7d616STomasz Nowicki goto out_redist_unmap; 2519ffa7d616STomasz Nowicki 25207327b16fSMarc Zyngier gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 25217327b16fSMarc Zyngier if (!gsi_domain_handle) { 2522ffa7d616STomasz Nowicki err = -ENOMEM; 2523ffa7d616STomasz Nowicki goto out_redist_unmap; 2524ffa7d616STomasz Nowicki } 2525ffa7d616STomasz Nowicki 252635727af2SShanker Donthineni err = gic_init_bases(dist->base_address, acpi_data.dist_base, 252735727af2SShanker Donthineni acpi_data.redist_regs, acpi_data.nr_redist_regions, 252835727af2SShanker Donthineni 0, gsi_domain_handle); 2529ffa7d616STomasz Nowicki if (err) 2530ffa7d616STomasz Nowicki goto out_fwhandle_free; 2531ffa7d616STomasz Nowicki 25327327b16fSMarc Zyngier acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id); 2533d33a3c8cSChristoffer Dall 2534d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 25351839e576SJulien Grall gic_acpi_setup_kvm_info(); 25361839e576SJulien Grall 2537ffa7d616STomasz Nowicki return 0; 2538ffa7d616STomasz Nowicki 2539ffa7d616STomasz Nowicki out_fwhandle_free: 25407327b16fSMarc Zyngier irq_domain_free_fwnode(gsi_domain_handle); 2541ffa7d616STomasz Nowicki out_redist_unmap: 2542611f039fSJulien Grall for (i = 0; i < acpi_data.nr_redist_regions; i++) 2543611f039fSJulien Grall if (acpi_data.redist_regs[i].redist_base) 2544611f039fSJulien Grall iounmap(acpi_data.redist_regs[i].redist_base); 2545611f039fSJulien Grall kfree(acpi_data.redist_regs); 2546ffa7d616STomasz Nowicki out_dist_unmap: 2547611f039fSJulien Grall iounmap(acpi_data.dist_base); 2548ffa7d616STomasz Nowicki return err; 2549ffa7d616STomasz Nowicki } 2550ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2551ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2552ffa7d616STomasz Nowicki gic_acpi_init); 2553ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2554ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2555ffa7d616STomasz Nowicki gic_acpi_init); 2556ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2557ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2558ffa7d616STomasz Nowicki gic_acpi_init); 2559ffa7d616STomasz Nowicki #endif 2560