1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2021f6537SMarc Zyngier /* 30edc23eaSMarc Zyngier * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4021f6537SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com> 5021f6537SMarc Zyngier */ 6021f6537SMarc Zyngier 768628bb8SJulien Grall #define pr_fmt(fmt) "GICv3: " fmt 868628bb8SJulien Grall 9ffa7d616STomasz Nowicki #include <linux/acpi.h> 10021f6537SMarc Zyngier #include <linux/cpu.h> 113708d52fSSudeep Holla #include <linux/cpu_pm.h> 12021f6537SMarc Zyngier #include <linux/delay.h> 13021f6537SMarc Zyngier #include <linux/interrupt.h> 14ffa7d616STomasz Nowicki #include <linux/irqdomain.h> 15021f6537SMarc Zyngier #include <linux/of.h> 16021f6537SMarc Zyngier #include <linux/of_address.h> 17021f6537SMarc Zyngier #include <linux/of_irq.h> 18021f6537SMarc Zyngier #include <linux/percpu.h> 19101b35f7SJulien Thierry #include <linux/refcount.h> 20021f6537SMarc Zyngier #include <linux/slab.h> 21021f6537SMarc Zyngier 2241a83e06SJoel Porquet #include <linux/irqchip.h> 231839e576SJulien Grall #include <linux/irqchip/arm-gic-common.h> 24021f6537SMarc Zyngier #include <linux/irqchip/arm-gic-v3.h> 25e3825ba1SMarc Zyngier #include <linux/irqchip/irq-partition-percpu.h> 26021f6537SMarc Zyngier 27021f6537SMarc Zyngier #include <asm/cputype.h> 28021f6537SMarc Zyngier #include <asm/exception.h> 29021f6537SMarc Zyngier #include <asm/smp_plat.h> 300b6a3da9SMarc Zyngier #include <asm/virt.h> 31021f6537SMarc Zyngier 32021f6537SMarc Zyngier #include "irq-gic-common.h" 33021f6537SMarc Zyngier 34f32c9266SJulien Thierry #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 35f32c9266SJulien Thierry 369c8114c2SSrinivas Kandagatla #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 37d01fd161SMarc Zyngier #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 389c8114c2SSrinivas Kandagatla 3964b499d8SMarc Zyngier #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 4064b499d8SMarc Zyngier 41f5c1434cSMarc Zyngier struct redist_region { 42f5c1434cSMarc Zyngier void __iomem *redist_base; 43f5c1434cSMarc Zyngier phys_addr_t phys_base; 44b70fb7afSTomasz Nowicki bool single_redist; 45f5c1434cSMarc Zyngier }; 46f5c1434cSMarc Zyngier 47021f6537SMarc Zyngier struct gic_chip_data { 48e3825ba1SMarc Zyngier struct fwnode_handle *fwnode; 49021f6537SMarc Zyngier void __iomem *dist_base; 50f5c1434cSMarc Zyngier struct redist_region *redist_regions; 51f5c1434cSMarc Zyngier struct rdists rdists; 52021f6537SMarc Zyngier struct irq_domain *domain; 53021f6537SMarc Zyngier u64 redist_stride; 54f5c1434cSMarc Zyngier u32 nr_redist_regions; 559c8114c2SSrinivas Kandagatla u64 flags; 56eda0d04aSShanker Donthineni bool has_rss; 571a60e1e6SMarc Zyngier unsigned int ppi_nr; 5852085d3fSMarc Zyngier struct partition_desc **ppi_descs; 59021f6537SMarc Zyngier }; 60021f6537SMarc Zyngier 61021f6537SMarc Zyngier static struct gic_chip_data gic_data __read_mostly; 62d01d3274SDavidlohr Bueso static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 63021f6537SMarc Zyngier 64211bddd2SMarc Zyngier #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 65c107d613SZenghui Yu #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 66211bddd2SMarc Zyngier #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 67211bddd2SMarc Zyngier 68d98d0a99SJulien Thierry /* 69d98d0a99SJulien Thierry * The behaviours of RPR and PMR registers differ depending on the value of 70d98d0a99SJulien Thierry * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 71d98d0a99SJulien Thierry * distributor and redistributors depends on whether security is enabled in the 72d98d0a99SJulien Thierry * GIC. 73d98d0a99SJulien Thierry * 74d98d0a99SJulien Thierry * When security is enabled, non-secure priority values from the (re)distributor 75d98d0a99SJulien Thierry * are presented to the GIC CPUIF as follow: 76d98d0a99SJulien Thierry * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 77d98d0a99SJulien Thierry * 78d4034114SLorenzo Pieralisi * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 79d98d0a99SJulien Thierry * EL1 are subject to a similar operation thus matching the priorities presented 8033678059SAlexandru Elisei * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 81d4034114SLorenzo Pieralisi * these values are unchanged by the GIC. 82d98d0a99SJulien Thierry * 83d98d0a99SJulien Thierry * see GICv3/GICv4 Architecture Specification (IHI0069D): 84d98d0a99SJulien Thierry * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 85d98d0a99SJulien Thierry * priorities. 86d98d0a99SJulien Thierry * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 87d98d0a99SJulien Thierry * interrupt. 88d98d0a99SJulien Thierry */ 89d98d0a99SJulien Thierry static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 90d98d0a99SJulien Thierry 91f2266504SMarc Zyngier /* 92f2266504SMarc Zyngier * Global static key controlling whether an update to PMR allowing more 93f2266504SMarc Zyngier * interrupts requires to be propagated to the redistributor (DSB SY). 94f2266504SMarc Zyngier * And this needs to be exported for modules to be able to enable 95f2266504SMarc Zyngier * interrupts... 96f2266504SMarc Zyngier */ 97f2266504SMarc Zyngier DEFINE_STATIC_KEY_FALSE(gic_pmr_sync); 98f2266504SMarc Zyngier EXPORT_SYMBOL(gic_pmr_sync); 99f2266504SMarc Zyngier 10033678059SAlexandru Elisei DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 10133678059SAlexandru Elisei EXPORT_SYMBOL(gic_nonsecure_priorities); 10233678059SAlexandru Elisei 1038d474deaSChen-Yu Tsai /* 1048d474deaSChen-Yu Tsai * When the Non-secure world has access to group 0 interrupts (as a 1058d474deaSChen-Yu Tsai * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will 1068d474deaSChen-Yu Tsai * return the Distributor's view of the interrupt priority. 1078d474deaSChen-Yu Tsai * 1088d474deaSChen-Yu Tsai * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority 1098d474deaSChen-Yu Tsai * written by software is moved to the Non-secure range by the Distributor. 1108d474deaSChen-Yu Tsai * 1118d474deaSChen-Yu Tsai * If both are true (which is when gic_nonsecure_priorities gets enabled), 1128d474deaSChen-Yu Tsai * we need to shift down the priority programmed by software to match it 1138d474deaSChen-Yu Tsai * against the value returned by ICC_RPR_EL1. 1148d474deaSChen-Yu Tsai */ 1158d474deaSChen-Yu Tsai #define GICD_INT_RPR_PRI(priority) \ 1168d474deaSChen-Yu Tsai ({ \ 1178d474deaSChen-Yu Tsai u32 __priority = (priority); \ 1188d474deaSChen-Yu Tsai if (static_branch_unlikely(&gic_nonsecure_priorities)) \ 1198d474deaSChen-Yu Tsai __priority = 0x80 | (__priority >> 1); \ 1208d474deaSChen-Yu Tsai \ 1218d474deaSChen-Yu Tsai __priority; \ 1228d474deaSChen-Yu Tsai }) 1238d474deaSChen-Yu Tsai 124101b35f7SJulien Thierry /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 12581a43273SMarc Zyngier static refcount_t *ppi_nmi_refs; 126101b35f7SJulien Thierry 1270e5cb777SMarc Zyngier static struct gic_kvm_info gic_v3_kvm_info __initdata; 128eda0d04aSShanker Donthineni static DEFINE_PER_CPU(bool, has_rss); 1291839e576SJulien Grall 130eda0d04aSShanker Donthineni #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 131f5c1434cSMarc Zyngier #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 132f5c1434cSMarc Zyngier #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 133021f6537SMarc Zyngier #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 134021f6537SMarc Zyngier 135021f6537SMarc Zyngier /* Our default, arbitrary priority value. Linux only uses one anyway. */ 136021f6537SMarc Zyngier #define DEFAULT_PMR_VALUE 0xf0 137021f6537SMarc Zyngier 138e91b036eSMarc Zyngier enum gic_intid_range { 13970a29c32SMarc Zyngier SGI_RANGE, 140e91b036eSMarc Zyngier PPI_RANGE, 141e91b036eSMarc Zyngier SPI_RANGE, 1425f51f803SMarc Zyngier EPPI_RANGE, 143211bddd2SMarc Zyngier ESPI_RANGE, 144e91b036eSMarc Zyngier LPI_RANGE, 145e91b036eSMarc Zyngier __INVALID_RANGE__ 146e91b036eSMarc Zyngier }; 147e91b036eSMarc Zyngier 148e91b036eSMarc Zyngier static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 149e91b036eSMarc Zyngier { 150e91b036eSMarc Zyngier switch (hwirq) { 15170a29c32SMarc Zyngier case 0 ... 15: 15270a29c32SMarc Zyngier return SGI_RANGE; 153e91b036eSMarc Zyngier case 16 ... 31: 154e91b036eSMarc Zyngier return PPI_RANGE; 155e91b036eSMarc Zyngier case 32 ... 1019: 156e91b036eSMarc Zyngier return SPI_RANGE; 1575f51f803SMarc Zyngier case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 1585f51f803SMarc Zyngier return EPPI_RANGE; 159211bddd2SMarc Zyngier case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 160211bddd2SMarc Zyngier return ESPI_RANGE; 161e91b036eSMarc Zyngier case 8192 ... GENMASK(23, 0): 162e91b036eSMarc Zyngier return LPI_RANGE; 163e91b036eSMarc Zyngier default: 164e91b036eSMarc Zyngier return __INVALID_RANGE__; 165e91b036eSMarc Zyngier } 166e91b036eSMarc Zyngier } 167e91b036eSMarc Zyngier 168e91b036eSMarc Zyngier static enum gic_intid_range get_intid_range(struct irq_data *d) 169e91b036eSMarc Zyngier { 170e91b036eSMarc Zyngier return __get_intid_range(d->hwirq); 171e91b036eSMarc Zyngier } 172e91b036eSMarc Zyngier 173021f6537SMarc Zyngier static inline unsigned int gic_irq(struct irq_data *d) 174021f6537SMarc Zyngier { 175021f6537SMarc Zyngier return d->hwirq; 176021f6537SMarc Zyngier } 177021f6537SMarc Zyngier 17870a29c32SMarc Zyngier static inline bool gic_irq_in_rdist(struct irq_data *d) 179021f6537SMarc Zyngier { 18070a29c32SMarc Zyngier switch (get_intid_range(d)) { 18170a29c32SMarc Zyngier case SGI_RANGE: 18270a29c32SMarc Zyngier case PPI_RANGE: 18370a29c32SMarc Zyngier case EPPI_RANGE: 18470a29c32SMarc Zyngier return true; 18570a29c32SMarc Zyngier default: 18670a29c32SMarc Zyngier return false; 18770a29c32SMarc Zyngier } 188021f6537SMarc Zyngier } 189021f6537SMarc Zyngier 190021f6537SMarc Zyngier static inline void __iomem *gic_dist_base(struct irq_data *d) 191021f6537SMarc Zyngier { 192e91b036eSMarc Zyngier switch (get_intid_range(d)) { 19370a29c32SMarc Zyngier case SGI_RANGE: 194e91b036eSMarc Zyngier case PPI_RANGE: 1955f51f803SMarc Zyngier case EPPI_RANGE: 196e91b036eSMarc Zyngier /* SGI+PPI -> SGI_base for this CPU */ 197021f6537SMarc Zyngier return gic_data_rdist_sgi_base(); 198021f6537SMarc Zyngier 199e91b036eSMarc Zyngier case SPI_RANGE: 200211bddd2SMarc Zyngier case ESPI_RANGE: 201e91b036eSMarc Zyngier /* SPI -> dist_base */ 202021f6537SMarc Zyngier return gic_data.dist_base; 203021f6537SMarc Zyngier 204e91b036eSMarc Zyngier default: 205021f6537SMarc Zyngier return NULL; 206021f6537SMarc Zyngier } 207e91b036eSMarc Zyngier } 208021f6537SMarc Zyngier 2090df66645SMarc Zyngier static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) 210021f6537SMarc Zyngier { 211021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 212021f6537SMarc Zyngier 2130df66645SMarc Zyngier while (readl_relaxed(base + GICD_CTLR) & bit) { 214021f6537SMarc Zyngier count--; 215021f6537SMarc Zyngier if (!count) { 216021f6537SMarc Zyngier pr_err_ratelimited("RWP timeout, gone fishing\n"); 217021f6537SMarc Zyngier return; 218021f6537SMarc Zyngier } 219021f6537SMarc Zyngier cpu_relax(); 220021f6537SMarc Zyngier udelay(1); 2212c542426SDaode Huang } 222021f6537SMarc Zyngier } 223021f6537SMarc Zyngier 224021f6537SMarc Zyngier /* Wait for completion of a distributor change */ 225021f6537SMarc Zyngier static void gic_dist_wait_for_rwp(void) 226021f6537SMarc Zyngier { 2270df66645SMarc Zyngier gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP); 228021f6537SMarc Zyngier } 229021f6537SMarc Zyngier 230021f6537SMarc Zyngier /* Wait for completion of a redistributor change */ 231021f6537SMarc Zyngier static void gic_redist_wait_for_rwp(void) 232021f6537SMarc Zyngier { 2330df66645SMarc Zyngier gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); 234021f6537SMarc Zyngier } 235021f6537SMarc Zyngier 2367936e914SJean-Philippe Brucker #ifdef CONFIG_ARM64 2376d4e11c5SRobert Richter 2386d4e11c5SRobert Richter static u64 __maybe_unused gic_read_iar(void) 2396d4e11c5SRobert Richter { 240a4023f68SSuzuki K Poulose if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 2416d4e11c5SRobert Richter return gic_read_iar_cavium_thunderx(); 2426d4e11c5SRobert Richter else 2436d4e11c5SRobert Richter return gic_read_iar_common(); 2446d4e11c5SRobert Richter } 2457936e914SJean-Philippe Brucker #endif 246021f6537SMarc Zyngier 247a2c22510SSudeep Holla static void gic_enable_redist(bool enable) 248021f6537SMarc Zyngier { 249021f6537SMarc Zyngier void __iomem *rbase; 250021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 251021f6537SMarc Zyngier u32 val; 252021f6537SMarc Zyngier 2539c8114c2SSrinivas Kandagatla if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 2549c8114c2SSrinivas Kandagatla return; 2559c8114c2SSrinivas Kandagatla 256021f6537SMarc Zyngier rbase = gic_data_rdist_rd_base(); 257021f6537SMarc Zyngier 258021f6537SMarc Zyngier val = readl_relaxed(rbase + GICR_WAKER); 259a2c22510SSudeep Holla if (enable) 260a2c22510SSudeep Holla /* Wake up this CPU redistributor */ 261021f6537SMarc Zyngier val &= ~GICR_WAKER_ProcessorSleep; 262a2c22510SSudeep Holla else 263a2c22510SSudeep Holla val |= GICR_WAKER_ProcessorSleep; 264021f6537SMarc Zyngier writel_relaxed(val, rbase + GICR_WAKER); 265021f6537SMarc Zyngier 266a2c22510SSudeep Holla if (!enable) { /* Check that GICR_WAKER is writeable */ 267a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 268a2c22510SSudeep Holla if (!(val & GICR_WAKER_ProcessorSleep)) 269a2c22510SSudeep Holla return; /* No PM support in this redistributor */ 270021f6537SMarc Zyngier } 271a2c22510SSudeep Holla 272d102eb5cSDan Carpenter while (--count) { 273a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 274cf1d9d11SAndrew Jones if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 275a2c22510SSudeep Holla break; 276021f6537SMarc Zyngier cpu_relax(); 277021f6537SMarc Zyngier udelay(1); 2782c542426SDaode Huang } 279a2c22510SSudeep Holla if (!count) 280a2c22510SSudeep Holla pr_err_ratelimited("redistributor failed to %s...\n", 281a2c22510SSudeep Holla enable ? "wakeup" : "sleep"); 282021f6537SMarc Zyngier } 283021f6537SMarc Zyngier 284021f6537SMarc Zyngier /* 285021f6537SMarc Zyngier * Routines to disable, enable, EOI and route interrupts 286021f6537SMarc Zyngier */ 287e91b036eSMarc Zyngier static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 288e91b036eSMarc Zyngier { 289e91b036eSMarc Zyngier switch (get_intid_range(d)) { 29070a29c32SMarc Zyngier case SGI_RANGE: 291e91b036eSMarc Zyngier case PPI_RANGE: 292e91b036eSMarc Zyngier case SPI_RANGE: 293e91b036eSMarc Zyngier *index = d->hwirq; 294e91b036eSMarc Zyngier return offset; 2955f51f803SMarc Zyngier case EPPI_RANGE: 2965f51f803SMarc Zyngier /* 2975f51f803SMarc Zyngier * Contrary to the ESPI range, the EPPI range is contiguous 2985f51f803SMarc Zyngier * to the PPI range in the registers, so let's adjust the 2995f51f803SMarc Zyngier * displacement accordingly. Consistency is overrated. 3005f51f803SMarc Zyngier */ 3015f51f803SMarc Zyngier *index = d->hwirq - EPPI_BASE_INTID + 32; 3025f51f803SMarc Zyngier return offset; 303211bddd2SMarc Zyngier case ESPI_RANGE: 304211bddd2SMarc Zyngier *index = d->hwirq - ESPI_BASE_INTID; 305211bddd2SMarc Zyngier switch (offset) { 306211bddd2SMarc Zyngier case GICD_ISENABLER: 307211bddd2SMarc Zyngier return GICD_ISENABLERnE; 308211bddd2SMarc Zyngier case GICD_ICENABLER: 309211bddd2SMarc Zyngier return GICD_ICENABLERnE; 310211bddd2SMarc Zyngier case GICD_ISPENDR: 311211bddd2SMarc Zyngier return GICD_ISPENDRnE; 312211bddd2SMarc Zyngier case GICD_ICPENDR: 313211bddd2SMarc Zyngier return GICD_ICPENDRnE; 314211bddd2SMarc Zyngier case GICD_ISACTIVER: 315211bddd2SMarc Zyngier return GICD_ISACTIVERnE; 316211bddd2SMarc Zyngier case GICD_ICACTIVER: 317211bddd2SMarc Zyngier return GICD_ICACTIVERnE; 318211bddd2SMarc Zyngier case GICD_IPRIORITYR: 319211bddd2SMarc Zyngier return GICD_IPRIORITYRnE; 320211bddd2SMarc Zyngier case GICD_ICFGR: 321211bddd2SMarc Zyngier return GICD_ICFGRnE; 322211bddd2SMarc Zyngier case GICD_IROUTER: 323211bddd2SMarc Zyngier return GICD_IROUTERnE; 324211bddd2SMarc Zyngier default: 325211bddd2SMarc Zyngier break; 326211bddd2SMarc Zyngier } 327211bddd2SMarc Zyngier break; 328e91b036eSMarc Zyngier default: 329e91b036eSMarc Zyngier break; 330e91b036eSMarc Zyngier } 331e91b036eSMarc Zyngier 332e91b036eSMarc Zyngier WARN_ON(1); 333e91b036eSMarc Zyngier *index = d->hwirq; 334e91b036eSMarc Zyngier return offset; 335e91b036eSMarc Zyngier } 336e91b036eSMarc Zyngier 337b594c6e2SMarc Zyngier static int gic_peek_irq(struct irq_data *d, u32 offset) 338b594c6e2SMarc Zyngier { 339b594c6e2SMarc Zyngier void __iomem *base; 340e91b036eSMarc Zyngier u32 index, mask; 341e91b036eSMarc Zyngier 342e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 343e91b036eSMarc Zyngier mask = 1 << (index % 32); 344b594c6e2SMarc Zyngier 345b594c6e2SMarc Zyngier if (gic_irq_in_rdist(d)) 346b594c6e2SMarc Zyngier base = gic_data_rdist_sgi_base(); 347b594c6e2SMarc Zyngier else 348b594c6e2SMarc Zyngier base = gic_data.dist_base; 349b594c6e2SMarc Zyngier 350e91b036eSMarc Zyngier return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 351b594c6e2SMarc Zyngier } 352b594c6e2SMarc Zyngier 353021f6537SMarc Zyngier static void gic_poke_irq(struct irq_data *d, u32 offset) 354021f6537SMarc Zyngier { 355021f6537SMarc Zyngier void __iomem *base; 356e91b036eSMarc Zyngier u32 index, mask; 357e91b036eSMarc Zyngier 358e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 359e91b036eSMarc Zyngier mask = 1 << (index % 32); 360021f6537SMarc Zyngier 36163f13483SMarc Zyngier if (gic_irq_in_rdist(d)) 362021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 36363f13483SMarc Zyngier else 364021f6537SMarc Zyngier base = gic_data.dist_base; 365021f6537SMarc Zyngier 366e91b036eSMarc Zyngier writel_relaxed(mask, base + offset + (index / 32) * 4); 367021f6537SMarc Zyngier } 368021f6537SMarc Zyngier 369021f6537SMarc Zyngier static void gic_mask_irq(struct irq_data *d) 370021f6537SMarc Zyngier { 371021f6537SMarc Zyngier gic_poke_irq(d, GICD_ICENABLER); 37263f13483SMarc Zyngier if (gic_irq_in_rdist(d)) 37363f13483SMarc Zyngier gic_redist_wait_for_rwp(); 37463f13483SMarc Zyngier else 37563f13483SMarc Zyngier gic_dist_wait_for_rwp(); 376021f6537SMarc Zyngier } 377021f6537SMarc Zyngier 3780b6a3da9SMarc Zyngier static void gic_eoimode1_mask_irq(struct irq_data *d) 3790b6a3da9SMarc Zyngier { 3800b6a3da9SMarc Zyngier gic_mask_irq(d); 381530bf353SMarc Zyngier /* 382530bf353SMarc Zyngier * When masking a forwarded interrupt, make sure it is 383530bf353SMarc Zyngier * deactivated as well. 384530bf353SMarc Zyngier * 385530bf353SMarc Zyngier * This ensures that an interrupt that is getting 386530bf353SMarc Zyngier * disabled/masked will not get "stuck", because there is 387530bf353SMarc Zyngier * noone to deactivate it (guest is being terminated). 388530bf353SMarc Zyngier */ 3894df7f54dSThomas Gleixner if (irqd_is_forwarded_to_vcpu(d)) 390530bf353SMarc Zyngier gic_poke_irq(d, GICD_ICACTIVER); 3910b6a3da9SMarc Zyngier } 3920b6a3da9SMarc Zyngier 393021f6537SMarc Zyngier static void gic_unmask_irq(struct irq_data *d) 394021f6537SMarc Zyngier { 395021f6537SMarc Zyngier gic_poke_irq(d, GICD_ISENABLER); 396021f6537SMarc Zyngier } 397021f6537SMarc Zyngier 398d98d0a99SJulien Thierry static inline bool gic_supports_nmi(void) 399d98d0a99SJulien Thierry { 400d98d0a99SJulien Thierry return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 401d98d0a99SJulien Thierry static_branch_likely(&supports_pseudo_nmis); 402d98d0a99SJulien Thierry } 403d98d0a99SJulien Thierry 404b594c6e2SMarc Zyngier static int gic_irq_set_irqchip_state(struct irq_data *d, 405b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool val) 406b594c6e2SMarc Zyngier { 407b594c6e2SMarc Zyngier u32 reg; 408b594c6e2SMarc Zyngier 40964b499d8SMarc Zyngier if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 410b594c6e2SMarc Zyngier return -EINVAL; 411b594c6e2SMarc Zyngier 412b594c6e2SMarc Zyngier switch (which) { 413b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 414b594c6e2SMarc Zyngier reg = val ? GICD_ISPENDR : GICD_ICPENDR; 415b594c6e2SMarc Zyngier break; 416b594c6e2SMarc Zyngier 417b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 418b594c6e2SMarc Zyngier reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 419b594c6e2SMarc Zyngier break; 420b594c6e2SMarc Zyngier 421b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 42263f13483SMarc Zyngier if (val) { 42363f13483SMarc Zyngier gic_mask_irq(d); 42463f13483SMarc Zyngier return 0; 42563f13483SMarc Zyngier } 42663f13483SMarc Zyngier reg = GICD_ISENABLER; 427b594c6e2SMarc Zyngier break; 428b594c6e2SMarc Zyngier 429b594c6e2SMarc Zyngier default: 430b594c6e2SMarc Zyngier return -EINVAL; 431b594c6e2SMarc Zyngier } 432b594c6e2SMarc Zyngier 433b594c6e2SMarc Zyngier gic_poke_irq(d, reg); 434b594c6e2SMarc Zyngier return 0; 435b594c6e2SMarc Zyngier } 436b594c6e2SMarc Zyngier 437b594c6e2SMarc Zyngier static int gic_irq_get_irqchip_state(struct irq_data *d, 438b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool *val) 439b594c6e2SMarc Zyngier { 440211bddd2SMarc Zyngier if (d->hwirq >= 8192) /* PPI/SPI only */ 441b594c6e2SMarc Zyngier return -EINVAL; 442b594c6e2SMarc Zyngier 443b594c6e2SMarc Zyngier switch (which) { 444b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 445b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISPENDR); 446b594c6e2SMarc Zyngier break; 447b594c6e2SMarc Zyngier 448b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 449b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISACTIVER); 450b594c6e2SMarc Zyngier break; 451b594c6e2SMarc Zyngier 452b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 453b594c6e2SMarc Zyngier *val = !gic_peek_irq(d, GICD_ISENABLER); 454b594c6e2SMarc Zyngier break; 455b594c6e2SMarc Zyngier 456b594c6e2SMarc Zyngier default: 457b594c6e2SMarc Zyngier return -EINVAL; 458b594c6e2SMarc Zyngier } 459b594c6e2SMarc Zyngier 460b594c6e2SMarc Zyngier return 0; 461b594c6e2SMarc Zyngier } 462b594c6e2SMarc Zyngier 463101b35f7SJulien Thierry static void gic_irq_set_prio(struct irq_data *d, u8 prio) 464101b35f7SJulien Thierry { 465101b35f7SJulien Thierry void __iomem *base = gic_dist_base(d); 466e91b036eSMarc Zyngier u32 offset, index; 467101b35f7SJulien Thierry 468e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 469e91b036eSMarc Zyngier 470e91b036eSMarc Zyngier writeb_relaxed(prio, base + offset + index); 471101b35f7SJulien Thierry } 472101b35f7SJulien Thierry 473bfa80ee9SJames Morse static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 47481a43273SMarc Zyngier { 475bfa80ee9SJames Morse switch (__get_intid_range(hwirq)) { 47681a43273SMarc Zyngier case PPI_RANGE: 477bfa80ee9SJames Morse return hwirq - 16; 4785f51f803SMarc Zyngier case EPPI_RANGE: 479bfa80ee9SJames Morse return hwirq - EPPI_BASE_INTID + 16; 48081a43273SMarc Zyngier default: 48181a43273SMarc Zyngier unreachable(); 48281a43273SMarc Zyngier } 48381a43273SMarc Zyngier } 48481a43273SMarc Zyngier 485bfa80ee9SJames Morse static u32 gic_get_ppi_index(struct irq_data *d) 486bfa80ee9SJames Morse { 487bfa80ee9SJames Morse return __gic_get_ppi_index(d->hwirq); 488bfa80ee9SJames Morse } 489bfa80ee9SJames Morse 490101b35f7SJulien Thierry static int gic_irq_nmi_setup(struct irq_data *d) 491101b35f7SJulien Thierry { 492101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 493101b35f7SJulien Thierry 494101b35f7SJulien Thierry if (!gic_supports_nmi()) 495101b35f7SJulien Thierry return -EINVAL; 496101b35f7SJulien Thierry 497101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 498101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 499101b35f7SJulien Thierry return -EINVAL; 500101b35f7SJulien Thierry } 501101b35f7SJulien Thierry 502101b35f7SJulien Thierry /* 503101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 504101b35f7SJulien Thierry * it should not be possible to get there 505101b35f7SJulien Thierry */ 506101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 507101b35f7SJulien Thierry return -EINVAL; 508101b35f7SJulien Thierry 509101b35f7SJulien Thierry /* desc lock should already be held */ 51081a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 51181a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 51281a43273SMarc Zyngier 513101b35f7SJulien Thierry /* Setting up PPI as NMI, only switch handler for first NMI */ 51481a43273SMarc Zyngier if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 51581a43273SMarc Zyngier refcount_set(&ppi_nmi_refs[idx], 1); 516101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 517101b35f7SJulien Thierry } 518101b35f7SJulien Thierry } else { 519101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_nmi; 520101b35f7SJulien Thierry } 521101b35f7SJulien Thierry 522101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_NMI_PRI); 523101b35f7SJulien Thierry 524101b35f7SJulien Thierry return 0; 525101b35f7SJulien Thierry } 526101b35f7SJulien Thierry 527101b35f7SJulien Thierry static void gic_irq_nmi_teardown(struct irq_data *d) 528101b35f7SJulien Thierry { 529101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 530101b35f7SJulien Thierry 531101b35f7SJulien Thierry if (WARN_ON(!gic_supports_nmi())) 532101b35f7SJulien Thierry return; 533101b35f7SJulien Thierry 534101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 535101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 536101b35f7SJulien Thierry return; 537101b35f7SJulien Thierry } 538101b35f7SJulien Thierry 539101b35f7SJulien Thierry /* 540101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 541101b35f7SJulien Thierry * it should not be possible to get there 542101b35f7SJulien Thierry */ 543101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 544101b35f7SJulien Thierry return; 545101b35f7SJulien Thierry 546101b35f7SJulien Thierry /* desc lock should already be held */ 54781a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 54881a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 54981a43273SMarc Zyngier 550101b35f7SJulien Thierry /* Tearing down NMI, only switch handler for last NMI */ 55181a43273SMarc Zyngier if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 552101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_irq; 553101b35f7SJulien Thierry } else { 554101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_irq; 555101b35f7SJulien Thierry } 556101b35f7SJulien Thierry 557101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_DEF_PRI); 558101b35f7SJulien Thierry } 559101b35f7SJulien Thierry 560021f6537SMarc Zyngier static void gic_eoi_irq(struct irq_data *d) 561021f6537SMarc Zyngier { 5626efb5092SMark Rutland write_gicreg(gic_irq(d), ICC_EOIR1_EL1); 5636efb5092SMark Rutland isb(); 564021f6537SMarc Zyngier } 565021f6537SMarc Zyngier 5660b6a3da9SMarc Zyngier static void gic_eoimode1_eoi_irq(struct irq_data *d) 5670b6a3da9SMarc Zyngier { 5680b6a3da9SMarc Zyngier /* 569530bf353SMarc Zyngier * No need to deactivate an LPI, or an interrupt that 570530bf353SMarc Zyngier * is is getting forwarded to a vcpu. 5710b6a3da9SMarc Zyngier */ 5724df7f54dSThomas Gleixner if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 5730b6a3da9SMarc Zyngier return; 5740b6a3da9SMarc Zyngier gic_write_dir(gic_irq(d)); 5750b6a3da9SMarc Zyngier } 5760b6a3da9SMarc Zyngier 577021f6537SMarc Zyngier static int gic_set_type(struct irq_data *d, unsigned int type) 578021f6537SMarc Zyngier { 5795f51f803SMarc Zyngier enum gic_intid_range range; 580021f6537SMarc Zyngier unsigned int irq = gic_irq(d); 581021f6537SMarc Zyngier void __iomem *base; 582e91b036eSMarc Zyngier u32 offset, index; 58313d22e2eSMarc Zyngier int ret; 584021f6537SMarc Zyngier 5855f51f803SMarc Zyngier range = get_intid_range(d); 5865f51f803SMarc Zyngier 58764b499d8SMarc Zyngier /* Interrupt configuration for SGIs can't be changed */ 58864b499d8SMarc Zyngier if (range == SGI_RANGE) 58964b499d8SMarc Zyngier return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 59064b499d8SMarc Zyngier 591fb7e7debSLiviu Dudau /* SPIs have restrictions on the supported types */ 5925f51f803SMarc Zyngier if ((range == SPI_RANGE || range == ESPI_RANGE) && 5935f51f803SMarc Zyngier type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 594021f6537SMarc Zyngier return -EINVAL; 595021f6537SMarc Zyngier 59663f13483SMarc Zyngier if (gic_irq_in_rdist(d)) 597021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 59863f13483SMarc Zyngier else 599021f6537SMarc Zyngier base = gic_data.dist_base; 600021f6537SMarc Zyngier 601e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_ICFGR, &index); 60213d22e2eSMarc Zyngier 60363f13483SMarc Zyngier ret = gic_configure_irq(index, type, base + offset, NULL); 6045f51f803SMarc Zyngier if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 60513d22e2eSMarc Zyngier /* Misconfigured PPIs are usually not fatal */ 6065f51f803SMarc Zyngier pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 60713d22e2eSMarc Zyngier ret = 0; 60813d22e2eSMarc Zyngier } 60913d22e2eSMarc Zyngier 61013d22e2eSMarc Zyngier return ret; 611021f6537SMarc Zyngier } 612021f6537SMarc Zyngier 613530bf353SMarc Zyngier static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 614530bf353SMarc Zyngier { 61564b499d8SMarc Zyngier if (get_intid_range(d) == SGI_RANGE) 61664b499d8SMarc Zyngier return -EINVAL; 61764b499d8SMarc Zyngier 6184df7f54dSThomas Gleixner if (vcpu) 6194df7f54dSThomas Gleixner irqd_set_forwarded_to_vcpu(d); 6204df7f54dSThomas Gleixner else 6214df7f54dSThomas Gleixner irqd_clr_forwarded_to_vcpu(d); 622530bf353SMarc Zyngier return 0; 623530bf353SMarc Zyngier } 624530bf353SMarc Zyngier 625f6c86a41SJean-Philippe Brucker static u64 gic_mpidr_to_affinity(unsigned long mpidr) 626021f6537SMarc Zyngier { 627021f6537SMarc Zyngier u64 aff; 628021f6537SMarc Zyngier 629f6c86a41SJean-Philippe Brucker aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 630021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 631021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 632021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 633021f6537SMarc Zyngier 634021f6537SMarc Zyngier return aff; 635021f6537SMarc Zyngier } 636021f6537SMarc Zyngier 637f32c9266SJulien Thierry static void gic_deactivate_unhandled(u32 irqnr) 638f32c9266SJulien Thierry { 639f32c9266SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) { 640f32c9266SJulien Thierry if (irqnr < 8192) 641f32c9266SJulien Thierry gic_write_dir(irqnr); 642f32c9266SJulien Thierry } else { 6436efb5092SMark Rutland write_gicreg(irqnr, ICC_EOIR1_EL1); 6446efb5092SMark Rutland isb(); 645f32c9266SJulien Thierry } 646f32c9266SJulien Thierry } 647f32c9266SJulien Thierry 648f32c9266SJulien Thierry /* 6496efb5092SMark Rutland * Follow a read of the IAR with any HW maintenance that needs to happen prior 6506efb5092SMark Rutland * to invoking the relevant IRQ handler. We must do two things: 6516efb5092SMark Rutland * 6526efb5092SMark Rutland * (1) Ensure instruction ordering between a read of IAR and subsequent 6536efb5092SMark Rutland * instructions in the IRQ handler using an ISB. 6546efb5092SMark Rutland * 6556efb5092SMark Rutland * It is possible for the IAR to report an IRQ which was signalled *after* 6566efb5092SMark Rutland * the CPU took an IRQ exception as multiple interrupts can race to be 6576efb5092SMark Rutland * recognized by the GIC, earlier interrupts could be withdrawn, and/or 6586efb5092SMark Rutland * later interrupts could be prioritized by the GIC. 6596efb5092SMark Rutland * 6606efb5092SMark Rutland * For devices which are tightly coupled to the CPU, such as PMUs, a 6616efb5092SMark Rutland * context synchronization event is necessary to ensure that system 6626efb5092SMark Rutland * register state is not stale, as these may have been indirectly written 6636efb5092SMark Rutland * *after* exception entry. 6646efb5092SMark Rutland * 6656efb5092SMark Rutland * (2) Deactivate the interrupt when EOI mode 1 is in use. 666f32c9266SJulien Thierry */ 6676efb5092SMark Rutland static inline void gic_complete_ack(u32 irqnr) 6686efb5092SMark Rutland { 6696efb5092SMark Rutland if (static_branch_likely(&supports_deactivate_key)) 6706efb5092SMark Rutland write_gicreg(irqnr, ICC_EOIR1_EL1); 67117ce302fSJulien Thierry 6726efb5092SMark Rutland isb(); 6736efb5092SMark Rutland } 6746efb5092SMark Rutland 675614ab80cSMark Rutland static bool gic_rpr_is_nmi_prio(void) 676f32c9266SJulien Thierry { 677614ab80cSMark Rutland if (!gic_supports_nmi()) 678614ab80cSMark Rutland return false; 679f32c9266SJulien Thierry 680614ab80cSMark Rutland return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI)); 681614ab80cSMark Rutland } 682614ab80cSMark Rutland 683614ab80cSMark Rutland static bool gic_irqnr_is_special(u32 irqnr) 684614ab80cSMark Rutland { 685614ab80cSMark Rutland return irqnr >= 1020 && irqnr <= 1023; 686614ab80cSMark Rutland } 687614ab80cSMark Rutland 688614ab80cSMark Rutland static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs) 689614ab80cSMark Rutland { 690614ab80cSMark Rutland if (gic_irqnr_is_special(irqnr)) 691614ab80cSMark Rutland return; 692f32c9266SJulien Thierry 6936efb5092SMark Rutland gic_complete_ack(irqnr); 694adf14453SMark Rutland 695614ab80cSMark Rutland if (generic_handle_domain_irq(gic_data.domain, irqnr)) { 696614ab80cSMark Rutland WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr); 697f32c9266SJulien Thierry gic_deactivate_unhandled(irqnr); 698614ab80cSMark Rutland } 699614ab80cSMark Rutland } 70017ce302fSJulien Thierry 701614ab80cSMark Rutland static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 702614ab80cSMark Rutland { 703614ab80cSMark Rutland if (gic_irqnr_is_special(irqnr)) 704614ab80cSMark Rutland return; 705614ab80cSMark Rutland 706614ab80cSMark Rutland gic_complete_ack(irqnr); 707614ab80cSMark Rutland 708614ab80cSMark Rutland if (generic_handle_domain_nmi(gic_data.domain, irqnr)) { 709614ab80cSMark Rutland WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr); 710614ab80cSMark Rutland gic_deactivate_unhandled(irqnr); 711614ab80cSMark Rutland } 712614ab80cSMark Rutland } 713614ab80cSMark Rutland 714614ab80cSMark Rutland /* 715614ab80cSMark Rutland * An exception has been taken from a context with IRQs enabled, and this could 716614ab80cSMark Rutland * be an IRQ or an NMI. 717614ab80cSMark Rutland * 718614ab80cSMark Rutland * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear 719614ab80cSMark Rutland * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning, 720614ab80cSMark Rutland * after handling any NMI but before handling any IRQ. 721614ab80cSMark Rutland * 722614ab80cSMark Rutland * The entry code has performed IRQ entry, and if an NMI is detected we must 723614ab80cSMark Rutland * perform NMI entry/exit around invoking the handler. 724614ab80cSMark Rutland */ 725614ab80cSMark Rutland static void __gic_handle_irq_from_irqson(struct pt_regs *regs) 726614ab80cSMark Rutland { 727614ab80cSMark Rutland bool is_nmi; 728614ab80cSMark Rutland u32 irqnr; 729614ab80cSMark Rutland 730614ab80cSMark Rutland irqnr = gic_read_iar(); 731614ab80cSMark Rutland 732614ab80cSMark Rutland is_nmi = gic_rpr_is_nmi_prio(); 733614ab80cSMark Rutland 734614ab80cSMark Rutland if (is_nmi) { 735614ab80cSMark Rutland nmi_enter(); 736614ab80cSMark Rutland __gic_handle_nmi(irqnr, regs); 73717ce302fSJulien Thierry nmi_exit(); 738f32c9266SJulien Thierry } 739f32c9266SJulien Thierry 740614ab80cSMark Rutland if (gic_prio_masking_enabled()) { 741614ab80cSMark Rutland gic_pmr_mask_irqs(); 742614ab80cSMark Rutland gic_arch_enable_irqs(); 743614ab80cSMark Rutland } 744382e6e17SMarc Zyngier 745614ab80cSMark Rutland if (!is_nmi) 746614ab80cSMark Rutland __gic_handle_irq(irqnr, regs); 747614ab80cSMark Rutland } 748614ab80cSMark Rutland 749614ab80cSMark Rutland /* 750614ab80cSMark Rutland * An exception has been taken from a context with IRQs disabled, which can only 751614ab80cSMark Rutland * be an NMI. 752614ab80cSMark Rutland * 753614ab80cSMark Rutland * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave 754614ab80cSMark Rutland * DAIF.IF (and ICC_PMR_EL1) unchanged. 755614ab80cSMark Rutland * 756614ab80cSMark Rutland * The entry code has performed NMI entry. 757614ab80cSMark Rutland */ 758614ab80cSMark Rutland static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) 759614ab80cSMark Rutland { 760382e6e17SMarc Zyngier u64 pmr; 761614ab80cSMark Rutland u32 irqnr; 762382e6e17SMarc Zyngier 763382e6e17SMarc Zyngier /* 764382e6e17SMarc Zyngier * We were in a context with IRQs disabled. However, the 765382e6e17SMarc Zyngier * entry code has set PMR to a value that allows any 766382e6e17SMarc Zyngier * interrupt to be acknowledged, and not just NMIs. This can 767382e6e17SMarc Zyngier * lead to surprising effects if the NMI has been retired in 768382e6e17SMarc Zyngier * the meantime, and that there is an IRQ pending. The IRQ 769382e6e17SMarc Zyngier * would then be taken in NMI context, something that nobody 770382e6e17SMarc Zyngier * wants to debug twice. 771382e6e17SMarc Zyngier * 772382e6e17SMarc Zyngier * Until we sort this, drop PMR again to a level that will 773382e6e17SMarc Zyngier * actually only allow NMIs before reading IAR, and then 774382e6e17SMarc Zyngier * restore it to what it was. 775382e6e17SMarc Zyngier */ 776382e6e17SMarc Zyngier pmr = gic_read_pmr(); 777382e6e17SMarc Zyngier gic_pmr_mask_irqs(); 778382e6e17SMarc Zyngier isb(); 779614ab80cSMark Rutland irqnr = gic_read_iar(); 780382e6e17SMarc Zyngier gic_write_pmr(pmr); 781382e6e17SMarc Zyngier 782614ab80cSMark Rutland __gic_handle_nmi(irqnr, regs); 783382e6e17SMarc Zyngier } 784382e6e17SMarc Zyngier 785021f6537SMarc Zyngier static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 786021f6537SMarc Zyngier { 787614ab80cSMark Rutland if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) 788614ab80cSMark Rutland __gic_handle_irq_from_irqsoff(regs); 78939a06b67SWill Deacon else 790614ab80cSMark Rutland __gic_handle_irq_from_irqson(regs); 791021f6537SMarc Zyngier } 792021f6537SMarc Zyngier 793b5cf6073SJulien Thierry static u32 gic_get_pribits(void) 794b5cf6073SJulien Thierry { 795b5cf6073SJulien Thierry u32 pribits; 796b5cf6073SJulien Thierry 797b5cf6073SJulien Thierry pribits = gic_read_ctlr(); 798b5cf6073SJulien Thierry pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 799b5cf6073SJulien Thierry pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 800b5cf6073SJulien Thierry pribits++; 801b5cf6073SJulien Thierry 802b5cf6073SJulien Thierry return pribits; 803b5cf6073SJulien Thierry } 804b5cf6073SJulien Thierry 805b5cf6073SJulien Thierry static bool gic_has_group0(void) 806b5cf6073SJulien Thierry { 807b5cf6073SJulien Thierry u32 val; 808e7932188SJulien Thierry u32 old_pmr; 809e7932188SJulien Thierry 810e7932188SJulien Thierry old_pmr = gic_read_pmr(); 811b5cf6073SJulien Thierry 812b5cf6073SJulien Thierry /* 813b5cf6073SJulien Thierry * Let's find out if Group0 is under control of EL3 or not by 814b5cf6073SJulien Thierry * setting the highest possible, non-zero priority in PMR. 815b5cf6073SJulien Thierry * 816b5cf6073SJulien Thierry * If SCR_EL3.FIQ is set, the priority gets shifted down in 817b5cf6073SJulien Thierry * order for the CPU interface to set bit 7, and keep the 818b5cf6073SJulien Thierry * actual priority in the non-secure range. In the process, it 819b5cf6073SJulien Thierry * looses the least significant bit and the actual priority 820b5cf6073SJulien Thierry * becomes 0x80. Reading it back returns 0, indicating that 821b5cf6073SJulien Thierry * we're don't have access to Group0. 822b5cf6073SJulien Thierry */ 823b5cf6073SJulien Thierry gic_write_pmr(BIT(8 - gic_get_pribits())); 824b5cf6073SJulien Thierry val = gic_read_pmr(); 825b5cf6073SJulien Thierry 826e7932188SJulien Thierry gic_write_pmr(old_pmr); 827e7932188SJulien Thierry 828b5cf6073SJulien Thierry return val != 0; 829b5cf6073SJulien Thierry } 830b5cf6073SJulien Thierry 831021f6537SMarc Zyngier static void __init gic_dist_init(void) 832021f6537SMarc Zyngier { 833021f6537SMarc Zyngier unsigned int i; 834021f6537SMarc Zyngier u64 affinity; 835021f6537SMarc Zyngier void __iomem *base = gic_data.dist_base; 8360b04758bSMarc Zyngier u32 val; 837021f6537SMarc Zyngier 838021f6537SMarc Zyngier /* Disable the distributor */ 839021f6537SMarc Zyngier writel_relaxed(0, base + GICD_CTLR); 840021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 841021f6537SMarc Zyngier 8427c9b9730SMarc Zyngier /* 8437c9b9730SMarc Zyngier * Configure SPIs as non-secure Group-1. This will only matter 8447c9b9730SMarc Zyngier * if the GIC only has a single security state. This will not 8457c9b9730SMarc Zyngier * do the right thing if the kernel is running in secure mode, 8467c9b9730SMarc Zyngier * but that's not the intended use case anyway. 8477c9b9730SMarc Zyngier */ 848211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i += 32) 8497c9b9730SMarc Zyngier writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 8507c9b9730SMarc Zyngier 851211bddd2SMarc Zyngier /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 852211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) { 853211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 854211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 855211bddd2SMarc Zyngier } 856211bddd2SMarc Zyngier 857211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) 858211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 859211bddd2SMarc Zyngier 860211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 16) 861211bddd2SMarc Zyngier writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 862211bddd2SMarc Zyngier 863211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 4) 864211bddd2SMarc Zyngier writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 865211bddd2SMarc Zyngier 86663f13483SMarc Zyngier /* Now do the common stuff */ 86763f13483SMarc Zyngier gic_dist_config(base, GIC_LINE_NR, NULL); 868021f6537SMarc Zyngier 8690b04758bSMarc Zyngier val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 8700b04758bSMarc Zyngier if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 8710b04758bSMarc Zyngier pr_info("Enabling SGIs without active state\n"); 8720b04758bSMarc Zyngier val |= GICD_CTLR_nASSGIreq; 8730b04758bSMarc Zyngier } 8740b04758bSMarc Zyngier 87563f13483SMarc Zyngier /* Enable distributor with ARE, Group1, and wait for it to drain */ 8760b04758bSMarc Zyngier writel_relaxed(val, base + GICD_CTLR); 87763f13483SMarc Zyngier gic_dist_wait_for_rwp(); 878021f6537SMarc Zyngier 879021f6537SMarc Zyngier /* 880021f6537SMarc Zyngier * Set all global interrupts to the boot CPU only. ARE must be 881021f6537SMarc Zyngier * enabled. 882021f6537SMarc Zyngier */ 883021f6537SMarc Zyngier affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 884211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i++) 88572c97126SJean-Philippe Brucker gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 886211bddd2SMarc Zyngier 887211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i++) 888211bddd2SMarc Zyngier gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 889021f6537SMarc Zyngier } 890021f6537SMarc Zyngier 8910d94ded2SMarc Zyngier static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 892021f6537SMarc Zyngier { 8930d94ded2SMarc Zyngier int ret = -ENODEV; 894021f6537SMarc Zyngier int i; 895021f6537SMarc Zyngier 896f5c1434cSMarc Zyngier for (i = 0; i < gic_data.nr_redist_regions; i++) { 897f5c1434cSMarc Zyngier void __iomem *ptr = gic_data.redist_regions[i].redist_base; 8980d94ded2SMarc Zyngier u64 typer; 899021f6537SMarc Zyngier u32 reg; 900021f6537SMarc Zyngier 901021f6537SMarc Zyngier reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 902021f6537SMarc Zyngier if (reg != GIC_PIDR2_ARCH_GICv3 && 903021f6537SMarc Zyngier reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 904021f6537SMarc Zyngier pr_warn("No redistributor present @%p\n", ptr); 905021f6537SMarc Zyngier break; 906021f6537SMarc Zyngier } 907021f6537SMarc Zyngier 908021f6537SMarc Zyngier do { 90972c97126SJean-Philippe Brucker typer = gic_read_typer(ptr + GICR_TYPER); 9100d94ded2SMarc Zyngier ret = fn(gic_data.redist_regions + i, ptr); 9110d94ded2SMarc Zyngier if (!ret) 912021f6537SMarc Zyngier return 0; 913021f6537SMarc Zyngier 914b70fb7afSTomasz Nowicki if (gic_data.redist_regions[i].single_redist) 915b70fb7afSTomasz Nowicki break; 916b70fb7afSTomasz Nowicki 917021f6537SMarc Zyngier if (gic_data.redist_stride) { 918021f6537SMarc Zyngier ptr += gic_data.redist_stride; 919021f6537SMarc Zyngier } else { 920021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 921021f6537SMarc Zyngier if (typer & GICR_TYPER_VLPIS) 922021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 923021f6537SMarc Zyngier } 924021f6537SMarc Zyngier } while (!(typer & GICR_TYPER_LAST)); 925021f6537SMarc Zyngier } 926021f6537SMarc Zyngier 9270d94ded2SMarc Zyngier return ret ? -ENODEV : 0; 9280d94ded2SMarc Zyngier } 9290d94ded2SMarc Zyngier 9300d94ded2SMarc Zyngier static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 9310d94ded2SMarc Zyngier { 9320d94ded2SMarc Zyngier unsigned long mpidr = cpu_logical_map(smp_processor_id()); 9330d94ded2SMarc Zyngier u64 typer; 9340d94ded2SMarc Zyngier u32 aff; 9350d94ded2SMarc Zyngier 9360d94ded2SMarc Zyngier /* 9370d94ded2SMarc Zyngier * Convert affinity to a 32bit value that can be matched to 9380d94ded2SMarc Zyngier * GICR_TYPER bits [63:32]. 9390d94ded2SMarc Zyngier */ 9400d94ded2SMarc Zyngier aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 9410d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 9420d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 9430d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 9440d94ded2SMarc Zyngier 9450d94ded2SMarc Zyngier typer = gic_read_typer(ptr + GICR_TYPER); 9460d94ded2SMarc Zyngier if ((typer >> 32) == aff) { 9470d94ded2SMarc Zyngier u64 offset = ptr - region->redist_base; 9489058a4e9SMarc Zyngier raw_spin_lock_init(&gic_data_rdist()->rd_lock); 9490d94ded2SMarc Zyngier gic_data_rdist_rd_base() = ptr; 9500d94ded2SMarc Zyngier gic_data_rdist()->phys_base = region->phys_base + offset; 9510d94ded2SMarc Zyngier 9520d94ded2SMarc Zyngier pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 9530d94ded2SMarc Zyngier smp_processor_id(), mpidr, 9540d94ded2SMarc Zyngier (int)(region - gic_data.redist_regions), 9550d94ded2SMarc Zyngier &gic_data_rdist()->phys_base); 9560d94ded2SMarc Zyngier return 0; 9570d94ded2SMarc Zyngier } 9580d94ded2SMarc Zyngier 9590d94ded2SMarc Zyngier /* Try next one */ 9600d94ded2SMarc Zyngier return 1; 9610d94ded2SMarc Zyngier } 9620d94ded2SMarc Zyngier 9630d94ded2SMarc Zyngier static int gic_populate_rdist(void) 9640d94ded2SMarc Zyngier { 9650d94ded2SMarc Zyngier if (gic_iterate_rdists(__gic_populate_rdist) == 0) 9660d94ded2SMarc Zyngier return 0; 9670d94ded2SMarc Zyngier 968021f6537SMarc Zyngier /* We couldn't even deal with ourselves... */ 969f6c86a41SJean-Philippe Brucker WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 9700d94ded2SMarc Zyngier smp_processor_id(), 9710d94ded2SMarc Zyngier (unsigned long)cpu_logical_map(smp_processor_id())); 972021f6537SMarc Zyngier return -ENODEV; 973021f6537SMarc Zyngier } 974021f6537SMarc Zyngier 9751a60e1e6SMarc Zyngier static int __gic_update_rdist_properties(struct redist_region *region, 9760edc23eaSMarc Zyngier void __iomem *ptr) 9770edc23eaSMarc Zyngier { 9780edc23eaSMarc Zyngier u64 typer = gic_read_typer(ptr + GICR_TYPER); 979a837ed36SMarc Zyngier u32 ctlr = readl_relaxed(ptr + GICR_CTLR); 980b25319d2SMarc Zyngier 98179a7f77bSMarc Zyngier /* Boot-time cleanip */ 98279a7f77bSMarc Zyngier if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { 98379a7f77bSMarc Zyngier u64 val; 98479a7f77bSMarc Zyngier 98579a7f77bSMarc Zyngier /* Deactivate any present vPE */ 98679a7f77bSMarc Zyngier val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); 98779a7f77bSMarc Zyngier if (val & GICR_VPENDBASER_Valid) 98879a7f77bSMarc Zyngier gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 98979a7f77bSMarc Zyngier ptr + SZ_128K + GICR_VPENDBASER); 99079a7f77bSMarc Zyngier 99179a7f77bSMarc Zyngier /* Mark the VPE table as invalid */ 99279a7f77bSMarc Zyngier val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); 99379a7f77bSMarc Zyngier val &= ~GICR_VPROPBASER_4_1_VALID; 99479a7f77bSMarc Zyngier gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); 99579a7f77bSMarc Zyngier } 99679a7f77bSMarc Zyngier 9970edc23eaSMarc Zyngier gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 998b25319d2SMarc Zyngier 999a837ed36SMarc Zyngier /* 1000a837ed36SMarc Zyngier * TYPER.RVPEID implies some form of DirectLPI, no matter what the 1001a837ed36SMarc Zyngier * doc says... :-/ And CTLR.IR implies another subset of DirectLPI 1002a837ed36SMarc Zyngier * that the ITS driver can make use of for LPIs (and not VLPIs). 1003a837ed36SMarc Zyngier * 1004a837ed36SMarc Zyngier * These are 3 different ways to express the same thing, depending 1005a837ed36SMarc Zyngier * on the revision of the architecture and its relaxations over 1006a837ed36SMarc Zyngier * time. Just group them under the 'direct_lpi' banner. 1007a837ed36SMarc Zyngier */ 1008b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 1009b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 1010a837ed36SMarc Zyngier !!(ctlr & GICR_CTLR_IR) | 1011b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid); 101296806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 1013b25319d2SMarc Zyngier 1014b25319d2SMarc Zyngier /* Detect non-sensical configurations */ 1015b25319d2SMarc Zyngier if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 1016b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi = false; 1017b25319d2SMarc Zyngier gic_data.rdists.has_vlpis = false; 1018b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = false; 1019b25319d2SMarc Zyngier } 1020b25319d2SMarc Zyngier 10215f51f803SMarc Zyngier gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 10220edc23eaSMarc Zyngier 10230edc23eaSMarc Zyngier return 1; 10240edc23eaSMarc Zyngier } 10250edc23eaSMarc Zyngier 10261a60e1e6SMarc Zyngier static void gic_update_rdist_properties(void) 10270edc23eaSMarc Zyngier { 10281a60e1e6SMarc Zyngier gic_data.ppi_nr = UINT_MAX; 10291a60e1e6SMarc Zyngier gic_iterate_rdists(__gic_update_rdist_properties); 10301a60e1e6SMarc Zyngier if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 10311a60e1e6SMarc Zyngier gic_data.ppi_nr = 0; 1032a837ed36SMarc Zyngier pr_info("GICv3 features: %d PPIs%s%s\n", 1033a837ed36SMarc Zyngier gic_data.ppi_nr, 1034a837ed36SMarc Zyngier gic_data.has_rss ? ", RSS" : "", 1035a837ed36SMarc Zyngier gic_data.rdists.has_direct_lpi ? ", DirectLPI" : ""); 1036a837ed36SMarc Zyngier 103796806229SMarc Zyngier if (gic_data.rdists.has_vlpis) 103896806229SMarc Zyngier pr_info("GICv4 features: %s%s%s\n", 103996806229SMarc Zyngier gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 104096806229SMarc Zyngier gic_data.rdists.has_rvpeid ? "RVPEID " : "", 104196806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 10420edc23eaSMarc Zyngier } 10430edc23eaSMarc Zyngier 1044d98d0a99SJulien Thierry /* Check whether it's single security state view */ 1045d98d0a99SJulien Thierry static inline bool gic_dist_security_disabled(void) 1046d98d0a99SJulien Thierry { 1047d98d0a99SJulien Thierry return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 1048d98d0a99SJulien Thierry } 1049d98d0a99SJulien Thierry 10503708d52fSSudeep Holla static void gic_cpu_sys_reg_init(void) 1051021f6537SMarc Zyngier { 1052eda0d04aSShanker Donthineni int i, cpu = smp_processor_id(); 1053eda0d04aSShanker Donthineni u64 mpidr = cpu_logical_map(cpu); 1054eda0d04aSShanker Donthineni u64 need_rss = MPIDR_RS(mpidr); 105533625282SMarc Zyngier bool group0; 1056b5cf6073SJulien Thierry u32 pribits; 1057eda0d04aSShanker Donthineni 10587cabd008SMarc Zyngier /* 10597cabd008SMarc Zyngier * Need to check that the SRE bit has actually been set. If 10607cabd008SMarc Zyngier * not, it means that SRE is disabled at EL2. We're going to 10617cabd008SMarc Zyngier * die painfully, and there is nothing we can do about it. 10627cabd008SMarc Zyngier * 10637cabd008SMarc Zyngier * Kindly inform the luser. 10647cabd008SMarc Zyngier */ 10657cabd008SMarc Zyngier if (!gic_enable_sre()) 10667cabd008SMarc Zyngier pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 1067021f6537SMarc Zyngier 1068b5cf6073SJulien Thierry pribits = gic_get_pribits(); 106933625282SMarc Zyngier 1070b5cf6073SJulien Thierry group0 = gic_has_group0(); 107133625282SMarc Zyngier 1072021f6537SMarc Zyngier /* Set priority mask register */ 1073d98d0a99SJulien Thierry if (!gic_prio_masking_enabled()) { 107433625282SMarc Zyngier write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 107533678059SAlexandru Elisei } else if (gic_supports_nmi()) { 1076d98d0a99SJulien Thierry /* 1077d98d0a99SJulien Thierry * Mismatch configuration with boot CPU, the system is likely 1078d98d0a99SJulien Thierry * to die as interrupt masking will not work properly on all 1079d98d0a99SJulien Thierry * CPUs 108033678059SAlexandru Elisei * 108133678059SAlexandru Elisei * The boot CPU calls this function before enabling NMI support, 108233678059SAlexandru Elisei * and as a result we'll never see this warning in the boot path 108333678059SAlexandru Elisei * for that CPU. 1084d98d0a99SJulien Thierry */ 108533678059SAlexandru Elisei if (static_branch_unlikely(&gic_nonsecure_priorities)) 108633678059SAlexandru Elisei WARN_ON(!group0 || gic_dist_security_disabled()); 108733678059SAlexandru Elisei else 108833678059SAlexandru Elisei WARN_ON(group0 && !gic_dist_security_disabled()); 1089d98d0a99SJulien Thierry } 1090021f6537SMarc Zyngier 109191ef8442SDaniel Thompson /* 109291ef8442SDaniel Thompson * Some firmwares hand over to the kernel with the BPR changed from 109391ef8442SDaniel Thompson * its reset value (and with a value large enough to prevent 109491ef8442SDaniel Thompson * any pre-emptive interrupts from working at all). Writing a zero 109591ef8442SDaniel Thompson * to BPR restores is reset value. 109691ef8442SDaniel Thompson */ 109791ef8442SDaniel Thompson gic_write_bpr1(0); 109891ef8442SDaniel Thompson 1099d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) { 11000b6a3da9SMarc Zyngier /* EOI drops priority only (mode 1) */ 11010b6a3da9SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 11020b6a3da9SMarc Zyngier } else { 1103021f6537SMarc Zyngier /* EOI deactivates interrupt too (mode 0) */ 1104021f6537SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 11050b6a3da9SMarc Zyngier } 1106021f6537SMarc Zyngier 110733625282SMarc Zyngier /* Always whack Group0 before Group1 */ 110833625282SMarc Zyngier if (group0) { 110933625282SMarc Zyngier switch(pribits) { 111033625282SMarc Zyngier case 8: 111133625282SMarc Zyngier case 7: 111233625282SMarc Zyngier write_gicreg(0, ICC_AP0R3_EL1); 111333625282SMarc Zyngier write_gicreg(0, ICC_AP0R2_EL1); 1114df561f66SGustavo A. R. Silva fallthrough; 111533625282SMarc Zyngier case 6: 111633625282SMarc Zyngier write_gicreg(0, ICC_AP0R1_EL1); 1117df561f66SGustavo A. R. Silva fallthrough; 111833625282SMarc Zyngier case 5: 111933625282SMarc Zyngier case 4: 112033625282SMarc Zyngier write_gicreg(0, ICC_AP0R0_EL1); 112133625282SMarc Zyngier } 1122d6062a6dSMarc Zyngier 112333625282SMarc Zyngier isb(); 112433625282SMarc Zyngier } 112533625282SMarc Zyngier 112633625282SMarc Zyngier switch(pribits) { 1127d6062a6dSMarc Zyngier case 8: 1128d6062a6dSMarc Zyngier case 7: 1129d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R3_EL1); 1130d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R2_EL1); 1131df561f66SGustavo A. R. Silva fallthrough; 1132d6062a6dSMarc Zyngier case 6: 1133d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R1_EL1); 1134df561f66SGustavo A. R. Silva fallthrough; 1135d6062a6dSMarc Zyngier case 5: 1136d6062a6dSMarc Zyngier case 4: 1137d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R0_EL1); 1138d6062a6dSMarc Zyngier } 1139d6062a6dSMarc Zyngier 1140d6062a6dSMarc Zyngier isb(); 1141d6062a6dSMarc Zyngier 1142021f6537SMarc Zyngier /* ... and let's hit the road... */ 1143021f6537SMarc Zyngier gic_write_grpen1(1); 1144eda0d04aSShanker Donthineni 1145eda0d04aSShanker Donthineni /* Keep the RSS capability status in per_cpu variable */ 1146eda0d04aSShanker Donthineni per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1147eda0d04aSShanker Donthineni 1148eda0d04aSShanker Donthineni /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1149eda0d04aSShanker Donthineni for_each_online_cpu(i) { 1150eda0d04aSShanker Donthineni bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1151eda0d04aSShanker Donthineni 1152eda0d04aSShanker Donthineni need_rss |= MPIDR_RS(cpu_logical_map(i)); 1153eda0d04aSShanker Donthineni if (need_rss && (!have_rss)) 1154eda0d04aSShanker Donthineni pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1155eda0d04aSShanker Donthineni cpu, (unsigned long)mpidr, 1156eda0d04aSShanker Donthineni i, (unsigned long)cpu_logical_map(i)); 1157eda0d04aSShanker Donthineni } 1158eda0d04aSShanker Donthineni 1159eda0d04aSShanker Donthineni /** 1160eda0d04aSShanker Donthineni * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1161eda0d04aSShanker Donthineni * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1162eda0d04aSShanker Donthineni * UNPREDICTABLE choice of : 1163eda0d04aSShanker Donthineni * - The write is ignored. 1164eda0d04aSShanker Donthineni * - The RS field is treated as 0. 1165eda0d04aSShanker Donthineni */ 1166eda0d04aSShanker Donthineni if (need_rss && (!gic_data.has_rss)) 1167eda0d04aSShanker Donthineni pr_crit_once("RSS is required but GICD doesn't support it\n"); 1168021f6537SMarc Zyngier } 1169021f6537SMarc Zyngier 1170f736d65dSMarc Zyngier static bool gicv3_nolpi; 1171f736d65dSMarc Zyngier 1172f736d65dSMarc Zyngier static int __init gicv3_nolpi_cfg(char *buf) 1173f736d65dSMarc Zyngier { 1174f736d65dSMarc Zyngier return strtobool(buf, &gicv3_nolpi); 1175f736d65dSMarc Zyngier } 1176f736d65dSMarc Zyngier early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1177f736d65dSMarc Zyngier 1178da33f31dSMarc Zyngier static int gic_dist_supports_lpis(void) 1179da33f31dSMarc Zyngier { 1180d38a71c5SMarc Zyngier return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1181d38a71c5SMarc Zyngier !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1182d38a71c5SMarc Zyngier !gicv3_nolpi); 1183da33f31dSMarc Zyngier } 1184da33f31dSMarc Zyngier 1185021f6537SMarc Zyngier static void gic_cpu_init(void) 1186021f6537SMarc Zyngier { 1187021f6537SMarc Zyngier void __iomem *rbase; 11881a60e1e6SMarc Zyngier int i; 1189021f6537SMarc Zyngier 1190021f6537SMarc Zyngier /* Register ourselves with the rest of the world */ 1191021f6537SMarc Zyngier if (gic_populate_rdist()) 1192021f6537SMarc Zyngier return; 1193021f6537SMarc Zyngier 1194a2c22510SSudeep Holla gic_enable_redist(true); 1195021f6537SMarc Zyngier 1196ad5a78d3SMarc Zyngier WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1197ad5a78d3SMarc Zyngier !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1198ad5a78d3SMarc Zyngier "Distributor has extended ranges, but CPU%d doesn't\n", 1199ad5a78d3SMarc Zyngier smp_processor_id()); 1200ad5a78d3SMarc Zyngier 1201021f6537SMarc Zyngier rbase = gic_data_rdist_sgi_base(); 1202021f6537SMarc Zyngier 12037c9b9730SMarc Zyngier /* Configure SGIs/PPIs as non-secure Group-1 */ 12041a60e1e6SMarc Zyngier for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 12051a60e1e6SMarc Zyngier writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 12067c9b9730SMarc Zyngier 12071a60e1e6SMarc Zyngier gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1208021f6537SMarc Zyngier 12093708d52fSSudeep Holla /* initialise system registers */ 12103708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1211021f6537SMarc Zyngier } 1212021f6537SMarc Zyngier 1213021f6537SMarc Zyngier #ifdef CONFIG_SMP 1214021f6537SMarc Zyngier 1215eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1216eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1217eda0d04aSShanker Donthineni 12186670a6d8SRichard Cochran static int gic_starting_cpu(unsigned int cpu) 12196670a6d8SRichard Cochran { 12206670a6d8SRichard Cochran gic_cpu_init(); 1221d38a71c5SMarc Zyngier 1222d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) 1223d38a71c5SMarc Zyngier its_cpu_init(); 1224d38a71c5SMarc Zyngier 12256670a6d8SRichard Cochran return 0; 12266670a6d8SRichard Cochran } 1227021f6537SMarc Zyngier 1228021f6537SMarc Zyngier static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1229f6c86a41SJean-Philippe Brucker unsigned long cluster_id) 1230021f6537SMarc Zyngier { 1231727653d6SJames Morse int next_cpu, cpu = *base_cpu; 1232f6c86a41SJean-Philippe Brucker unsigned long mpidr = cpu_logical_map(cpu); 1233021f6537SMarc Zyngier u16 tlist = 0; 1234021f6537SMarc Zyngier 1235021f6537SMarc Zyngier while (cpu < nr_cpu_ids) { 1236021f6537SMarc Zyngier tlist |= 1 << (mpidr & 0xf); 1237021f6537SMarc Zyngier 1238727653d6SJames Morse next_cpu = cpumask_next(cpu, mask); 1239727653d6SJames Morse if (next_cpu >= nr_cpu_ids) 1240021f6537SMarc Zyngier goto out; 1241727653d6SJames Morse cpu = next_cpu; 1242021f6537SMarc Zyngier 1243021f6537SMarc Zyngier mpidr = cpu_logical_map(cpu); 1244021f6537SMarc Zyngier 1245eda0d04aSShanker Donthineni if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1246021f6537SMarc Zyngier cpu--; 1247021f6537SMarc Zyngier goto out; 1248021f6537SMarc Zyngier } 1249021f6537SMarc Zyngier } 1250021f6537SMarc Zyngier out: 1251021f6537SMarc Zyngier *base_cpu = cpu; 1252021f6537SMarc Zyngier return tlist; 1253021f6537SMarc Zyngier } 1254021f6537SMarc Zyngier 12557e580278SAndre Przywara #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 12567e580278SAndre Przywara (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 12577e580278SAndre Przywara << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 12587e580278SAndre Przywara 1259021f6537SMarc Zyngier static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1260021f6537SMarc Zyngier { 1261021f6537SMarc Zyngier u64 val; 1262021f6537SMarc Zyngier 12637e580278SAndre Przywara val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 12647e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 12657e580278SAndre Przywara irq << ICC_SGI1R_SGI_ID_SHIFT | 12667e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1267eda0d04aSShanker Donthineni MPIDR_TO_SGI_RS(cluster_id) | 12687e580278SAndre Przywara tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1269021f6537SMarc Zyngier 1270b6dd4d83SMark Salter pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1271021f6537SMarc Zyngier gic_write_sgi1r(val); 1272021f6537SMarc Zyngier } 1273021f6537SMarc Zyngier 127464b499d8SMarc Zyngier static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1275021f6537SMarc Zyngier { 1276021f6537SMarc Zyngier int cpu; 1277021f6537SMarc Zyngier 127864b499d8SMarc Zyngier if (WARN_ON(d->hwirq >= 16)) 1279021f6537SMarc Zyngier return; 1280021f6537SMarc Zyngier 1281021f6537SMarc Zyngier /* 1282021f6537SMarc Zyngier * Ensure that stores to Normal memory are visible to the 1283021f6537SMarc Zyngier * other CPUs before issuing the IPI. 1284021f6537SMarc Zyngier */ 128580e4e1f4SBarry Song dsb(ishst); 1286021f6537SMarc Zyngier 1287f9b531feSRusty Russell for_each_cpu(cpu, mask) { 1288eda0d04aSShanker Donthineni u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1289021f6537SMarc Zyngier u16 tlist; 1290021f6537SMarc Zyngier 1291021f6537SMarc Zyngier tlist = gic_compute_target_list(&cpu, mask, cluster_id); 129264b499d8SMarc Zyngier gic_send_sgi(cluster_id, tlist, d->hwirq); 1293021f6537SMarc Zyngier } 1294021f6537SMarc Zyngier 1295021f6537SMarc Zyngier /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1296021f6537SMarc Zyngier isb(); 1297021f6537SMarc Zyngier } 1298021f6537SMarc Zyngier 12998a94c1abSIngo Rohloff static void __init gic_smp_init(void) 1300021f6537SMarc Zyngier { 130164b499d8SMarc Zyngier struct irq_fwspec sgi_fwspec = { 130264b499d8SMarc Zyngier .fwnode = gic_data.fwnode, 130364b499d8SMarc Zyngier .param_count = 1, 130464b499d8SMarc Zyngier }; 130564b499d8SMarc Zyngier int base_sgi; 130664b499d8SMarc Zyngier 13076896bcd1SThomas Gleixner cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 130873c1b41eSThomas Gleixner "irqchip/arm/gicv3:starting", 130973c1b41eSThomas Gleixner gic_starting_cpu, NULL); 131064b499d8SMarc Zyngier 131164b499d8SMarc Zyngier /* Register all 8 non-secure SGIs */ 131264b499d8SMarc Zyngier base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8, 131364b499d8SMarc Zyngier NUMA_NO_NODE, &sgi_fwspec, 131464b499d8SMarc Zyngier false, NULL); 131564b499d8SMarc Zyngier if (WARN_ON(base_sgi <= 0)) 131664b499d8SMarc Zyngier return; 131764b499d8SMarc Zyngier 131864b499d8SMarc Zyngier set_smp_ipi_range(base_sgi, 8); 1319021f6537SMarc Zyngier } 1320021f6537SMarc Zyngier 1321021f6537SMarc Zyngier static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1322021f6537SMarc Zyngier bool force) 1323021f6537SMarc Zyngier { 132465a30f8bSSuzuki K Poulose unsigned int cpu; 1325e91b036eSMarc Zyngier u32 offset, index; 1326021f6537SMarc Zyngier void __iomem *reg; 1327021f6537SMarc Zyngier int enabled; 1328021f6537SMarc Zyngier u64 val; 1329021f6537SMarc Zyngier 133065a30f8bSSuzuki K Poulose if (force) 133165a30f8bSSuzuki K Poulose cpu = cpumask_first(mask_val); 133265a30f8bSSuzuki K Poulose else 133365a30f8bSSuzuki K Poulose cpu = cpumask_any_and(mask_val, cpu_online_mask); 133465a30f8bSSuzuki K Poulose 1335866d7c1bSSuzuki K Poulose if (cpu >= nr_cpu_ids) 1336866d7c1bSSuzuki K Poulose return -EINVAL; 1337866d7c1bSSuzuki K Poulose 1338021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) 1339021f6537SMarc Zyngier return -EINVAL; 1340021f6537SMarc Zyngier 1341021f6537SMarc Zyngier /* If interrupt was enabled, disable it first */ 1342021f6537SMarc Zyngier enabled = gic_peek_irq(d, GICD_ISENABLER); 1343021f6537SMarc Zyngier if (enabled) 1344021f6537SMarc Zyngier gic_mask_irq(d); 1345021f6537SMarc Zyngier 1346e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IROUTER, &index); 1347e91b036eSMarc Zyngier reg = gic_dist_base(d) + offset + (index * 8); 1348021f6537SMarc Zyngier val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 1349021f6537SMarc Zyngier 135072c97126SJean-Philippe Brucker gic_write_irouter(val, reg); 1351021f6537SMarc Zyngier 1352021f6537SMarc Zyngier /* 1353021f6537SMarc Zyngier * If the interrupt was enabled, enabled it again. Otherwise, 1354021f6537SMarc Zyngier * just wait for the distributor to have digested our changes. 1355021f6537SMarc Zyngier */ 1356021f6537SMarc Zyngier if (enabled) 1357021f6537SMarc Zyngier gic_unmask_irq(d); 1358021f6537SMarc Zyngier 1359956ae91aSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1360956ae91aSMarc Zyngier 13610fc6fa29SAntoine Tenart return IRQ_SET_MASK_OK_DONE; 1362021f6537SMarc Zyngier } 1363021f6537SMarc Zyngier #else 1364021f6537SMarc Zyngier #define gic_set_affinity NULL 136564b499d8SMarc Zyngier #define gic_ipi_send_mask NULL 1366021f6537SMarc Zyngier #define gic_smp_init() do { } while(0) 1367021f6537SMarc Zyngier #endif 1368021f6537SMarc Zyngier 136917f644e9SValentin Schneider static int gic_retrigger(struct irq_data *data) 137017f644e9SValentin Schneider { 137117f644e9SValentin Schneider return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 137217f644e9SValentin Schneider } 137317f644e9SValentin Schneider 13743708d52fSSudeep Holla #ifdef CONFIG_CPU_PM 13753708d52fSSudeep Holla static int gic_cpu_pm_notifier(struct notifier_block *self, 13763708d52fSSudeep Holla unsigned long cmd, void *v) 13773708d52fSSudeep Holla { 13783708d52fSSudeep Holla if (cmd == CPU_PM_EXIT) { 1379ccd9432aSSudeep Holla if (gic_dist_security_disabled()) 13803708d52fSSudeep Holla gic_enable_redist(true); 13813708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1382ccd9432aSSudeep Holla } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 13833708d52fSSudeep Holla gic_write_grpen1(0); 13843708d52fSSudeep Holla gic_enable_redist(false); 13853708d52fSSudeep Holla } 13863708d52fSSudeep Holla return NOTIFY_OK; 13873708d52fSSudeep Holla } 13883708d52fSSudeep Holla 13893708d52fSSudeep Holla static struct notifier_block gic_cpu_pm_notifier_block = { 13903708d52fSSudeep Holla .notifier_call = gic_cpu_pm_notifier, 13913708d52fSSudeep Holla }; 13923708d52fSSudeep Holla 13933708d52fSSudeep Holla static void gic_cpu_pm_init(void) 13943708d52fSSudeep Holla { 13953708d52fSSudeep Holla cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 13963708d52fSSudeep Holla } 13973708d52fSSudeep Holla 13983708d52fSSudeep Holla #else 13993708d52fSSudeep Holla static inline void gic_cpu_pm_init(void) { } 14003708d52fSSudeep Holla #endif /* CONFIG_CPU_PM */ 14013708d52fSSudeep Holla 1402021f6537SMarc Zyngier static struct irq_chip gic_chip = { 1403021f6537SMarc Zyngier .name = "GICv3", 1404021f6537SMarc Zyngier .irq_mask = gic_mask_irq, 1405021f6537SMarc Zyngier .irq_unmask = gic_unmask_irq, 1406021f6537SMarc Zyngier .irq_eoi = gic_eoi_irq, 1407021f6537SMarc Zyngier .irq_set_type = gic_set_type, 1408021f6537SMarc Zyngier .irq_set_affinity = gic_set_affinity, 140917f644e9SValentin Schneider .irq_retrigger = gic_retrigger, 1410b594c6e2SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1411b594c6e2SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1412101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1413101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 141464b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask, 14154110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 14164110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 14174110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 1418021f6537SMarc Zyngier }; 1419021f6537SMarc Zyngier 14200b6a3da9SMarc Zyngier static struct irq_chip gic_eoimode1_chip = { 14210b6a3da9SMarc Zyngier .name = "GICv3", 14220b6a3da9SMarc Zyngier .irq_mask = gic_eoimode1_mask_irq, 14230b6a3da9SMarc Zyngier .irq_unmask = gic_unmask_irq, 14240b6a3da9SMarc Zyngier .irq_eoi = gic_eoimode1_eoi_irq, 14250b6a3da9SMarc Zyngier .irq_set_type = gic_set_type, 14260b6a3da9SMarc Zyngier .irq_set_affinity = gic_set_affinity, 142717f644e9SValentin Schneider .irq_retrigger = gic_retrigger, 14280b6a3da9SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 14290b6a3da9SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1430530bf353SMarc Zyngier .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1431101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1432101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 143364b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask, 14344110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 14354110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 14364110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 14370b6a3da9SMarc Zyngier }; 14380b6a3da9SMarc Zyngier 1439021f6537SMarc Zyngier static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1440021f6537SMarc Zyngier irq_hw_number_t hw) 1441021f6537SMarc Zyngier { 14420b6a3da9SMarc Zyngier struct irq_chip *chip = &gic_chip; 14431b57d91bSValentin Schneider struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 14440b6a3da9SMarc Zyngier 1445d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 14460b6a3da9SMarc Zyngier chip = &gic_eoimode1_chip; 14470b6a3da9SMarc Zyngier 1448e91b036eSMarc Zyngier switch (__get_intid_range(hw)) { 144970a29c32SMarc Zyngier case SGI_RANGE: 1450e91b036eSMarc Zyngier case PPI_RANGE: 14515f51f803SMarc Zyngier case EPPI_RANGE: 1452021f6537SMarc Zyngier irq_set_percpu_devid(irq); 14530b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1454443acc4fSMarc Zyngier handle_percpu_devid_irq, NULL, NULL); 1455e91b036eSMarc Zyngier break; 1456e91b036eSMarc Zyngier 1457e91b036eSMarc Zyngier case SPI_RANGE: 1458211bddd2SMarc Zyngier case ESPI_RANGE: 14590b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1460443acc4fSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1461d17cab44SRob Herring irq_set_probe(irq); 14621b57d91bSValentin Schneider irqd_set_single_target(irqd); 1463e91b036eSMarc Zyngier break; 1464e91b036eSMarc Zyngier 1465e91b036eSMarc Zyngier case LPI_RANGE: 1466da33f31dSMarc Zyngier if (!gic_dist_supports_lpis()) 1467da33f31dSMarc Zyngier return -EPERM; 14680b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1469da33f31dSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1470e91b036eSMarc Zyngier break; 1471e91b036eSMarc Zyngier 1472e91b036eSMarc Zyngier default: 1473e91b036eSMarc Zyngier return -EPERM; 1474da33f31dSMarc Zyngier } 1475da33f31dSMarc Zyngier 14761b57d91bSValentin Schneider /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 14771b57d91bSValentin Schneider irqd_set_handle_enforce_irqctx(irqd); 1478021f6537SMarc Zyngier return 0; 1479021f6537SMarc Zyngier } 1480021f6537SMarc Zyngier 1481f833f57fSMarc Zyngier static int gic_irq_domain_translate(struct irq_domain *d, 1482f833f57fSMarc Zyngier struct irq_fwspec *fwspec, 1483f833f57fSMarc Zyngier unsigned long *hwirq, 1484f833f57fSMarc Zyngier unsigned int *type) 1485021f6537SMarc Zyngier { 148664b499d8SMarc Zyngier if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 148764b499d8SMarc Zyngier *hwirq = fwspec->param[0]; 148864b499d8SMarc Zyngier *type = IRQ_TYPE_EDGE_RISING; 148964b499d8SMarc Zyngier return 0; 149064b499d8SMarc Zyngier } 149164b499d8SMarc Zyngier 1492f833f57fSMarc Zyngier if (is_of_node(fwspec->fwnode)) { 1493f833f57fSMarc Zyngier if (fwspec->param_count < 3) 1494021f6537SMarc Zyngier return -EINVAL; 1495021f6537SMarc Zyngier 1496db8c70ecSMarc Zyngier switch (fwspec->param[0]) { 1497db8c70ecSMarc Zyngier case 0: /* SPI */ 1498db8c70ecSMarc Zyngier *hwirq = fwspec->param[1] + 32; 1499db8c70ecSMarc Zyngier break; 1500db8c70ecSMarc Zyngier case 1: /* PPI */ 1501f833f57fSMarc Zyngier *hwirq = fwspec->param[1] + 16; 1502db8c70ecSMarc Zyngier break; 1503211bddd2SMarc Zyngier case 2: /* ESPI */ 1504211bddd2SMarc Zyngier *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1505211bddd2SMarc Zyngier break; 15065f51f803SMarc Zyngier case 3: /* EPPI */ 15075f51f803SMarc Zyngier *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 15085f51f803SMarc Zyngier break; 1509db8c70ecSMarc Zyngier case GIC_IRQ_TYPE_LPI: /* LPI */ 1510db8c70ecSMarc Zyngier *hwirq = fwspec->param[1]; 1511db8c70ecSMarc Zyngier break; 15125f51f803SMarc Zyngier case GIC_IRQ_TYPE_PARTITION: 15135f51f803SMarc Zyngier *hwirq = fwspec->param[1]; 15145f51f803SMarc Zyngier if (fwspec->param[1] >= 16) 15155f51f803SMarc Zyngier *hwirq += EPPI_BASE_INTID - 16; 15165f51f803SMarc Zyngier else 15175f51f803SMarc Zyngier *hwirq += 16; 15185f51f803SMarc Zyngier break; 1519db8c70ecSMarc Zyngier default: 1520db8c70ecSMarc Zyngier return -EINVAL; 1521db8c70ecSMarc Zyngier } 1522f833f57fSMarc Zyngier 1523f833f57fSMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 15246ef6386eSMarc Zyngier 152565da7d19SMarc Zyngier /* 152665da7d19SMarc Zyngier * Make it clear that broken DTs are... broken. 1527a359f757SIngo Molnar * Partitioned PPIs are an unfortunate exception. 152865da7d19SMarc Zyngier */ 152965da7d19SMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE && 153065da7d19SMarc Zyngier fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1531f833f57fSMarc Zyngier return 0; 1532021f6537SMarc Zyngier } 1533021f6537SMarc Zyngier 1534ffa7d616STomasz Nowicki if (is_fwnode_irqchip(fwspec->fwnode)) { 1535ffa7d616STomasz Nowicki if(fwspec->param_count != 2) 1536ffa7d616STomasz Nowicki return -EINVAL; 1537ffa7d616STomasz Nowicki 1538544808f7SAndre Przywara if (fwspec->param[0] < 16) { 1539544808f7SAndre Przywara pr_err(FW_BUG "Illegal GSI%d translation request\n", 1540544808f7SAndre Przywara fwspec->param[0]); 1541544808f7SAndre Przywara return -EINVAL; 1542544808f7SAndre Przywara } 1543544808f7SAndre Przywara 1544ffa7d616STomasz Nowicki *hwirq = fwspec->param[0]; 1545ffa7d616STomasz Nowicki *type = fwspec->param[1]; 15466ef6386eSMarc Zyngier 15476ef6386eSMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE); 1548ffa7d616STomasz Nowicki return 0; 1549ffa7d616STomasz Nowicki } 1550ffa7d616STomasz Nowicki 1551f833f57fSMarc Zyngier return -EINVAL; 1552021f6537SMarc Zyngier } 1553021f6537SMarc Zyngier 1554443acc4fSMarc Zyngier static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1555443acc4fSMarc Zyngier unsigned int nr_irqs, void *arg) 1556443acc4fSMarc Zyngier { 1557443acc4fSMarc Zyngier int i, ret; 1558443acc4fSMarc Zyngier irq_hw_number_t hwirq; 1559443acc4fSMarc Zyngier unsigned int type = IRQ_TYPE_NONE; 1560f833f57fSMarc Zyngier struct irq_fwspec *fwspec = arg; 1561443acc4fSMarc Zyngier 1562f833f57fSMarc Zyngier ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1563443acc4fSMarc Zyngier if (ret) 1564443acc4fSMarc Zyngier return ret; 1565443acc4fSMarc Zyngier 156663c16c6eSSuzuki K Poulose for (i = 0; i < nr_irqs; i++) { 156763c16c6eSSuzuki K Poulose ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 156863c16c6eSSuzuki K Poulose if (ret) 156963c16c6eSSuzuki K Poulose return ret; 157063c16c6eSSuzuki K Poulose } 1571443acc4fSMarc Zyngier 1572443acc4fSMarc Zyngier return 0; 1573443acc4fSMarc Zyngier } 1574443acc4fSMarc Zyngier 1575443acc4fSMarc Zyngier static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1576443acc4fSMarc Zyngier unsigned int nr_irqs) 1577443acc4fSMarc Zyngier { 1578443acc4fSMarc Zyngier int i; 1579443acc4fSMarc Zyngier 1580443acc4fSMarc Zyngier for (i = 0; i < nr_irqs; i++) { 1581443acc4fSMarc Zyngier struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1582443acc4fSMarc Zyngier irq_set_handler(virq + i, NULL); 1583443acc4fSMarc Zyngier irq_domain_reset_irq_data(d); 1584443acc4fSMarc Zyngier } 1585443acc4fSMarc Zyngier } 1586443acc4fSMarc Zyngier 1587d753f849SJames Morse static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, 1588d753f849SJames Morse irq_hw_number_t hwirq) 1589d753f849SJames Morse { 1590d753f849SJames Morse enum gic_intid_range range; 1591d753f849SJames Morse 1592d753f849SJames Morse if (!gic_data.ppi_descs) 1593d753f849SJames Morse return false; 1594d753f849SJames Morse 1595d753f849SJames Morse if (!is_of_node(fwspec->fwnode)) 1596d753f849SJames Morse return false; 1597d753f849SJames Morse 1598d753f849SJames Morse if (fwspec->param_count < 4 || !fwspec->param[3]) 1599d753f849SJames Morse return false; 1600d753f849SJames Morse 1601d753f849SJames Morse range = __get_intid_range(hwirq); 1602d753f849SJames Morse if (range != PPI_RANGE && range != EPPI_RANGE) 1603d753f849SJames Morse return false; 1604d753f849SJames Morse 1605d753f849SJames Morse return true; 1606d753f849SJames Morse } 1607d753f849SJames Morse 1608e3825ba1SMarc Zyngier static int gic_irq_domain_select(struct irq_domain *d, 1609e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1610e3825ba1SMarc Zyngier enum irq_domain_bus_token bus_token) 1611e3825ba1SMarc Zyngier { 1612d753f849SJames Morse unsigned int type, ret, ppi_idx; 1613d753f849SJames Morse irq_hw_number_t hwirq; 1614d753f849SJames Morse 1615e3825ba1SMarc Zyngier /* Not for us */ 1616e3825ba1SMarc Zyngier if (fwspec->fwnode != d->fwnode) 1617e3825ba1SMarc Zyngier return 0; 1618e3825ba1SMarc Zyngier 1619e3825ba1SMarc Zyngier /* If this is not DT, then we have a single domain */ 1620e3825ba1SMarc Zyngier if (!is_of_node(fwspec->fwnode)) 1621e3825ba1SMarc Zyngier return 1; 1622e3825ba1SMarc Zyngier 1623d753f849SJames Morse ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type); 1624d753f849SJames Morse if (WARN_ON_ONCE(ret)) 1625d753f849SJames Morse return 0; 1626d753f849SJames Morse 1627d753f849SJames Morse if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) 1628d753f849SJames Morse return d == gic_data.domain; 1629d753f849SJames Morse 1630e3825ba1SMarc Zyngier /* 1631e3825ba1SMarc Zyngier * If this is a PPI and we have a 4th (non-null) parameter, 1632e3825ba1SMarc Zyngier * then we need to match the partition domain. 1633e3825ba1SMarc Zyngier */ 1634d753f849SJames Morse ppi_idx = __gic_get_ppi_index(hwirq); 1635d753f849SJames Morse return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); 1636e3825ba1SMarc Zyngier } 1637e3825ba1SMarc Zyngier 1638021f6537SMarc Zyngier static const struct irq_domain_ops gic_irq_domain_ops = { 1639f833f57fSMarc Zyngier .translate = gic_irq_domain_translate, 1640443acc4fSMarc Zyngier .alloc = gic_irq_domain_alloc, 1641443acc4fSMarc Zyngier .free = gic_irq_domain_free, 1642e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1643e3825ba1SMarc Zyngier }; 1644e3825ba1SMarc Zyngier 1645e3825ba1SMarc Zyngier static int partition_domain_translate(struct irq_domain *d, 1646e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1647e3825ba1SMarc Zyngier unsigned long *hwirq, 1648e3825ba1SMarc Zyngier unsigned int *type) 1649e3825ba1SMarc Zyngier { 1650d753f849SJames Morse unsigned long ppi_intid; 1651e3825ba1SMarc Zyngier struct device_node *np; 1652d753f849SJames Morse unsigned int ppi_idx; 1653e3825ba1SMarc Zyngier int ret; 1654e3825ba1SMarc Zyngier 165552085d3fSMarc Zyngier if (!gic_data.ppi_descs) 165652085d3fSMarc Zyngier return -ENOMEM; 165752085d3fSMarc Zyngier 1658e3825ba1SMarc Zyngier np = of_find_node_by_phandle(fwspec->param[3]); 1659e3825ba1SMarc Zyngier if (WARN_ON(!np)) 1660e3825ba1SMarc Zyngier return -EINVAL; 1661e3825ba1SMarc Zyngier 1662d753f849SJames Morse ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); 1663d753f849SJames Morse if (WARN_ON_ONCE(ret)) 1664d753f849SJames Morse return 0; 1665d753f849SJames Morse 1666d753f849SJames Morse ppi_idx = __gic_get_ppi_index(ppi_intid); 1667d753f849SJames Morse ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], 1668e3825ba1SMarc Zyngier of_node_to_fwnode(np)); 1669e3825ba1SMarc Zyngier if (ret < 0) 1670e3825ba1SMarc Zyngier return ret; 1671e3825ba1SMarc Zyngier 1672e3825ba1SMarc Zyngier *hwirq = ret; 1673e3825ba1SMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1674e3825ba1SMarc Zyngier 1675e3825ba1SMarc Zyngier return 0; 1676e3825ba1SMarc Zyngier } 1677e3825ba1SMarc Zyngier 1678e3825ba1SMarc Zyngier static const struct irq_domain_ops partition_domain_ops = { 1679e3825ba1SMarc Zyngier .translate = partition_domain_translate, 1680e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1681021f6537SMarc Zyngier }; 1682021f6537SMarc Zyngier 16839c8114c2SSrinivas Kandagatla static bool gic_enable_quirk_msm8996(void *data) 16849c8114c2SSrinivas Kandagatla { 16859c8114c2SSrinivas Kandagatla struct gic_chip_data *d = data; 16869c8114c2SSrinivas Kandagatla 16879c8114c2SSrinivas Kandagatla d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 16889c8114c2SSrinivas Kandagatla 16899c8114c2SSrinivas Kandagatla return true; 16909c8114c2SSrinivas Kandagatla } 16919c8114c2SSrinivas Kandagatla 1692d01fd161SMarc Zyngier static bool gic_enable_quirk_cavium_38539(void *data) 1693d01fd161SMarc Zyngier { 1694d01fd161SMarc Zyngier struct gic_chip_data *d = data; 1695d01fd161SMarc Zyngier 1696d01fd161SMarc Zyngier d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1697d01fd161SMarc Zyngier 1698d01fd161SMarc Zyngier return true; 1699d01fd161SMarc Zyngier } 1700d01fd161SMarc Zyngier 17017f2481b3SMarc Zyngier static bool gic_enable_quirk_hip06_07(void *data) 17027f2481b3SMarc Zyngier { 17037f2481b3SMarc Zyngier struct gic_chip_data *d = data; 17047f2481b3SMarc Zyngier 17057f2481b3SMarc Zyngier /* 17067f2481b3SMarc Zyngier * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 17077f2481b3SMarc Zyngier * not being an actual ARM implementation). The saving grace is 17087f2481b3SMarc Zyngier * that GIC-600 doesn't have ESPI, so nothing to do in that case. 17097f2481b3SMarc Zyngier * HIP07 doesn't even have a proper IIDR, and still pretends to 17107f2481b3SMarc Zyngier * have ESPI. In both cases, put them right. 17117f2481b3SMarc Zyngier */ 17127f2481b3SMarc Zyngier if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 17137f2481b3SMarc Zyngier /* Zero both ESPI and the RES0 field next to it... */ 17147f2481b3SMarc Zyngier d->rdists.gicd_typer &= ~GENMASK(9, 8); 17157f2481b3SMarc Zyngier return true; 17167f2481b3SMarc Zyngier } 17177f2481b3SMarc Zyngier 17187f2481b3SMarc Zyngier return false; 17197f2481b3SMarc Zyngier } 17207f2481b3SMarc Zyngier 17217f2481b3SMarc Zyngier static const struct gic_quirk gic_quirks[] = { 17227f2481b3SMarc Zyngier { 17237f2481b3SMarc Zyngier .desc = "GICv3: Qualcomm MSM8996 broken firmware", 17247f2481b3SMarc Zyngier .compatible = "qcom,msm8996-gic-v3", 17257f2481b3SMarc Zyngier .init = gic_enable_quirk_msm8996, 17267f2481b3SMarc Zyngier }, 17277f2481b3SMarc Zyngier { 17287f2481b3SMarc Zyngier .desc = "GICv3: HIP06 erratum 161010803", 17297f2481b3SMarc Zyngier .iidr = 0x0204043b, 17307f2481b3SMarc Zyngier .mask = 0xffffffff, 17317f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 17327f2481b3SMarc Zyngier }, 17337f2481b3SMarc Zyngier { 17347f2481b3SMarc Zyngier .desc = "GICv3: HIP07 erratum 161010803", 17357f2481b3SMarc Zyngier .iidr = 0x00000000, 17367f2481b3SMarc Zyngier .mask = 0xffffffff, 17377f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 17387f2481b3SMarc Zyngier }, 17397f2481b3SMarc Zyngier { 1740d01fd161SMarc Zyngier /* 1741d01fd161SMarc Zyngier * Reserved register accesses generate a Synchronous 1742d01fd161SMarc Zyngier * External Abort. This erratum applies to: 1743d01fd161SMarc Zyngier * - ThunderX: CN88xx 1744d01fd161SMarc Zyngier * - OCTEON TX: CN83xx, CN81xx 1745d01fd161SMarc Zyngier * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1746d01fd161SMarc Zyngier */ 1747d01fd161SMarc Zyngier .desc = "GICv3: Cavium erratum 38539", 1748d01fd161SMarc Zyngier .iidr = 0xa000034c, 1749d01fd161SMarc Zyngier .mask = 0xe8f00fff, 1750d01fd161SMarc Zyngier .init = gic_enable_quirk_cavium_38539, 1751d01fd161SMarc Zyngier }, 1752d01fd161SMarc Zyngier { 17537f2481b3SMarc Zyngier } 17547f2481b3SMarc Zyngier }; 17557f2481b3SMarc Zyngier 1756d98d0a99SJulien Thierry static void gic_enable_nmi_support(void) 1757d98d0a99SJulien Thierry { 1758101b35f7SJulien Thierry int i; 1759101b35f7SJulien Thierry 176081a43273SMarc Zyngier if (!gic_prio_masking_enabled()) 176181a43273SMarc Zyngier return; 176281a43273SMarc Zyngier 176381a43273SMarc Zyngier ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 176481a43273SMarc Zyngier if (!ppi_nmi_refs) 176581a43273SMarc Zyngier return; 176681a43273SMarc Zyngier 176781a43273SMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) 1768101b35f7SJulien Thierry refcount_set(&ppi_nmi_refs[i], 0); 1769101b35f7SJulien Thierry 1770f2266504SMarc Zyngier /* 1771f2266504SMarc Zyngier * Linux itself doesn't use 1:N distribution, so has no need to 1772f2266504SMarc Zyngier * set PMHE. The only reason to have it set is if EL3 requires it 1773f2266504SMarc Zyngier * (and we can't change it). 1774f2266504SMarc Zyngier */ 1775f2266504SMarc Zyngier if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) 1776f2266504SMarc Zyngier static_branch_enable(&gic_pmr_sync); 1777f2266504SMarc Zyngier 17784e594ad1SAlexandru Elisei pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 17794e594ad1SAlexandru Elisei static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed"); 1780f2266504SMarc Zyngier 178133678059SAlexandru Elisei /* 178233678059SAlexandru Elisei * How priority values are used by the GIC depends on two things: 178333678059SAlexandru Elisei * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 178433678059SAlexandru Elisei * and if Group 0 interrupts can be delivered to Linux in the non-secure 178533678059SAlexandru Elisei * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 178633678059SAlexandru Elisei * the ICC_PMR_EL1 register and the priority that software assigns to 178733678059SAlexandru Elisei * interrupts: 178833678059SAlexandru Elisei * 178933678059SAlexandru Elisei * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 179033678059SAlexandru Elisei * ----------------------------------------------------------- 179133678059SAlexandru Elisei * 1 | - | unchanged | unchanged 179233678059SAlexandru Elisei * ----------------------------------------------------------- 179333678059SAlexandru Elisei * 0 | 1 | non-secure | non-secure 179433678059SAlexandru Elisei * ----------------------------------------------------------- 179533678059SAlexandru Elisei * 0 | 0 | unchanged | non-secure 179633678059SAlexandru Elisei * 179733678059SAlexandru Elisei * where non-secure means that the value is right-shifted by one and the 179833678059SAlexandru Elisei * MSB bit set, to make it fit in the non-secure priority range. 179933678059SAlexandru Elisei * 180033678059SAlexandru Elisei * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 180133678059SAlexandru Elisei * are both either modified or unchanged, we can use the same set of 180233678059SAlexandru Elisei * priorities. 180333678059SAlexandru Elisei * 180433678059SAlexandru Elisei * In the last case, where only the interrupt priorities are modified to 180533678059SAlexandru Elisei * be in the non-secure range, we use a different PMR value to mask IRQs 180633678059SAlexandru Elisei * and the rest of the values that we use remain unchanged. 180733678059SAlexandru Elisei */ 180833678059SAlexandru Elisei if (gic_has_group0() && !gic_dist_security_disabled()) 180933678059SAlexandru Elisei static_branch_enable(&gic_nonsecure_priorities); 181033678059SAlexandru Elisei 1811d98d0a99SJulien Thierry static_branch_enable(&supports_pseudo_nmis); 1812101b35f7SJulien Thierry 1813101b35f7SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) 1814101b35f7SJulien Thierry gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1815101b35f7SJulien Thierry else 1816101b35f7SJulien Thierry gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1817d98d0a99SJulien Thierry } 1818d98d0a99SJulien Thierry 1819db57d746STomasz Nowicki static int __init gic_init_bases(void __iomem *dist_base, 1820db57d746STomasz Nowicki struct redist_region *rdist_regs, 1821db57d746STomasz Nowicki u32 nr_redist_regions, 1822db57d746STomasz Nowicki u64 redist_stride, 1823db57d746STomasz Nowicki struct fwnode_handle *handle) 1824db57d746STomasz Nowicki { 1825db57d746STomasz Nowicki u32 typer; 1826db57d746STomasz Nowicki int err; 1827db57d746STomasz Nowicki 1828db57d746STomasz Nowicki if (!is_hyp_mode_available()) 1829d01d3274SDavidlohr Bueso static_branch_disable(&supports_deactivate_key); 1830db57d746STomasz Nowicki 1831d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 1832db57d746STomasz Nowicki pr_info("GIC: Using split EOI/Deactivate mode\n"); 1833db57d746STomasz Nowicki 1834e3825ba1SMarc Zyngier gic_data.fwnode = handle; 1835db57d746STomasz Nowicki gic_data.dist_base = dist_base; 1836db57d746STomasz Nowicki gic_data.redist_regions = rdist_regs; 1837db57d746STomasz Nowicki gic_data.nr_redist_regions = nr_redist_regions; 1838db57d746STomasz Nowicki gic_data.redist_stride = redist_stride; 1839db57d746STomasz Nowicki 1840db57d746STomasz Nowicki /* 1841db57d746STomasz Nowicki * Find out how many interrupts are supported. 1842db57d746STomasz Nowicki */ 1843db57d746STomasz Nowicki typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1844a4f9edb2SMarc Zyngier gic_data.rdists.gicd_typer = typer; 18457f2481b3SMarc Zyngier 18467f2481b3SMarc Zyngier gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 18477f2481b3SMarc Zyngier gic_quirks, &gic_data); 18487f2481b3SMarc Zyngier 1849211bddd2SMarc Zyngier pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 1850211bddd2SMarc Zyngier pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 1851f2d83409SMarc Zyngier 1852d01fd161SMarc Zyngier /* 1853d01fd161SMarc Zyngier * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 1854d01fd161SMarc Zyngier * architecture spec (which says that reserved registers are RES0). 1855d01fd161SMarc Zyngier */ 1856d01fd161SMarc Zyngier if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 1857f2d83409SMarc Zyngier gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 1858f2d83409SMarc Zyngier 1859db57d746STomasz Nowicki gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1860db57d746STomasz Nowicki &gic_data); 1861db57d746STomasz Nowicki gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 1862b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = true; 18630edc23eaSMarc Zyngier gic_data.rdists.has_vlpis = true; 18640edc23eaSMarc Zyngier gic_data.rdists.has_direct_lpi = true; 186596806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty = true; 1866db57d746STomasz Nowicki 1867db57d746STomasz Nowicki if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1868db57d746STomasz Nowicki err = -ENOMEM; 1869db57d746STomasz Nowicki goto out_free; 1870db57d746STomasz Nowicki } 1871db57d746STomasz Nowicki 1872eeaa4b24Sluanshi irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1873eeaa4b24Sluanshi 1874eda0d04aSShanker Donthineni gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1875eda0d04aSShanker Donthineni 187650528752SMarc Zyngier if (typer & GICD_TYPER_MBIS) { 187750528752SMarc Zyngier err = mbi_init(handle, gic_data.domain); 187850528752SMarc Zyngier if (err) 187950528752SMarc Zyngier pr_err("Failed to initialize MBIs\n"); 188050528752SMarc Zyngier } 188150528752SMarc Zyngier 1882db57d746STomasz Nowicki set_handle_irq(gic_handle_irq); 1883db57d746STomasz Nowicki 18841a60e1e6SMarc Zyngier gic_update_rdist_properties(); 18850edc23eaSMarc Zyngier 1886db57d746STomasz Nowicki gic_dist_init(); 1887db57d746STomasz Nowicki gic_cpu_init(); 188864b499d8SMarc Zyngier gic_smp_init(); 1889db57d746STomasz Nowicki gic_cpu_pm_init(); 1890db57d746STomasz Nowicki 1891d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) { 1892d38a71c5SMarc Zyngier its_init(handle, &gic_data.rdists, gic_data.domain); 1893d38a71c5SMarc Zyngier its_cpu_init(); 1894d23bc2bcSValentin Schneider its_lpi_memreserve_init(); 189590b4c555SZeev Zilberman } else { 189690b4c555SZeev Zilberman if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 189790b4c555SZeev Zilberman gicv2m_init(handle, gic_data.domain); 1898d38a71c5SMarc Zyngier } 1899d38a71c5SMarc Zyngier 1900d98d0a99SJulien Thierry gic_enable_nmi_support(); 1901d98d0a99SJulien Thierry 1902db57d746STomasz Nowicki return 0; 1903db57d746STomasz Nowicki 1904db57d746STomasz Nowicki out_free: 1905db57d746STomasz Nowicki if (gic_data.domain) 1906db57d746STomasz Nowicki irq_domain_remove(gic_data.domain); 1907db57d746STomasz Nowicki free_percpu(gic_data.rdists.rdist); 1908db57d746STomasz Nowicki return err; 1909db57d746STomasz Nowicki } 1910db57d746STomasz Nowicki 1911db57d746STomasz Nowicki static int __init gic_validate_dist_version(void __iomem *dist_base) 1912db57d746STomasz Nowicki { 1913db57d746STomasz Nowicki u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1914db57d746STomasz Nowicki 1915db57d746STomasz Nowicki if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 1916db57d746STomasz Nowicki return -ENODEV; 1917db57d746STomasz Nowicki 1918db57d746STomasz Nowicki return 0; 1919db57d746STomasz Nowicki } 1920db57d746STomasz Nowicki 1921e3825ba1SMarc Zyngier /* Create all possible partitions at boot time */ 19227beaa24bSLinus Torvalds static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 1923e3825ba1SMarc Zyngier { 1924e3825ba1SMarc Zyngier struct device_node *parts_node, *child_part; 1925e3825ba1SMarc Zyngier int part_idx = 0, i; 1926e3825ba1SMarc Zyngier int nr_parts; 1927e3825ba1SMarc Zyngier struct partition_affinity *parts; 1928e3825ba1SMarc Zyngier 192900ee9a1cSJohan Hovold parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 1930e3825ba1SMarc Zyngier if (!parts_node) 1931e3825ba1SMarc Zyngier return; 1932e3825ba1SMarc Zyngier 193352085d3fSMarc Zyngier gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 193452085d3fSMarc Zyngier if (!gic_data.ppi_descs) 1935ec8401a4SMiaoqian Lin goto out_put_node; 193652085d3fSMarc Zyngier 1937e3825ba1SMarc Zyngier nr_parts = of_get_child_count(parts_node); 1938e3825ba1SMarc Zyngier 1939e3825ba1SMarc Zyngier if (!nr_parts) 194000ee9a1cSJohan Hovold goto out_put_node; 1941e3825ba1SMarc Zyngier 19426396bb22SKees Cook parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 1943e3825ba1SMarc Zyngier if (WARN_ON(!parts)) 194400ee9a1cSJohan Hovold goto out_put_node; 1945e3825ba1SMarc Zyngier 1946e3825ba1SMarc Zyngier for_each_child_of_node(parts_node, child_part) { 1947e3825ba1SMarc Zyngier struct partition_affinity *part; 1948e3825ba1SMarc Zyngier int n; 1949e3825ba1SMarc Zyngier 1950e3825ba1SMarc Zyngier part = &parts[part_idx]; 1951e3825ba1SMarc Zyngier 1952e3825ba1SMarc Zyngier part->partition_id = of_node_to_fwnode(child_part); 1953e3825ba1SMarc Zyngier 19542ef790dcSRob Herring pr_info("GIC: PPI partition %pOFn[%d] { ", 19552ef790dcSRob Herring child_part, part_idx); 1956e3825ba1SMarc Zyngier 1957e3825ba1SMarc Zyngier n = of_property_count_elems_of_size(child_part, "affinity", 1958e3825ba1SMarc Zyngier sizeof(u32)); 1959e3825ba1SMarc Zyngier WARN_ON(n <= 0); 1960e3825ba1SMarc Zyngier 1961e3825ba1SMarc Zyngier for (i = 0; i < n; i++) { 1962e3825ba1SMarc Zyngier int err, cpu; 1963e3825ba1SMarc Zyngier u32 cpu_phandle; 1964e3825ba1SMarc Zyngier struct device_node *cpu_node; 1965e3825ba1SMarc Zyngier 1966e3825ba1SMarc Zyngier err = of_property_read_u32_index(child_part, "affinity", 1967e3825ba1SMarc Zyngier i, &cpu_phandle); 1968e3825ba1SMarc Zyngier if (WARN_ON(err)) 1969e3825ba1SMarc Zyngier continue; 1970e3825ba1SMarc Zyngier 1971e3825ba1SMarc Zyngier cpu_node = of_find_node_by_phandle(cpu_phandle); 1972e3825ba1SMarc Zyngier if (WARN_ON(!cpu_node)) 1973e3825ba1SMarc Zyngier continue; 1974e3825ba1SMarc Zyngier 1975c08ec7daSSuzuki K Poulose cpu = of_cpu_node_to_id(cpu_node); 1976fa1ad9d4SMiaoqian Lin if (WARN_ON(cpu < 0)) { 1977fa1ad9d4SMiaoqian Lin of_node_put(cpu_node); 1978e3825ba1SMarc Zyngier continue; 1979fa1ad9d4SMiaoqian Lin } 1980e3825ba1SMarc Zyngier 1981e81f54c6SRob Herring pr_cont("%pOF[%d] ", cpu_node, cpu); 1982e3825ba1SMarc Zyngier 1983e3825ba1SMarc Zyngier cpumask_set_cpu(cpu, &part->mask); 1984fa1ad9d4SMiaoqian Lin of_node_put(cpu_node); 1985e3825ba1SMarc Zyngier } 1986e3825ba1SMarc Zyngier 1987e3825ba1SMarc Zyngier pr_cont("}\n"); 1988e3825ba1SMarc Zyngier part_idx++; 1989e3825ba1SMarc Zyngier } 1990e3825ba1SMarc Zyngier 199152085d3fSMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) { 1992e3825ba1SMarc Zyngier unsigned int irq; 1993e3825ba1SMarc Zyngier struct partition_desc *desc; 1994e3825ba1SMarc Zyngier struct irq_fwspec ppi_fwspec = { 1995e3825ba1SMarc Zyngier .fwnode = gic_data.fwnode, 1996e3825ba1SMarc Zyngier .param_count = 3, 1997e3825ba1SMarc Zyngier .param = { 199865da7d19SMarc Zyngier [0] = GIC_IRQ_TYPE_PARTITION, 1999e3825ba1SMarc Zyngier [1] = i, 2000e3825ba1SMarc Zyngier [2] = IRQ_TYPE_NONE, 2001e3825ba1SMarc Zyngier }, 2002e3825ba1SMarc Zyngier }; 2003e3825ba1SMarc Zyngier 2004e3825ba1SMarc Zyngier irq = irq_create_fwspec_mapping(&ppi_fwspec); 2005e3825ba1SMarc Zyngier if (WARN_ON(!irq)) 2006e3825ba1SMarc Zyngier continue; 2007e3825ba1SMarc Zyngier desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 2008e3825ba1SMarc Zyngier irq, &partition_domain_ops); 2009e3825ba1SMarc Zyngier if (WARN_ON(!desc)) 2010e3825ba1SMarc Zyngier continue; 2011e3825ba1SMarc Zyngier 2012e3825ba1SMarc Zyngier gic_data.ppi_descs[i] = desc; 2013e3825ba1SMarc Zyngier } 201400ee9a1cSJohan Hovold 201500ee9a1cSJohan Hovold out_put_node: 201600ee9a1cSJohan Hovold of_node_put(parts_node); 2017e3825ba1SMarc Zyngier } 2018e3825ba1SMarc Zyngier 20191839e576SJulien Grall static void __init gic_of_setup_kvm_info(struct device_node *node) 20201839e576SJulien Grall { 20211839e576SJulien Grall int ret; 20221839e576SJulien Grall struct resource r; 20231839e576SJulien Grall u32 gicv_idx; 20241839e576SJulien Grall 20251839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 20261839e576SJulien Grall 20271839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 20281839e576SJulien Grall if (!gic_v3_kvm_info.maint_irq) 20291839e576SJulien Grall return; 20301839e576SJulien Grall 20311839e576SJulien Grall if (of_property_read_u32(node, "#redistributor-regions", 20321839e576SJulien Grall &gicv_idx)) 20331839e576SJulien Grall gicv_idx = 1; 20341839e576SJulien Grall 20351839e576SJulien Grall gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 20361839e576SJulien Grall ret = of_address_to_resource(node, gicv_idx, &r); 20371839e576SJulien Grall if (!ret) 20381839e576SJulien Grall gic_v3_kvm_info.vcpu = r; 20391839e576SJulien Grall 20404bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 20413c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 20420e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info); 20431839e576SJulien Grall } 20441839e576SJulien Grall 2045021f6537SMarc Zyngier static int __init gic_of_init(struct device_node *node, struct device_node *parent) 2046021f6537SMarc Zyngier { 2047021f6537SMarc Zyngier void __iomem *dist_base; 2048f5c1434cSMarc Zyngier struct redist_region *rdist_regs; 2049021f6537SMarc Zyngier u64 redist_stride; 2050f5c1434cSMarc Zyngier u32 nr_redist_regions; 2051db57d746STomasz Nowicki int err, i; 2052021f6537SMarc Zyngier 20532b2cd74aSRobin Murphy dist_base = of_io_request_and_map(node, 0, "GICD"); 20542b2cd74aSRobin Murphy if (IS_ERR(dist_base)) { 2055e81f54c6SRob Herring pr_err("%pOF: unable to map gic dist registers\n", node); 20562b2cd74aSRobin Murphy return PTR_ERR(dist_base); 2057021f6537SMarc Zyngier } 2058021f6537SMarc Zyngier 2059db57d746STomasz Nowicki err = gic_validate_dist_version(dist_base); 2060db57d746STomasz Nowicki if (err) { 2061e81f54c6SRob Herring pr_err("%pOF: no distributor detected, giving up\n", node); 2062021f6537SMarc Zyngier goto out_unmap_dist; 2063021f6537SMarc Zyngier } 2064021f6537SMarc Zyngier 2065f5c1434cSMarc Zyngier if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 2066f5c1434cSMarc Zyngier nr_redist_regions = 1; 2067021f6537SMarc Zyngier 20686396bb22SKees Cook rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 20696396bb22SKees Cook GFP_KERNEL); 2070f5c1434cSMarc Zyngier if (!rdist_regs) { 2071021f6537SMarc Zyngier err = -ENOMEM; 2072021f6537SMarc Zyngier goto out_unmap_dist; 2073021f6537SMarc Zyngier } 2074021f6537SMarc Zyngier 2075f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) { 2076f5c1434cSMarc Zyngier struct resource res; 2077f5c1434cSMarc Zyngier int ret; 2078f5c1434cSMarc Zyngier 2079f5c1434cSMarc Zyngier ret = of_address_to_resource(node, 1 + i, &res); 20802b2cd74aSRobin Murphy rdist_regs[i].redist_base = of_io_request_and_map(node, 1 + i, "GICR"); 20812b2cd74aSRobin Murphy if (ret || IS_ERR(rdist_regs[i].redist_base)) { 2082e81f54c6SRob Herring pr_err("%pOF: couldn't map region %d\n", node, i); 2083021f6537SMarc Zyngier err = -ENODEV; 2084021f6537SMarc Zyngier goto out_unmap_rdist; 2085021f6537SMarc Zyngier } 2086f5c1434cSMarc Zyngier rdist_regs[i].phys_base = res.start; 2087021f6537SMarc Zyngier } 2088021f6537SMarc Zyngier 2089021f6537SMarc Zyngier if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 2090021f6537SMarc Zyngier redist_stride = 0; 2091021f6537SMarc Zyngier 2092f70fdb42SSrinivas Kandagatla gic_enable_of_quirks(node, gic_quirks, &gic_data); 2093f70fdb42SSrinivas Kandagatla 2094db57d746STomasz Nowicki err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions, 2095db57d746STomasz Nowicki redist_stride, &node->fwnode); 2096e3825ba1SMarc Zyngier if (err) 2097e3825ba1SMarc Zyngier goto out_unmap_rdist; 2098e3825ba1SMarc Zyngier 2099e3825ba1SMarc Zyngier gic_populate_ppi_partitions(node); 2100d33a3c8cSChristoffer Dall 2101d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 21021839e576SJulien Grall gic_of_setup_kvm_info(node); 2103021f6537SMarc Zyngier return 0; 2104021f6537SMarc Zyngier 2105021f6537SMarc Zyngier out_unmap_rdist: 2106f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) 21072b2cd74aSRobin Murphy if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base)) 2108f5c1434cSMarc Zyngier iounmap(rdist_regs[i].redist_base); 2109f5c1434cSMarc Zyngier kfree(rdist_regs); 2110021f6537SMarc Zyngier out_unmap_dist: 2111021f6537SMarc Zyngier iounmap(dist_base); 2112021f6537SMarc Zyngier return err; 2113021f6537SMarc Zyngier } 2114021f6537SMarc Zyngier 2115021f6537SMarc Zyngier IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 2116ffa7d616STomasz Nowicki 2117ffa7d616STomasz Nowicki #ifdef CONFIG_ACPI 2118611f039fSJulien Grall static struct 2119611f039fSJulien Grall { 2120611f039fSJulien Grall void __iomem *dist_base; 2121611f039fSJulien Grall struct redist_region *redist_regs; 2122611f039fSJulien Grall u32 nr_redist_regions; 2123611f039fSJulien Grall bool single_redist; 2124926b5dfaSMarc Zyngier int enabled_rdists; 21251839e576SJulien Grall u32 maint_irq; 21261839e576SJulien Grall int maint_irq_mode; 21271839e576SJulien Grall phys_addr_t vcpu_base; 2128611f039fSJulien Grall } acpi_data __initdata; 2129b70fb7afSTomasz Nowicki 2130b70fb7afSTomasz Nowicki static void __init 2131b70fb7afSTomasz Nowicki gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 2132b70fb7afSTomasz Nowicki { 2133b70fb7afSTomasz Nowicki static int count = 0; 2134b70fb7afSTomasz Nowicki 2135611f039fSJulien Grall acpi_data.redist_regs[count].phys_base = phys_base; 2136611f039fSJulien Grall acpi_data.redist_regs[count].redist_base = redist_base; 2137611f039fSJulien Grall acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 2138b70fb7afSTomasz Nowicki count++; 2139b70fb7afSTomasz Nowicki } 2140ffa7d616STomasz Nowicki 2141ffa7d616STomasz Nowicki static int __init 214260574d1eSKeith Busch gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 2143ffa7d616STomasz Nowicki const unsigned long end) 2144ffa7d616STomasz Nowicki { 2145ffa7d616STomasz Nowicki struct acpi_madt_generic_redistributor *redist = 2146ffa7d616STomasz Nowicki (struct acpi_madt_generic_redistributor *)header; 2147ffa7d616STomasz Nowicki void __iomem *redist_base; 2148ffa7d616STomasz Nowicki 2149ffa7d616STomasz Nowicki redist_base = ioremap(redist->base_address, redist->length); 2150ffa7d616STomasz Nowicki if (!redist_base) { 2151ffa7d616STomasz Nowicki pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2152ffa7d616STomasz Nowicki return -ENOMEM; 2153ffa7d616STomasz Nowicki } 21542b2cd74aSRobin Murphy request_mem_region(redist->base_address, redist->length, "GICR"); 2155ffa7d616STomasz Nowicki 2156b70fb7afSTomasz Nowicki gic_acpi_register_redist(redist->base_address, redist_base); 2157ffa7d616STomasz Nowicki return 0; 2158ffa7d616STomasz Nowicki } 2159ffa7d616STomasz Nowicki 2160b70fb7afSTomasz Nowicki static int __init 216160574d1eSKeith Busch gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2162b70fb7afSTomasz Nowicki const unsigned long end) 2163b70fb7afSTomasz Nowicki { 2164b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 2165b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 2166611f039fSJulien Grall u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2167b70fb7afSTomasz Nowicki u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2168b70fb7afSTomasz Nowicki void __iomem *redist_base; 2169b70fb7afSTomasz Nowicki 2170ebe2f871SShanker Donthineni /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 2171ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 2172ebe2f871SShanker Donthineni return 0; 2173ebe2f871SShanker Donthineni 2174b70fb7afSTomasz Nowicki redist_base = ioremap(gicc->gicr_base_address, size); 2175b70fb7afSTomasz Nowicki if (!redist_base) 2176b70fb7afSTomasz Nowicki return -ENOMEM; 21772b2cd74aSRobin Murphy request_mem_region(gicc->gicr_base_address, size, "GICR"); 2178b70fb7afSTomasz Nowicki 2179b70fb7afSTomasz Nowicki gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2180b70fb7afSTomasz Nowicki return 0; 2181b70fb7afSTomasz Nowicki } 2182b70fb7afSTomasz Nowicki 2183b70fb7afSTomasz Nowicki static int __init gic_acpi_collect_gicr_base(void) 2184b70fb7afSTomasz Nowicki { 2185b70fb7afSTomasz Nowicki acpi_tbl_entry_handler redist_parser; 2186b70fb7afSTomasz Nowicki enum acpi_madt_type type; 2187b70fb7afSTomasz Nowicki 2188611f039fSJulien Grall if (acpi_data.single_redist) { 2189b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2190b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_gicc; 2191b70fb7afSTomasz Nowicki } else { 2192b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2193b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_redist; 2194b70fb7afSTomasz Nowicki } 2195b70fb7afSTomasz Nowicki 2196b70fb7afSTomasz Nowicki /* Collect redistributor base addresses in GICR entries */ 2197b70fb7afSTomasz Nowicki if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2198b70fb7afSTomasz Nowicki return 0; 2199b70fb7afSTomasz Nowicki 2200b70fb7afSTomasz Nowicki pr_info("No valid GICR entries exist\n"); 2201b70fb7afSTomasz Nowicki return -ENODEV; 2202b70fb7afSTomasz Nowicki } 2203b70fb7afSTomasz Nowicki 220460574d1eSKeith Busch static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2205ffa7d616STomasz Nowicki const unsigned long end) 2206ffa7d616STomasz Nowicki { 2207ffa7d616STomasz Nowicki /* Subtable presence means that redist exists, that's it */ 2208ffa7d616STomasz Nowicki return 0; 2209ffa7d616STomasz Nowicki } 2210ffa7d616STomasz Nowicki 221160574d1eSKeith Busch static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2212b70fb7afSTomasz Nowicki const unsigned long end) 2213b70fb7afSTomasz Nowicki { 2214b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 2215b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 2216b70fb7afSTomasz Nowicki 2217b70fb7afSTomasz Nowicki /* 2218b70fb7afSTomasz Nowicki * If GICC is enabled and has valid gicr base address, then it means 2219b70fb7afSTomasz Nowicki * GICR base is presented via GICC 2220b70fb7afSTomasz Nowicki */ 2221926b5dfaSMarc Zyngier if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2222926b5dfaSMarc Zyngier acpi_data.enabled_rdists++; 2223b70fb7afSTomasz Nowicki return 0; 2224926b5dfaSMarc Zyngier } 2225b70fb7afSTomasz Nowicki 2226ebe2f871SShanker Donthineni /* 2227ebe2f871SShanker Donthineni * It's perfectly valid firmware can pass disabled GICC entry, driver 2228ebe2f871SShanker Donthineni * should not treat as errors, skip the entry instead of probe fail. 2229ebe2f871SShanker Donthineni */ 2230ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 2231ebe2f871SShanker Donthineni return 0; 2232ebe2f871SShanker Donthineni 2233b70fb7afSTomasz Nowicki return -ENODEV; 2234b70fb7afSTomasz Nowicki } 2235b70fb7afSTomasz Nowicki 2236b70fb7afSTomasz Nowicki static int __init gic_acpi_count_gicr_regions(void) 2237b70fb7afSTomasz Nowicki { 2238b70fb7afSTomasz Nowicki int count; 2239b70fb7afSTomasz Nowicki 2240b70fb7afSTomasz Nowicki /* 2241b70fb7afSTomasz Nowicki * Count how many redistributor regions we have. It is not allowed 2242b70fb7afSTomasz Nowicki * to mix redistributor description, GICR and GICC subtables have to be 2243b70fb7afSTomasz Nowicki * mutually exclusive. 2244b70fb7afSTomasz Nowicki */ 2245b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2246b70fb7afSTomasz Nowicki gic_acpi_match_gicr, 0); 2247b70fb7afSTomasz Nowicki if (count > 0) { 2248611f039fSJulien Grall acpi_data.single_redist = false; 2249b70fb7afSTomasz Nowicki return count; 2250b70fb7afSTomasz Nowicki } 2251b70fb7afSTomasz Nowicki 2252b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2253b70fb7afSTomasz Nowicki gic_acpi_match_gicc, 0); 2254926b5dfaSMarc Zyngier if (count > 0) { 2255611f039fSJulien Grall acpi_data.single_redist = true; 2256926b5dfaSMarc Zyngier count = acpi_data.enabled_rdists; 2257926b5dfaSMarc Zyngier } 2258b70fb7afSTomasz Nowicki 2259b70fb7afSTomasz Nowicki return count; 2260b70fb7afSTomasz Nowicki } 2261b70fb7afSTomasz Nowicki 2262ffa7d616STomasz Nowicki static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2263ffa7d616STomasz Nowicki struct acpi_probe_entry *ape) 2264ffa7d616STomasz Nowicki { 2265ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2266ffa7d616STomasz Nowicki int count; 2267ffa7d616STomasz Nowicki 2268ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2269ffa7d616STomasz Nowicki if (dist->version != ape->driver_data) 2270ffa7d616STomasz Nowicki return false; 2271ffa7d616STomasz Nowicki 2272ffa7d616STomasz Nowicki /* We need to do that exercise anyway, the sooner the better */ 2273b70fb7afSTomasz Nowicki count = gic_acpi_count_gicr_regions(); 2274ffa7d616STomasz Nowicki if (count <= 0) 2275ffa7d616STomasz Nowicki return false; 2276ffa7d616STomasz Nowicki 2277611f039fSJulien Grall acpi_data.nr_redist_regions = count; 2278ffa7d616STomasz Nowicki return true; 2279ffa7d616STomasz Nowicki } 2280ffa7d616STomasz Nowicki 228160574d1eSKeith Busch static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 22821839e576SJulien Grall const unsigned long end) 22831839e576SJulien Grall { 22841839e576SJulien Grall struct acpi_madt_generic_interrupt *gicc = 22851839e576SJulien Grall (struct acpi_madt_generic_interrupt *)header; 22861839e576SJulien Grall int maint_irq_mode; 22871839e576SJulien Grall static int first_madt = true; 22881839e576SJulien Grall 22891839e576SJulien Grall /* Skip unusable CPUs */ 22901839e576SJulien Grall if (!(gicc->flags & ACPI_MADT_ENABLED)) 22911839e576SJulien Grall return 0; 22921839e576SJulien Grall 22931839e576SJulien Grall maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 22941839e576SJulien Grall ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 22951839e576SJulien Grall 22961839e576SJulien Grall if (first_madt) { 22971839e576SJulien Grall first_madt = false; 22981839e576SJulien Grall 22991839e576SJulien Grall acpi_data.maint_irq = gicc->vgic_interrupt; 23001839e576SJulien Grall acpi_data.maint_irq_mode = maint_irq_mode; 23011839e576SJulien Grall acpi_data.vcpu_base = gicc->gicv_base_address; 23021839e576SJulien Grall 23031839e576SJulien Grall return 0; 23041839e576SJulien Grall } 23051839e576SJulien Grall 23061839e576SJulien Grall /* 23071839e576SJulien Grall * The maintenance interrupt and GICV should be the same for every CPU 23081839e576SJulien Grall */ 23091839e576SJulien Grall if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 23101839e576SJulien Grall (acpi_data.maint_irq_mode != maint_irq_mode) || 23111839e576SJulien Grall (acpi_data.vcpu_base != gicc->gicv_base_address)) 23121839e576SJulien Grall return -EINVAL; 23131839e576SJulien Grall 23141839e576SJulien Grall return 0; 23151839e576SJulien Grall } 23161839e576SJulien Grall 23171839e576SJulien Grall static bool __init gic_acpi_collect_virt_info(void) 23181839e576SJulien Grall { 23191839e576SJulien Grall int count; 23201839e576SJulien Grall 23211839e576SJulien Grall count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 23221839e576SJulien Grall gic_acpi_parse_virt_madt_gicc, 0); 23231839e576SJulien Grall 23241839e576SJulien Grall return (count > 0); 23251839e576SJulien Grall } 23261839e576SJulien Grall 2327ffa7d616STomasz Nowicki #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 23281839e576SJulien Grall #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 23291839e576SJulien Grall #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 23301839e576SJulien Grall 23311839e576SJulien Grall static void __init gic_acpi_setup_kvm_info(void) 23321839e576SJulien Grall { 23331839e576SJulien Grall int irq; 23341839e576SJulien Grall 23351839e576SJulien Grall if (!gic_acpi_collect_virt_info()) { 23361839e576SJulien Grall pr_warn("Unable to get hardware information used for virtualization\n"); 23371839e576SJulien Grall return; 23381839e576SJulien Grall } 23391839e576SJulien Grall 23401839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 23411839e576SJulien Grall 23421839e576SJulien Grall irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 23431839e576SJulien Grall acpi_data.maint_irq_mode, 23441839e576SJulien Grall ACPI_ACTIVE_HIGH); 23451839e576SJulien Grall if (irq <= 0) 23461839e576SJulien Grall return; 23471839e576SJulien Grall 23481839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq; 23491839e576SJulien Grall 23501839e576SJulien Grall if (acpi_data.vcpu_base) { 23511839e576SJulien Grall struct resource *vcpu = &gic_v3_kvm_info.vcpu; 23521839e576SJulien Grall 23531839e576SJulien Grall vcpu->flags = IORESOURCE_MEM; 23541839e576SJulien Grall vcpu->start = acpi_data.vcpu_base; 23551839e576SJulien Grall vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 23561839e576SJulien Grall } 23571839e576SJulien Grall 23584bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 23593c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 23600e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info); 23611839e576SJulien Grall } 2362ffa7d616STomasz Nowicki 2363*7327b16fSMarc Zyngier static struct fwnode_handle *gsi_domain_handle; 2364*7327b16fSMarc Zyngier 2365*7327b16fSMarc Zyngier static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi) 2366*7327b16fSMarc Zyngier { 2367*7327b16fSMarc Zyngier return gsi_domain_handle; 2368*7327b16fSMarc Zyngier } 2369*7327b16fSMarc Zyngier 2370ffa7d616STomasz Nowicki static int __init 2371aba3c7edSOscar Carter gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2372ffa7d616STomasz Nowicki { 2373ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2374611f039fSJulien Grall size_t size; 2375b70fb7afSTomasz Nowicki int i, err; 2376ffa7d616STomasz Nowicki 2377ffa7d616STomasz Nowicki /* Get distributor base address */ 2378ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2379611f039fSJulien Grall acpi_data.dist_base = ioremap(dist->base_address, 2380611f039fSJulien Grall ACPI_GICV3_DIST_MEM_SIZE); 2381611f039fSJulien Grall if (!acpi_data.dist_base) { 2382ffa7d616STomasz Nowicki pr_err("Unable to map GICD registers\n"); 2383ffa7d616STomasz Nowicki return -ENOMEM; 2384ffa7d616STomasz Nowicki } 23852b2cd74aSRobin Murphy request_mem_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD"); 2386ffa7d616STomasz Nowicki 2387611f039fSJulien Grall err = gic_validate_dist_version(acpi_data.dist_base); 2388ffa7d616STomasz Nowicki if (err) { 238971192a68SArvind Yadav pr_err("No distributor detected at @%p, giving up\n", 2390611f039fSJulien Grall acpi_data.dist_base); 2391ffa7d616STomasz Nowicki goto out_dist_unmap; 2392ffa7d616STomasz Nowicki } 2393ffa7d616STomasz Nowicki 2394611f039fSJulien Grall size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2395611f039fSJulien Grall acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2396611f039fSJulien Grall if (!acpi_data.redist_regs) { 2397ffa7d616STomasz Nowicki err = -ENOMEM; 2398ffa7d616STomasz Nowicki goto out_dist_unmap; 2399ffa7d616STomasz Nowicki } 2400ffa7d616STomasz Nowicki 2401b70fb7afSTomasz Nowicki err = gic_acpi_collect_gicr_base(); 2402b70fb7afSTomasz Nowicki if (err) 2403ffa7d616STomasz Nowicki goto out_redist_unmap; 2404ffa7d616STomasz Nowicki 2405*7327b16fSMarc Zyngier gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2406*7327b16fSMarc Zyngier if (!gsi_domain_handle) { 2407ffa7d616STomasz Nowicki err = -ENOMEM; 2408ffa7d616STomasz Nowicki goto out_redist_unmap; 2409ffa7d616STomasz Nowicki } 2410ffa7d616STomasz Nowicki 2411611f039fSJulien Grall err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs, 2412*7327b16fSMarc Zyngier acpi_data.nr_redist_regions, 0, gsi_domain_handle); 2413ffa7d616STomasz Nowicki if (err) 2414ffa7d616STomasz Nowicki goto out_fwhandle_free; 2415ffa7d616STomasz Nowicki 2416*7327b16fSMarc Zyngier acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id); 2417d33a3c8cSChristoffer Dall 2418d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 24191839e576SJulien Grall gic_acpi_setup_kvm_info(); 24201839e576SJulien Grall 2421ffa7d616STomasz Nowicki return 0; 2422ffa7d616STomasz Nowicki 2423ffa7d616STomasz Nowicki out_fwhandle_free: 2424*7327b16fSMarc Zyngier irq_domain_free_fwnode(gsi_domain_handle); 2425ffa7d616STomasz Nowicki out_redist_unmap: 2426611f039fSJulien Grall for (i = 0; i < acpi_data.nr_redist_regions; i++) 2427611f039fSJulien Grall if (acpi_data.redist_regs[i].redist_base) 2428611f039fSJulien Grall iounmap(acpi_data.redist_regs[i].redist_base); 2429611f039fSJulien Grall kfree(acpi_data.redist_regs); 2430ffa7d616STomasz Nowicki out_dist_unmap: 2431611f039fSJulien Grall iounmap(acpi_data.dist_base); 2432ffa7d616STomasz Nowicki return err; 2433ffa7d616STomasz Nowicki } 2434ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2435ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2436ffa7d616STomasz Nowicki gic_acpi_init); 2437ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2438ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2439ffa7d616STomasz Nowicki gic_acpi_init); 2440ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2441ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2442ffa7d616STomasz Nowicki gic_acpi_init); 2443ffa7d616STomasz Nowicki #endif 2444