1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2021f6537SMarc Zyngier /* 30edc23eaSMarc Zyngier * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4021f6537SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com> 5021f6537SMarc Zyngier */ 6021f6537SMarc Zyngier 768628bb8SJulien Grall #define pr_fmt(fmt) "GICv3: " fmt 868628bb8SJulien Grall 9ffa7d616STomasz Nowicki #include <linux/acpi.h> 10021f6537SMarc Zyngier #include <linux/cpu.h> 113708d52fSSudeep Holla #include <linux/cpu_pm.h> 12021f6537SMarc Zyngier #include <linux/delay.h> 13021f6537SMarc Zyngier #include <linux/interrupt.h> 14ffa7d616STomasz Nowicki #include <linux/irqdomain.h> 15021f6537SMarc Zyngier #include <linux/of.h> 16021f6537SMarc Zyngier #include <linux/of_address.h> 17021f6537SMarc Zyngier #include <linux/of_irq.h> 18021f6537SMarc Zyngier #include <linux/percpu.h> 19101b35f7SJulien Thierry #include <linux/refcount.h> 20021f6537SMarc Zyngier #include <linux/slab.h> 21021f6537SMarc Zyngier 2241a83e06SJoel Porquet #include <linux/irqchip.h> 231839e576SJulien Grall #include <linux/irqchip/arm-gic-common.h> 24021f6537SMarc Zyngier #include <linux/irqchip/arm-gic-v3.h> 25e3825ba1SMarc Zyngier #include <linux/irqchip/irq-partition-percpu.h> 26021f6537SMarc Zyngier 27021f6537SMarc Zyngier #include <asm/cputype.h> 28021f6537SMarc Zyngier #include <asm/exception.h> 29021f6537SMarc Zyngier #include <asm/smp_plat.h> 300b6a3da9SMarc Zyngier #include <asm/virt.h> 31021f6537SMarc Zyngier 32021f6537SMarc Zyngier #include "irq-gic-common.h" 33021f6537SMarc Zyngier 34f32c9266SJulien Thierry #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 35f32c9266SJulien Thierry 369c8114c2SSrinivas Kandagatla #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 37d01fd161SMarc Zyngier #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 389c8114c2SSrinivas Kandagatla 39f5c1434cSMarc Zyngier struct redist_region { 40f5c1434cSMarc Zyngier void __iomem *redist_base; 41f5c1434cSMarc Zyngier phys_addr_t phys_base; 42b70fb7afSTomasz Nowicki bool single_redist; 43f5c1434cSMarc Zyngier }; 44f5c1434cSMarc Zyngier 45021f6537SMarc Zyngier struct gic_chip_data { 46e3825ba1SMarc Zyngier struct fwnode_handle *fwnode; 47021f6537SMarc Zyngier void __iomem *dist_base; 48f5c1434cSMarc Zyngier struct redist_region *redist_regions; 49f5c1434cSMarc Zyngier struct rdists rdists; 50021f6537SMarc Zyngier struct irq_domain *domain; 51021f6537SMarc Zyngier u64 redist_stride; 52f5c1434cSMarc Zyngier u32 nr_redist_regions; 539c8114c2SSrinivas Kandagatla u64 flags; 54eda0d04aSShanker Donthineni bool has_rss; 551a60e1e6SMarc Zyngier unsigned int ppi_nr; 5652085d3fSMarc Zyngier struct partition_desc **ppi_descs; 57021f6537SMarc Zyngier }; 58021f6537SMarc Zyngier 59021f6537SMarc Zyngier static struct gic_chip_data gic_data __read_mostly; 60d01d3274SDavidlohr Bueso static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 61021f6537SMarc Zyngier 62211bddd2SMarc Zyngier #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 63c107d613SZenghui Yu #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 64211bddd2SMarc Zyngier #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 65211bddd2SMarc Zyngier 66d98d0a99SJulien Thierry /* 67d98d0a99SJulien Thierry * The behaviours of RPR and PMR registers differ depending on the value of 68d98d0a99SJulien Thierry * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 69d98d0a99SJulien Thierry * distributor and redistributors depends on whether security is enabled in the 70d98d0a99SJulien Thierry * GIC. 71d98d0a99SJulien Thierry * 72d98d0a99SJulien Thierry * When security is enabled, non-secure priority values from the (re)distributor 73d98d0a99SJulien Thierry * are presented to the GIC CPUIF as follow: 74d98d0a99SJulien Thierry * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 75d98d0a99SJulien Thierry * 76d98d0a99SJulien Thierry * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure 77d98d0a99SJulien Thierry * EL1 are subject to a similar operation thus matching the priorities presented 7833678059SAlexandru Elisei * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 7933678059SAlexandru Elisei * these values are unchanched by the GIC. 80d98d0a99SJulien Thierry * 81d98d0a99SJulien Thierry * see GICv3/GICv4 Architecture Specification (IHI0069D): 82d98d0a99SJulien Thierry * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 83d98d0a99SJulien Thierry * priorities. 84d98d0a99SJulien Thierry * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 85d98d0a99SJulien Thierry * interrupt. 86d98d0a99SJulien Thierry */ 87d98d0a99SJulien Thierry static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 88d98d0a99SJulien Thierry 89f2266504SMarc Zyngier /* 90f2266504SMarc Zyngier * Global static key controlling whether an update to PMR allowing more 91f2266504SMarc Zyngier * interrupts requires to be propagated to the redistributor (DSB SY). 92f2266504SMarc Zyngier * And this needs to be exported for modules to be able to enable 93f2266504SMarc Zyngier * interrupts... 94f2266504SMarc Zyngier */ 95f2266504SMarc Zyngier DEFINE_STATIC_KEY_FALSE(gic_pmr_sync); 96f2266504SMarc Zyngier EXPORT_SYMBOL(gic_pmr_sync); 97f2266504SMarc Zyngier 9833678059SAlexandru Elisei DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 9933678059SAlexandru Elisei EXPORT_SYMBOL(gic_nonsecure_priorities); 10033678059SAlexandru Elisei 101101b35f7SJulien Thierry /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 10281a43273SMarc Zyngier static refcount_t *ppi_nmi_refs; 103101b35f7SJulien Thierry 1041839e576SJulien Grall static struct gic_kvm_info gic_v3_kvm_info; 105eda0d04aSShanker Donthineni static DEFINE_PER_CPU(bool, has_rss); 1061839e576SJulien Grall 107eda0d04aSShanker Donthineni #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 108f5c1434cSMarc Zyngier #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 109f5c1434cSMarc Zyngier #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 110021f6537SMarc Zyngier #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 111021f6537SMarc Zyngier 112021f6537SMarc Zyngier /* Our default, arbitrary priority value. Linux only uses one anyway. */ 113021f6537SMarc Zyngier #define DEFAULT_PMR_VALUE 0xf0 114021f6537SMarc Zyngier 115e91b036eSMarc Zyngier enum gic_intid_range { 116e91b036eSMarc Zyngier PPI_RANGE, 117e91b036eSMarc Zyngier SPI_RANGE, 1185f51f803SMarc Zyngier EPPI_RANGE, 119211bddd2SMarc Zyngier ESPI_RANGE, 120e91b036eSMarc Zyngier LPI_RANGE, 121e91b036eSMarc Zyngier __INVALID_RANGE__ 122e91b036eSMarc Zyngier }; 123e91b036eSMarc Zyngier 124e91b036eSMarc Zyngier static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 125e91b036eSMarc Zyngier { 126e91b036eSMarc Zyngier switch (hwirq) { 127e91b036eSMarc Zyngier case 16 ... 31: 128e91b036eSMarc Zyngier return PPI_RANGE; 129e91b036eSMarc Zyngier case 32 ... 1019: 130e91b036eSMarc Zyngier return SPI_RANGE; 1315f51f803SMarc Zyngier case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 1325f51f803SMarc Zyngier return EPPI_RANGE; 133211bddd2SMarc Zyngier case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 134211bddd2SMarc Zyngier return ESPI_RANGE; 135e91b036eSMarc Zyngier case 8192 ... GENMASK(23, 0): 136e91b036eSMarc Zyngier return LPI_RANGE; 137e91b036eSMarc Zyngier default: 138e91b036eSMarc Zyngier return __INVALID_RANGE__; 139e91b036eSMarc Zyngier } 140e91b036eSMarc Zyngier } 141e91b036eSMarc Zyngier 142e91b036eSMarc Zyngier static enum gic_intid_range get_intid_range(struct irq_data *d) 143e91b036eSMarc Zyngier { 144e91b036eSMarc Zyngier return __get_intid_range(d->hwirq); 145e91b036eSMarc Zyngier } 146e91b036eSMarc Zyngier 147021f6537SMarc Zyngier static inline unsigned int gic_irq(struct irq_data *d) 148021f6537SMarc Zyngier { 149021f6537SMarc Zyngier return d->hwirq; 150021f6537SMarc Zyngier } 151021f6537SMarc Zyngier 152021f6537SMarc Zyngier static inline int gic_irq_in_rdist(struct irq_data *d) 153021f6537SMarc Zyngier { 1545f51f803SMarc Zyngier enum gic_intid_range range = get_intid_range(d); 1555f51f803SMarc Zyngier return range == PPI_RANGE || range == EPPI_RANGE; 156021f6537SMarc Zyngier } 157021f6537SMarc Zyngier 158021f6537SMarc Zyngier static inline void __iomem *gic_dist_base(struct irq_data *d) 159021f6537SMarc Zyngier { 160e91b036eSMarc Zyngier switch (get_intid_range(d)) { 161e91b036eSMarc Zyngier case PPI_RANGE: 1625f51f803SMarc Zyngier case EPPI_RANGE: 163e91b036eSMarc Zyngier /* SGI+PPI -> SGI_base for this CPU */ 164021f6537SMarc Zyngier return gic_data_rdist_sgi_base(); 165021f6537SMarc Zyngier 166e91b036eSMarc Zyngier case SPI_RANGE: 167211bddd2SMarc Zyngier case ESPI_RANGE: 168e91b036eSMarc Zyngier /* SPI -> dist_base */ 169021f6537SMarc Zyngier return gic_data.dist_base; 170021f6537SMarc Zyngier 171e91b036eSMarc Zyngier default: 172021f6537SMarc Zyngier return NULL; 173021f6537SMarc Zyngier } 174e91b036eSMarc Zyngier } 175021f6537SMarc Zyngier 176021f6537SMarc Zyngier static void gic_do_wait_for_rwp(void __iomem *base) 177021f6537SMarc Zyngier { 178021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 179021f6537SMarc Zyngier 180021f6537SMarc Zyngier while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) { 181021f6537SMarc Zyngier count--; 182021f6537SMarc Zyngier if (!count) { 183021f6537SMarc Zyngier pr_err_ratelimited("RWP timeout, gone fishing\n"); 184021f6537SMarc Zyngier return; 185021f6537SMarc Zyngier } 186021f6537SMarc Zyngier cpu_relax(); 187021f6537SMarc Zyngier udelay(1); 1882c542426SDaode Huang } 189021f6537SMarc Zyngier } 190021f6537SMarc Zyngier 191021f6537SMarc Zyngier /* Wait for completion of a distributor change */ 192021f6537SMarc Zyngier static void gic_dist_wait_for_rwp(void) 193021f6537SMarc Zyngier { 194021f6537SMarc Zyngier gic_do_wait_for_rwp(gic_data.dist_base); 195021f6537SMarc Zyngier } 196021f6537SMarc Zyngier 197021f6537SMarc Zyngier /* Wait for completion of a redistributor change */ 198021f6537SMarc Zyngier static void gic_redist_wait_for_rwp(void) 199021f6537SMarc Zyngier { 200021f6537SMarc Zyngier gic_do_wait_for_rwp(gic_data_rdist_rd_base()); 201021f6537SMarc Zyngier } 202021f6537SMarc Zyngier 2037936e914SJean-Philippe Brucker #ifdef CONFIG_ARM64 2046d4e11c5SRobert Richter 2056d4e11c5SRobert Richter static u64 __maybe_unused gic_read_iar(void) 2066d4e11c5SRobert Richter { 207a4023f68SSuzuki K Poulose if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 2086d4e11c5SRobert Richter return gic_read_iar_cavium_thunderx(); 2096d4e11c5SRobert Richter else 2106d4e11c5SRobert Richter return gic_read_iar_common(); 2116d4e11c5SRobert Richter } 2127936e914SJean-Philippe Brucker #endif 213021f6537SMarc Zyngier 214a2c22510SSudeep Holla static void gic_enable_redist(bool enable) 215021f6537SMarc Zyngier { 216021f6537SMarc Zyngier void __iomem *rbase; 217021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 218021f6537SMarc Zyngier u32 val; 219021f6537SMarc Zyngier 2209c8114c2SSrinivas Kandagatla if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 2219c8114c2SSrinivas Kandagatla return; 2229c8114c2SSrinivas Kandagatla 223021f6537SMarc Zyngier rbase = gic_data_rdist_rd_base(); 224021f6537SMarc Zyngier 225021f6537SMarc Zyngier val = readl_relaxed(rbase + GICR_WAKER); 226a2c22510SSudeep Holla if (enable) 227a2c22510SSudeep Holla /* Wake up this CPU redistributor */ 228021f6537SMarc Zyngier val &= ~GICR_WAKER_ProcessorSleep; 229a2c22510SSudeep Holla else 230a2c22510SSudeep Holla val |= GICR_WAKER_ProcessorSleep; 231021f6537SMarc Zyngier writel_relaxed(val, rbase + GICR_WAKER); 232021f6537SMarc Zyngier 233a2c22510SSudeep Holla if (!enable) { /* Check that GICR_WAKER is writeable */ 234a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 235a2c22510SSudeep Holla if (!(val & GICR_WAKER_ProcessorSleep)) 236a2c22510SSudeep Holla return; /* No PM support in this redistributor */ 237021f6537SMarc Zyngier } 238a2c22510SSudeep Holla 239d102eb5cSDan Carpenter while (--count) { 240a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 241cf1d9d11SAndrew Jones if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 242a2c22510SSudeep Holla break; 243021f6537SMarc Zyngier cpu_relax(); 244021f6537SMarc Zyngier udelay(1); 2452c542426SDaode Huang } 246a2c22510SSudeep Holla if (!count) 247a2c22510SSudeep Holla pr_err_ratelimited("redistributor failed to %s...\n", 248a2c22510SSudeep Holla enable ? "wakeup" : "sleep"); 249021f6537SMarc Zyngier } 250021f6537SMarc Zyngier 251021f6537SMarc Zyngier /* 252021f6537SMarc Zyngier * Routines to disable, enable, EOI and route interrupts 253021f6537SMarc Zyngier */ 254e91b036eSMarc Zyngier static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 255e91b036eSMarc Zyngier { 256e91b036eSMarc Zyngier switch (get_intid_range(d)) { 257e91b036eSMarc Zyngier case PPI_RANGE: 258e91b036eSMarc Zyngier case SPI_RANGE: 259e91b036eSMarc Zyngier *index = d->hwirq; 260e91b036eSMarc Zyngier return offset; 2615f51f803SMarc Zyngier case EPPI_RANGE: 2625f51f803SMarc Zyngier /* 2635f51f803SMarc Zyngier * Contrary to the ESPI range, the EPPI range is contiguous 2645f51f803SMarc Zyngier * to the PPI range in the registers, so let's adjust the 2655f51f803SMarc Zyngier * displacement accordingly. Consistency is overrated. 2665f51f803SMarc Zyngier */ 2675f51f803SMarc Zyngier *index = d->hwirq - EPPI_BASE_INTID + 32; 2685f51f803SMarc Zyngier return offset; 269211bddd2SMarc Zyngier case ESPI_RANGE: 270211bddd2SMarc Zyngier *index = d->hwirq - ESPI_BASE_INTID; 271211bddd2SMarc Zyngier switch (offset) { 272211bddd2SMarc Zyngier case GICD_ISENABLER: 273211bddd2SMarc Zyngier return GICD_ISENABLERnE; 274211bddd2SMarc Zyngier case GICD_ICENABLER: 275211bddd2SMarc Zyngier return GICD_ICENABLERnE; 276211bddd2SMarc Zyngier case GICD_ISPENDR: 277211bddd2SMarc Zyngier return GICD_ISPENDRnE; 278211bddd2SMarc Zyngier case GICD_ICPENDR: 279211bddd2SMarc Zyngier return GICD_ICPENDRnE; 280211bddd2SMarc Zyngier case GICD_ISACTIVER: 281211bddd2SMarc Zyngier return GICD_ISACTIVERnE; 282211bddd2SMarc Zyngier case GICD_ICACTIVER: 283211bddd2SMarc Zyngier return GICD_ICACTIVERnE; 284211bddd2SMarc Zyngier case GICD_IPRIORITYR: 285211bddd2SMarc Zyngier return GICD_IPRIORITYRnE; 286211bddd2SMarc Zyngier case GICD_ICFGR: 287211bddd2SMarc Zyngier return GICD_ICFGRnE; 288211bddd2SMarc Zyngier case GICD_IROUTER: 289211bddd2SMarc Zyngier return GICD_IROUTERnE; 290211bddd2SMarc Zyngier default: 291211bddd2SMarc Zyngier break; 292211bddd2SMarc Zyngier } 293211bddd2SMarc Zyngier break; 294e91b036eSMarc Zyngier default: 295e91b036eSMarc Zyngier break; 296e91b036eSMarc Zyngier } 297e91b036eSMarc Zyngier 298e91b036eSMarc Zyngier WARN_ON(1); 299e91b036eSMarc Zyngier *index = d->hwirq; 300e91b036eSMarc Zyngier return offset; 301e91b036eSMarc Zyngier } 302e91b036eSMarc Zyngier 303b594c6e2SMarc Zyngier static int gic_peek_irq(struct irq_data *d, u32 offset) 304b594c6e2SMarc Zyngier { 305b594c6e2SMarc Zyngier void __iomem *base; 306e91b036eSMarc Zyngier u32 index, mask; 307e91b036eSMarc Zyngier 308e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 309e91b036eSMarc Zyngier mask = 1 << (index % 32); 310b594c6e2SMarc Zyngier 311b594c6e2SMarc Zyngier if (gic_irq_in_rdist(d)) 312b594c6e2SMarc Zyngier base = gic_data_rdist_sgi_base(); 313b594c6e2SMarc Zyngier else 314b594c6e2SMarc Zyngier base = gic_data.dist_base; 315b594c6e2SMarc Zyngier 316e91b036eSMarc Zyngier return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 317b594c6e2SMarc Zyngier } 318b594c6e2SMarc Zyngier 319021f6537SMarc Zyngier static void gic_poke_irq(struct irq_data *d, u32 offset) 320021f6537SMarc Zyngier { 321021f6537SMarc Zyngier void (*rwp_wait)(void); 322021f6537SMarc Zyngier void __iomem *base; 323e91b036eSMarc Zyngier u32 index, mask; 324e91b036eSMarc Zyngier 325e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 326e91b036eSMarc Zyngier mask = 1 << (index % 32); 327021f6537SMarc Zyngier 328021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) { 329021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 330021f6537SMarc Zyngier rwp_wait = gic_redist_wait_for_rwp; 331021f6537SMarc Zyngier } else { 332021f6537SMarc Zyngier base = gic_data.dist_base; 333021f6537SMarc Zyngier rwp_wait = gic_dist_wait_for_rwp; 334021f6537SMarc Zyngier } 335021f6537SMarc Zyngier 336e91b036eSMarc Zyngier writel_relaxed(mask, base + offset + (index / 32) * 4); 337021f6537SMarc Zyngier rwp_wait(); 338021f6537SMarc Zyngier } 339021f6537SMarc Zyngier 340021f6537SMarc Zyngier static void gic_mask_irq(struct irq_data *d) 341021f6537SMarc Zyngier { 342021f6537SMarc Zyngier gic_poke_irq(d, GICD_ICENABLER); 343021f6537SMarc Zyngier } 344021f6537SMarc Zyngier 3450b6a3da9SMarc Zyngier static void gic_eoimode1_mask_irq(struct irq_data *d) 3460b6a3da9SMarc Zyngier { 3470b6a3da9SMarc Zyngier gic_mask_irq(d); 348530bf353SMarc Zyngier /* 349530bf353SMarc Zyngier * When masking a forwarded interrupt, make sure it is 350530bf353SMarc Zyngier * deactivated as well. 351530bf353SMarc Zyngier * 352530bf353SMarc Zyngier * This ensures that an interrupt that is getting 353530bf353SMarc Zyngier * disabled/masked will not get "stuck", because there is 354530bf353SMarc Zyngier * noone to deactivate it (guest is being terminated). 355530bf353SMarc Zyngier */ 3564df7f54dSThomas Gleixner if (irqd_is_forwarded_to_vcpu(d)) 357530bf353SMarc Zyngier gic_poke_irq(d, GICD_ICACTIVER); 3580b6a3da9SMarc Zyngier } 3590b6a3da9SMarc Zyngier 360021f6537SMarc Zyngier static void gic_unmask_irq(struct irq_data *d) 361021f6537SMarc Zyngier { 362021f6537SMarc Zyngier gic_poke_irq(d, GICD_ISENABLER); 363021f6537SMarc Zyngier } 364021f6537SMarc Zyngier 365d98d0a99SJulien Thierry static inline bool gic_supports_nmi(void) 366d98d0a99SJulien Thierry { 367d98d0a99SJulien Thierry return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 368d98d0a99SJulien Thierry static_branch_likely(&supports_pseudo_nmis); 369d98d0a99SJulien Thierry } 370d98d0a99SJulien Thierry 371b594c6e2SMarc Zyngier static int gic_irq_set_irqchip_state(struct irq_data *d, 372b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool val) 373b594c6e2SMarc Zyngier { 374b594c6e2SMarc Zyngier u32 reg; 375b594c6e2SMarc Zyngier 376211bddd2SMarc Zyngier if (d->hwirq >= 8192) /* PPI/SPI only */ 377b594c6e2SMarc Zyngier return -EINVAL; 378b594c6e2SMarc Zyngier 379b594c6e2SMarc Zyngier switch (which) { 380b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 381b594c6e2SMarc Zyngier reg = val ? GICD_ISPENDR : GICD_ICPENDR; 382b594c6e2SMarc Zyngier break; 383b594c6e2SMarc Zyngier 384b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 385b594c6e2SMarc Zyngier reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 386b594c6e2SMarc Zyngier break; 387b594c6e2SMarc Zyngier 388b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 389b594c6e2SMarc Zyngier reg = val ? GICD_ICENABLER : GICD_ISENABLER; 390b594c6e2SMarc Zyngier break; 391b594c6e2SMarc Zyngier 392b594c6e2SMarc Zyngier default: 393b594c6e2SMarc Zyngier return -EINVAL; 394b594c6e2SMarc Zyngier } 395b594c6e2SMarc Zyngier 396b594c6e2SMarc Zyngier gic_poke_irq(d, reg); 397b594c6e2SMarc Zyngier return 0; 398b594c6e2SMarc Zyngier } 399b594c6e2SMarc Zyngier 400b594c6e2SMarc Zyngier static int gic_irq_get_irqchip_state(struct irq_data *d, 401b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool *val) 402b594c6e2SMarc Zyngier { 403211bddd2SMarc Zyngier if (d->hwirq >= 8192) /* PPI/SPI only */ 404b594c6e2SMarc Zyngier return -EINVAL; 405b594c6e2SMarc Zyngier 406b594c6e2SMarc Zyngier switch (which) { 407b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 408b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISPENDR); 409b594c6e2SMarc Zyngier break; 410b594c6e2SMarc Zyngier 411b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 412b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISACTIVER); 413b594c6e2SMarc Zyngier break; 414b594c6e2SMarc Zyngier 415b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 416b594c6e2SMarc Zyngier *val = !gic_peek_irq(d, GICD_ISENABLER); 417b594c6e2SMarc Zyngier break; 418b594c6e2SMarc Zyngier 419b594c6e2SMarc Zyngier default: 420b594c6e2SMarc Zyngier return -EINVAL; 421b594c6e2SMarc Zyngier } 422b594c6e2SMarc Zyngier 423b594c6e2SMarc Zyngier return 0; 424b594c6e2SMarc Zyngier } 425b594c6e2SMarc Zyngier 426101b35f7SJulien Thierry static void gic_irq_set_prio(struct irq_data *d, u8 prio) 427101b35f7SJulien Thierry { 428101b35f7SJulien Thierry void __iomem *base = gic_dist_base(d); 429e91b036eSMarc Zyngier u32 offset, index; 430101b35f7SJulien Thierry 431e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 432e91b036eSMarc Zyngier 433e91b036eSMarc Zyngier writeb_relaxed(prio, base + offset + index); 434101b35f7SJulien Thierry } 435101b35f7SJulien Thierry 43681a43273SMarc Zyngier static u32 gic_get_ppi_index(struct irq_data *d) 43781a43273SMarc Zyngier { 43881a43273SMarc Zyngier switch (get_intid_range(d)) { 43981a43273SMarc Zyngier case PPI_RANGE: 44081a43273SMarc Zyngier return d->hwirq - 16; 4415f51f803SMarc Zyngier case EPPI_RANGE: 4425f51f803SMarc Zyngier return d->hwirq - EPPI_BASE_INTID + 16; 44381a43273SMarc Zyngier default: 44481a43273SMarc Zyngier unreachable(); 44581a43273SMarc Zyngier } 44681a43273SMarc Zyngier } 44781a43273SMarc Zyngier 448101b35f7SJulien Thierry static int gic_irq_nmi_setup(struct irq_data *d) 449101b35f7SJulien Thierry { 450101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 451101b35f7SJulien Thierry 452101b35f7SJulien Thierry if (!gic_supports_nmi()) 453101b35f7SJulien Thierry return -EINVAL; 454101b35f7SJulien Thierry 455101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 456101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 457101b35f7SJulien Thierry return -EINVAL; 458101b35f7SJulien Thierry } 459101b35f7SJulien Thierry 460101b35f7SJulien Thierry /* 461101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 462101b35f7SJulien Thierry * it should not be possible to get there 463101b35f7SJulien Thierry */ 464101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 465101b35f7SJulien Thierry return -EINVAL; 466101b35f7SJulien Thierry 467101b35f7SJulien Thierry /* desc lock should already be held */ 46881a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 46981a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 47081a43273SMarc Zyngier 471101b35f7SJulien Thierry /* Setting up PPI as NMI, only switch handler for first NMI */ 47281a43273SMarc Zyngier if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 47381a43273SMarc Zyngier refcount_set(&ppi_nmi_refs[idx], 1); 474101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 475101b35f7SJulien Thierry } 476101b35f7SJulien Thierry } else { 477101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_nmi; 478101b35f7SJulien Thierry } 479101b35f7SJulien Thierry 480101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_NMI_PRI); 481101b35f7SJulien Thierry 482101b35f7SJulien Thierry return 0; 483101b35f7SJulien Thierry } 484101b35f7SJulien Thierry 485101b35f7SJulien Thierry static void gic_irq_nmi_teardown(struct irq_data *d) 486101b35f7SJulien Thierry { 487101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 488101b35f7SJulien Thierry 489101b35f7SJulien Thierry if (WARN_ON(!gic_supports_nmi())) 490101b35f7SJulien Thierry return; 491101b35f7SJulien Thierry 492101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 493101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 494101b35f7SJulien Thierry return; 495101b35f7SJulien Thierry } 496101b35f7SJulien Thierry 497101b35f7SJulien Thierry /* 498101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 499101b35f7SJulien Thierry * it should not be possible to get there 500101b35f7SJulien Thierry */ 501101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 502101b35f7SJulien Thierry return; 503101b35f7SJulien Thierry 504101b35f7SJulien Thierry /* desc lock should already be held */ 50581a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 50681a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 50781a43273SMarc Zyngier 508101b35f7SJulien Thierry /* Tearing down NMI, only switch handler for last NMI */ 50981a43273SMarc Zyngier if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 510101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_irq; 511101b35f7SJulien Thierry } else { 512101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_irq; 513101b35f7SJulien Thierry } 514101b35f7SJulien Thierry 515101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_DEF_PRI); 516101b35f7SJulien Thierry } 517101b35f7SJulien Thierry 518021f6537SMarc Zyngier static void gic_eoi_irq(struct irq_data *d) 519021f6537SMarc Zyngier { 520021f6537SMarc Zyngier gic_write_eoir(gic_irq(d)); 521021f6537SMarc Zyngier } 522021f6537SMarc Zyngier 5230b6a3da9SMarc Zyngier static void gic_eoimode1_eoi_irq(struct irq_data *d) 5240b6a3da9SMarc Zyngier { 5250b6a3da9SMarc Zyngier /* 526530bf353SMarc Zyngier * No need to deactivate an LPI, or an interrupt that 527530bf353SMarc Zyngier * is is getting forwarded to a vcpu. 5280b6a3da9SMarc Zyngier */ 5294df7f54dSThomas Gleixner if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 5300b6a3da9SMarc Zyngier return; 5310b6a3da9SMarc Zyngier gic_write_dir(gic_irq(d)); 5320b6a3da9SMarc Zyngier } 5330b6a3da9SMarc Zyngier 534021f6537SMarc Zyngier static int gic_set_type(struct irq_data *d, unsigned int type) 535021f6537SMarc Zyngier { 5365f51f803SMarc Zyngier enum gic_intid_range range; 537021f6537SMarc Zyngier unsigned int irq = gic_irq(d); 538021f6537SMarc Zyngier void (*rwp_wait)(void); 539021f6537SMarc Zyngier void __iomem *base; 540e91b036eSMarc Zyngier u32 offset, index; 54113d22e2eSMarc Zyngier int ret; 542021f6537SMarc Zyngier 543021f6537SMarc Zyngier /* Interrupt configuration for SGIs can't be changed */ 544021f6537SMarc Zyngier if (irq < 16) 545021f6537SMarc Zyngier return -EINVAL; 546021f6537SMarc Zyngier 5475f51f803SMarc Zyngier range = get_intid_range(d); 5485f51f803SMarc Zyngier 549fb7e7debSLiviu Dudau /* SPIs have restrictions on the supported types */ 5505f51f803SMarc Zyngier if ((range == SPI_RANGE || range == ESPI_RANGE) && 5515f51f803SMarc Zyngier type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 552021f6537SMarc Zyngier return -EINVAL; 553021f6537SMarc Zyngier 554021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) { 555021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 556021f6537SMarc Zyngier rwp_wait = gic_redist_wait_for_rwp; 557021f6537SMarc Zyngier } else { 558021f6537SMarc Zyngier base = gic_data.dist_base; 559021f6537SMarc Zyngier rwp_wait = gic_dist_wait_for_rwp; 560021f6537SMarc Zyngier } 561021f6537SMarc Zyngier 562e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_ICFGR, &index); 56313d22e2eSMarc Zyngier 564e91b036eSMarc Zyngier ret = gic_configure_irq(index, type, base + offset, rwp_wait); 5655f51f803SMarc Zyngier if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 56613d22e2eSMarc Zyngier /* Misconfigured PPIs are usually not fatal */ 5675f51f803SMarc Zyngier pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 56813d22e2eSMarc Zyngier ret = 0; 56913d22e2eSMarc Zyngier } 57013d22e2eSMarc Zyngier 57113d22e2eSMarc Zyngier return ret; 572021f6537SMarc Zyngier } 573021f6537SMarc Zyngier 574530bf353SMarc Zyngier static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 575530bf353SMarc Zyngier { 5764df7f54dSThomas Gleixner if (vcpu) 5774df7f54dSThomas Gleixner irqd_set_forwarded_to_vcpu(d); 5784df7f54dSThomas Gleixner else 5794df7f54dSThomas Gleixner irqd_clr_forwarded_to_vcpu(d); 580530bf353SMarc Zyngier return 0; 581530bf353SMarc Zyngier } 582530bf353SMarc Zyngier 583f6c86a41SJean-Philippe Brucker static u64 gic_mpidr_to_affinity(unsigned long mpidr) 584021f6537SMarc Zyngier { 585021f6537SMarc Zyngier u64 aff; 586021f6537SMarc Zyngier 587f6c86a41SJean-Philippe Brucker aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 588021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 589021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 590021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 591021f6537SMarc Zyngier 592021f6537SMarc Zyngier return aff; 593021f6537SMarc Zyngier } 594021f6537SMarc Zyngier 595f32c9266SJulien Thierry static void gic_deactivate_unhandled(u32 irqnr) 596f32c9266SJulien Thierry { 597f32c9266SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) { 598f32c9266SJulien Thierry if (irqnr < 8192) 599f32c9266SJulien Thierry gic_write_dir(irqnr); 600f32c9266SJulien Thierry } else { 601f32c9266SJulien Thierry gic_write_eoir(irqnr); 602f32c9266SJulien Thierry } 603f32c9266SJulien Thierry } 604f32c9266SJulien Thierry 605f32c9266SJulien Thierry static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 606f32c9266SJulien Thierry { 60717ce302fSJulien Thierry bool irqs_enabled = interrupts_enabled(regs); 608f32c9266SJulien Thierry int err; 609f32c9266SJulien Thierry 61017ce302fSJulien Thierry if (irqs_enabled) 61117ce302fSJulien Thierry nmi_enter(); 61217ce302fSJulien Thierry 613f32c9266SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) 614f32c9266SJulien Thierry gic_write_eoir(irqnr); 615f32c9266SJulien Thierry /* 616f32c9266SJulien Thierry * Leave the PSR.I bit set to prevent other NMIs to be 617f32c9266SJulien Thierry * received while handling this one. 618f32c9266SJulien Thierry * PSR.I will be restored when we ERET to the 619f32c9266SJulien Thierry * interrupted context. 620f32c9266SJulien Thierry */ 621f32c9266SJulien Thierry err = handle_domain_nmi(gic_data.domain, irqnr, regs); 622f32c9266SJulien Thierry if (err) 623f32c9266SJulien Thierry gic_deactivate_unhandled(irqnr); 62417ce302fSJulien Thierry 62517ce302fSJulien Thierry if (irqs_enabled) 62617ce302fSJulien Thierry nmi_exit(); 627f32c9266SJulien Thierry } 628f32c9266SJulien Thierry 629021f6537SMarc Zyngier static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 630021f6537SMarc Zyngier { 631f6c86a41SJean-Philippe Brucker u32 irqnr; 632021f6537SMarc Zyngier 633021f6537SMarc Zyngier irqnr = gic_read_iar(); 634021f6537SMarc Zyngier 635f32c9266SJulien Thierry if (gic_supports_nmi() && 636f32c9266SJulien Thierry unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) { 637f32c9266SJulien Thierry gic_handle_nmi(irqnr, regs); 638f32c9266SJulien Thierry return; 639f32c9266SJulien Thierry } 640f32c9266SJulien Thierry 6413f1f3234SJulien Thierry if (gic_prio_masking_enabled()) { 6423f1f3234SJulien Thierry gic_pmr_mask_irqs(); 6433f1f3234SJulien Thierry gic_arch_enable_irqs(); 6443f1f3234SJulien Thierry } 6453f1f3234SJulien Thierry 646211bddd2SMarc Zyngier /* Check for special IDs first */ 647211bddd2SMarc Zyngier if ((irqnr >= 1020 && irqnr <= 1023)) 648211bddd2SMarc Zyngier return; 649211bddd2SMarc Zyngier 650211bddd2SMarc Zyngier /* Treat anything but SGIs in a uniform way */ 651211bddd2SMarc Zyngier if (likely(irqnr > 15)) { 652ebc6de00SMarc Zyngier int err; 6530b6a3da9SMarc Zyngier 654d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 6550b6a3da9SMarc Zyngier gic_write_eoir(irqnr); 65639a06b67SWill Deacon else 65739a06b67SWill Deacon isb(); 6580b6a3da9SMarc Zyngier 659ebc6de00SMarc Zyngier err = handle_domain_irq(gic_data.domain, irqnr, regs); 660ebc6de00SMarc Zyngier if (err) { 661da33f31dSMarc Zyngier WARN_ONCE(true, "Unexpected interrupt received!\n"); 662f32c9266SJulien Thierry gic_deactivate_unhandled(irqnr); 6630b6a3da9SMarc Zyngier } 664342677d7SJulien Thierry return; 665ebc6de00SMarc Zyngier } 666021f6537SMarc Zyngier if (irqnr < 16) { 667021f6537SMarc Zyngier gic_write_eoir(irqnr); 668d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 6690b6a3da9SMarc Zyngier gic_write_dir(irqnr); 670021f6537SMarc Zyngier #ifdef CONFIG_SMP 671f86c4fbdSWill Deacon /* 672f86c4fbdSWill Deacon * Unlike GICv2, we don't need an smp_rmb() here. 673f86c4fbdSWill Deacon * The control dependency from gic_read_iar to 674f86c4fbdSWill Deacon * the ISB in gic_write_eoir is enough to ensure 675f86c4fbdSWill Deacon * that any shared data read by handle_IPI will 676f86c4fbdSWill Deacon * be read after the ACK. 677f86c4fbdSWill Deacon */ 678021f6537SMarc Zyngier handle_IPI(irqnr, regs); 679021f6537SMarc Zyngier #else 680021f6537SMarc Zyngier WARN_ONCE(true, "Unexpected SGI received!\n"); 681021f6537SMarc Zyngier #endif 682021f6537SMarc Zyngier } 683021f6537SMarc Zyngier } 684021f6537SMarc Zyngier 685b5cf6073SJulien Thierry static u32 gic_get_pribits(void) 686b5cf6073SJulien Thierry { 687b5cf6073SJulien Thierry u32 pribits; 688b5cf6073SJulien Thierry 689b5cf6073SJulien Thierry pribits = gic_read_ctlr(); 690b5cf6073SJulien Thierry pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 691b5cf6073SJulien Thierry pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 692b5cf6073SJulien Thierry pribits++; 693b5cf6073SJulien Thierry 694b5cf6073SJulien Thierry return pribits; 695b5cf6073SJulien Thierry } 696b5cf6073SJulien Thierry 697b5cf6073SJulien Thierry static bool gic_has_group0(void) 698b5cf6073SJulien Thierry { 699b5cf6073SJulien Thierry u32 val; 700e7932188SJulien Thierry u32 old_pmr; 701e7932188SJulien Thierry 702e7932188SJulien Thierry old_pmr = gic_read_pmr(); 703b5cf6073SJulien Thierry 704b5cf6073SJulien Thierry /* 705b5cf6073SJulien Thierry * Let's find out if Group0 is under control of EL3 or not by 706b5cf6073SJulien Thierry * setting the highest possible, non-zero priority in PMR. 707b5cf6073SJulien Thierry * 708b5cf6073SJulien Thierry * If SCR_EL3.FIQ is set, the priority gets shifted down in 709b5cf6073SJulien Thierry * order for the CPU interface to set bit 7, and keep the 710b5cf6073SJulien Thierry * actual priority in the non-secure range. In the process, it 711b5cf6073SJulien Thierry * looses the least significant bit and the actual priority 712b5cf6073SJulien Thierry * becomes 0x80. Reading it back returns 0, indicating that 713b5cf6073SJulien Thierry * we're don't have access to Group0. 714b5cf6073SJulien Thierry */ 715b5cf6073SJulien Thierry gic_write_pmr(BIT(8 - gic_get_pribits())); 716b5cf6073SJulien Thierry val = gic_read_pmr(); 717b5cf6073SJulien Thierry 718e7932188SJulien Thierry gic_write_pmr(old_pmr); 719e7932188SJulien Thierry 720b5cf6073SJulien Thierry return val != 0; 721b5cf6073SJulien Thierry } 722b5cf6073SJulien Thierry 723021f6537SMarc Zyngier static void __init gic_dist_init(void) 724021f6537SMarc Zyngier { 725021f6537SMarc Zyngier unsigned int i; 726021f6537SMarc Zyngier u64 affinity; 727021f6537SMarc Zyngier void __iomem *base = gic_data.dist_base; 7280b04758bSMarc Zyngier u32 val; 729021f6537SMarc Zyngier 730021f6537SMarc Zyngier /* Disable the distributor */ 731021f6537SMarc Zyngier writel_relaxed(0, base + GICD_CTLR); 732021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 733021f6537SMarc Zyngier 7347c9b9730SMarc Zyngier /* 7357c9b9730SMarc Zyngier * Configure SPIs as non-secure Group-1. This will only matter 7367c9b9730SMarc Zyngier * if the GIC only has a single security state. This will not 7377c9b9730SMarc Zyngier * do the right thing if the kernel is running in secure mode, 7387c9b9730SMarc Zyngier * but that's not the intended use case anyway. 7397c9b9730SMarc Zyngier */ 740211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i += 32) 7417c9b9730SMarc Zyngier writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 7427c9b9730SMarc Zyngier 743211bddd2SMarc Zyngier /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 744211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) { 745211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 746211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 747211bddd2SMarc Zyngier } 748211bddd2SMarc Zyngier 749211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) 750211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 751211bddd2SMarc Zyngier 752211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 16) 753211bddd2SMarc Zyngier writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 754211bddd2SMarc Zyngier 755211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 4) 756211bddd2SMarc Zyngier writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 757211bddd2SMarc Zyngier 758211bddd2SMarc Zyngier /* Now do the common stuff, and wait for the distributor to drain */ 759211bddd2SMarc Zyngier gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp); 760021f6537SMarc Zyngier 7610b04758bSMarc Zyngier val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 7620b04758bSMarc Zyngier if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 7630b04758bSMarc Zyngier pr_info("Enabling SGIs without active state\n"); 7640b04758bSMarc Zyngier val |= GICD_CTLR_nASSGIreq; 7650b04758bSMarc Zyngier } 7660b04758bSMarc Zyngier 767021f6537SMarc Zyngier /* Enable distributor with ARE, Group1 */ 7680b04758bSMarc Zyngier writel_relaxed(val, base + GICD_CTLR); 769021f6537SMarc Zyngier 770021f6537SMarc Zyngier /* 771021f6537SMarc Zyngier * Set all global interrupts to the boot CPU only. ARE must be 772021f6537SMarc Zyngier * enabled. 773021f6537SMarc Zyngier */ 774021f6537SMarc Zyngier affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 775211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i++) 77672c97126SJean-Philippe Brucker gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 777211bddd2SMarc Zyngier 778211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i++) 779211bddd2SMarc Zyngier gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 780021f6537SMarc Zyngier } 781021f6537SMarc Zyngier 7820d94ded2SMarc Zyngier static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 783021f6537SMarc Zyngier { 7840d94ded2SMarc Zyngier int ret = -ENODEV; 785021f6537SMarc Zyngier int i; 786021f6537SMarc Zyngier 787f5c1434cSMarc Zyngier for (i = 0; i < gic_data.nr_redist_regions; i++) { 788f5c1434cSMarc Zyngier void __iomem *ptr = gic_data.redist_regions[i].redist_base; 7890d94ded2SMarc Zyngier u64 typer; 790021f6537SMarc Zyngier u32 reg; 791021f6537SMarc Zyngier 792021f6537SMarc Zyngier reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 793021f6537SMarc Zyngier if (reg != GIC_PIDR2_ARCH_GICv3 && 794021f6537SMarc Zyngier reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 795021f6537SMarc Zyngier pr_warn("No redistributor present @%p\n", ptr); 796021f6537SMarc Zyngier break; 797021f6537SMarc Zyngier } 798021f6537SMarc Zyngier 799021f6537SMarc Zyngier do { 80072c97126SJean-Philippe Brucker typer = gic_read_typer(ptr + GICR_TYPER); 8010d94ded2SMarc Zyngier ret = fn(gic_data.redist_regions + i, ptr); 8020d94ded2SMarc Zyngier if (!ret) 803021f6537SMarc Zyngier return 0; 804021f6537SMarc Zyngier 805b70fb7afSTomasz Nowicki if (gic_data.redist_regions[i].single_redist) 806b70fb7afSTomasz Nowicki break; 807b70fb7afSTomasz Nowicki 808021f6537SMarc Zyngier if (gic_data.redist_stride) { 809021f6537SMarc Zyngier ptr += gic_data.redist_stride; 810021f6537SMarc Zyngier } else { 811021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 812021f6537SMarc Zyngier if (typer & GICR_TYPER_VLPIS) 813021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 814021f6537SMarc Zyngier } 815021f6537SMarc Zyngier } while (!(typer & GICR_TYPER_LAST)); 816021f6537SMarc Zyngier } 817021f6537SMarc Zyngier 8180d94ded2SMarc Zyngier return ret ? -ENODEV : 0; 8190d94ded2SMarc Zyngier } 8200d94ded2SMarc Zyngier 8210d94ded2SMarc Zyngier static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 8220d94ded2SMarc Zyngier { 8230d94ded2SMarc Zyngier unsigned long mpidr = cpu_logical_map(smp_processor_id()); 8240d94ded2SMarc Zyngier u64 typer; 8250d94ded2SMarc Zyngier u32 aff; 8260d94ded2SMarc Zyngier 8270d94ded2SMarc Zyngier /* 8280d94ded2SMarc Zyngier * Convert affinity to a 32bit value that can be matched to 8290d94ded2SMarc Zyngier * GICR_TYPER bits [63:32]. 8300d94ded2SMarc Zyngier */ 8310d94ded2SMarc Zyngier aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 8320d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 8330d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 8340d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 8350d94ded2SMarc Zyngier 8360d94ded2SMarc Zyngier typer = gic_read_typer(ptr + GICR_TYPER); 8370d94ded2SMarc Zyngier if ((typer >> 32) == aff) { 8380d94ded2SMarc Zyngier u64 offset = ptr - region->redist_base; 8399058a4e9SMarc Zyngier raw_spin_lock_init(&gic_data_rdist()->rd_lock); 8400d94ded2SMarc Zyngier gic_data_rdist_rd_base() = ptr; 8410d94ded2SMarc Zyngier gic_data_rdist()->phys_base = region->phys_base + offset; 8420d94ded2SMarc Zyngier 8430d94ded2SMarc Zyngier pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 8440d94ded2SMarc Zyngier smp_processor_id(), mpidr, 8450d94ded2SMarc Zyngier (int)(region - gic_data.redist_regions), 8460d94ded2SMarc Zyngier &gic_data_rdist()->phys_base); 8470d94ded2SMarc Zyngier return 0; 8480d94ded2SMarc Zyngier } 8490d94ded2SMarc Zyngier 8500d94ded2SMarc Zyngier /* Try next one */ 8510d94ded2SMarc Zyngier return 1; 8520d94ded2SMarc Zyngier } 8530d94ded2SMarc Zyngier 8540d94ded2SMarc Zyngier static int gic_populate_rdist(void) 8550d94ded2SMarc Zyngier { 8560d94ded2SMarc Zyngier if (gic_iterate_rdists(__gic_populate_rdist) == 0) 8570d94ded2SMarc Zyngier return 0; 8580d94ded2SMarc Zyngier 859021f6537SMarc Zyngier /* We couldn't even deal with ourselves... */ 860f6c86a41SJean-Philippe Brucker WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 8610d94ded2SMarc Zyngier smp_processor_id(), 8620d94ded2SMarc Zyngier (unsigned long)cpu_logical_map(smp_processor_id())); 863021f6537SMarc Zyngier return -ENODEV; 864021f6537SMarc Zyngier } 865021f6537SMarc Zyngier 8661a60e1e6SMarc Zyngier static int __gic_update_rdist_properties(struct redist_region *region, 8670edc23eaSMarc Zyngier void __iomem *ptr) 8680edc23eaSMarc Zyngier { 8690edc23eaSMarc Zyngier u64 typer = gic_read_typer(ptr + GICR_TYPER); 870b25319d2SMarc Zyngier 8710edc23eaSMarc Zyngier gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 872b25319d2SMarc Zyngier 873b25319d2SMarc Zyngier /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */ 874b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 875b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 876b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid); 87796806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 878b25319d2SMarc Zyngier 879b25319d2SMarc Zyngier /* Detect non-sensical configurations */ 880b25319d2SMarc Zyngier if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 881b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi = false; 882b25319d2SMarc Zyngier gic_data.rdists.has_vlpis = false; 883b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = false; 884b25319d2SMarc Zyngier } 885b25319d2SMarc Zyngier 8865f51f803SMarc Zyngier gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 8870edc23eaSMarc Zyngier 8880edc23eaSMarc Zyngier return 1; 8890edc23eaSMarc Zyngier } 8900edc23eaSMarc Zyngier 8911a60e1e6SMarc Zyngier static void gic_update_rdist_properties(void) 8920edc23eaSMarc Zyngier { 8931a60e1e6SMarc Zyngier gic_data.ppi_nr = UINT_MAX; 8941a60e1e6SMarc Zyngier gic_iterate_rdists(__gic_update_rdist_properties); 8951a60e1e6SMarc Zyngier if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 8961a60e1e6SMarc Zyngier gic_data.ppi_nr = 0; 8971a60e1e6SMarc Zyngier pr_info("%d PPIs implemented\n", gic_data.ppi_nr); 89896806229SMarc Zyngier if (gic_data.rdists.has_vlpis) 89996806229SMarc Zyngier pr_info("GICv4 features: %s%s%s\n", 90096806229SMarc Zyngier gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 90196806229SMarc Zyngier gic_data.rdists.has_rvpeid ? "RVPEID " : "", 90296806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 9030edc23eaSMarc Zyngier } 9040edc23eaSMarc Zyngier 905d98d0a99SJulien Thierry /* Check whether it's single security state view */ 906d98d0a99SJulien Thierry static inline bool gic_dist_security_disabled(void) 907d98d0a99SJulien Thierry { 908d98d0a99SJulien Thierry return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 909d98d0a99SJulien Thierry } 910d98d0a99SJulien Thierry 9113708d52fSSudeep Holla static void gic_cpu_sys_reg_init(void) 912021f6537SMarc Zyngier { 913eda0d04aSShanker Donthineni int i, cpu = smp_processor_id(); 914eda0d04aSShanker Donthineni u64 mpidr = cpu_logical_map(cpu); 915eda0d04aSShanker Donthineni u64 need_rss = MPIDR_RS(mpidr); 91633625282SMarc Zyngier bool group0; 917b5cf6073SJulien Thierry u32 pribits; 918eda0d04aSShanker Donthineni 9197cabd008SMarc Zyngier /* 9207cabd008SMarc Zyngier * Need to check that the SRE bit has actually been set. If 9217cabd008SMarc Zyngier * not, it means that SRE is disabled at EL2. We're going to 9227cabd008SMarc Zyngier * die painfully, and there is nothing we can do about it. 9237cabd008SMarc Zyngier * 9247cabd008SMarc Zyngier * Kindly inform the luser. 9257cabd008SMarc Zyngier */ 9267cabd008SMarc Zyngier if (!gic_enable_sre()) 9277cabd008SMarc Zyngier pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 928021f6537SMarc Zyngier 929b5cf6073SJulien Thierry pribits = gic_get_pribits(); 93033625282SMarc Zyngier 931b5cf6073SJulien Thierry group0 = gic_has_group0(); 93233625282SMarc Zyngier 933021f6537SMarc Zyngier /* Set priority mask register */ 934d98d0a99SJulien Thierry if (!gic_prio_masking_enabled()) { 93533625282SMarc Zyngier write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 93633678059SAlexandru Elisei } else if (gic_supports_nmi()) { 937d98d0a99SJulien Thierry /* 938d98d0a99SJulien Thierry * Mismatch configuration with boot CPU, the system is likely 939d98d0a99SJulien Thierry * to die as interrupt masking will not work properly on all 940d98d0a99SJulien Thierry * CPUs 94133678059SAlexandru Elisei * 94233678059SAlexandru Elisei * The boot CPU calls this function before enabling NMI support, 94333678059SAlexandru Elisei * and as a result we'll never see this warning in the boot path 94433678059SAlexandru Elisei * for that CPU. 945d98d0a99SJulien Thierry */ 94633678059SAlexandru Elisei if (static_branch_unlikely(&gic_nonsecure_priorities)) 94733678059SAlexandru Elisei WARN_ON(!group0 || gic_dist_security_disabled()); 94833678059SAlexandru Elisei else 94933678059SAlexandru Elisei WARN_ON(group0 && !gic_dist_security_disabled()); 950d98d0a99SJulien Thierry } 951021f6537SMarc Zyngier 95291ef8442SDaniel Thompson /* 95391ef8442SDaniel Thompson * Some firmwares hand over to the kernel with the BPR changed from 95491ef8442SDaniel Thompson * its reset value (and with a value large enough to prevent 95591ef8442SDaniel Thompson * any pre-emptive interrupts from working at all). Writing a zero 95691ef8442SDaniel Thompson * to BPR restores is reset value. 95791ef8442SDaniel Thompson */ 95891ef8442SDaniel Thompson gic_write_bpr1(0); 95991ef8442SDaniel Thompson 960d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) { 9610b6a3da9SMarc Zyngier /* EOI drops priority only (mode 1) */ 9620b6a3da9SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 9630b6a3da9SMarc Zyngier } else { 964021f6537SMarc Zyngier /* EOI deactivates interrupt too (mode 0) */ 965021f6537SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 9660b6a3da9SMarc Zyngier } 967021f6537SMarc Zyngier 96833625282SMarc Zyngier /* Always whack Group0 before Group1 */ 96933625282SMarc Zyngier if (group0) { 97033625282SMarc Zyngier switch(pribits) { 97133625282SMarc Zyngier case 8: 97233625282SMarc Zyngier case 7: 97333625282SMarc Zyngier write_gicreg(0, ICC_AP0R3_EL1); 97433625282SMarc Zyngier write_gicreg(0, ICC_AP0R2_EL1); 975df561f66SGustavo A. R. Silva fallthrough; 97633625282SMarc Zyngier case 6: 97733625282SMarc Zyngier write_gicreg(0, ICC_AP0R1_EL1); 978df561f66SGustavo A. R. Silva fallthrough; 97933625282SMarc Zyngier case 5: 98033625282SMarc Zyngier case 4: 98133625282SMarc Zyngier write_gicreg(0, ICC_AP0R0_EL1); 98233625282SMarc Zyngier } 983d6062a6dSMarc Zyngier 98433625282SMarc Zyngier isb(); 98533625282SMarc Zyngier } 98633625282SMarc Zyngier 98733625282SMarc Zyngier switch(pribits) { 988d6062a6dSMarc Zyngier case 8: 989d6062a6dSMarc Zyngier case 7: 990d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R3_EL1); 991d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R2_EL1); 992df561f66SGustavo A. R. Silva fallthrough; 993d6062a6dSMarc Zyngier case 6: 994d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R1_EL1); 995df561f66SGustavo A. R. Silva fallthrough; 996d6062a6dSMarc Zyngier case 5: 997d6062a6dSMarc Zyngier case 4: 998d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R0_EL1); 999d6062a6dSMarc Zyngier } 1000d6062a6dSMarc Zyngier 1001d6062a6dSMarc Zyngier isb(); 1002d6062a6dSMarc Zyngier 1003021f6537SMarc Zyngier /* ... and let's hit the road... */ 1004021f6537SMarc Zyngier gic_write_grpen1(1); 1005eda0d04aSShanker Donthineni 1006eda0d04aSShanker Donthineni /* Keep the RSS capability status in per_cpu variable */ 1007eda0d04aSShanker Donthineni per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1008eda0d04aSShanker Donthineni 1009eda0d04aSShanker Donthineni /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1010eda0d04aSShanker Donthineni for_each_online_cpu(i) { 1011eda0d04aSShanker Donthineni bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1012eda0d04aSShanker Donthineni 1013eda0d04aSShanker Donthineni need_rss |= MPIDR_RS(cpu_logical_map(i)); 1014eda0d04aSShanker Donthineni if (need_rss && (!have_rss)) 1015eda0d04aSShanker Donthineni pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1016eda0d04aSShanker Donthineni cpu, (unsigned long)mpidr, 1017eda0d04aSShanker Donthineni i, (unsigned long)cpu_logical_map(i)); 1018eda0d04aSShanker Donthineni } 1019eda0d04aSShanker Donthineni 1020eda0d04aSShanker Donthineni /** 1021eda0d04aSShanker Donthineni * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1022eda0d04aSShanker Donthineni * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1023eda0d04aSShanker Donthineni * UNPREDICTABLE choice of : 1024eda0d04aSShanker Donthineni * - The write is ignored. 1025eda0d04aSShanker Donthineni * - The RS field is treated as 0. 1026eda0d04aSShanker Donthineni */ 1027eda0d04aSShanker Donthineni if (need_rss && (!gic_data.has_rss)) 1028eda0d04aSShanker Donthineni pr_crit_once("RSS is required but GICD doesn't support it\n"); 1029021f6537SMarc Zyngier } 1030021f6537SMarc Zyngier 1031f736d65dSMarc Zyngier static bool gicv3_nolpi; 1032f736d65dSMarc Zyngier 1033f736d65dSMarc Zyngier static int __init gicv3_nolpi_cfg(char *buf) 1034f736d65dSMarc Zyngier { 1035f736d65dSMarc Zyngier return strtobool(buf, &gicv3_nolpi); 1036f736d65dSMarc Zyngier } 1037f736d65dSMarc Zyngier early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1038f736d65dSMarc Zyngier 1039da33f31dSMarc Zyngier static int gic_dist_supports_lpis(void) 1040da33f31dSMarc Zyngier { 1041d38a71c5SMarc Zyngier return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1042d38a71c5SMarc Zyngier !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1043d38a71c5SMarc Zyngier !gicv3_nolpi); 1044da33f31dSMarc Zyngier } 1045da33f31dSMarc Zyngier 1046021f6537SMarc Zyngier static void gic_cpu_init(void) 1047021f6537SMarc Zyngier { 1048021f6537SMarc Zyngier void __iomem *rbase; 10491a60e1e6SMarc Zyngier int i; 1050021f6537SMarc Zyngier 1051021f6537SMarc Zyngier /* Register ourselves with the rest of the world */ 1052021f6537SMarc Zyngier if (gic_populate_rdist()) 1053021f6537SMarc Zyngier return; 1054021f6537SMarc Zyngier 1055a2c22510SSudeep Holla gic_enable_redist(true); 1056021f6537SMarc Zyngier 1057ad5a78d3SMarc Zyngier WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1058ad5a78d3SMarc Zyngier !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1059ad5a78d3SMarc Zyngier "Distributor has extended ranges, but CPU%d doesn't\n", 1060ad5a78d3SMarc Zyngier smp_processor_id()); 1061ad5a78d3SMarc Zyngier 1062021f6537SMarc Zyngier rbase = gic_data_rdist_sgi_base(); 1063021f6537SMarc Zyngier 10647c9b9730SMarc Zyngier /* Configure SGIs/PPIs as non-secure Group-1 */ 10651a60e1e6SMarc Zyngier for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 10661a60e1e6SMarc Zyngier writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 10677c9b9730SMarc Zyngier 10681a60e1e6SMarc Zyngier gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1069021f6537SMarc Zyngier 10703708d52fSSudeep Holla /* initialise system registers */ 10713708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1072021f6537SMarc Zyngier } 1073021f6537SMarc Zyngier 1074021f6537SMarc Zyngier #ifdef CONFIG_SMP 1075021f6537SMarc Zyngier 1076eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1077eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1078eda0d04aSShanker Donthineni 10796670a6d8SRichard Cochran static int gic_starting_cpu(unsigned int cpu) 10806670a6d8SRichard Cochran { 10816670a6d8SRichard Cochran gic_cpu_init(); 1082d38a71c5SMarc Zyngier 1083d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) 1084d38a71c5SMarc Zyngier its_cpu_init(); 1085d38a71c5SMarc Zyngier 10866670a6d8SRichard Cochran return 0; 10876670a6d8SRichard Cochran } 1088021f6537SMarc Zyngier 1089021f6537SMarc Zyngier static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1090f6c86a41SJean-Philippe Brucker unsigned long cluster_id) 1091021f6537SMarc Zyngier { 1092727653d6SJames Morse int next_cpu, cpu = *base_cpu; 1093f6c86a41SJean-Philippe Brucker unsigned long mpidr = cpu_logical_map(cpu); 1094021f6537SMarc Zyngier u16 tlist = 0; 1095021f6537SMarc Zyngier 1096021f6537SMarc Zyngier while (cpu < nr_cpu_ids) { 1097021f6537SMarc Zyngier tlist |= 1 << (mpidr & 0xf); 1098021f6537SMarc Zyngier 1099727653d6SJames Morse next_cpu = cpumask_next(cpu, mask); 1100727653d6SJames Morse if (next_cpu >= nr_cpu_ids) 1101021f6537SMarc Zyngier goto out; 1102727653d6SJames Morse cpu = next_cpu; 1103021f6537SMarc Zyngier 1104021f6537SMarc Zyngier mpidr = cpu_logical_map(cpu); 1105021f6537SMarc Zyngier 1106eda0d04aSShanker Donthineni if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1107021f6537SMarc Zyngier cpu--; 1108021f6537SMarc Zyngier goto out; 1109021f6537SMarc Zyngier } 1110021f6537SMarc Zyngier } 1111021f6537SMarc Zyngier out: 1112021f6537SMarc Zyngier *base_cpu = cpu; 1113021f6537SMarc Zyngier return tlist; 1114021f6537SMarc Zyngier } 1115021f6537SMarc Zyngier 11167e580278SAndre Przywara #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 11177e580278SAndre Przywara (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 11187e580278SAndre Przywara << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 11197e580278SAndre Przywara 1120021f6537SMarc Zyngier static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1121021f6537SMarc Zyngier { 1122021f6537SMarc Zyngier u64 val; 1123021f6537SMarc Zyngier 11247e580278SAndre Przywara val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 11257e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 11267e580278SAndre Przywara irq << ICC_SGI1R_SGI_ID_SHIFT | 11277e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1128eda0d04aSShanker Donthineni MPIDR_TO_SGI_RS(cluster_id) | 11297e580278SAndre Przywara tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1130021f6537SMarc Zyngier 1131b6dd4d83SMark Salter pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1132021f6537SMarc Zyngier gic_write_sgi1r(val); 1133021f6537SMarc Zyngier } 1134021f6537SMarc Zyngier 1135021f6537SMarc Zyngier static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) 1136021f6537SMarc Zyngier { 1137021f6537SMarc Zyngier int cpu; 1138021f6537SMarc Zyngier 1139021f6537SMarc Zyngier if (WARN_ON(irq >= 16)) 1140021f6537SMarc Zyngier return; 1141021f6537SMarc Zyngier 1142021f6537SMarc Zyngier /* 1143021f6537SMarc Zyngier * Ensure that stores to Normal memory are visible to the 1144021f6537SMarc Zyngier * other CPUs before issuing the IPI. 1145021f6537SMarc Zyngier */ 114621ec30c0SShanker Donthineni wmb(); 1147021f6537SMarc Zyngier 1148f9b531feSRusty Russell for_each_cpu(cpu, mask) { 1149eda0d04aSShanker Donthineni u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1150021f6537SMarc Zyngier u16 tlist; 1151021f6537SMarc Zyngier 1152021f6537SMarc Zyngier tlist = gic_compute_target_list(&cpu, mask, cluster_id); 1153021f6537SMarc Zyngier gic_send_sgi(cluster_id, tlist, irq); 1154021f6537SMarc Zyngier } 1155021f6537SMarc Zyngier 1156021f6537SMarc Zyngier /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1157021f6537SMarc Zyngier isb(); 1158021f6537SMarc Zyngier } 1159021f6537SMarc Zyngier 11608a94c1abSIngo Rohloff static void __init gic_smp_init(void) 1161021f6537SMarc Zyngier { 1162021f6537SMarc Zyngier set_smp_cross_call(gic_raise_softirq); 11636896bcd1SThomas Gleixner cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 116473c1b41eSThomas Gleixner "irqchip/arm/gicv3:starting", 116573c1b41eSThomas Gleixner gic_starting_cpu, NULL); 1166021f6537SMarc Zyngier } 1167021f6537SMarc Zyngier 1168021f6537SMarc Zyngier static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1169021f6537SMarc Zyngier bool force) 1170021f6537SMarc Zyngier { 117165a30f8bSSuzuki K Poulose unsigned int cpu; 1172e91b036eSMarc Zyngier u32 offset, index; 1173021f6537SMarc Zyngier void __iomem *reg; 1174021f6537SMarc Zyngier int enabled; 1175021f6537SMarc Zyngier u64 val; 1176021f6537SMarc Zyngier 117765a30f8bSSuzuki K Poulose if (force) 117865a30f8bSSuzuki K Poulose cpu = cpumask_first(mask_val); 117965a30f8bSSuzuki K Poulose else 118065a30f8bSSuzuki K Poulose cpu = cpumask_any_and(mask_val, cpu_online_mask); 118165a30f8bSSuzuki K Poulose 1182866d7c1bSSuzuki K Poulose if (cpu >= nr_cpu_ids) 1183866d7c1bSSuzuki K Poulose return -EINVAL; 1184866d7c1bSSuzuki K Poulose 1185021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) 1186021f6537SMarc Zyngier return -EINVAL; 1187021f6537SMarc Zyngier 1188021f6537SMarc Zyngier /* If interrupt was enabled, disable it first */ 1189021f6537SMarc Zyngier enabled = gic_peek_irq(d, GICD_ISENABLER); 1190021f6537SMarc Zyngier if (enabled) 1191021f6537SMarc Zyngier gic_mask_irq(d); 1192021f6537SMarc Zyngier 1193e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IROUTER, &index); 1194e91b036eSMarc Zyngier reg = gic_dist_base(d) + offset + (index * 8); 1195021f6537SMarc Zyngier val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 1196021f6537SMarc Zyngier 119772c97126SJean-Philippe Brucker gic_write_irouter(val, reg); 1198021f6537SMarc Zyngier 1199021f6537SMarc Zyngier /* 1200021f6537SMarc Zyngier * If the interrupt was enabled, enabled it again. Otherwise, 1201021f6537SMarc Zyngier * just wait for the distributor to have digested our changes. 1202021f6537SMarc Zyngier */ 1203021f6537SMarc Zyngier if (enabled) 1204021f6537SMarc Zyngier gic_unmask_irq(d); 1205021f6537SMarc Zyngier else 1206021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 1207021f6537SMarc Zyngier 1208956ae91aSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1209956ae91aSMarc Zyngier 12100fc6fa29SAntoine Tenart return IRQ_SET_MASK_OK_DONE; 1211021f6537SMarc Zyngier } 1212021f6537SMarc Zyngier #else 1213021f6537SMarc Zyngier #define gic_set_affinity NULL 1214021f6537SMarc Zyngier #define gic_smp_init() do { } while(0) 1215021f6537SMarc Zyngier #endif 1216021f6537SMarc Zyngier 12173708d52fSSudeep Holla #ifdef CONFIG_CPU_PM 12183708d52fSSudeep Holla static int gic_cpu_pm_notifier(struct notifier_block *self, 12193708d52fSSudeep Holla unsigned long cmd, void *v) 12203708d52fSSudeep Holla { 12213708d52fSSudeep Holla if (cmd == CPU_PM_EXIT) { 1222ccd9432aSSudeep Holla if (gic_dist_security_disabled()) 12233708d52fSSudeep Holla gic_enable_redist(true); 12243708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1225ccd9432aSSudeep Holla } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 12263708d52fSSudeep Holla gic_write_grpen1(0); 12273708d52fSSudeep Holla gic_enable_redist(false); 12283708d52fSSudeep Holla } 12293708d52fSSudeep Holla return NOTIFY_OK; 12303708d52fSSudeep Holla } 12313708d52fSSudeep Holla 12323708d52fSSudeep Holla static struct notifier_block gic_cpu_pm_notifier_block = { 12333708d52fSSudeep Holla .notifier_call = gic_cpu_pm_notifier, 12343708d52fSSudeep Holla }; 12353708d52fSSudeep Holla 12363708d52fSSudeep Holla static void gic_cpu_pm_init(void) 12373708d52fSSudeep Holla { 12383708d52fSSudeep Holla cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 12393708d52fSSudeep Holla } 12403708d52fSSudeep Holla 12413708d52fSSudeep Holla #else 12423708d52fSSudeep Holla static inline void gic_cpu_pm_init(void) { } 12433708d52fSSudeep Holla #endif /* CONFIG_CPU_PM */ 12443708d52fSSudeep Holla 1245021f6537SMarc Zyngier static struct irq_chip gic_chip = { 1246021f6537SMarc Zyngier .name = "GICv3", 1247021f6537SMarc Zyngier .irq_mask = gic_mask_irq, 1248021f6537SMarc Zyngier .irq_unmask = gic_unmask_irq, 1249021f6537SMarc Zyngier .irq_eoi = gic_eoi_irq, 1250021f6537SMarc Zyngier .irq_set_type = gic_set_type, 1251021f6537SMarc Zyngier .irq_set_affinity = gic_set_affinity, 1252b594c6e2SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1253b594c6e2SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1254101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1255101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 12564110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 12574110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 12584110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 1259021f6537SMarc Zyngier }; 1260021f6537SMarc Zyngier 12610b6a3da9SMarc Zyngier static struct irq_chip gic_eoimode1_chip = { 12620b6a3da9SMarc Zyngier .name = "GICv3", 12630b6a3da9SMarc Zyngier .irq_mask = gic_eoimode1_mask_irq, 12640b6a3da9SMarc Zyngier .irq_unmask = gic_unmask_irq, 12650b6a3da9SMarc Zyngier .irq_eoi = gic_eoimode1_eoi_irq, 12660b6a3da9SMarc Zyngier .irq_set_type = gic_set_type, 12670b6a3da9SMarc Zyngier .irq_set_affinity = gic_set_affinity, 12680b6a3da9SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 12690b6a3da9SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1270530bf353SMarc Zyngier .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1271101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1272101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 12734110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 12744110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 12754110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 12760b6a3da9SMarc Zyngier }; 12770b6a3da9SMarc Zyngier 1278021f6537SMarc Zyngier static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1279021f6537SMarc Zyngier irq_hw_number_t hw) 1280021f6537SMarc Zyngier { 12810b6a3da9SMarc Zyngier struct irq_chip *chip = &gic_chip; 12820b6a3da9SMarc Zyngier 1283d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 12840b6a3da9SMarc Zyngier chip = &gic_eoimode1_chip; 12850b6a3da9SMarc Zyngier 1286e91b036eSMarc Zyngier switch (__get_intid_range(hw)) { 1287e91b036eSMarc Zyngier case PPI_RANGE: 12885f51f803SMarc Zyngier case EPPI_RANGE: 1289021f6537SMarc Zyngier irq_set_percpu_devid(irq); 12900b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1291443acc4fSMarc Zyngier handle_percpu_devid_irq, NULL, NULL); 1292e91b036eSMarc Zyngier break; 1293e91b036eSMarc Zyngier 1294e91b036eSMarc Zyngier case SPI_RANGE: 1295211bddd2SMarc Zyngier case ESPI_RANGE: 12960b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1297443acc4fSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1298d17cab44SRob Herring irq_set_probe(irq); 1299956ae91aSMarc Zyngier irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); 1300e91b036eSMarc Zyngier break; 1301e91b036eSMarc Zyngier 1302e91b036eSMarc Zyngier case LPI_RANGE: 1303da33f31dSMarc Zyngier if (!gic_dist_supports_lpis()) 1304da33f31dSMarc Zyngier return -EPERM; 13050b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1306da33f31dSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1307e91b036eSMarc Zyngier break; 1308e91b036eSMarc Zyngier 1309e91b036eSMarc Zyngier default: 1310e91b036eSMarc Zyngier return -EPERM; 1311da33f31dSMarc Zyngier } 1312da33f31dSMarc Zyngier 1313021f6537SMarc Zyngier return 0; 1314021f6537SMarc Zyngier } 1315021f6537SMarc Zyngier 131665da7d19SMarc Zyngier #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 131765da7d19SMarc Zyngier 1318f833f57fSMarc Zyngier static int gic_irq_domain_translate(struct irq_domain *d, 1319f833f57fSMarc Zyngier struct irq_fwspec *fwspec, 1320f833f57fSMarc Zyngier unsigned long *hwirq, 1321f833f57fSMarc Zyngier unsigned int *type) 1322021f6537SMarc Zyngier { 1323f833f57fSMarc Zyngier if (is_of_node(fwspec->fwnode)) { 1324f833f57fSMarc Zyngier if (fwspec->param_count < 3) 1325021f6537SMarc Zyngier return -EINVAL; 1326021f6537SMarc Zyngier 1327db8c70ecSMarc Zyngier switch (fwspec->param[0]) { 1328db8c70ecSMarc Zyngier case 0: /* SPI */ 1329db8c70ecSMarc Zyngier *hwirq = fwspec->param[1] + 32; 1330db8c70ecSMarc Zyngier break; 1331db8c70ecSMarc Zyngier case 1: /* PPI */ 1332f833f57fSMarc Zyngier *hwirq = fwspec->param[1] + 16; 1333db8c70ecSMarc Zyngier break; 1334211bddd2SMarc Zyngier case 2: /* ESPI */ 1335211bddd2SMarc Zyngier *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1336211bddd2SMarc Zyngier break; 13375f51f803SMarc Zyngier case 3: /* EPPI */ 13385f51f803SMarc Zyngier *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 13395f51f803SMarc Zyngier break; 1340db8c70ecSMarc Zyngier case GIC_IRQ_TYPE_LPI: /* LPI */ 1341db8c70ecSMarc Zyngier *hwirq = fwspec->param[1]; 1342db8c70ecSMarc Zyngier break; 13435f51f803SMarc Zyngier case GIC_IRQ_TYPE_PARTITION: 13445f51f803SMarc Zyngier *hwirq = fwspec->param[1]; 13455f51f803SMarc Zyngier if (fwspec->param[1] >= 16) 13465f51f803SMarc Zyngier *hwirq += EPPI_BASE_INTID - 16; 13475f51f803SMarc Zyngier else 13485f51f803SMarc Zyngier *hwirq += 16; 13495f51f803SMarc Zyngier break; 1350db8c70ecSMarc Zyngier default: 1351db8c70ecSMarc Zyngier return -EINVAL; 1352db8c70ecSMarc Zyngier } 1353f833f57fSMarc Zyngier 1354f833f57fSMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 13556ef6386eSMarc Zyngier 135665da7d19SMarc Zyngier /* 135765da7d19SMarc Zyngier * Make it clear that broken DTs are... broken. 135865da7d19SMarc Zyngier * Partitionned PPIs are an unfortunate exception. 135965da7d19SMarc Zyngier */ 136065da7d19SMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE && 136165da7d19SMarc Zyngier fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1362f833f57fSMarc Zyngier return 0; 1363021f6537SMarc Zyngier } 1364021f6537SMarc Zyngier 1365ffa7d616STomasz Nowicki if (is_fwnode_irqchip(fwspec->fwnode)) { 1366ffa7d616STomasz Nowicki if(fwspec->param_count != 2) 1367ffa7d616STomasz Nowicki return -EINVAL; 1368ffa7d616STomasz Nowicki 1369ffa7d616STomasz Nowicki *hwirq = fwspec->param[0]; 1370ffa7d616STomasz Nowicki *type = fwspec->param[1]; 13716ef6386eSMarc Zyngier 13726ef6386eSMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE); 1373ffa7d616STomasz Nowicki return 0; 1374ffa7d616STomasz Nowicki } 1375ffa7d616STomasz Nowicki 1376f833f57fSMarc Zyngier return -EINVAL; 1377021f6537SMarc Zyngier } 1378021f6537SMarc Zyngier 1379443acc4fSMarc Zyngier static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1380443acc4fSMarc Zyngier unsigned int nr_irqs, void *arg) 1381443acc4fSMarc Zyngier { 1382443acc4fSMarc Zyngier int i, ret; 1383443acc4fSMarc Zyngier irq_hw_number_t hwirq; 1384443acc4fSMarc Zyngier unsigned int type = IRQ_TYPE_NONE; 1385f833f57fSMarc Zyngier struct irq_fwspec *fwspec = arg; 1386443acc4fSMarc Zyngier 1387f833f57fSMarc Zyngier ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1388443acc4fSMarc Zyngier if (ret) 1389443acc4fSMarc Zyngier return ret; 1390443acc4fSMarc Zyngier 139163c16c6eSSuzuki K Poulose for (i = 0; i < nr_irqs; i++) { 139263c16c6eSSuzuki K Poulose ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 139363c16c6eSSuzuki K Poulose if (ret) 139463c16c6eSSuzuki K Poulose return ret; 139563c16c6eSSuzuki K Poulose } 1396443acc4fSMarc Zyngier 1397443acc4fSMarc Zyngier return 0; 1398443acc4fSMarc Zyngier } 1399443acc4fSMarc Zyngier 1400443acc4fSMarc Zyngier static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1401443acc4fSMarc Zyngier unsigned int nr_irqs) 1402443acc4fSMarc Zyngier { 1403443acc4fSMarc Zyngier int i; 1404443acc4fSMarc Zyngier 1405443acc4fSMarc Zyngier for (i = 0; i < nr_irqs; i++) { 1406443acc4fSMarc Zyngier struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1407443acc4fSMarc Zyngier irq_set_handler(virq + i, NULL); 1408443acc4fSMarc Zyngier irq_domain_reset_irq_data(d); 1409443acc4fSMarc Zyngier } 1410443acc4fSMarc Zyngier } 1411443acc4fSMarc Zyngier 1412e3825ba1SMarc Zyngier static int gic_irq_domain_select(struct irq_domain *d, 1413e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1414e3825ba1SMarc Zyngier enum irq_domain_bus_token bus_token) 1415e3825ba1SMarc Zyngier { 1416e3825ba1SMarc Zyngier /* Not for us */ 1417e3825ba1SMarc Zyngier if (fwspec->fwnode != d->fwnode) 1418e3825ba1SMarc Zyngier return 0; 1419e3825ba1SMarc Zyngier 1420e3825ba1SMarc Zyngier /* If this is not DT, then we have a single domain */ 1421e3825ba1SMarc Zyngier if (!is_of_node(fwspec->fwnode)) 1422e3825ba1SMarc Zyngier return 1; 1423e3825ba1SMarc Zyngier 1424e3825ba1SMarc Zyngier /* 1425e3825ba1SMarc Zyngier * If this is a PPI and we have a 4th (non-null) parameter, 1426e3825ba1SMarc Zyngier * then we need to match the partition domain. 1427e3825ba1SMarc Zyngier */ 1428e3825ba1SMarc Zyngier if (fwspec->param_count >= 4 && 142952085d3fSMarc Zyngier fwspec->param[0] == 1 && fwspec->param[3] != 0 && 143052085d3fSMarc Zyngier gic_data.ppi_descs) 1431e3825ba1SMarc Zyngier return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]); 1432e3825ba1SMarc Zyngier 1433e3825ba1SMarc Zyngier return d == gic_data.domain; 1434e3825ba1SMarc Zyngier } 1435e3825ba1SMarc Zyngier 1436021f6537SMarc Zyngier static const struct irq_domain_ops gic_irq_domain_ops = { 1437f833f57fSMarc Zyngier .translate = gic_irq_domain_translate, 1438443acc4fSMarc Zyngier .alloc = gic_irq_domain_alloc, 1439443acc4fSMarc Zyngier .free = gic_irq_domain_free, 1440e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1441e3825ba1SMarc Zyngier }; 1442e3825ba1SMarc Zyngier 1443e3825ba1SMarc Zyngier static int partition_domain_translate(struct irq_domain *d, 1444e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1445e3825ba1SMarc Zyngier unsigned long *hwirq, 1446e3825ba1SMarc Zyngier unsigned int *type) 1447e3825ba1SMarc Zyngier { 1448e3825ba1SMarc Zyngier struct device_node *np; 1449e3825ba1SMarc Zyngier int ret; 1450e3825ba1SMarc Zyngier 145152085d3fSMarc Zyngier if (!gic_data.ppi_descs) 145252085d3fSMarc Zyngier return -ENOMEM; 145352085d3fSMarc Zyngier 1454e3825ba1SMarc Zyngier np = of_find_node_by_phandle(fwspec->param[3]); 1455e3825ba1SMarc Zyngier if (WARN_ON(!np)) 1456e3825ba1SMarc Zyngier return -EINVAL; 1457e3825ba1SMarc Zyngier 1458e3825ba1SMarc Zyngier ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]], 1459e3825ba1SMarc Zyngier of_node_to_fwnode(np)); 1460e3825ba1SMarc Zyngier if (ret < 0) 1461e3825ba1SMarc Zyngier return ret; 1462e3825ba1SMarc Zyngier 1463e3825ba1SMarc Zyngier *hwirq = ret; 1464e3825ba1SMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1465e3825ba1SMarc Zyngier 1466e3825ba1SMarc Zyngier return 0; 1467e3825ba1SMarc Zyngier } 1468e3825ba1SMarc Zyngier 1469e3825ba1SMarc Zyngier static const struct irq_domain_ops partition_domain_ops = { 1470e3825ba1SMarc Zyngier .translate = partition_domain_translate, 1471e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1472021f6537SMarc Zyngier }; 1473021f6537SMarc Zyngier 14749c8114c2SSrinivas Kandagatla static bool gic_enable_quirk_msm8996(void *data) 14759c8114c2SSrinivas Kandagatla { 14769c8114c2SSrinivas Kandagatla struct gic_chip_data *d = data; 14779c8114c2SSrinivas Kandagatla 14789c8114c2SSrinivas Kandagatla d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 14799c8114c2SSrinivas Kandagatla 14809c8114c2SSrinivas Kandagatla return true; 14819c8114c2SSrinivas Kandagatla } 14829c8114c2SSrinivas Kandagatla 1483d01fd161SMarc Zyngier static bool gic_enable_quirk_cavium_38539(void *data) 1484d01fd161SMarc Zyngier { 1485d01fd161SMarc Zyngier struct gic_chip_data *d = data; 1486d01fd161SMarc Zyngier 1487d01fd161SMarc Zyngier d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1488d01fd161SMarc Zyngier 1489d01fd161SMarc Zyngier return true; 1490d01fd161SMarc Zyngier } 1491d01fd161SMarc Zyngier 14927f2481b3SMarc Zyngier static bool gic_enable_quirk_hip06_07(void *data) 14937f2481b3SMarc Zyngier { 14947f2481b3SMarc Zyngier struct gic_chip_data *d = data; 14957f2481b3SMarc Zyngier 14967f2481b3SMarc Zyngier /* 14977f2481b3SMarc Zyngier * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 14987f2481b3SMarc Zyngier * not being an actual ARM implementation). The saving grace is 14997f2481b3SMarc Zyngier * that GIC-600 doesn't have ESPI, so nothing to do in that case. 15007f2481b3SMarc Zyngier * HIP07 doesn't even have a proper IIDR, and still pretends to 15017f2481b3SMarc Zyngier * have ESPI. In both cases, put them right. 15027f2481b3SMarc Zyngier */ 15037f2481b3SMarc Zyngier if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 15047f2481b3SMarc Zyngier /* Zero both ESPI and the RES0 field next to it... */ 15057f2481b3SMarc Zyngier d->rdists.gicd_typer &= ~GENMASK(9, 8); 15067f2481b3SMarc Zyngier return true; 15077f2481b3SMarc Zyngier } 15087f2481b3SMarc Zyngier 15097f2481b3SMarc Zyngier return false; 15107f2481b3SMarc Zyngier } 15117f2481b3SMarc Zyngier 15127f2481b3SMarc Zyngier static const struct gic_quirk gic_quirks[] = { 15137f2481b3SMarc Zyngier { 15147f2481b3SMarc Zyngier .desc = "GICv3: Qualcomm MSM8996 broken firmware", 15157f2481b3SMarc Zyngier .compatible = "qcom,msm8996-gic-v3", 15167f2481b3SMarc Zyngier .init = gic_enable_quirk_msm8996, 15177f2481b3SMarc Zyngier }, 15187f2481b3SMarc Zyngier { 15197f2481b3SMarc Zyngier .desc = "GICv3: HIP06 erratum 161010803", 15207f2481b3SMarc Zyngier .iidr = 0x0204043b, 15217f2481b3SMarc Zyngier .mask = 0xffffffff, 15227f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 15237f2481b3SMarc Zyngier }, 15247f2481b3SMarc Zyngier { 15257f2481b3SMarc Zyngier .desc = "GICv3: HIP07 erratum 161010803", 15267f2481b3SMarc Zyngier .iidr = 0x00000000, 15277f2481b3SMarc Zyngier .mask = 0xffffffff, 15287f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 15297f2481b3SMarc Zyngier }, 15307f2481b3SMarc Zyngier { 1531d01fd161SMarc Zyngier /* 1532d01fd161SMarc Zyngier * Reserved register accesses generate a Synchronous 1533d01fd161SMarc Zyngier * External Abort. This erratum applies to: 1534d01fd161SMarc Zyngier * - ThunderX: CN88xx 1535d01fd161SMarc Zyngier * - OCTEON TX: CN83xx, CN81xx 1536d01fd161SMarc Zyngier * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1537d01fd161SMarc Zyngier */ 1538d01fd161SMarc Zyngier .desc = "GICv3: Cavium erratum 38539", 1539d01fd161SMarc Zyngier .iidr = 0xa000034c, 1540d01fd161SMarc Zyngier .mask = 0xe8f00fff, 1541d01fd161SMarc Zyngier .init = gic_enable_quirk_cavium_38539, 1542d01fd161SMarc Zyngier }, 1543d01fd161SMarc Zyngier { 15447f2481b3SMarc Zyngier } 15457f2481b3SMarc Zyngier }; 15467f2481b3SMarc Zyngier 1547d98d0a99SJulien Thierry static void gic_enable_nmi_support(void) 1548d98d0a99SJulien Thierry { 1549101b35f7SJulien Thierry int i; 1550101b35f7SJulien Thierry 155181a43273SMarc Zyngier if (!gic_prio_masking_enabled()) 155281a43273SMarc Zyngier return; 155381a43273SMarc Zyngier 155481a43273SMarc Zyngier ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 155581a43273SMarc Zyngier if (!ppi_nmi_refs) 155681a43273SMarc Zyngier return; 155781a43273SMarc Zyngier 155881a43273SMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) 1559101b35f7SJulien Thierry refcount_set(&ppi_nmi_refs[i], 0); 1560101b35f7SJulien Thierry 1561f2266504SMarc Zyngier /* 1562f2266504SMarc Zyngier * Linux itself doesn't use 1:N distribution, so has no need to 1563f2266504SMarc Zyngier * set PMHE. The only reason to have it set is if EL3 requires it 1564f2266504SMarc Zyngier * (and we can't change it). 1565f2266504SMarc Zyngier */ 1566f2266504SMarc Zyngier if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) 1567f2266504SMarc Zyngier static_branch_enable(&gic_pmr_sync); 1568f2266504SMarc Zyngier 15694e594ad1SAlexandru Elisei pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 15704e594ad1SAlexandru Elisei static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed"); 1571f2266504SMarc Zyngier 157233678059SAlexandru Elisei /* 157333678059SAlexandru Elisei * How priority values are used by the GIC depends on two things: 157433678059SAlexandru Elisei * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 157533678059SAlexandru Elisei * and if Group 0 interrupts can be delivered to Linux in the non-secure 157633678059SAlexandru Elisei * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 157733678059SAlexandru Elisei * the ICC_PMR_EL1 register and the priority that software assigns to 157833678059SAlexandru Elisei * interrupts: 157933678059SAlexandru Elisei * 158033678059SAlexandru Elisei * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 158133678059SAlexandru Elisei * ----------------------------------------------------------- 158233678059SAlexandru Elisei * 1 | - | unchanged | unchanged 158333678059SAlexandru Elisei * ----------------------------------------------------------- 158433678059SAlexandru Elisei * 0 | 1 | non-secure | non-secure 158533678059SAlexandru Elisei * ----------------------------------------------------------- 158633678059SAlexandru Elisei * 0 | 0 | unchanged | non-secure 158733678059SAlexandru Elisei * 158833678059SAlexandru Elisei * where non-secure means that the value is right-shifted by one and the 158933678059SAlexandru Elisei * MSB bit set, to make it fit in the non-secure priority range. 159033678059SAlexandru Elisei * 159133678059SAlexandru Elisei * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 159233678059SAlexandru Elisei * are both either modified or unchanged, we can use the same set of 159333678059SAlexandru Elisei * priorities. 159433678059SAlexandru Elisei * 159533678059SAlexandru Elisei * In the last case, where only the interrupt priorities are modified to 159633678059SAlexandru Elisei * be in the non-secure range, we use a different PMR value to mask IRQs 159733678059SAlexandru Elisei * and the rest of the values that we use remain unchanged. 159833678059SAlexandru Elisei */ 159933678059SAlexandru Elisei if (gic_has_group0() && !gic_dist_security_disabled()) 160033678059SAlexandru Elisei static_branch_enable(&gic_nonsecure_priorities); 160133678059SAlexandru Elisei 1602d98d0a99SJulien Thierry static_branch_enable(&supports_pseudo_nmis); 1603101b35f7SJulien Thierry 1604101b35f7SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) 1605101b35f7SJulien Thierry gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1606101b35f7SJulien Thierry else 1607101b35f7SJulien Thierry gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1608d98d0a99SJulien Thierry } 1609d98d0a99SJulien Thierry 1610db57d746STomasz Nowicki static int __init gic_init_bases(void __iomem *dist_base, 1611db57d746STomasz Nowicki struct redist_region *rdist_regs, 1612db57d746STomasz Nowicki u32 nr_redist_regions, 1613db57d746STomasz Nowicki u64 redist_stride, 1614db57d746STomasz Nowicki struct fwnode_handle *handle) 1615db57d746STomasz Nowicki { 1616db57d746STomasz Nowicki u32 typer; 1617db57d746STomasz Nowicki int err; 1618db57d746STomasz Nowicki 1619db57d746STomasz Nowicki if (!is_hyp_mode_available()) 1620d01d3274SDavidlohr Bueso static_branch_disable(&supports_deactivate_key); 1621db57d746STomasz Nowicki 1622d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 1623db57d746STomasz Nowicki pr_info("GIC: Using split EOI/Deactivate mode\n"); 1624db57d746STomasz Nowicki 1625e3825ba1SMarc Zyngier gic_data.fwnode = handle; 1626db57d746STomasz Nowicki gic_data.dist_base = dist_base; 1627db57d746STomasz Nowicki gic_data.redist_regions = rdist_regs; 1628db57d746STomasz Nowicki gic_data.nr_redist_regions = nr_redist_regions; 1629db57d746STomasz Nowicki gic_data.redist_stride = redist_stride; 1630db57d746STomasz Nowicki 1631db57d746STomasz Nowicki /* 1632db57d746STomasz Nowicki * Find out how many interrupts are supported. 1633db57d746STomasz Nowicki */ 1634db57d746STomasz Nowicki typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1635a4f9edb2SMarc Zyngier gic_data.rdists.gicd_typer = typer; 16367f2481b3SMarc Zyngier 16377f2481b3SMarc Zyngier gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 16387f2481b3SMarc Zyngier gic_quirks, &gic_data); 16397f2481b3SMarc Zyngier 1640211bddd2SMarc Zyngier pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 1641211bddd2SMarc Zyngier pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 1642f2d83409SMarc Zyngier 1643d01fd161SMarc Zyngier /* 1644d01fd161SMarc Zyngier * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 1645d01fd161SMarc Zyngier * architecture spec (which says that reserved registers are RES0). 1646d01fd161SMarc Zyngier */ 1647d01fd161SMarc Zyngier if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 1648f2d83409SMarc Zyngier gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 1649f2d83409SMarc Zyngier 1650db57d746STomasz Nowicki gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1651db57d746STomasz Nowicki &gic_data); 1652db57d746STomasz Nowicki gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 1653b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = true; 16540edc23eaSMarc Zyngier gic_data.rdists.has_vlpis = true; 16550edc23eaSMarc Zyngier gic_data.rdists.has_direct_lpi = true; 165696806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty = true; 1657db57d746STomasz Nowicki 1658db57d746STomasz Nowicki if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1659db57d746STomasz Nowicki err = -ENOMEM; 1660db57d746STomasz Nowicki goto out_free; 1661db57d746STomasz Nowicki } 1662db57d746STomasz Nowicki 1663eeaa4b24Sluanshi irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1664eeaa4b24Sluanshi 1665eda0d04aSShanker Donthineni gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1666eda0d04aSShanker Donthineni pr_info("Distributor has %sRange Selector support\n", 1667eda0d04aSShanker Donthineni gic_data.has_rss ? "" : "no "); 1668eda0d04aSShanker Donthineni 166950528752SMarc Zyngier if (typer & GICD_TYPER_MBIS) { 167050528752SMarc Zyngier err = mbi_init(handle, gic_data.domain); 167150528752SMarc Zyngier if (err) 167250528752SMarc Zyngier pr_err("Failed to initialize MBIs\n"); 167350528752SMarc Zyngier } 167450528752SMarc Zyngier 1675db57d746STomasz Nowicki set_handle_irq(gic_handle_irq); 1676db57d746STomasz Nowicki 16771a60e1e6SMarc Zyngier gic_update_rdist_properties(); 16780edc23eaSMarc Zyngier 1679db57d746STomasz Nowicki gic_smp_init(); 1680db57d746STomasz Nowicki gic_dist_init(); 1681db57d746STomasz Nowicki gic_cpu_init(); 1682db57d746STomasz Nowicki gic_cpu_pm_init(); 1683db57d746STomasz Nowicki 1684d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) { 1685d38a71c5SMarc Zyngier its_init(handle, &gic_data.rdists, gic_data.domain); 1686d38a71c5SMarc Zyngier its_cpu_init(); 168790b4c555SZeev Zilberman } else { 168890b4c555SZeev Zilberman if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 168990b4c555SZeev Zilberman gicv2m_init(handle, gic_data.domain); 1690d38a71c5SMarc Zyngier } 1691d38a71c5SMarc Zyngier 1692d98d0a99SJulien Thierry gic_enable_nmi_support(); 1693d98d0a99SJulien Thierry 1694db57d746STomasz Nowicki return 0; 1695db57d746STomasz Nowicki 1696db57d746STomasz Nowicki out_free: 1697db57d746STomasz Nowicki if (gic_data.domain) 1698db57d746STomasz Nowicki irq_domain_remove(gic_data.domain); 1699db57d746STomasz Nowicki free_percpu(gic_data.rdists.rdist); 1700db57d746STomasz Nowicki return err; 1701db57d746STomasz Nowicki } 1702db57d746STomasz Nowicki 1703db57d746STomasz Nowicki static int __init gic_validate_dist_version(void __iomem *dist_base) 1704db57d746STomasz Nowicki { 1705db57d746STomasz Nowicki u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1706db57d746STomasz Nowicki 1707db57d746STomasz Nowicki if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 1708db57d746STomasz Nowicki return -ENODEV; 1709db57d746STomasz Nowicki 1710db57d746STomasz Nowicki return 0; 1711db57d746STomasz Nowicki } 1712db57d746STomasz Nowicki 1713e3825ba1SMarc Zyngier /* Create all possible partitions at boot time */ 17147beaa24bSLinus Torvalds static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 1715e3825ba1SMarc Zyngier { 1716e3825ba1SMarc Zyngier struct device_node *parts_node, *child_part; 1717e3825ba1SMarc Zyngier int part_idx = 0, i; 1718e3825ba1SMarc Zyngier int nr_parts; 1719e3825ba1SMarc Zyngier struct partition_affinity *parts; 1720e3825ba1SMarc Zyngier 172100ee9a1cSJohan Hovold parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 1722e3825ba1SMarc Zyngier if (!parts_node) 1723e3825ba1SMarc Zyngier return; 1724e3825ba1SMarc Zyngier 172552085d3fSMarc Zyngier gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 172652085d3fSMarc Zyngier if (!gic_data.ppi_descs) 172752085d3fSMarc Zyngier return; 172852085d3fSMarc Zyngier 1729e3825ba1SMarc Zyngier nr_parts = of_get_child_count(parts_node); 1730e3825ba1SMarc Zyngier 1731e3825ba1SMarc Zyngier if (!nr_parts) 173200ee9a1cSJohan Hovold goto out_put_node; 1733e3825ba1SMarc Zyngier 17346396bb22SKees Cook parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 1735e3825ba1SMarc Zyngier if (WARN_ON(!parts)) 173600ee9a1cSJohan Hovold goto out_put_node; 1737e3825ba1SMarc Zyngier 1738e3825ba1SMarc Zyngier for_each_child_of_node(parts_node, child_part) { 1739e3825ba1SMarc Zyngier struct partition_affinity *part; 1740e3825ba1SMarc Zyngier int n; 1741e3825ba1SMarc Zyngier 1742e3825ba1SMarc Zyngier part = &parts[part_idx]; 1743e3825ba1SMarc Zyngier 1744e3825ba1SMarc Zyngier part->partition_id = of_node_to_fwnode(child_part); 1745e3825ba1SMarc Zyngier 17462ef790dcSRob Herring pr_info("GIC: PPI partition %pOFn[%d] { ", 17472ef790dcSRob Herring child_part, part_idx); 1748e3825ba1SMarc Zyngier 1749e3825ba1SMarc Zyngier n = of_property_count_elems_of_size(child_part, "affinity", 1750e3825ba1SMarc Zyngier sizeof(u32)); 1751e3825ba1SMarc Zyngier WARN_ON(n <= 0); 1752e3825ba1SMarc Zyngier 1753e3825ba1SMarc Zyngier for (i = 0; i < n; i++) { 1754e3825ba1SMarc Zyngier int err, cpu; 1755e3825ba1SMarc Zyngier u32 cpu_phandle; 1756e3825ba1SMarc Zyngier struct device_node *cpu_node; 1757e3825ba1SMarc Zyngier 1758e3825ba1SMarc Zyngier err = of_property_read_u32_index(child_part, "affinity", 1759e3825ba1SMarc Zyngier i, &cpu_phandle); 1760e3825ba1SMarc Zyngier if (WARN_ON(err)) 1761e3825ba1SMarc Zyngier continue; 1762e3825ba1SMarc Zyngier 1763e3825ba1SMarc Zyngier cpu_node = of_find_node_by_phandle(cpu_phandle); 1764e3825ba1SMarc Zyngier if (WARN_ON(!cpu_node)) 1765e3825ba1SMarc Zyngier continue; 1766e3825ba1SMarc Zyngier 1767c08ec7daSSuzuki K Poulose cpu = of_cpu_node_to_id(cpu_node); 1768c08ec7daSSuzuki K Poulose if (WARN_ON(cpu < 0)) 1769e3825ba1SMarc Zyngier continue; 1770e3825ba1SMarc Zyngier 1771e81f54c6SRob Herring pr_cont("%pOF[%d] ", cpu_node, cpu); 1772e3825ba1SMarc Zyngier 1773e3825ba1SMarc Zyngier cpumask_set_cpu(cpu, &part->mask); 1774e3825ba1SMarc Zyngier } 1775e3825ba1SMarc Zyngier 1776e3825ba1SMarc Zyngier pr_cont("}\n"); 1777e3825ba1SMarc Zyngier part_idx++; 1778e3825ba1SMarc Zyngier } 1779e3825ba1SMarc Zyngier 178052085d3fSMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) { 1781e3825ba1SMarc Zyngier unsigned int irq; 1782e3825ba1SMarc Zyngier struct partition_desc *desc; 1783e3825ba1SMarc Zyngier struct irq_fwspec ppi_fwspec = { 1784e3825ba1SMarc Zyngier .fwnode = gic_data.fwnode, 1785e3825ba1SMarc Zyngier .param_count = 3, 1786e3825ba1SMarc Zyngier .param = { 178765da7d19SMarc Zyngier [0] = GIC_IRQ_TYPE_PARTITION, 1788e3825ba1SMarc Zyngier [1] = i, 1789e3825ba1SMarc Zyngier [2] = IRQ_TYPE_NONE, 1790e3825ba1SMarc Zyngier }, 1791e3825ba1SMarc Zyngier }; 1792e3825ba1SMarc Zyngier 1793e3825ba1SMarc Zyngier irq = irq_create_fwspec_mapping(&ppi_fwspec); 1794e3825ba1SMarc Zyngier if (WARN_ON(!irq)) 1795e3825ba1SMarc Zyngier continue; 1796e3825ba1SMarc Zyngier desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 1797e3825ba1SMarc Zyngier irq, &partition_domain_ops); 1798e3825ba1SMarc Zyngier if (WARN_ON(!desc)) 1799e3825ba1SMarc Zyngier continue; 1800e3825ba1SMarc Zyngier 1801e3825ba1SMarc Zyngier gic_data.ppi_descs[i] = desc; 1802e3825ba1SMarc Zyngier } 180300ee9a1cSJohan Hovold 180400ee9a1cSJohan Hovold out_put_node: 180500ee9a1cSJohan Hovold of_node_put(parts_node); 1806e3825ba1SMarc Zyngier } 1807e3825ba1SMarc Zyngier 18081839e576SJulien Grall static void __init gic_of_setup_kvm_info(struct device_node *node) 18091839e576SJulien Grall { 18101839e576SJulien Grall int ret; 18111839e576SJulien Grall struct resource r; 18121839e576SJulien Grall u32 gicv_idx; 18131839e576SJulien Grall 18141839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 18151839e576SJulien Grall 18161839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 18171839e576SJulien Grall if (!gic_v3_kvm_info.maint_irq) 18181839e576SJulien Grall return; 18191839e576SJulien Grall 18201839e576SJulien Grall if (of_property_read_u32(node, "#redistributor-regions", 18211839e576SJulien Grall &gicv_idx)) 18221839e576SJulien Grall gicv_idx = 1; 18231839e576SJulien Grall 18241839e576SJulien Grall gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 18251839e576SJulien Grall ret = of_address_to_resource(node, gicv_idx, &r); 18261839e576SJulien Grall if (!ret) 18271839e576SJulien Grall gic_v3_kvm_info.vcpu = r; 18281839e576SJulien Grall 18294bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 18303c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 18311839e576SJulien Grall gic_set_kvm_info(&gic_v3_kvm_info); 18321839e576SJulien Grall } 18331839e576SJulien Grall 1834021f6537SMarc Zyngier static int __init gic_of_init(struct device_node *node, struct device_node *parent) 1835021f6537SMarc Zyngier { 1836021f6537SMarc Zyngier void __iomem *dist_base; 1837f5c1434cSMarc Zyngier struct redist_region *rdist_regs; 1838021f6537SMarc Zyngier u64 redist_stride; 1839f5c1434cSMarc Zyngier u32 nr_redist_regions; 1840db57d746STomasz Nowicki int err, i; 1841021f6537SMarc Zyngier 1842021f6537SMarc Zyngier dist_base = of_iomap(node, 0); 1843021f6537SMarc Zyngier if (!dist_base) { 1844e81f54c6SRob Herring pr_err("%pOF: unable to map gic dist registers\n", node); 1845021f6537SMarc Zyngier return -ENXIO; 1846021f6537SMarc Zyngier } 1847021f6537SMarc Zyngier 1848db57d746STomasz Nowicki err = gic_validate_dist_version(dist_base); 1849db57d746STomasz Nowicki if (err) { 1850e81f54c6SRob Herring pr_err("%pOF: no distributor detected, giving up\n", node); 1851021f6537SMarc Zyngier goto out_unmap_dist; 1852021f6537SMarc Zyngier } 1853021f6537SMarc Zyngier 1854f5c1434cSMarc Zyngier if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 1855f5c1434cSMarc Zyngier nr_redist_regions = 1; 1856021f6537SMarc Zyngier 18576396bb22SKees Cook rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 18586396bb22SKees Cook GFP_KERNEL); 1859f5c1434cSMarc Zyngier if (!rdist_regs) { 1860021f6537SMarc Zyngier err = -ENOMEM; 1861021f6537SMarc Zyngier goto out_unmap_dist; 1862021f6537SMarc Zyngier } 1863021f6537SMarc Zyngier 1864f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) { 1865f5c1434cSMarc Zyngier struct resource res; 1866f5c1434cSMarc Zyngier int ret; 1867f5c1434cSMarc Zyngier 1868f5c1434cSMarc Zyngier ret = of_address_to_resource(node, 1 + i, &res); 1869f5c1434cSMarc Zyngier rdist_regs[i].redist_base = of_iomap(node, 1 + i); 1870f5c1434cSMarc Zyngier if (ret || !rdist_regs[i].redist_base) { 1871e81f54c6SRob Herring pr_err("%pOF: couldn't map region %d\n", node, i); 1872021f6537SMarc Zyngier err = -ENODEV; 1873021f6537SMarc Zyngier goto out_unmap_rdist; 1874021f6537SMarc Zyngier } 1875f5c1434cSMarc Zyngier rdist_regs[i].phys_base = res.start; 1876021f6537SMarc Zyngier } 1877021f6537SMarc Zyngier 1878021f6537SMarc Zyngier if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 1879021f6537SMarc Zyngier redist_stride = 0; 1880021f6537SMarc Zyngier 1881f70fdb42SSrinivas Kandagatla gic_enable_of_quirks(node, gic_quirks, &gic_data); 1882f70fdb42SSrinivas Kandagatla 1883db57d746STomasz Nowicki err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions, 1884db57d746STomasz Nowicki redist_stride, &node->fwnode); 1885e3825ba1SMarc Zyngier if (err) 1886e3825ba1SMarc Zyngier goto out_unmap_rdist; 1887e3825ba1SMarc Zyngier 1888e3825ba1SMarc Zyngier gic_populate_ppi_partitions(node); 1889d33a3c8cSChristoffer Dall 1890d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 18911839e576SJulien Grall gic_of_setup_kvm_info(node); 1892021f6537SMarc Zyngier return 0; 1893021f6537SMarc Zyngier 1894021f6537SMarc Zyngier out_unmap_rdist: 1895f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) 1896f5c1434cSMarc Zyngier if (rdist_regs[i].redist_base) 1897f5c1434cSMarc Zyngier iounmap(rdist_regs[i].redist_base); 1898f5c1434cSMarc Zyngier kfree(rdist_regs); 1899021f6537SMarc Zyngier out_unmap_dist: 1900021f6537SMarc Zyngier iounmap(dist_base); 1901021f6537SMarc Zyngier return err; 1902021f6537SMarc Zyngier } 1903021f6537SMarc Zyngier 1904021f6537SMarc Zyngier IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 1905ffa7d616STomasz Nowicki 1906ffa7d616STomasz Nowicki #ifdef CONFIG_ACPI 1907611f039fSJulien Grall static struct 1908611f039fSJulien Grall { 1909611f039fSJulien Grall void __iomem *dist_base; 1910611f039fSJulien Grall struct redist_region *redist_regs; 1911611f039fSJulien Grall u32 nr_redist_regions; 1912611f039fSJulien Grall bool single_redist; 1913926b5dfaSMarc Zyngier int enabled_rdists; 19141839e576SJulien Grall u32 maint_irq; 19151839e576SJulien Grall int maint_irq_mode; 19161839e576SJulien Grall phys_addr_t vcpu_base; 1917611f039fSJulien Grall } acpi_data __initdata; 1918b70fb7afSTomasz Nowicki 1919b70fb7afSTomasz Nowicki static void __init 1920b70fb7afSTomasz Nowicki gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 1921b70fb7afSTomasz Nowicki { 1922b70fb7afSTomasz Nowicki static int count = 0; 1923b70fb7afSTomasz Nowicki 1924611f039fSJulien Grall acpi_data.redist_regs[count].phys_base = phys_base; 1925611f039fSJulien Grall acpi_data.redist_regs[count].redist_base = redist_base; 1926611f039fSJulien Grall acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 1927b70fb7afSTomasz Nowicki count++; 1928b70fb7afSTomasz Nowicki } 1929ffa7d616STomasz Nowicki 1930ffa7d616STomasz Nowicki static int __init 193160574d1eSKeith Busch gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 1932ffa7d616STomasz Nowicki const unsigned long end) 1933ffa7d616STomasz Nowicki { 1934ffa7d616STomasz Nowicki struct acpi_madt_generic_redistributor *redist = 1935ffa7d616STomasz Nowicki (struct acpi_madt_generic_redistributor *)header; 1936ffa7d616STomasz Nowicki void __iomem *redist_base; 1937ffa7d616STomasz Nowicki 1938ffa7d616STomasz Nowicki redist_base = ioremap(redist->base_address, redist->length); 1939ffa7d616STomasz Nowicki if (!redist_base) { 1940ffa7d616STomasz Nowicki pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 1941ffa7d616STomasz Nowicki return -ENOMEM; 1942ffa7d616STomasz Nowicki } 1943ffa7d616STomasz Nowicki 1944b70fb7afSTomasz Nowicki gic_acpi_register_redist(redist->base_address, redist_base); 1945ffa7d616STomasz Nowicki return 0; 1946ffa7d616STomasz Nowicki } 1947ffa7d616STomasz Nowicki 1948b70fb7afSTomasz Nowicki static int __init 194960574d1eSKeith Busch gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 1950b70fb7afSTomasz Nowicki const unsigned long end) 1951b70fb7afSTomasz Nowicki { 1952b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 1953b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 1954611f039fSJulien Grall u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1955b70fb7afSTomasz Nowicki u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 1956b70fb7afSTomasz Nowicki void __iomem *redist_base; 1957b70fb7afSTomasz Nowicki 1958ebe2f871SShanker Donthineni /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 1959ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 1960ebe2f871SShanker Donthineni return 0; 1961ebe2f871SShanker Donthineni 1962b70fb7afSTomasz Nowicki redist_base = ioremap(gicc->gicr_base_address, size); 1963b70fb7afSTomasz Nowicki if (!redist_base) 1964b70fb7afSTomasz Nowicki return -ENOMEM; 1965b70fb7afSTomasz Nowicki 1966b70fb7afSTomasz Nowicki gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 1967b70fb7afSTomasz Nowicki return 0; 1968b70fb7afSTomasz Nowicki } 1969b70fb7afSTomasz Nowicki 1970b70fb7afSTomasz Nowicki static int __init gic_acpi_collect_gicr_base(void) 1971b70fb7afSTomasz Nowicki { 1972b70fb7afSTomasz Nowicki acpi_tbl_entry_handler redist_parser; 1973b70fb7afSTomasz Nowicki enum acpi_madt_type type; 1974b70fb7afSTomasz Nowicki 1975611f039fSJulien Grall if (acpi_data.single_redist) { 1976b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 1977b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_gicc; 1978b70fb7afSTomasz Nowicki } else { 1979b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 1980b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_redist; 1981b70fb7afSTomasz Nowicki } 1982b70fb7afSTomasz Nowicki 1983b70fb7afSTomasz Nowicki /* Collect redistributor base addresses in GICR entries */ 1984b70fb7afSTomasz Nowicki if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 1985b70fb7afSTomasz Nowicki return 0; 1986b70fb7afSTomasz Nowicki 1987b70fb7afSTomasz Nowicki pr_info("No valid GICR entries exist\n"); 1988b70fb7afSTomasz Nowicki return -ENODEV; 1989b70fb7afSTomasz Nowicki } 1990b70fb7afSTomasz Nowicki 199160574d1eSKeith Busch static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 1992ffa7d616STomasz Nowicki const unsigned long end) 1993ffa7d616STomasz Nowicki { 1994ffa7d616STomasz Nowicki /* Subtable presence means that redist exists, that's it */ 1995ffa7d616STomasz Nowicki return 0; 1996ffa7d616STomasz Nowicki } 1997ffa7d616STomasz Nowicki 199860574d1eSKeith Busch static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 1999b70fb7afSTomasz Nowicki const unsigned long end) 2000b70fb7afSTomasz Nowicki { 2001b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 2002b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 2003b70fb7afSTomasz Nowicki 2004b70fb7afSTomasz Nowicki /* 2005b70fb7afSTomasz Nowicki * If GICC is enabled and has valid gicr base address, then it means 2006b70fb7afSTomasz Nowicki * GICR base is presented via GICC 2007b70fb7afSTomasz Nowicki */ 2008926b5dfaSMarc Zyngier if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2009926b5dfaSMarc Zyngier acpi_data.enabled_rdists++; 2010b70fb7afSTomasz Nowicki return 0; 2011926b5dfaSMarc Zyngier } 2012b70fb7afSTomasz Nowicki 2013ebe2f871SShanker Donthineni /* 2014ebe2f871SShanker Donthineni * It's perfectly valid firmware can pass disabled GICC entry, driver 2015ebe2f871SShanker Donthineni * should not treat as errors, skip the entry instead of probe fail. 2016ebe2f871SShanker Donthineni */ 2017ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 2018ebe2f871SShanker Donthineni return 0; 2019ebe2f871SShanker Donthineni 2020b70fb7afSTomasz Nowicki return -ENODEV; 2021b70fb7afSTomasz Nowicki } 2022b70fb7afSTomasz Nowicki 2023b70fb7afSTomasz Nowicki static int __init gic_acpi_count_gicr_regions(void) 2024b70fb7afSTomasz Nowicki { 2025b70fb7afSTomasz Nowicki int count; 2026b70fb7afSTomasz Nowicki 2027b70fb7afSTomasz Nowicki /* 2028b70fb7afSTomasz Nowicki * Count how many redistributor regions we have. It is not allowed 2029b70fb7afSTomasz Nowicki * to mix redistributor description, GICR and GICC subtables have to be 2030b70fb7afSTomasz Nowicki * mutually exclusive. 2031b70fb7afSTomasz Nowicki */ 2032b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2033b70fb7afSTomasz Nowicki gic_acpi_match_gicr, 0); 2034b70fb7afSTomasz Nowicki if (count > 0) { 2035611f039fSJulien Grall acpi_data.single_redist = false; 2036b70fb7afSTomasz Nowicki return count; 2037b70fb7afSTomasz Nowicki } 2038b70fb7afSTomasz Nowicki 2039b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2040b70fb7afSTomasz Nowicki gic_acpi_match_gicc, 0); 2041926b5dfaSMarc Zyngier if (count > 0) { 2042611f039fSJulien Grall acpi_data.single_redist = true; 2043926b5dfaSMarc Zyngier count = acpi_data.enabled_rdists; 2044926b5dfaSMarc Zyngier } 2045b70fb7afSTomasz Nowicki 2046b70fb7afSTomasz Nowicki return count; 2047b70fb7afSTomasz Nowicki } 2048b70fb7afSTomasz Nowicki 2049ffa7d616STomasz Nowicki static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2050ffa7d616STomasz Nowicki struct acpi_probe_entry *ape) 2051ffa7d616STomasz Nowicki { 2052ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2053ffa7d616STomasz Nowicki int count; 2054ffa7d616STomasz Nowicki 2055ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2056ffa7d616STomasz Nowicki if (dist->version != ape->driver_data) 2057ffa7d616STomasz Nowicki return false; 2058ffa7d616STomasz Nowicki 2059ffa7d616STomasz Nowicki /* We need to do that exercise anyway, the sooner the better */ 2060b70fb7afSTomasz Nowicki count = gic_acpi_count_gicr_regions(); 2061ffa7d616STomasz Nowicki if (count <= 0) 2062ffa7d616STomasz Nowicki return false; 2063ffa7d616STomasz Nowicki 2064611f039fSJulien Grall acpi_data.nr_redist_regions = count; 2065ffa7d616STomasz Nowicki return true; 2066ffa7d616STomasz Nowicki } 2067ffa7d616STomasz Nowicki 206860574d1eSKeith Busch static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 20691839e576SJulien Grall const unsigned long end) 20701839e576SJulien Grall { 20711839e576SJulien Grall struct acpi_madt_generic_interrupt *gicc = 20721839e576SJulien Grall (struct acpi_madt_generic_interrupt *)header; 20731839e576SJulien Grall int maint_irq_mode; 20741839e576SJulien Grall static int first_madt = true; 20751839e576SJulien Grall 20761839e576SJulien Grall /* Skip unusable CPUs */ 20771839e576SJulien Grall if (!(gicc->flags & ACPI_MADT_ENABLED)) 20781839e576SJulien Grall return 0; 20791839e576SJulien Grall 20801839e576SJulien Grall maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 20811839e576SJulien Grall ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 20821839e576SJulien Grall 20831839e576SJulien Grall if (first_madt) { 20841839e576SJulien Grall first_madt = false; 20851839e576SJulien Grall 20861839e576SJulien Grall acpi_data.maint_irq = gicc->vgic_interrupt; 20871839e576SJulien Grall acpi_data.maint_irq_mode = maint_irq_mode; 20881839e576SJulien Grall acpi_data.vcpu_base = gicc->gicv_base_address; 20891839e576SJulien Grall 20901839e576SJulien Grall return 0; 20911839e576SJulien Grall } 20921839e576SJulien Grall 20931839e576SJulien Grall /* 20941839e576SJulien Grall * The maintenance interrupt and GICV should be the same for every CPU 20951839e576SJulien Grall */ 20961839e576SJulien Grall if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 20971839e576SJulien Grall (acpi_data.maint_irq_mode != maint_irq_mode) || 20981839e576SJulien Grall (acpi_data.vcpu_base != gicc->gicv_base_address)) 20991839e576SJulien Grall return -EINVAL; 21001839e576SJulien Grall 21011839e576SJulien Grall return 0; 21021839e576SJulien Grall } 21031839e576SJulien Grall 21041839e576SJulien Grall static bool __init gic_acpi_collect_virt_info(void) 21051839e576SJulien Grall { 21061839e576SJulien Grall int count; 21071839e576SJulien Grall 21081839e576SJulien Grall count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 21091839e576SJulien Grall gic_acpi_parse_virt_madt_gicc, 0); 21101839e576SJulien Grall 21111839e576SJulien Grall return (count > 0); 21121839e576SJulien Grall } 21131839e576SJulien Grall 2114ffa7d616STomasz Nowicki #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 21151839e576SJulien Grall #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 21161839e576SJulien Grall #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 21171839e576SJulien Grall 21181839e576SJulien Grall static void __init gic_acpi_setup_kvm_info(void) 21191839e576SJulien Grall { 21201839e576SJulien Grall int irq; 21211839e576SJulien Grall 21221839e576SJulien Grall if (!gic_acpi_collect_virt_info()) { 21231839e576SJulien Grall pr_warn("Unable to get hardware information used for virtualization\n"); 21241839e576SJulien Grall return; 21251839e576SJulien Grall } 21261839e576SJulien Grall 21271839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 21281839e576SJulien Grall 21291839e576SJulien Grall irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 21301839e576SJulien Grall acpi_data.maint_irq_mode, 21311839e576SJulien Grall ACPI_ACTIVE_HIGH); 21321839e576SJulien Grall if (irq <= 0) 21331839e576SJulien Grall return; 21341839e576SJulien Grall 21351839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq; 21361839e576SJulien Grall 21371839e576SJulien Grall if (acpi_data.vcpu_base) { 21381839e576SJulien Grall struct resource *vcpu = &gic_v3_kvm_info.vcpu; 21391839e576SJulien Grall 21401839e576SJulien Grall vcpu->flags = IORESOURCE_MEM; 21411839e576SJulien Grall vcpu->start = acpi_data.vcpu_base; 21421839e576SJulien Grall vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 21431839e576SJulien Grall } 21441839e576SJulien Grall 21454bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 21463c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 21471839e576SJulien Grall gic_set_kvm_info(&gic_v3_kvm_info); 21481839e576SJulien Grall } 2149ffa7d616STomasz Nowicki 2150ffa7d616STomasz Nowicki static int __init 2151aba3c7edSOscar Carter gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2152ffa7d616STomasz Nowicki { 2153ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2154ffa7d616STomasz Nowicki struct fwnode_handle *domain_handle; 2155611f039fSJulien Grall size_t size; 2156b70fb7afSTomasz Nowicki int i, err; 2157ffa7d616STomasz Nowicki 2158ffa7d616STomasz Nowicki /* Get distributor base address */ 2159ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2160611f039fSJulien Grall acpi_data.dist_base = ioremap(dist->base_address, 2161611f039fSJulien Grall ACPI_GICV3_DIST_MEM_SIZE); 2162611f039fSJulien Grall if (!acpi_data.dist_base) { 2163ffa7d616STomasz Nowicki pr_err("Unable to map GICD registers\n"); 2164ffa7d616STomasz Nowicki return -ENOMEM; 2165ffa7d616STomasz Nowicki } 2166ffa7d616STomasz Nowicki 2167611f039fSJulien Grall err = gic_validate_dist_version(acpi_data.dist_base); 2168ffa7d616STomasz Nowicki if (err) { 216971192a68SArvind Yadav pr_err("No distributor detected at @%p, giving up\n", 2170611f039fSJulien Grall acpi_data.dist_base); 2171ffa7d616STomasz Nowicki goto out_dist_unmap; 2172ffa7d616STomasz Nowicki } 2173ffa7d616STomasz Nowicki 2174611f039fSJulien Grall size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2175611f039fSJulien Grall acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2176611f039fSJulien Grall if (!acpi_data.redist_regs) { 2177ffa7d616STomasz Nowicki err = -ENOMEM; 2178ffa7d616STomasz Nowicki goto out_dist_unmap; 2179ffa7d616STomasz Nowicki } 2180ffa7d616STomasz Nowicki 2181b70fb7afSTomasz Nowicki err = gic_acpi_collect_gicr_base(); 2182b70fb7afSTomasz Nowicki if (err) 2183ffa7d616STomasz Nowicki goto out_redist_unmap; 2184ffa7d616STomasz Nowicki 2185eeee0d09SMarc Zyngier domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2186ffa7d616STomasz Nowicki if (!domain_handle) { 2187ffa7d616STomasz Nowicki err = -ENOMEM; 2188ffa7d616STomasz Nowicki goto out_redist_unmap; 2189ffa7d616STomasz Nowicki } 2190ffa7d616STomasz Nowicki 2191611f039fSJulien Grall err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs, 2192611f039fSJulien Grall acpi_data.nr_redist_regions, 0, domain_handle); 2193ffa7d616STomasz Nowicki if (err) 2194ffa7d616STomasz Nowicki goto out_fwhandle_free; 2195ffa7d616STomasz Nowicki 2196ffa7d616STomasz Nowicki acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle); 2197d33a3c8cSChristoffer Dall 2198d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 21991839e576SJulien Grall gic_acpi_setup_kvm_info(); 22001839e576SJulien Grall 2201ffa7d616STomasz Nowicki return 0; 2202ffa7d616STomasz Nowicki 2203ffa7d616STomasz Nowicki out_fwhandle_free: 2204ffa7d616STomasz Nowicki irq_domain_free_fwnode(domain_handle); 2205ffa7d616STomasz Nowicki out_redist_unmap: 2206611f039fSJulien Grall for (i = 0; i < acpi_data.nr_redist_regions; i++) 2207611f039fSJulien Grall if (acpi_data.redist_regs[i].redist_base) 2208611f039fSJulien Grall iounmap(acpi_data.redist_regs[i].redist_base); 2209611f039fSJulien Grall kfree(acpi_data.redist_regs); 2210ffa7d616STomasz Nowicki out_dist_unmap: 2211611f039fSJulien Grall iounmap(acpi_data.dist_base); 2212ffa7d616STomasz Nowicki return err; 2213ffa7d616STomasz Nowicki } 2214ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2215ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2216ffa7d616STomasz Nowicki gic_acpi_init); 2217ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2218ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2219ffa7d616STomasz Nowicki gic_acpi_init); 2220ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2221ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2222ffa7d616STomasz Nowicki gic_acpi_init); 2223ffa7d616STomasz Nowicki #endif 2224