1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2021f6537SMarc Zyngier /* 30edc23eaSMarc Zyngier * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4021f6537SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com> 5021f6537SMarc Zyngier */ 6021f6537SMarc Zyngier 768628bb8SJulien Grall #define pr_fmt(fmt) "GICv3: " fmt 868628bb8SJulien Grall 9ffa7d616STomasz Nowicki #include <linux/acpi.h> 10021f6537SMarc Zyngier #include <linux/cpu.h> 113708d52fSSudeep Holla #include <linux/cpu_pm.h> 12021f6537SMarc Zyngier #include <linux/delay.h> 13021f6537SMarc Zyngier #include <linux/interrupt.h> 14ffa7d616STomasz Nowicki #include <linux/irqdomain.h> 15021f6537SMarc Zyngier #include <linux/of.h> 16021f6537SMarc Zyngier #include <linux/of_address.h> 17021f6537SMarc Zyngier #include <linux/of_irq.h> 18021f6537SMarc Zyngier #include <linux/percpu.h> 19101b35f7SJulien Thierry #include <linux/refcount.h> 20021f6537SMarc Zyngier #include <linux/slab.h> 21021f6537SMarc Zyngier 2241a83e06SJoel Porquet #include <linux/irqchip.h> 231839e576SJulien Grall #include <linux/irqchip/arm-gic-common.h> 24021f6537SMarc Zyngier #include <linux/irqchip/arm-gic-v3.h> 25e3825ba1SMarc Zyngier #include <linux/irqchip/irq-partition-percpu.h> 26021f6537SMarc Zyngier 27021f6537SMarc Zyngier #include <asm/cputype.h> 28021f6537SMarc Zyngier #include <asm/exception.h> 29021f6537SMarc Zyngier #include <asm/smp_plat.h> 300b6a3da9SMarc Zyngier #include <asm/virt.h> 31021f6537SMarc Zyngier 32021f6537SMarc Zyngier #include "irq-gic-common.h" 33021f6537SMarc Zyngier 34f32c9266SJulien Thierry #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 35f32c9266SJulien Thierry 369c8114c2SSrinivas Kandagatla #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 37d01fd161SMarc Zyngier #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 389c8114c2SSrinivas Kandagatla 3964b499d8SMarc Zyngier #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 4064b499d8SMarc Zyngier 41f5c1434cSMarc Zyngier struct redist_region { 42f5c1434cSMarc Zyngier void __iomem *redist_base; 43f5c1434cSMarc Zyngier phys_addr_t phys_base; 44b70fb7afSTomasz Nowicki bool single_redist; 45f5c1434cSMarc Zyngier }; 46f5c1434cSMarc Zyngier 47021f6537SMarc Zyngier struct gic_chip_data { 48e3825ba1SMarc Zyngier struct fwnode_handle *fwnode; 49021f6537SMarc Zyngier void __iomem *dist_base; 50f5c1434cSMarc Zyngier struct redist_region *redist_regions; 51f5c1434cSMarc Zyngier struct rdists rdists; 52021f6537SMarc Zyngier struct irq_domain *domain; 53021f6537SMarc Zyngier u64 redist_stride; 54f5c1434cSMarc Zyngier u32 nr_redist_regions; 559c8114c2SSrinivas Kandagatla u64 flags; 56eda0d04aSShanker Donthineni bool has_rss; 571a60e1e6SMarc Zyngier unsigned int ppi_nr; 5852085d3fSMarc Zyngier struct partition_desc **ppi_descs; 59021f6537SMarc Zyngier }; 60021f6537SMarc Zyngier 61021f6537SMarc Zyngier static struct gic_chip_data gic_data __read_mostly; 62d01d3274SDavidlohr Bueso static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 63021f6537SMarc Zyngier 64211bddd2SMarc Zyngier #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 65c107d613SZenghui Yu #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 66211bddd2SMarc Zyngier #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 67211bddd2SMarc Zyngier 68d98d0a99SJulien Thierry /* 69d98d0a99SJulien Thierry * The behaviours of RPR and PMR registers differ depending on the value of 70d98d0a99SJulien Thierry * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 71d98d0a99SJulien Thierry * distributor and redistributors depends on whether security is enabled in the 72d98d0a99SJulien Thierry * GIC. 73d98d0a99SJulien Thierry * 74d98d0a99SJulien Thierry * When security is enabled, non-secure priority values from the (re)distributor 75d98d0a99SJulien Thierry * are presented to the GIC CPUIF as follow: 76d98d0a99SJulien Thierry * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 77d98d0a99SJulien Thierry * 78d4034114SLorenzo Pieralisi * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 79d98d0a99SJulien Thierry * EL1 are subject to a similar operation thus matching the priorities presented 8033678059SAlexandru Elisei * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 81d4034114SLorenzo Pieralisi * these values are unchanged by the GIC. 82d98d0a99SJulien Thierry * 83d98d0a99SJulien Thierry * see GICv3/GICv4 Architecture Specification (IHI0069D): 84d98d0a99SJulien Thierry * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 85d98d0a99SJulien Thierry * priorities. 86d98d0a99SJulien Thierry * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 87d98d0a99SJulien Thierry * interrupt. 88d98d0a99SJulien Thierry */ 89d98d0a99SJulien Thierry static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 90d98d0a99SJulien Thierry 91f2266504SMarc Zyngier /* 92f2266504SMarc Zyngier * Global static key controlling whether an update to PMR allowing more 93f2266504SMarc Zyngier * interrupts requires to be propagated to the redistributor (DSB SY). 94f2266504SMarc Zyngier * And this needs to be exported for modules to be able to enable 95f2266504SMarc Zyngier * interrupts... 96f2266504SMarc Zyngier */ 97f2266504SMarc Zyngier DEFINE_STATIC_KEY_FALSE(gic_pmr_sync); 98f2266504SMarc Zyngier EXPORT_SYMBOL(gic_pmr_sync); 99f2266504SMarc Zyngier 10033678059SAlexandru Elisei DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 10133678059SAlexandru Elisei EXPORT_SYMBOL(gic_nonsecure_priorities); 10233678059SAlexandru Elisei 103101b35f7SJulien Thierry /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 10481a43273SMarc Zyngier static refcount_t *ppi_nmi_refs; 105101b35f7SJulien Thierry 1060e5cb777SMarc Zyngier static struct gic_kvm_info gic_v3_kvm_info __initdata; 107eda0d04aSShanker Donthineni static DEFINE_PER_CPU(bool, has_rss); 1081839e576SJulien Grall 109eda0d04aSShanker Donthineni #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 110f5c1434cSMarc Zyngier #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 111f5c1434cSMarc Zyngier #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 112021f6537SMarc Zyngier #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 113021f6537SMarc Zyngier 114021f6537SMarc Zyngier /* Our default, arbitrary priority value. Linux only uses one anyway. */ 115021f6537SMarc Zyngier #define DEFAULT_PMR_VALUE 0xf0 116021f6537SMarc Zyngier 117e91b036eSMarc Zyngier enum gic_intid_range { 11870a29c32SMarc Zyngier SGI_RANGE, 119e91b036eSMarc Zyngier PPI_RANGE, 120e91b036eSMarc Zyngier SPI_RANGE, 1215f51f803SMarc Zyngier EPPI_RANGE, 122211bddd2SMarc Zyngier ESPI_RANGE, 123e91b036eSMarc Zyngier LPI_RANGE, 124e91b036eSMarc Zyngier __INVALID_RANGE__ 125e91b036eSMarc Zyngier }; 126e91b036eSMarc Zyngier 127e91b036eSMarc Zyngier static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 128e91b036eSMarc Zyngier { 129e91b036eSMarc Zyngier switch (hwirq) { 13070a29c32SMarc Zyngier case 0 ... 15: 13170a29c32SMarc Zyngier return SGI_RANGE; 132e91b036eSMarc Zyngier case 16 ... 31: 133e91b036eSMarc Zyngier return PPI_RANGE; 134e91b036eSMarc Zyngier case 32 ... 1019: 135e91b036eSMarc Zyngier return SPI_RANGE; 1365f51f803SMarc Zyngier case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 1375f51f803SMarc Zyngier return EPPI_RANGE; 138211bddd2SMarc Zyngier case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 139211bddd2SMarc Zyngier return ESPI_RANGE; 140e91b036eSMarc Zyngier case 8192 ... GENMASK(23, 0): 141e91b036eSMarc Zyngier return LPI_RANGE; 142e91b036eSMarc Zyngier default: 143e91b036eSMarc Zyngier return __INVALID_RANGE__; 144e91b036eSMarc Zyngier } 145e91b036eSMarc Zyngier } 146e91b036eSMarc Zyngier 147e91b036eSMarc Zyngier static enum gic_intid_range get_intid_range(struct irq_data *d) 148e91b036eSMarc Zyngier { 149e91b036eSMarc Zyngier return __get_intid_range(d->hwirq); 150e91b036eSMarc Zyngier } 151e91b036eSMarc Zyngier 152021f6537SMarc Zyngier static inline unsigned int gic_irq(struct irq_data *d) 153021f6537SMarc Zyngier { 154021f6537SMarc Zyngier return d->hwirq; 155021f6537SMarc Zyngier } 156021f6537SMarc Zyngier 15770a29c32SMarc Zyngier static inline bool gic_irq_in_rdist(struct irq_data *d) 158021f6537SMarc Zyngier { 15970a29c32SMarc Zyngier switch (get_intid_range(d)) { 16070a29c32SMarc Zyngier case SGI_RANGE: 16170a29c32SMarc Zyngier case PPI_RANGE: 16270a29c32SMarc Zyngier case EPPI_RANGE: 16370a29c32SMarc Zyngier return true; 16470a29c32SMarc Zyngier default: 16570a29c32SMarc Zyngier return false; 16670a29c32SMarc Zyngier } 167021f6537SMarc Zyngier } 168021f6537SMarc Zyngier 169021f6537SMarc Zyngier static inline void __iomem *gic_dist_base(struct irq_data *d) 170021f6537SMarc Zyngier { 171e91b036eSMarc Zyngier switch (get_intid_range(d)) { 17270a29c32SMarc Zyngier case SGI_RANGE: 173e91b036eSMarc Zyngier case PPI_RANGE: 1745f51f803SMarc Zyngier case EPPI_RANGE: 175e91b036eSMarc Zyngier /* SGI+PPI -> SGI_base for this CPU */ 176021f6537SMarc Zyngier return gic_data_rdist_sgi_base(); 177021f6537SMarc Zyngier 178e91b036eSMarc Zyngier case SPI_RANGE: 179211bddd2SMarc Zyngier case ESPI_RANGE: 180e91b036eSMarc Zyngier /* SPI -> dist_base */ 181021f6537SMarc Zyngier return gic_data.dist_base; 182021f6537SMarc Zyngier 183e91b036eSMarc Zyngier default: 184021f6537SMarc Zyngier return NULL; 185021f6537SMarc Zyngier } 186e91b036eSMarc Zyngier } 187021f6537SMarc Zyngier 188021f6537SMarc Zyngier static void gic_do_wait_for_rwp(void __iomem *base) 189021f6537SMarc Zyngier { 190021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 191021f6537SMarc Zyngier 192021f6537SMarc Zyngier while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) { 193021f6537SMarc Zyngier count--; 194021f6537SMarc Zyngier if (!count) { 195021f6537SMarc Zyngier pr_err_ratelimited("RWP timeout, gone fishing\n"); 196021f6537SMarc Zyngier return; 197021f6537SMarc Zyngier } 198021f6537SMarc Zyngier cpu_relax(); 199021f6537SMarc Zyngier udelay(1); 2002c542426SDaode Huang } 201021f6537SMarc Zyngier } 202021f6537SMarc Zyngier 203021f6537SMarc Zyngier /* Wait for completion of a distributor change */ 204021f6537SMarc Zyngier static void gic_dist_wait_for_rwp(void) 205021f6537SMarc Zyngier { 206021f6537SMarc Zyngier gic_do_wait_for_rwp(gic_data.dist_base); 207021f6537SMarc Zyngier } 208021f6537SMarc Zyngier 209021f6537SMarc Zyngier /* Wait for completion of a redistributor change */ 210021f6537SMarc Zyngier static void gic_redist_wait_for_rwp(void) 211021f6537SMarc Zyngier { 212021f6537SMarc Zyngier gic_do_wait_for_rwp(gic_data_rdist_rd_base()); 213021f6537SMarc Zyngier } 214021f6537SMarc Zyngier 2157936e914SJean-Philippe Brucker #ifdef CONFIG_ARM64 2166d4e11c5SRobert Richter 2176d4e11c5SRobert Richter static u64 __maybe_unused gic_read_iar(void) 2186d4e11c5SRobert Richter { 219a4023f68SSuzuki K Poulose if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 2206d4e11c5SRobert Richter return gic_read_iar_cavium_thunderx(); 2216d4e11c5SRobert Richter else 2226d4e11c5SRobert Richter return gic_read_iar_common(); 2236d4e11c5SRobert Richter } 2247936e914SJean-Philippe Brucker #endif 225021f6537SMarc Zyngier 226a2c22510SSudeep Holla static void gic_enable_redist(bool enable) 227021f6537SMarc Zyngier { 228021f6537SMarc Zyngier void __iomem *rbase; 229021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 230021f6537SMarc Zyngier u32 val; 231021f6537SMarc Zyngier 2329c8114c2SSrinivas Kandagatla if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 2339c8114c2SSrinivas Kandagatla return; 2349c8114c2SSrinivas Kandagatla 235021f6537SMarc Zyngier rbase = gic_data_rdist_rd_base(); 236021f6537SMarc Zyngier 237021f6537SMarc Zyngier val = readl_relaxed(rbase + GICR_WAKER); 238a2c22510SSudeep Holla if (enable) 239a2c22510SSudeep Holla /* Wake up this CPU redistributor */ 240021f6537SMarc Zyngier val &= ~GICR_WAKER_ProcessorSleep; 241a2c22510SSudeep Holla else 242a2c22510SSudeep Holla val |= GICR_WAKER_ProcessorSleep; 243021f6537SMarc Zyngier writel_relaxed(val, rbase + GICR_WAKER); 244021f6537SMarc Zyngier 245a2c22510SSudeep Holla if (!enable) { /* Check that GICR_WAKER is writeable */ 246a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 247a2c22510SSudeep Holla if (!(val & GICR_WAKER_ProcessorSleep)) 248a2c22510SSudeep Holla return; /* No PM support in this redistributor */ 249021f6537SMarc Zyngier } 250a2c22510SSudeep Holla 251d102eb5cSDan Carpenter while (--count) { 252a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 253cf1d9d11SAndrew Jones if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 254a2c22510SSudeep Holla break; 255021f6537SMarc Zyngier cpu_relax(); 256021f6537SMarc Zyngier udelay(1); 2572c542426SDaode Huang } 258a2c22510SSudeep Holla if (!count) 259a2c22510SSudeep Holla pr_err_ratelimited("redistributor failed to %s...\n", 260a2c22510SSudeep Holla enable ? "wakeup" : "sleep"); 261021f6537SMarc Zyngier } 262021f6537SMarc Zyngier 263021f6537SMarc Zyngier /* 264021f6537SMarc Zyngier * Routines to disable, enable, EOI and route interrupts 265021f6537SMarc Zyngier */ 266e91b036eSMarc Zyngier static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 267e91b036eSMarc Zyngier { 268e91b036eSMarc Zyngier switch (get_intid_range(d)) { 26970a29c32SMarc Zyngier case SGI_RANGE: 270e91b036eSMarc Zyngier case PPI_RANGE: 271e91b036eSMarc Zyngier case SPI_RANGE: 272e91b036eSMarc Zyngier *index = d->hwirq; 273e91b036eSMarc Zyngier return offset; 2745f51f803SMarc Zyngier case EPPI_RANGE: 2755f51f803SMarc Zyngier /* 2765f51f803SMarc Zyngier * Contrary to the ESPI range, the EPPI range is contiguous 2775f51f803SMarc Zyngier * to the PPI range in the registers, so let's adjust the 2785f51f803SMarc Zyngier * displacement accordingly. Consistency is overrated. 2795f51f803SMarc Zyngier */ 2805f51f803SMarc Zyngier *index = d->hwirq - EPPI_BASE_INTID + 32; 2815f51f803SMarc Zyngier return offset; 282211bddd2SMarc Zyngier case ESPI_RANGE: 283211bddd2SMarc Zyngier *index = d->hwirq - ESPI_BASE_INTID; 284211bddd2SMarc Zyngier switch (offset) { 285211bddd2SMarc Zyngier case GICD_ISENABLER: 286211bddd2SMarc Zyngier return GICD_ISENABLERnE; 287211bddd2SMarc Zyngier case GICD_ICENABLER: 288211bddd2SMarc Zyngier return GICD_ICENABLERnE; 289211bddd2SMarc Zyngier case GICD_ISPENDR: 290211bddd2SMarc Zyngier return GICD_ISPENDRnE; 291211bddd2SMarc Zyngier case GICD_ICPENDR: 292211bddd2SMarc Zyngier return GICD_ICPENDRnE; 293211bddd2SMarc Zyngier case GICD_ISACTIVER: 294211bddd2SMarc Zyngier return GICD_ISACTIVERnE; 295211bddd2SMarc Zyngier case GICD_ICACTIVER: 296211bddd2SMarc Zyngier return GICD_ICACTIVERnE; 297211bddd2SMarc Zyngier case GICD_IPRIORITYR: 298211bddd2SMarc Zyngier return GICD_IPRIORITYRnE; 299211bddd2SMarc Zyngier case GICD_ICFGR: 300211bddd2SMarc Zyngier return GICD_ICFGRnE; 301211bddd2SMarc Zyngier case GICD_IROUTER: 302211bddd2SMarc Zyngier return GICD_IROUTERnE; 303211bddd2SMarc Zyngier default: 304211bddd2SMarc Zyngier break; 305211bddd2SMarc Zyngier } 306211bddd2SMarc Zyngier break; 307e91b036eSMarc Zyngier default: 308e91b036eSMarc Zyngier break; 309e91b036eSMarc Zyngier } 310e91b036eSMarc Zyngier 311e91b036eSMarc Zyngier WARN_ON(1); 312e91b036eSMarc Zyngier *index = d->hwirq; 313e91b036eSMarc Zyngier return offset; 314e91b036eSMarc Zyngier } 315e91b036eSMarc Zyngier 316b594c6e2SMarc Zyngier static int gic_peek_irq(struct irq_data *d, u32 offset) 317b594c6e2SMarc Zyngier { 318b594c6e2SMarc Zyngier void __iomem *base; 319e91b036eSMarc Zyngier u32 index, mask; 320e91b036eSMarc Zyngier 321e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 322e91b036eSMarc Zyngier mask = 1 << (index % 32); 323b594c6e2SMarc Zyngier 324b594c6e2SMarc Zyngier if (gic_irq_in_rdist(d)) 325b594c6e2SMarc Zyngier base = gic_data_rdist_sgi_base(); 326b594c6e2SMarc Zyngier else 327b594c6e2SMarc Zyngier base = gic_data.dist_base; 328b594c6e2SMarc Zyngier 329e91b036eSMarc Zyngier return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 330b594c6e2SMarc Zyngier } 331b594c6e2SMarc Zyngier 332021f6537SMarc Zyngier static void gic_poke_irq(struct irq_data *d, u32 offset) 333021f6537SMarc Zyngier { 334021f6537SMarc Zyngier void (*rwp_wait)(void); 335021f6537SMarc Zyngier void __iomem *base; 336e91b036eSMarc Zyngier u32 index, mask; 337e91b036eSMarc Zyngier 338e91b036eSMarc Zyngier offset = convert_offset_index(d, offset, &index); 339e91b036eSMarc Zyngier mask = 1 << (index % 32); 340021f6537SMarc Zyngier 341021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) { 342021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 343021f6537SMarc Zyngier rwp_wait = gic_redist_wait_for_rwp; 344021f6537SMarc Zyngier } else { 345021f6537SMarc Zyngier base = gic_data.dist_base; 346021f6537SMarc Zyngier rwp_wait = gic_dist_wait_for_rwp; 347021f6537SMarc Zyngier } 348021f6537SMarc Zyngier 349e91b036eSMarc Zyngier writel_relaxed(mask, base + offset + (index / 32) * 4); 350021f6537SMarc Zyngier rwp_wait(); 351021f6537SMarc Zyngier } 352021f6537SMarc Zyngier 353021f6537SMarc Zyngier static void gic_mask_irq(struct irq_data *d) 354021f6537SMarc Zyngier { 355021f6537SMarc Zyngier gic_poke_irq(d, GICD_ICENABLER); 356021f6537SMarc Zyngier } 357021f6537SMarc Zyngier 3580b6a3da9SMarc Zyngier static void gic_eoimode1_mask_irq(struct irq_data *d) 3590b6a3da9SMarc Zyngier { 3600b6a3da9SMarc Zyngier gic_mask_irq(d); 361530bf353SMarc Zyngier /* 362530bf353SMarc Zyngier * When masking a forwarded interrupt, make sure it is 363530bf353SMarc Zyngier * deactivated as well. 364530bf353SMarc Zyngier * 365530bf353SMarc Zyngier * This ensures that an interrupt that is getting 366530bf353SMarc Zyngier * disabled/masked will not get "stuck", because there is 367530bf353SMarc Zyngier * noone to deactivate it (guest is being terminated). 368530bf353SMarc Zyngier */ 3694df7f54dSThomas Gleixner if (irqd_is_forwarded_to_vcpu(d)) 370530bf353SMarc Zyngier gic_poke_irq(d, GICD_ICACTIVER); 3710b6a3da9SMarc Zyngier } 3720b6a3da9SMarc Zyngier 373021f6537SMarc Zyngier static void gic_unmask_irq(struct irq_data *d) 374021f6537SMarc Zyngier { 375021f6537SMarc Zyngier gic_poke_irq(d, GICD_ISENABLER); 376021f6537SMarc Zyngier } 377021f6537SMarc Zyngier 378d98d0a99SJulien Thierry static inline bool gic_supports_nmi(void) 379d98d0a99SJulien Thierry { 380d98d0a99SJulien Thierry return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 381d98d0a99SJulien Thierry static_branch_likely(&supports_pseudo_nmis); 382d98d0a99SJulien Thierry } 383d98d0a99SJulien Thierry 384b594c6e2SMarc Zyngier static int gic_irq_set_irqchip_state(struct irq_data *d, 385b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool val) 386b594c6e2SMarc Zyngier { 387b594c6e2SMarc Zyngier u32 reg; 388b594c6e2SMarc Zyngier 38964b499d8SMarc Zyngier if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 390b594c6e2SMarc Zyngier return -EINVAL; 391b594c6e2SMarc Zyngier 392b594c6e2SMarc Zyngier switch (which) { 393b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 394b594c6e2SMarc Zyngier reg = val ? GICD_ISPENDR : GICD_ICPENDR; 395b594c6e2SMarc Zyngier break; 396b594c6e2SMarc Zyngier 397b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 398b594c6e2SMarc Zyngier reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 399b594c6e2SMarc Zyngier break; 400b594c6e2SMarc Zyngier 401b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 402b594c6e2SMarc Zyngier reg = val ? GICD_ICENABLER : GICD_ISENABLER; 403b594c6e2SMarc Zyngier break; 404b594c6e2SMarc Zyngier 405b594c6e2SMarc Zyngier default: 406b594c6e2SMarc Zyngier return -EINVAL; 407b594c6e2SMarc Zyngier } 408b594c6e2SMarc Zyngier 409b594c6e2SMarc Zyngier gic_poke_irq(d, reg); 410b594c6e2SMarc Zyngier return 0; 411b594c6e2SMarc Zyngier } 412b594c6e2SMarc Zyngier 413b594c6e2SMarc Zyngier static int gic_irq_get_irqchip_state(struct irq_data *d, 414b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool *val) 415b594c6e2SMarc Zyngier { 416211bddd2SMarc Zyngier if (d->hwirq >= 8192) /* PPI/SPI only */ 417b594c6e2SMarc Zyngier return -EINVAL; 418b594c6e2SMarc Zyngier 419b594c6e2SMarc Zyngier switch (which) { 420b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 421b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISPENDR); 422b594c6e2SMarc Zyngier break; 423b594c6e2SMarc Zyngier 424b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 425b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISACTIVER); 426b594c6e2SMarc Zyngier break; 427b594c6e2SMarc Zyngier 428b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 429b594c6e2SMarc Zyngier *val = !gic_peek_irq(d, GICD_ISENABLER); 430b594c6e2SMarc Zyngier break; 431b594c6e2SMarc Zyngier 432b594c6e2SMarc Zyngier default: 433b594c6e2SMarc Zyngier return -EINVAL; 434b594c6e2SMarc Zyngier } 435b594c6e2SMarc Zyngier 436b594c6e2SMarc Zyngier return 0; 437b594c6e2SMarc Zyngier } 438b594c6e2SMarc Zyngier 439101b35f7SJulien Thierry static void gic_irq_set_prio(struct irq_data *d, u8 prio) 440101b35f7SJulien Thierry { 441101b35f7SJulien Thierry void __iomem *base = gic_dist_base(d); 442e91b036eSMarc Zyngier u32 offset, index; 443101b35f7SJulien Thierry 444e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 445e91b036eSMarc Zyngier 446e91b036eSMarc Zyngier writeb_relaxed(prio, base + offset + index); 447101b35f7SJulien Thierry } 448101b35f7SJulien Thierry 449*bfa80ee9SJames Morse static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 45081a43273SMarc Zyngier { 451*bfa80ee9SJames Morse switch (__get_intid_range(hwirq)) { 45281a43273SMarc Zyngier case PPI_RANGE: 453*bfa80ee9SJames Morse return hwirq - 16; 4545f51f803SMarc Zyngier case EPPI_RANGE: 455*bfa80ee9SJames Morse return hwirq - EPPI_BASE_INTID + 16; 45681a43273SMarc Zyngier default: 45781a43273SMarc Zyngier unreachable(); 45881a43273SMarc Zyngier } 45981a43273SMarc Zyngier } 46081a43273SMarc Zyngier 461*bfa80ee9SJames Morse static u32 gic_get_ppi_index(struct irq_data *d) 462*bfa80ee9SJames Morse { 463*bfa80ee9SJames Morse return __gic_get_ppi_index(d->hwirq); 464*bfa80ee9SJames Morse } 465*bfa80ee9SJames Morse 466101b35f7SJulien Thierry static int gic_irq_nmi_setup(struct irq_data *d) 467101b35f7SJulien Thierry { 468101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 469101b35f7SJulien Thierry 470101b35f7SJulien Thierry if (!gic_supports_nmi()) 471101b35f7SJulien Thierry return -EINVAL; 472101b35f7SJulien Thierry 473101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 474101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 475101b35f7SJulien Thierry return -EINVAL; 476101b35f7SJulien Thierry } 477101b35f7SJulien Thierry 478101b35f7SJulien Thierry /* 479101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 480101b35f7SJulien Thierry * it should not be possible to get there 481101b35f7SJulien Thierry */ 482101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 483101b35f7SJulien Thierry return -EINVAL; 484101b35f7SJulien Thierry 485101b35f7SJulien Thierry /* desc lock should already be held */ 48681a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 48781a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 48881a43273SMarc Zyngier 489101b35f7SJulien Thierry /* Setting up PPI as NMI, only switch handler for first NMI */ 49081a43273SMarc Zyngier if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 49181a43273SMarc Zyngier refcount_set(&ppi_nmi_refs[idx], 1); 492101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 493101b35f7SJulien Thierry } 494101b35f7SJulien Thierry } else { 495101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_nmi; 496101b35f7SJulien Thierry } 497101b35f7SJulien Thierry 498101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_NMI_PRI); 499101b35f7SJulien Thierry 500101b35f7SJulien Thierry return 0; 501101b35f7SJulien Thierry } 502101b35f7SJulien Thierry 503101b35f7SJulien Thierry static void gic_irq_nmi_teardown(struct irq_data *d) 504101b35f7SJulien Thierry { 505101b35f7SJulien Thierry struct irq_desc *desc = irq_to_desc(d->irq); 506101b35f7SJulien Thierry 507101b35f7SJulien Thierry if (WARN_ON(!gic_supports_nmi())) 508101b35f7SJulien Thierry return; 509101b35f7SJulien Thierry 510101b35f7SJulien Thierry if (gic_peek_irq(d, GICD_ISENABLER)) { 511101b35f7SJulien Thierry pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 512101b35f7SJulien Thierry return; 513101b35f7SJulien Thierry } 514101b35f7SJulien Thierry 515101b35f7SJulien Thierry /* 516101b35f7SJulien Thierry * A secondary irq_chip should be in charge of LPI request, 517101b35f7SJulien Thierry * it should not be possible to get there 518101b35f7SJulien Thierry */ 519101b35f7SJulien Thierry if (WARN_ON(gic_irq(d) >= 8192)) 520101b35f7SJulien Thierry return; 521101b35f7SJulien Thierry 522101b35f7SJulien Thierry /* desc lock should already be held */ 52381a43273SMarc Zyngier if (gic_irq_in_rdist(d)) { 52481a43273SMarc Zyngier u32 idx = gic_get_ppi_index(d); 52581a43273SMarc Zyngier 526101b35f7SJulien Thierry /* Tearing down NMI, only switch handler for last NMI */ 52781a43273SMarc Zyngier if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 528101b35f7SJulien Thierry desc->handle_irq = handle_percpu_devid_irq; 529101b35f7SJulien Thierry } else { 530101b35f7SJulien Thierry desc->handle_irq = handle_fasteoi_irq; 531101b35f7SJulien Thierry } 532101b35f7SJulien Thierry 533101b35f7SJulien Thierry gic_irq_set_prio(d, GICD_INT_DEF_PRI); 534101b35f7SJulien Thierry } 535101b35f7SJulien Thierry 536021f6537SMarc Zyngier static void gic_eoi_irq(struct irq_data *d) 537021f6537SMarc Zyngier { 538021f6537SMarc Zyngier gic_write_eoir(gic_irq(d)); 539021f6537SMarc Zyngier } 540021f6537SMarc Zyngier 5410b6a3da9SMarc Zyngier static void gic_eoimode1_eoi_irq(struct irq_data *d) 5420b6a3da9SMarc Zyngier { 5430b6a3da9SMarc Zyngier /* 544530bf353SMarc Zyngier * No need to deactivate an LPI, or an interrupt that 545530bf353SMarc Zyngier * is is getting forwarded to a vcpu. 5460b6a3da9SMarc Zyngier */ 5474df7f54dSThomas Gleixner if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 5480b6a3da9SMarc Zyngier return; 5490b6a3da9SMarc Zyngier gic_write_dir(gic_irq(d)); 5500b6a3da9SMarc Zyngier } 5510b6a3da9SMarc Zyngier 552021f6537SMarc Zyngier static int gic_set_type(struct irq_data *d, unsigned int type) 553021f6537SMarc Zyngier { 5545f51f803SMarc Zyngier enum gic_intid_range range; 555021f6537SMarc Zyngier unsigned int irq = gic_irq(d); 556021f6537SMarc Zyngier void (*rwp_wait)(void); 557021f6537SMarc Zyngier void __iomem *base; 558e91b036eSMarc Zyngier u32 offset, index; 55913d22e2eSMarc Zyngier int ret; 560021f6537SMarc Zyngier 5615f51f803SMarc Zyngier range = get_intid_range(d); 5625f51f803SMarc Zyngier 56364b499d8SMarc Zyngier /* Interrupt configuration for SGIs can't be changed */ 56464b499d8SMarc Zyngier if (range == SGI_RANGE) 56564b499d8SMarc Zyngier return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 56664b499d8SMarc Zyngier 567fb7e7debSLiviu Dudau /* SPIs have restrictions on the supported types */ 5685f51f803SMarc Zyngier if ((range == SPI_RANGE || range == ESPI_RANGE) && 5695f51f803SMarc Zyngier type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 570021f6537SMarc Zyngier return -EINVAL; 571021f6537SMarc Zyngier 572021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) { 573021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 574021f6537SMarc Zyngier rwp_wait = gic_redist_wait_for_rwp; 575021f6537SMarc Zyngier } else { 576021f6537SMarc Zyngier base = gic_data.dist_base; 577021f6537SMarc Zyngier rwp_wait = gic_dist_wait_for_rwp; 578021f6537SMarc Zyngier } 579021f6537SMarc Zyngier 580e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_ICFGR, &index); 58113d22e2eSMarc Zyngier 582e91b036eSMarc Zyngier ret = gic_configure_irq(index, type, base + offset, rwp_wait); 5835f51f803SMarc Zyngier if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 58413d22e2eSMarc Zyngier /* Misconfigured PPIs are usually not fatal */ 5855f51f803SMarc Zyngier pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 58613d22e2eSMarc Zyngier ret = 0; 58713d22e2eSMarc Zyngier } 58813d22e2eSMarc Zyngier 58913d22e2eSMarc Zyngier return ret; 590021f6537SMarc Zyngier } 591021f6537SMarc Zyngier 592530bf353SMarc Zyngier static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 593530bf353SMarc Zyngier { 59464b499d8SMarc Zyngier if (get_intid_range(d) == SGI_RANGE) 59564b499d8SMarc Zyngier return -EINVAL; 59664b499d8SMarc Zyngier 5974df7f54dSThomas Gleixner if (vcpu) 5984df7f54dSThomas Gleixner irqd_set_forwarded_to_vcpu(d); 5994df7f54dSThomas Gleixner else 6004df7f54dSThomas Gleixner irqd_clr_forwarded_to_vcpu(d); 601530bf353SMarc Zyngier return 0; 602530bf353SMarc Zyngier } 603530bf353SMarc Zyngier 604f6c86a41SJean-Philippe Brucker static u64 gic_mpidr_to_affinity(unsigned long mpidr) 605021f6537SMarc Zyngier { 606021f6537SMarc Zyngier u64 aff; 607021f6537SMarc Zyngier 608f6c86a41SJean-Philippe Brucker aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 609021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 610021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 611021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 612021f6537SMarc Zyngier 613021f6537SMarc Zyngier return aff; 614021f6537SMarc Zyngier } 615021f6537SMarc Zyngier 616f32c9266SJulien Thierry static void gic_deactivate_unhandled(u32 irqnr) 617f32c9266SJulien Thierry { 618f32c9266SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) { 619f32c9266SJulien Thierry if (irqnr < 8192) 620f32c9266SJulien Thierry gic_write_dir(irqnr); 621f32c9266SJulien Thierry } else { 622f32c9266SJulien Thierry gic_write_eoir(irqnr); 623f32c9266SJulien Thierry } 624f32c9266SJulien Thierry } 625f32c9266SJulien Thierry 626f32c9266SJulien Thierry static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 627f32c9266SJulien Thierry { 62817ce302fSJulien Thierry bool irqs_enabled = interrupts_enabled(regs); 629f32c9266SJulien Thierry int err; 630f32c9266SJulien Thierry 63117ce302fSJulien Thierry if (irqs_enabled) 63217ce302fSJulien Thierry nmi_enter(); 63317ce302fSJulien Thierry 634f32c9266SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) 635f32c9266SJulien Thierry gic_write_eoir(irqnr); 636f32c9266SJulien Thierry /* 637f32c9266SJulien Thierry * Leave the PSR.I bit set to prevent other NMIs to be 638f32c9266SJulien Thierry * received while handling this one. 639f32c9266SJulien Thierry * PSR.I will be restored when we ERET to the 640f32c9266SJulien Thierry * interrupted context. 641f32c9266SJulien Thierry */ 642f32c9266SJulien Thierry err = handle_domain_nmi(gic_data.domain, irqnr, regs); 643f32c9266SJulien Thierry if (err) 644f32c9266SJulien Thierry gic_deactivate_unhandled(irqnr); 64517ce302fSJulien Thierry 64617ce302fSJulien Thierry if (irqs_enabled) 64717ce302fSJulien Thierry nmi_exit(); 648f32c9266SJulien Thierry } 649f32c9266SJulien Thierry 650382e6e17SMarc Zyngier static u32 do_read_iar(struct pt_regs *regs) 651382e6e17SMarc Zyngier { 652382e6e17SMarc Zyngier u32 iar; 653382e6e17SMarc Zyngier 654382e6e17SMarc Zyngier if (gic_supports_nmi() && unlikely(!interrupts_enabled(regs))) { 655382e6e17SMarc Zyngier u64 pmr; 656382e6e17SMarc Zyngier 657382e6e17SMarc Zyngier /* 658382e6e17SMarc Zyngier * We were in a context with IRQs disabled. However, the 659382e6e17SMarc Zyngier * entry code has set PMR to a value that allows any 660382e6e17SMarc Zyngier * interrupt to be acknowledged, and not just NMIs. This can 661382e6e17SMarc Zyngier * lead to surprising effects if the NMI has been retired in 662382e6e17SMarc Zyngier * the meantime, and that there is an IRQ pending. The IRQ 663382e6e17SMarc Zyngier * would then be taken in NMI context, something that nobody 664382e6e17SMarc Zyngier * wants to debug twice. 665382e6e17SMarc Zyngier * 666382e6e17SMarc Zyngier * Until we sort this, drop PMR again to a level that will 667382e6e17SMarc Zyngier * actually only allow NMIs before reading IAR, and then 668382e6e17SMarc Zyngier * restore it to what it was. 669382e6e17SMarc Zyngier */ 670382e6e17SMarc Zyngier pmr = gic_read_pmr(); 671382e6e17SMarc Zyngier gic_pmr_mask_irqs(); 672382e6e17SMarc Zyngier isb(); 673382e6e17SMarc Zyngier 674382e6e17SMarc Zyngier iar = gic_read_iar(); 675382e6e17SMarc Zyngier 676382e6e17SMarc Zyngier gic_write_pmr(pmr); 677382e6e17SMarc Zyngier } else { 678382e6e17SMarc Zyngier iar = gic_read_iar(); 679382e6e17SMarc Zyngier } 680382e6e17SMarc Zyngier 681382e6e17SMarc Zyngier return iar; 682382e6e17SMarc Zyngier } 683382e6e17SMarc Zyngier 684021f6537SMarc Zyngier static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 685021f6537SMarc Zyngier { 686f6c86a41SJean-Philippe Brucker u32 irqnr; 687021f6537SMarc Zyngier 688382e6e17SMarc Zyngier irqnr = do_read_iar(regs); 689021f6537SMarc Zyngier 690a97709f5SHe Ying /* Check for special IDs first */ 691a97709f5SHe Ying if ((irqnr >= 1020 && irqnr <= 1023)) 692a97709f5SHe Ying return; 693a97709f5SHe Ying 694f32c9266SJulien Thierry if (gic_supports_nmi() && 695f32c9266SJulien Thierry unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) { 696f32c9266SJulien Thierry gic_handle_nmi(irqnr, regs); 697f32c9266SJulien Thierry return; 698f32c9266SJulien Thierry } 699f32c9266SJulien Thierry 7003f1f3234SJulien Thierry if (gic_prio_masking_enabled()) { 7013f1f3234SJulien Thierry gic_pmr_mask_irqs(); 7023f1f3234SJulien Thierry gic_arch_enable_irqs(); 7033f1f3234SJulien Thierry } 7043f1f3234SJulien Thierry 705d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 7060b6a3da9SMarc Zyngier gic_write_eoir(irqnr); 70739a06b67SWill Deacon else 70839a06b67SWill Deacon isb(); 7090b6a3da9SMarc Zyngier 71064b499d8SMarc Zyngier if (handle_domain_irq(gic_data.domain, irqnr, regs)) { 711da33f31dSMarc Zyngier WARN_ONCE(true, "Unexpected interrupt received!\n"); 712f32c9266SJulien Thierry gic_deactivate_unhandled(irqnr); 7130b6a3da9SMarc Zyngier } 714021f6537SMarc Zyngier } 715021f6537SMarc Zyngier 716b5cf6073SJulien Thierry static u32 gic_get_pribits(void) 717b5cf6073SJulien Thierry { 718b5cf6073SJulien Thierry u32 pribits; 719b5cf6073SJulien Thierry 720b5cf6073SJulien Thierry pribits = gic_read_ctlr(); 721b5cf6073SJulien Thierry pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 722b5cf6073SJulien Thierry pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 723b5cf6073SJulien Thierry pribits++; 724b5cf6073SJulien Thierry 725b5cf6073SJulien Thierry return pribits; 726b5cf6073SJulien Thierry } 727b5cf6073SJulien Thierry 728b5cf6073SJulien Thierry static bool gic_has_group0(void) 729b5cf6073SJulien Thierry { 730b5cf6073SJulien Thierry u32 val; 731e7932188SJulien Thierry u32 old_pmr; 732e7932188SJulien Thierry 733e7932188SJulien Thierry old_pmr = gic_read_pmr(); 734b5cf6073SJulien Thierry 735b5cf6073SJulien Thierry /* 736b5cf6073SJulien Thierry * Let's find out if Group0 is under control of EL3 or not by 737b5cf6073SJulien Thierry * setting the highest possible, non-zero priority in PMR. 738b5cf6073SJulien Thierry * 739b5cf6073SJulien Thierry * If SCR_EL3.FIQ is set, the priority gets shifted down in 740b5cf6073SJulien Thierry * order for the CPU interface to set bit 7, and keep the 741b5cf6073SJulien Thierry * actual priority in the non-secure range. In the process, it 742b5cf6073SJulien Thierry * looses the least significant bit and the actual priority 743b5cf6073SJulien Thierry * becomes 0x80. Reading it back returns 0, indicating that 744b5cf6073SJulien Thierry * we're don't have access to Group0. 745b5cf6073SJulien Thierry */ 746b5cf6073SJulien Thierry gic_write_pmr(BIT(8 - gic_get_pribits())); 747b5cf6073SJulien Thierry val = gic_read_pmr(); 748b5cf6073SJulien Thierry 749e7932188SJulien Thierry gic_write_pmr(old_pmr); 750e7932188SJulien Thierry 751b5cf6073SJulien Thierry return val != 0; 752b5cf6073SJulien Thierry } 753b5cf6073SJulien Thierry 754021f6537SMarc Zyngier static void __init gic_dist_init(void) 755021f6537SMarc Zyngier { 756021f6537SMarc Zyngier unsigned int i; 757021f6537SMarc Zyngier u64 affinity; 758021f6537SMarc Zyngier void __iomem *base = gic_data.dist_base; 7590b04758bSMarc Zyngier u32 val; 760021f6537SMarc Zyngier 761021f6537SMarc Zyngier /* Disable the distributor */ 762021f6537SMarc Zyngier writel_relaxed(0, base + GICD_CTLR); 763021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 764021f6537SMarc Zyngier 7657c9b9730SMarc Zyngier /* 7667c9b9730SMarc Zyngier * Configure SPIs as non-secure Group-1. This will only matter 7677c9b9730SMarc Zyngier * if the GIC only has a single security state. This will not 7687c9b9730SMarc Zyngier * do the right thing if the kernel is running in secure mode, 7697c9b9730SMarc Zyngier * but that's not the intended use case anyway. 7707c9b9730SMarc Zyngier */ 771211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i += 32) 7727c9b9730SMarc Zyngier writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 7737c9b9730SMarc Zyngier 774211bddd2SMarc Zyngier /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 775211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) { 776211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 777211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 778211bddd2SMarc Zyngier } 779211bddd2SMarc Zyngier 780211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 32) 781211bddd2SMarc Zyngier writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 782211bddd2SMarc Zyngier 783211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 16) 784211bddd2SMarc Zyngier writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 785211bddd2SMarc Zyngier 786211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i += 4) 787211bddd2SMarc Zyngier writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 788211bddd2SMarc Zyngier 789211bddd2SMarc Zyngier /* Now do the common stuff, and wait for the distributor to drain */ 790211bddd2SMarc Zyngier gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp); 791021f6537SMarc Zyngier 7920b04758bSMarc Zyngier val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 7930b04758bSMarc Zyngier if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 7940b04758bSMarc Zyngier pr_info("Enabling SGIs without active state\n"); 7950b04758bSMarc Zyngier val |= GICD_CTLR_nASSGIreq; 7960b04758bSMarc Zyngier } 7970b04758bSMarc Zyngier 798021f6537SMarc Zyngier /* Enable distributor with ARE, Group1 */ 7990b04758bSMarc Zyngier writel_relaxed(val, base + GICD_CTLR); 800021f6537SMarc Zyngier 801021f6537SMarc Zyngier /* 802021f6537SMarc Zyngier * Set all global interrupts to the boot CPU only. ARE must be 803021f6537SMarc Zyngier * enabled. 804021f6537SMarc Zyngier */ 805021f6537SMarc Zyngier affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 806211bddd2SMarc Zyngier for (i = 32; i < GIC_LINE_NR; i++) 80772c97126SJean-Philippe Brucker gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 808211bddd2SMarc Zyngier 809211bddd2SMarc Zyngier for (i = 0; i < GIC_ESPI_NR; i++) 810211bddd2SMarc Zyngier gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 811021f6537SMarc Zyngier } 812021f6537SMarc Zyngier 8130d94ded2SMarc Zyngier static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 814021f6537SMarc Zyngier { 8150d94ded2SMarc Zyngier int ret = -ENODEV; 816021f6537SMarc Zyngier int i; 817021f6537SMarc Zyngier 818f5c1434cSMarc Zyngier for (i = 0; i < gic_data.nr_redist_regions; i++) { 819f5c1434cSMarc Zyngier void __iomem *ptr = gic_data.redist_regions[i].redist_base; 8200d94ded2SMarc Zyngier u64 typer; 821021f6537SMarc Zyngier u32 reg; 822021f6537SMarc Zyngier 823021f6537SMarc Zyngier reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 824021f6537SMarc Zyngier if (reg != GIC_PIDR2_ARCH_GICv3 && 825021f6537SMarc Zyngier reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 826021f6537SMarc Zyngier pr_warn("No redistributor present @%p\n", ptr); 827021f6537SMarc Zyngier break; 828021f6537SMarc Zyngier } 829021f6537SMarc Zyngier 830021f6537SMarc Zyngier do { 83172c97126SJean-Philippe Brucker typer = gic_read_typer(ptr + GICR_TYPER); 8320d94ded2SMarc Zyngier ret = fn(gic_data.redist_regions + i, ptr); 8330d94ded2SMarc Zyngier if (!ret) 834021f6537SMarc Zyngier return 0; 835021f6537SMarc Zyngier 836b70fb7afSTomasz Nowicki if (gic_data.redist_regions[i].single_redist) 837b70fb7afSTomasz Nowicki break; 838b70fb7afSTomasz Nowicki 839021f6537SMarc Zyngier if (gic_data.redist_stride) { 840021f6537SMarc Zyngier ptr += gic_data.redist_stride; 841021f6537SMarc Zyngier } else { 842021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 843021f6537SMarc Zyngier if (typer & GICR_TYPER_VLPIS) 844021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 845021f6537SMarc Zyngier } 846021f6537SMarc Zyngier } while (!(typer & GICR_TYPER_LAST)); 847021f6537SMarc Zyngier } 848021f6537SMarc Zyngier 8490d94ded2SMarc Zyngier return ret ? -ENODEV : 0; 8500d94ded2SMarc Zyngier } 8510d94ded2SMarc Zyngier 8520d94ded2SMarc Zyngier static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 8530d94ded2SMarc Zyngier { 8540d94ded2SMarc Zyngier unsigned long mpidr = cpu_logical_map(smp_processor_id()); 8550d94ded2SMarc Zyngier u64 typer; 8560d94ded2SMarc Zyngier u32 aff; 8570d94ded2SMarc Zyngier 8580d94ded2SMarc Zyngier /* 8590d94ded2SMarc Zyngier * Convert affinity to a 32bit value that can be matched to 8600d94ded2SMarc Zyngier * GICR_TYPER bits [63:32]. 8610d94ded2SMarc Zyngier */ 8620d94ded2SMarc Zyngier aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 8630d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 8640d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 8650d94ded2SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 8660d94ded2SMarc Zyngier 8670d94ded2SMarc Zyngier typer = gic_read_typer(ptr + GICR_TYPER); 8680d94ded2SMarc Zyngier if ((typer >> 32) == aff) { 8690d94ded2SMarc Zyngier u64 offset = ptr - region->redist_base; 8709058a4e9SMarc Zyngier raw_spin_lock_init(&gic_data_rdist()->rd_lock); 8710d94ded2SMarc Zyngier gic_data_rdist_rd_base() = ptr; 8720d94ded2SMarc Zyngier gic_data_rdist()->phys_base = region->phys_base + offset; 8730d94ded2SMarc Zyngier 8740d94ded2SMarc Zyngier pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 8750d94ded2SMarc Zyngier smp_processor_id(), mpidr, 8760d94ded2SMarc Zyngier (int)(region - gic_data.redist_regions), 8770d94ded2SMarc Zyngier &gic_data_rdist()->phys_base); 8780d94ded2SMarc Zyngier return 0; 8790d94ded2SMarc Zyngier } 8800d94ded2SMarc Zyngier 8810d94ded2SMarc Zyngier /* Try next one */ 8820d94ded2SMarc Zyngier return 1; 8830d94ded2SMarc Zyngier } 8840d94ded2SMarc Zyngier 8850d94ded2SMarc Zyngier static int gic_populate_rdist(void) 8860d94ded2SMarc Zyngier { 8870d94ded2SMarc Zyngier if (gic_iterate_rdists(__gic_populate_rdist) == 0) 8880d94ded2SMarc Zyngier return 0; 8890d94ded2SMarc Zyngier 890021f6537SMarc Zyngier /* We couldn't even deal with ourselves... */ 891f6c86a41SJean-Philippe Brucker WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 8920d94ded2SMarc Zyngier smp_processor_id(), 8930d94ded2SMarc Zyngier (unsigned long)cpu_logical_map(smp_processor_id())); 894021f6537SMarc Zyngier return -ENODEV; 895021f6537SMarc Zyngier } 896021f6537SMarc Zyngier 8971a60e1e6SMarc Zyngier static int __gic_update_rdist_properties(struct redist_region *region, 8980edc23eaSMarc Zyngier void __iomem *ptr) 8990edc23eaSMarc Zyngier { 9000edc23eaSMarc Zyngier u64 typer = gic_read_typer(ptr + GICR_TYPER); 901b25319d2SMarc Zyngier 9020edc23eaSMarc Zyngier gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 903b25319d2SMarc Zyngier 904b25319d2SMarc Zyngier /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */ 905b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 906b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 907b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid); 90896806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 909b25319d2SMarc Zyngier 910b25319d2SMarc Zyngier /* Detect non-sensical configurations */ 911b25319d2SMarc Zyngier if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 912b25319d2SMarc Zyngier gic_data.rdists.has_direct_lpi = false; 913b25319d2SMarc Zyngier gic_data.rdists.has_vlpis = false; 914b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = false; 915b25319d2SMarc Zyngier } 916b25319d2SMarc Zyngier 9175f51f803SMarc Zyngier gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 9180edc23eaSMarc Zyngier 9190edc23eaSMarc Zyngier return 1; 9200edc23eaSMarc Zyngier } 9210edc23eaSMarc Zyngier 9221a60e1e6SMarc Zyngier static void gic_update_rdist_properties(void) 9230edc23eaSMarc Zyngier { 9241a60e1e6SMarc Zyngier gic_data.ppi_nr = UINT_MAX; 9251a60e1e6SMarc Zyngier gic_iterate_rdists(__gic_update_rdist_properties); 9261a60e1e6SMarc Zyngier if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 9271a60e1e6SMarc Zyngier gic_data.ppi_nr = 0; 9281a60e1e6SMarc Zyngier pr_info("%d PPIs implemented\n", gic_data.ppi_nr); 92996806229SMarc Zyngier if (gic_data.rdists.has_vlpis) 93096806229SMarc Zyngier pr_info("GICv4 features: %s%s%s\n", 93196806229SMarc Zyngier gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 93296806229SMarc Zyngier gic_data.rdists.has_rvpeid ? "RVPEID " : "", 93396806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 9340edc23eaSMarc Zyngier } 9350edc23eaSMarc Zyngier 936d98d0a99SJulien Thierry /* Check whether it's single security state view */ 937d98d0a99SJulien Thierry static inline bool gic_dist_security_disabled(void) 938d98d0a99SJulien Thierry { 939d98d0a99SJulien Thierry return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 940d98d0a99SJulien Thierry } 941d98d0a99SJulien Thierry 9423708d52fSSudeep Holla static void gic_cpu_sys_reg_init(void) 943021f6537SMarc Zyngier { 944eda0d04aSShanker Donthineni int i, cpu = smp_processor_id(); 945eda0d04aSShanker Donthineni u64 mpidr = cpu_logical_map(cpu); 946eda0d04aSShanker Donthineni u64 need_rss = MPIDR_RS(mpidr); 94733625282SMarc Zyngier bool group0; 948b5cf6073SJulien Thierry u32 pribits; 949eda0d04aSShanker Donthineni 9507cabd008SMarc Zyngier /* 9517cabd008SMarc Zyngier * Need to check that the SRE bit has actually been set. If 9527cabd008SMarc Zyngier * not, it means that SRE is disabled at EL2. We're going to 9537cabd008SMarc Zyngier * die painfully, and there is nothing we can do about it. 9547cabd008SMarc Zyngier * 9557cabd008SMarc Zyngier * Kindly inform the luser. 9567cabd008SMarc Zyngier */ 9577cabd008SMarc Zyngier if (!gic_enable_sre()) 9587cabd008SMarc Zyngier pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 959021f6537SMarc Zyngier 960b5cf6073SJulien Thierry pribits = gic_get_pribits(); 96133625282SMarc Zyngier 962b5cf6073SJulien Thierry group0 = gic_has_group0(); 96333625282SMarc Zyngier 964021f6537SMarc Zyngier /* Set priority mask register */ 965d98d0a99SJulien Thierry if (!gic_prio_masking_enabled()) { 96633625282SMarc Zyngier write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 96733678059SAlexandru Elisei } else if (gic_supports_nmi()) { 968d98d0a99SJulien Thierry /* 969d98d0a99SJulien Thierry * Mismatch configuration with boot CPU, the system is likely 970d98d0a99SJulien Thierry * to die as interrupt masking will not work properly on all 971d98d0a99SJulien Thierry * CPUs 97233678059SAlexandru Elisei * 97333678059SAlexandru Elisei * The boot CPU calls this function before enabling NMI support, 97433678059SAlexandru Elisei * and as a result we'll never see this warning in the boot path 97533678059SAlexandru Elisei * for that CPU. 976d98d0a99SJulien Thierry */ 97733678059SAlexandru Elisei if (static_branch_unlikely(&gic_nonsecure_priorities)) 97833678059SAlexandru Elisei WARN_ON(!group0 || gic_dist_security_disabled()); 97933678059SAlexandru Elisei else 98033678059SAlexandru Elisei WARN_ON(group0 && !gic_dist_security_disabled()); 981d98d0a99SJulien Thierry } 982021f6537SMarc Zyngier 98391ef8442SDaniel Thompson /* 98491ef8442SDaniel Thompson * Some firmwares hand over to the kernel with the BPR changed from 98591ef8442SDaniel Thompson * its reset value (and with a value large enough to prevent 98691ef8442SDaniel Thompson * any pre-emptive interrupts from working at all). Writing a zero 98791ef8442SDaniel Thompson * to BPR restores is reset value. 98891ef8442SDaniel Thompson */ 98991ef8442SDaniel Thompson gic_write_bpr1(0); 99091ef8442SDaniel Thompson 991d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) { 9920b6a3da9SMarc Zyngier /* EOI drops priority only (mode 1) */ 9930b6a3da9SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 9940b6a3da9SMarc Zyngier } else { 995021f6537SMarc Zyngier /* EOI deactivates interrupt too (mode 0) */ 996021f6537SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 9970b6a3da9SMarc Zyngier } 998021f6537SMarc Zyngier 99933625282SMarc Zyngier /* Always whack Group0 before Group1 */ 100033625282SMarc Zyngier if (group0) { 100133625282SMarc Zyngier switch(pribits) { 100233625282SMarc Zyngier case 8: 100333625282SMarc Zyngier case 7: 100433625282SMarc Zyngier write_gicreg(0, ICC_AP0R3_EL1); 100533625282SMarc Zyngier write_gicreg(0, ICC_AP0R2_EL1); 1006df561f66SGustavo A. R. Silva fallthrough; 100733625282SMarc Zyngier case 6: 100833625282SMarc Zyngier write_gicreg(0, ICC_AP0R1_EL1); 1009df561f66SGustavo A. R. Silva fallthrough; 101033625282SMarc Zyngier case 5: 101133625282SMarc Zyngier case 4: 101233625282SMarc Zyngier write_gicreg(0, ICC_AP0R0_EL1); 101333625282SMarc Zyngier } 1014d6062a6dSMarc Zyngier 101533625282SMarc Zyngier isb(); 101633625282SMarc Zyngier } 101733625282SMarc Zyngier 101833625282SMarc Zyngier switch(pribits) { 1019d6062a6dSMarc Zyngier case 8: 1020d6062a6dSMarc Zyngier case 7: 1021d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R3_EL1); 1022d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R2_EL1); 1023df561f66SGustavo A. R. Silva fallthrough; 1024d6062a6dSMarc Zyngier case 6: 1025d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R1_EL1); 1026df561f66SGustavo A. R. Silva fallthrough; 1027d6062a6dSMarc Zyngier case 5: 1028d6062a6dSMarc Zyngier case 4: 1029d6062a6dSMarc Zyngier write_gicreg(0, ICC_AP1R0_EL1); 1030d6062a6dSMarc Zyngier } 1031d6062a6dSMarc Zyngier 1032d6062a6dSMarc Zyngier isb(); 1033d6062a6dSMarc Zyngier 1034021f6537SMarc Zyngier /* ... and let's hit the road... */ 1035021f6537SMarc Zyngier gic_write_grpen1(1); 1036eda0d04aSShanker Donthineni 1037eda0d04aSShanker Donthineni /* Keep the RSS capability status in per_cpu variable */ 1038eda0d04aSShanker Donthineni per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1039eda0d04aSShanker Donthineni 1040eda0d04aSShanker Donthineni /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1041eda0d04aSShanker Donthineni for_each_online_cpu(i) { 1042eda0d04aSShanker Donthineni bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1043eda0d04aSShanker Donthineni 1044eda0d04aSShanker Donthineni need_rss |= MPIDR_RS(cpu_logical_map(i)); 1045eda0d04aSShanker Donthineni if (need_rss && (!have_rss)) 1046eda0d04aSShanker Donthineni pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1047eda0d04aSShanker Donthineni cpu, (unsigned long)mpidr, 1048eda0d04aSShanker Donthineni i, (unsigned long)cpu_logical_map(i)); 1049eda0d04aSShanker Donthineni } 1050eda0d04aSShanker Donthineni 1051eda0d04aSShanker Donthineni /** 1052eda0d04aSShanker Donthineni * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1053eda0d04aSShanker Donthineni * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1054eda0d04aSShanker Donthineni * UNPREDICTABLE choice of : 1055eda0d04aSShanker Donthineni * - The write is ignored. 1056eda0d04aSShanker Donthineni * - The RS field is treated as 0. 1057eda0d04aSShanker Donthineni */ 1058eda0d04aSShanker Donthineni if (need_rss && (!gic_data.has_rss)) 1059eda0d04aSShanker Donthineni pr_crit_once("RSS is required but GICD doesn't support it\n"); 1060021f6537SMarc Zyngier } 1061021f6537SMarc Zyngier 1062f736d65dSMarc Zyngier static bool gicv3_nolpi; 1063f736d65dSMarc Zyngier 1064f736d65dSMarc Zyngier static int __init gicv3_nolpi_cfg(char *buf) 1065f736d65dSMarc Zyngier { 1066f736d65dSMarc Zyngier return strtobool(buf, &gicv3_nolpi); 1067f736d65dSMarc Zyngier } 1068f736d65dSMarc Zyngier early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1069f736d65dSMarc Zyngier 1070da33f31dSMarc Zyngier static int gic_dist_supports_lpis(void) 1071da33f31dSMarc Zyngier { 1072d38a71c5SMarc Zyngier return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1073d38a71c5SMarc Zyngier !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1074d38a71c5SMarc Zyngier !gicv3_nolpi); 1075da33f31dSMarc Zyngier } 1076da33f31dSMarc Zyngier 1077021f6537SMarc Zyngier static void gic_cpu_init(void) 1078021f6537SMarc Zyngier { 1079021f6537SMarc Zyngier void __iomem *rbase; 10801a60e1e6SMarc Zyngier int i; 1081021f6537SMarc Zyngier 1082021f6537SMarc Zyngier /* Register ourselves with the rest of the world */ 1083021f6537SMarc Zyngier if (gic_populate_rdist()) 1084021f6537SMarc Zyngier return; 1085021f6537SMarc Zyngier 1086a2c22510SSudeep Holla gic_enable_redist(true); 1087021f6537SMarc Zyngier 1088ad5a78d3SMarc Zyngier WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1089ad5a78d3SMarc Zyngier !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1090ad5a78d3SMarc Zyngier "Distributor has extended ranges, but CPU%d doesn't\n", 1091ad5a78d3SMarc Zyngier smp_processor_id()); 1092ad5a78d3SMarc Zyngier 1093021f6537SMarc Zyngier rbase = gic_data_rdist_sgi_base(); 1094021f6537SMarc Zyngier 10957c9b9730SMarc Zyngier /* Configure SGIs/PPIs as non-secure Group-1 */ 10961a60e1e6SMarc Zyngier for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 10971a60e1e6SMarc Zyngier writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 10987c9b9730SMarc Zyngier 10991a60e1e6SMarc Zyngier gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1100021f6537SMarc Zyngier 11013708d52fSSudeep Holla /* initialise system registers */ 11023708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1103021f6537SMarc Zyngier } 1104021f6537SMarc Zyngier 1105021f6537SMarc Zyngier #ifdef CONFIG_SMP 1106021f6537SMarc Zyngier 1107eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1108eda0d04aSShanker Donthineni #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1109eda0d04aSShanker Donthineni 11106670a6d8SRichard Cochran static int gic_starting_cpu(unsigned int cpu) 11116670a6d8SRichard Cochran { 11126670a6d8SRichard Cochran gic_cpu_init(); 1113d38a71c5SMarc Zyngier 1114d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) 1115d38a71c5SMarc Zyngier its_cpu_init(); 1116d38a71c5SMarc Zyngier 11176670a6d8SRichard Cochran return 0; 11186670a6d8SRichard Cochran } 1119021f6537SMarc Zyngier 1120021f6537SMarc Zyngier static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1121f6c86a41SJean-Philippe Brucker unsigned long cluster_id) 1122021f6537SMarc Zyngier { 1123727653d6SJames Morse int next_cpu, cpu = *base_cpu; 1124f6c86a41SJean-Philippe Brucker unsigned long mpidr = cpu_logical_map(cpu); 1125021f6537SMarc Zyngier u16 tlist = 0; 1126021f6537SMarc Zyngier 1127021f6537SMarc Zyngier while (cpu < nr_cpu_ids) { 1128021f6537SMarc Zyngier tlist |= 1 << (mpidr & 0xf); 1129021f6537SMarc Zyngier 1130727653d6SJames Morse next_cpu = cpumask_next(cpu, mask); 1131727653d6SJames Morse if (next_cpu >= nr_cpu_ids) 1132021f6537SMarc Zyngier goto out; 1133727653d6SJames Morse cpu = next_cpu; 1134021f6537SMarc Zyngier 1135021f6537SMarc Zyngier mpidr = cpu_logical_map(cpu); 1136021f6537SMarc Zyngier 1137eda0d04aSShanker Donthineni if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1138021f6537SMarc Zyngier cpu--; 1139021f6537SMarc Zyngier goto out; 1140021f6537SMarc Zyngier } 1141021f6537SMarc Zyngier } 1142021f6537SMarc Zyngier out: 1143021f6537SMarc Zyngier *base_cpu = cpu; 1144021f6537SMarc Zyngier return tlist; 1145021f6537SMarc Zyngier } 1146021f6537SMarc Zyngier 11477e580278SAndre Przywara #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 11487e580278SAndre Przywara (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 11497e580278SAndre Przywara << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 11507e580278SAndre Przywara 1151021f6537SMarc Zyngier static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1152021f6537SMarc Zyngier { 1153021f6537SMarc Zyngier u64 val; 1154021f6537SMarc Zyngier 11557e580278SAndre Przywara val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 11567e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 11577e580278SAndre Przywara irq << ICC_SGI1R_SGI_ID_SHIFT | 11587e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1159eda0d04aSShanker Donthineni MPIDR_TO_SGI_RS(cluster_id) | 11607e580278SAndre Przywara tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1161021f6537SMarc Zyngier 1162b6dd4d83SMark Salter pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1163021f6537SMarc Zyngier gic_write_sgi1r(val); 1164021f6537SMarc Zyngier } 1165021f6537SMarc Zyngier 116664b499d8SMarc Zyngier static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1167021f6537SMarc Zyngier { 1168021f6537SMarc Zyngier int cpu; 1169021f6537SMarc Zyngier 117064b499d8SMarc Zyngier if (WARN_ON(d->hwirq >= 16)) 1171021f6537SMarc Zyngier return; 1172021f6537SMarc Zyngier 1173021f6537SMarc Zyngier /* 1174021f6537SMarc Zyngier * Ensure that stores to Normal memory are visible to the 1175021f6537SMarc Zyngier * other CPUs before issuing the IPI. 1176021f6537SMarc Zyngier */ 117721ec30c0SShanker Donthineni wmb(); 1178021f6537SMarc Zyngier 1179f9b531feSRusty Russell for_each_cpu(cpu, mask) { 1180eda0d04aSShanker Donthineni u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1181021f6537SMarc Zyngier u16 tlist; 1182021f6537SMarc Zyngier 1183021f6537SMarc Zyngier tlist = gic_compute_target_list(&cpu, mask, cluster_id); 118464b499d8SMarc Zyngier gic_send_sgi(cluster_id, tlist, d->hwirq); 1185021f6537SMarc Zyngier } 1186021f6537SMarc Zyngier 1187021f6537SMarc Zyngier /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1188021f6537SMarc Zyngier isb(); 1189021f6537SMarc Zyngier } 1190021f6537SMarc Zyngier 11918a94c1abSIngo Rohloff static void __init gic_smp_init(void) 1192021f6537SMarc Zyngier { 119364b499d8SMarc Zyngier struct irq_fwspec sgi_fwspec = { 119464b499d8SMarc Zyngier .fwnode = gic_data.fwnode, 119564b499d8SMarc Zyngier .param_count = 1, 119664b499d8SMarc Zyngier }; 119764b499d8SMarc Zyngier int base_sgi; 119864b499d8SMarc Zyngier 11996896bcd1SThomas Gleixner cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 120073c1b41eSThomas Gleixner "irqchip/arm/gicv3:starting", 120173c1b41eSThomas Gleixner gic_starting_cpu, NULL); 120264b499d8SMarc Zyngier 120364b499d8SMarc Zyngier /* Register all 8 non-secure SGIs */ 120464b499d8SMarc Zyngier base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8, 120564b499d8SMarc Zyngier NUMA_NO_NODE, &sgi_fwspec, 120664b499d8SMarc Zyngier false, NULL); 120764b499d8SMarc Zyngier if (WARN_ON(base_sgi <= 0)) 120864b499d8SMarc Zyngier return; 120964b499d8SMarc Zyngier 121064b499d8SMarc Zyngier set_smp_ipi_range(base_sgi, 8); 1211021f6537SMarc Zyngier } 1212021f6537SMarc Zyngier 1213021f6537SMarc Zyngier static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1214021f6537SMarc Zyngier bool force) 1215021f6537SMarc Zyngier { 121665a30f8bSSuzuki K Poulose unsigned int cpu; 1217e91b036eSMarc Zyngier u32 offset, index; 1218021f6537SMarc Zyngier void __iomem *reg; 1219021f6537SMarc Zyngier int enabled; 1220021f6537SMarc Zyngier u64 val; 1221021f6537SMarc Zyngier 122265a30f8bSSuzuki K Poulose if (force) 122365a30f8bSSuzuki K Poulose cpu = cpumask_first(mask_val); 122465a30f8bSSuzuki K Poulose else 122565a30f8bSSuzuki K Poulose cpu = cpumask_any_and(mask_val, cpu_online_mask); 122665a30f8bSSuzuki K Poulose 1227866d7c1bSSuzuki K Poulose if (cpu >= nr_cpu_ids) 1228866d7c1bSSuzuki K Poulose return -EINVAL; 1229866d7c1bSSuzuki K Poulose 1230021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) 1231021f6537SMarc Zyngier return -EINVAL; 1232021f6537SMarc Zyngier 1233021f6537SMarc Zyngier /* If interrupt was enabled, disable it first */ 1234021f6537SMarc Zyngier enabled = gic_peek_irq(d, GICD_ISENABLER); 1235021f6537SMarc Zyngier if (enabled) 1236021f6537SMarc Zyngier gic_mask_irq(d); 1237021f6537SMarc Zyngier 1238e91b036eSMarc Zyngier offset = convert_offset_index(d, GICD_IROUTER, &index); 1239e91b036eSMarc Zyngier reg = gic_dist_base(d) + offset + (index * 8); 1240021f6537SMarc Zyngier val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 1241021f6537SMarc Zyngier 124272c97126SJean-Philippe Brucker gic_write_irouter(val, reg); 1243021f6537SMarc Zyngier 1244021f6537SMarc Zyngier /* 1245021f6537SMarc Zyngier * If the interrupt was enabled, enabled it again. Otherwise, 1246021f6537SMarc Zyngier * just wait for the distributor to have digested our changes. 1247021f6537SMarc Zyngier */ 1248021f6537SMarc Zyngier if (enabled) 1249021f6537SMarc Zyngier gic_unmask_irq(d); 1250021f6537SMarc Zyngier else 1251021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 1252021f6537SMarc Zyngier 1253956ae91aSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1254956ae91aSMarc Zyngier 12550fc6fa29SAntoine Tenart return IRQ_SET_MASK_OK_DONE; 1256021f6537SMarc Zyngier } 1257021f6537SMarc Zyngier #else 1258021f6537SMarc Zyngier #define gic_set_affinity NULL 125964b499d8SMarc Zyngier #define gic_ipi_send_mask NULL 1260021f6537SMarc Zyngier #define gic_smp_init() do { } while(0) 1261021f6537SMarc Zyngier #endif 1262021f6537SMarc Zyngier 126317f644e9SValentin Schneider static int gic_retrigger(struct irq_data *data) 126417f644e9SValentin Schneider { 126517f644e9SValentin Schneider return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 126617f644e9SValentin Schneider } 126717f644e9SValentin Schneider 12683708d52fSSudeep Holla #ifdef CONFIG_CPU_PM 12693708d52fSSudeep Holla static int gic_cpu_pm_notifier(struct notifier_block *self, 12703708d52fSSudeep Holla unsigned long cmd, void *v) 12713708d52fSSudeep Holla { 12723708d52fSSudeep Holla if (cmd == CPU_PM_EXIT) { 1273ccd9432aSSudeep Holla if (gic_dist_security_disabled()) 12743708d52fSSudeep Holla gic_enable_redist(true); 12753708d52fSSudeep Holla gic_cpu_sys_reg_init(); 1276ccd9432aSSudeep Holla } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 12773708d52fSSudeep Holla gic_write_grpen1(0); 12783708d52fSSudeep Holla gic_enable_redist(false); 12793708d52fSSudeep Holla } 12803708d52fSSudeep Holla return NOTIFY_OK; 12813708d52fSSudeep Holla } 12823708d52fSSudeep Holla 12833708d52fSSudeep Holla static struct notifier_block gic_cpu_pm_notifier_block = { 12843708d52fSSudeep Holla .notifier_call = gic_cpu_pm_notifier, 12853708d52fSSudeep Holla }; 12863708d52fSSudeep Holla 12873708d52fSSudeep Holla static void gic_cpu_pm_init(void) 12883708d52fSSudeep Holla { 12893708d52fSSudeep Holla cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 12903708d52fSSudeep Holla } 12913708d52fSSudeep Holla 12923708d52fSSudeep Holla #else 12933708d52fSSudeep Holla static inline void gic_cpu_pm_init(void) { } 12943708d52fSSudeep Holla #endif /* CONFIG_CPU_PM */ 12953708d52fSSudeep Holla 1296021f6537SMarc Zyngier static struct irq_chip gic_chip = { 1297021f6537SMarc Zyngier .name = "GICv3", 1298021f6537SMarc Zyngier .irq_mask = gic_mask_irq, 1299021f6537SMarc Zyngier .irq_unmask = gic_unmask_irq, 1300021f6537SMarc Zyngier .irq_eoi = gic_eoi_irq, 1301021f6537SMarc Zyngier .irq_set_type = gic_set_type, 1302021f6537SMarc Zyngier .irq_set_affinity = gic_set_affinity, 130317f644e9SValentin Schneider .irq_retrigger = gic_retrigger, 1304b594c6e2SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1305b594c6e2SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1306101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1307101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 130864b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask, 13094110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 13104110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 13114110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 1312021f6537SMarc Zyngier }; 1313021f6537SMarc Zyngier 13140b6a3da9SMarc Zyngier static struct irq_chip gic_eoimode1_chip = { 13150b6a3da9SMarc Zyngier .name = "GICv3", 13160b6a3da9SMarc Zyngier .irq_mask = gic_eoimode1_mask_irq, 13170b6a3da9SMarc Zyngier .irq_unmask = gic_unmask_irq, 13180b6a3da9SMarc Zyngier .irq_eoi = gic_eoimode1_eoi_irq, 13190b6a3da9SMarc Zyngier .irq_set_type = gic_set_type, 13200b6a3da9SMarc Zyngier .irq_set_affinity = gic_set_affinity, 132117f644e9SValentin Schneider .irq_retrigger = gic_retrigger, 13220b6a3da9SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 13230b6a3da9SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1324530bf353SMarc Zyngier .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1325101b35f7SJulien Thierry .irq_nmi_setup = gic_irq_nmi_setup, 1326101b35f7SJulien Thierry .irq_nmi_teardown = gic_irq_nmi_teardown, 132764b499d8SMarc Zyngier .ipi_send_mask = gic_ipi_send_mask, 13284110b5cbSMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED | 13294110b5cbSMarc Zyngier IRQCHIP_SKIP_SET_WAKE | 13304110b5cbSMarc Zyngier IRQCHIP_MASK_ON_SUSPEND, 13310b6a3da9SMarc Zyngier }; 13320b6a3da9SMarc Zyngier 1333021f6537SMarc Zyngier static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1334021f6537SMarc Zyngier irq_hw_number_t hw) 1335021f6537SMarc Zyngier { 13360b6a3da9SMarc Zyngier struct irq_chip *chip = &gic_chip; 13371b57d91bSValentin Schneider struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 13380b6a3da9SMarc Zyngier 1339d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 13400b6a3da9SMarc Zyngier chip = &gic_eoimode1_chip; 13410b6a3da9SMarc Zyngier 1342e91b036eSMarc Zyngier switch (__get_intid_range(hw)) { 134370a29c32SMarc Zyngier case SGI_RANGE: 1344e91b036eSMarc Zyngier case PPI_RANGE: 13455f51f803SMarc Zyngier case EPPI_RANGE: 1346021f6537SMarc Zyngier irq_set_percpu_devid(irq); 13470b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1348443acc4fSMarc Zyngier handle_percpu_devid_irq, NULL, NULL); 1349e91b036eSMarc Zyngier break; 1350e91b036eSMarc Zyngier 1351e91b036eSMarc Zyngier case SPI_RANGE: 1352211bddd2SMarc Zyngier case ESPI_RANGE: 13530b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1354443acc4fSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1355d17cab44SRob Herring irq_set_probe(irq); 13561b57d91bSValentin Schneider irqd_set_single_target(irqd); 1357e91b036eSMarc Zyngier break; 1358e91b036eSMarc Zyngier 1359e91b036eSMarc Zyngier case LPI_RANGE: 1360da33f31dSMarc Zyngier if (!gic_dist_supports_lpis()) 1361da33f31dSMarc Zyngier return -EPERM; 13620b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 1363da33f31dSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 1364e91b036eSMarc Zyngier break; 1365e91b036eSMarc Zyngier 1366e91b036eSMarc Zyngier default: 1367e91b036eSMarc Zyngier return -EPERM; 1368da33f31dSMarc Zyngier } 1369da33f31dSMarc Zyngier 13701b57d91bSValentin Schneider /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 13711b57d91bSValentin Schneider irqd_set_handle_enforce_irqctx(irqd); 1372021f6537SMarc Zyngier return 0; 1373021f6537SMarc Zyngier } 1374021f6537SMarc Zyngier 1375f833f57fSMarc Zyngier static int gic_irq_domain_translate(struct irq_domain *d, 1376f833f57fSMarc Zyngier struct irq_fwspec *fwspec, 1377f833f57fSMarc Zyngier unsigned long *hwirq, 1378f833f57fSMarc Zyngier unsigned int *type) 1379021f6537SMarc Zyngier { 138064b499d8SMarc Zyngier if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 138164b499d8SMarc Zyngier *hwirq = fwspec->param[0]; 138264b499d8SMarc Zyngier *type = IRQ_TYPE_EDGE_RISING; 138364b499d8SMarc Zyngier return 0; 138464b499d8SMarc Zyngier } 138564b499d8SMarc Zyngier 1386f833f57fSMarc Zyngier if (is_of_node(fwspec->fwnode)) { 1387f833f57fSMarc Zyngier if (fwspec->param_count < 3) 1388021f6537SMarc Zyngier return -EINVAL; 1389021f6537SMarc Zyngier 1390db8c70ecSMarc Zyngier switch (fwspec->param[0]) { 1391db8c70ecSMarc Zyngier case 0: /* SPI */ 1392db8c70ecSMarc Zyngier *hwirq = fwspec->param[1] + 32; 1393db8c70ecSMarc Zyngier break; 1394db8c70ecSMarc Zyngier case 1: /* PPI */ 1395f833f57fSMarc Zyngier *hwirq = fwspec->param[1] + 16; 1396db8c70ecSMarc Zyngier break; 1397211bddd2SMarc Zyngier case 2: /* ESPI */ 1398211bddd2SMarc Zyngier *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1399211bddd2SMarc Zyngier break; 14005f51f803SMarc Zyngier case 3: /* EPPI */ 14015f51f803SMarc Zyngier *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 14025f51f803SMarc Zyngier break; 1403db8c70ecSMarc Zyngier case GIC_IRQ_TYPE_LPI: /* LPI */ 1404db8c70ecSMarc Zyngier *hwirq = fwspec->param[1]; 1405db8c70ecSMarc Zyngier break; 14065f51f803SMarc Zyngier case GIC_IRQ_TYPE_PARTITION: 14075f51f803SMarc Zyngier *hwirq = fwspec->param[1]; 14085f51f803SMarc Zyngier if (fwspec->param[1] >= 16) 14095f51f803SMarc Zyngier *hwirq += EPPI_BASE_INTID - 16; 14105f51f803SMarc Zyngier else 14115f51f803SMarc Zyngier *hwirq += 16; 14125f51f803SMarc Zyngier break; 1413db8c70ecSMarc Zyngier default: 1414db8c70ecSMarc Zyngier return -EINVAL; 1415db8c70ecSMarc Zyngier } 1416f833f57fSMarc Zyngier 1417f833f57fSMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 14186ef6386eSMarc Zyngier 141965da7d19SMarc Zyngier /* 142065da7d19SMarc Zyngier * Make it clear that broken DTs are... broken. 1421a359f757SIngo Molnar * Partitioned PPIs are an unfortunate exception. 142265da7d19SMarc Zyngier */ 142365da7d19SMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE && 142465da7d19SMarc Zyngier fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1425f833f57fSMarc Zyngier return 0; 1426021f6537SMarc Zyngier } 1427021f6537SMarc Zyngier 1428ffa7d616STomasz Nowicki if (is_fwnode_irqchip(fwspec->fwnode)) { 1429ffa7d616STomasz Nowicki if(fwspec->param_count != 2) 1430ffa7d616STomasz Nowicki return -EINVAL; 1431ffa7d616STomasz Nowicki 1432ffa7d616STomasz Nowicki *hwirq = fwspec->param[0]; 1433ffa7d616STomasz Nowicki *type = fwspec->param[1]; 14346ef6386eSMarc Zyngier 14356ef6386eSMarc Zyngier WARN_ON(*type == IRQ_TYPE_NONE); 1436ffa7d616STomasz Nowicki return 0; 1437ffa7d616STomasz Nowicki } 1438ffa7d616STomasz Nowicki 1439f833f57fSMarc Zyngier return -EINVAL; 1440021f6537SMarc Zyngier } 1441021f6537SMarc Zyngier 1442443acc4fSMarc Zyngier static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1443443acc4fSMarc Zyngier unsigned int nr_irqs, void *arg) 1444443acc4fSMarc Zyngier { 1445443acc4fSMarc Zyngier int i, ret; 1446443acc4fSMarc Zyngier irq_hw_number_t hwirq; 1447443acc4fSMarc Zyngier unsigned int type = IRQ_TYPE_NONE; 1448f833f57fSMarc Zyngier struct irq_fwspec *fwspec = arg; 1449443acc4fSMarc Zyngier 1450f833f57fSMarc Zyngier ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1451443acc4fSMarc Zyngier if (ret) 1452443acc4fSMarc Zyngier return ret; 1453443acc4fSMarc Zyngier 145463c16c6eSSuzuki K Poulose for (i = 0; i < nr_irqs; i++) { 145563c16c6eSSuzuki K Poulose ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 145663c16c6eSSuzuki K Poulose if (ret) 145763c16c6eSSuzuki K Poulose return ret; 145863c16c6eSSuzuki K Poulose } 1459443acc4fSMarc Zyngier 1460443acc4fSMarc Zyngier return 0; 1461443acc4fSMarc Zyngier } 1462443acc4fSMarc Zyngier 1463443acc4fSMarc Zyngier static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1464443acc4fSMarc Zyngier unsigned int nr_irqs) 1465443acc4fSMarc Zyngier { 1466443acc4fSMarc Zyngier int i; 1467443acc4fSMarc Zyngier 1468443acc4fSMarc Zyngier for (i = 0; i < nr_irqs; i++) { 1469443acc4fSMarc Zyngier struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1470443acc4fSMarc Zyngier irq_set_handler(virq + i, NULL); 1471443acc4fSMarc Zyngier irq_domain_reset_irq_data(d); 1472443acc4fSMarc Zyngier } 1473443acc4fSMarc Zyngier } 1474443acc4fSMarc Zyngier 1475e3825ba1SMarc Zyngier static int gic_irq_domain_select(struct irq_domain *d, 1476e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1477e3825ba1SMarc Zyngier enum irq_domain_bus_token bus_token) 1478e3825ba1SMarc Zyngier { 1479e3825ba1SMarc Zyngier /* Not for us */ 1480e3825ba1SMarc Zyngier if (fwspec->fwnode != d->fwnode) 1481e3825ba1SMarc Zyngier return 0; 1482e3825ba1SMarc Zyngier 1483e3825ba1SMarc Zyngier /* If this is not DT, then we have a single domain */ 1484e3825ba1SMarc Zyngier if (!is_of_node(fwspec->fwnode)) 1485e3825ba1SMarc Zyngier return 1; 1486e3825ba1SMarc Zyngier 1487e3825ba1SMarc Zyngier /* 1488e3825ba1SMarc Zyngier * If this is a PPI and we have a 4th (non-null) parameter, 1489e3825ba1SMarc Zyngier * then we need to match the partition domain. 1490e3825ba1SMarc Zyngier */ 1491e3825ba1SMarc Zyngier if (fwspec->param_count >= 4 && 149252085d3fSMarc Zyngier fwspec->param[0] == 1 && fwspec->param[3] != 0 && 149352085d3fSMarc Zyngier gic_data.ppi_descs) 1494e3825ba1SMarc Zyngier return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]); 1495e3825ba1SMarc Zyngier 1496e3825ba1SMarc Zyngier return d == gic_data.domain; 1497e3825ba1SMarc Zyngier } 1498e3825ba1SMarc Zyngier 1499021f6537SMarc Zyngier static const struct irq_domain_ops gic_irq_domain_ops = { 1500f833f57fSMarc Zyngier .translate = gic_irq_domain_translate, 1501443acc4fSMarc Zyngier .alloc = gic_irq_domain_alloc, 1502443acc4fSMarc Zyngier .free = gic_irq_domain_free, 1503e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1504e3825ba1SMarc Zyngier }; 1505e3825ba1SMarc Zyngier 1506e3825ba1SMarc Zyngier static int partition_domain_translate(struct irq_domain *d, 1507e3825ba1SMarc Zyngier struct irq_fwspec *fwspec, 1508e3825ba1SMarc Zyngier unsigned long *hwirq, 1509e3825ba1SMarc Zyngier unsigned int *type) 1510e3825ba1SMarc Zyngier { 1511e3825ba1SMarc Zyngier struct device_node *np; 1512e3825ba1SMarc Zyngier int ret; 1513e3825ba1SMarc Zyngier 151452085d3fSMarc Zyngier if (!gic_data.ppi_descs) 151552085d3fSMarc Zyngier return -ENOMEM; 151652085d3fSMarc Zyngier 1517e3825ba1SMarc Zyngier np = of_find_node_by_phandle(fwspec->param[3]); 1518e3825ba1SMarc Zyngier if (WARN_ON(!np)) 1519e3825ba1SMarc Zyngier return -EINVAL; 1520e3825ba1SMarc Zyngier 1521e3825ba1SMarc Zyngier ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]], 1522e3825ba1SMarc Zyngier of_node_to_fwnode(np)); 1523e3825ba1SMarc Zyngier if (ret < 0) 1524e3825ba1SMarc Zyngier return ret; 1525e3825ba1SMarc Zyngier 1526e3825ba1SMarc Zyngier *hwirq = ret; 1527e3825ba1SMarc Zyngier *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1528e3825ba1SMarc Zyngier 1529e3825ba1SMarc Zyngier return 0; 1530e3825ba1SMarc Zyngier } 1531e3825ba1SMarc Zyngier 1532e3825ba1SMarc Zyngier static const struct irq_domain_ops partition_domain_ops = { 1533e3825ba1SMarc Zyngier .translate = partition_domain_translate, 1534e3825ba1SMarc Zyngier .select = gic_irq_domain_select, 1535021f6537SMarc Zyngier }; 1536021f6537SMarc Zyngier 15379c8114c2SSrinivas Kandagatla static bool gic_enable_quirk_msm8996(void *data) 15389c8114c2SSrinivas Kandagatla { 15399c8114c2SSrinivas Kandagatla struct gic_chip_data *d = data; 15409c8114c2SSrinivas Kandagatla 15419c8114c2SSrinivas Kandagatla d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 15429c8114c2SSrinivas Kandagatla 15439c8114c2SSrinivas Kandagatla return true; 15449c8114c2SSrinivas Kandagatla } 15459c8114c2SSrinivas Kandagatla 1546d01fd161SMarc Zyngier static bool gic_enable_quirk_cavium_38539(void *data) 1547d01fd161SMarc Zyngier { 1548d01fd161SMarc Zyngier struct gic_chip_data *d = data; 1549d01fd161SMarc Zyngier 1550d01fd161SMarc Zyngier d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1551d01fd161SMarc Zyngier 1552d01fd161SMarc Zyngier return true; 1553d01fd161SMarc Zyngier } 1554d01fd161SMarc Zyngier 15557f2481b3SMarc Zyngier static bool gic_enable_quirk_hip06_07(void *data) 15567f2481b3SMarc Zyngier { 15577f2481b3SMarc Zyngier struct gic_chip_data *d = data; 15587f2481b3SMarc Zyngier 15597f2481b3SMarc Zyngier /* 15607f2481b3SMarc Zyngier * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 15617f2481b3SMarc Zyngier * not being an actual ARM implementation). The saving grace is 15627f2481b3SMarc Zyngier * that GIC-600 doesn't have ESPI, so nothing to do in that case. 15637f2481b3SMarc Zyngier * HIP07 doesn't even have a proper IIDR, and still pretends to 15647f2481b3SMarc Zyngier * have ESPI. In both cases, put them right. 15657f2481b3SMarc Zyngier */ 15667f2481b3SMarc Zyngier if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 15677f2481b3SMarc Zyngier /* Zero both ESPI and the RES0 field next to it... */ 15687f2481b3SMarc Zyngier d->rdists.gicd_typer &= ~GENMASK(9, 8); 15697f2481b3SMarc Zyngier return true; 15707f2481b3SMarc Zyngier } 15717f2481b3SMarc Zyngier 15727f2481b3SMarc Zyngier return false; 15737f2481b3SMarc Zyngier } 15747f2481b3SMarc Zyngier 15757f2481b3SMarc Zyngier static const struct gic_quirk gic_quirks[] = { 15767f2481b3SMarc Zyngier { 15777f2481b3SMarc Zyngier .desc = "GICv3: Qualcomm MSM8996 broken firmware", 15787f2481b3SMarc Zyngier .compatible = "qcom,msm8996-gic-v3", 15797f2481b3SMarc Zyngier .init = gic_enable_quirk_msm8996, 15807f2481b3SMarc Zyngier }, 15817f2481b3SMarc Zyngier { 15827f2481b3SMarc Zyngier .desc = "GICv3: HIP06 erratum 161010803", 15837f2481b3SMarc Zyngier .iidr = 0x0204043b, 15847f2481b3SMarc Zyngier .mask = 0xffffffff, 15857f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 15867f2481b3SMarc Zyngier }, 15877f2481b3SMarc Zyngier { 15887f2481b3SMarc Zyngier .desc = "GICv3: HIP07 erratum 161010803", 15897f2481b3SMarc Zyngier .iidr = 0x00000000, 15907f2481b3SMarc Zyngier .mask = 0xffffffff, 15917f2481b3SMarc Zyngier .init = gic_enable_quirk_hip06_07, 15927f2481b3SMarc Zyngier }, 15937f2481b3SMarc Zyngier { 1594d01fd161SMarc Zyngier /* 1595d01fd161SMarc Zyngier * Reserved register accesses generate a Synchronous 1596d01fd161SMarc Zyngier * External Abort. This erratum applies to: 1597d01fd161SMarc Zyngier * - ThunderX: CN88xx 1598d01fd161SMarc Zyngier * - OCTEON TX: CN83xx, CN81xx 1599d01fd161SMarc Zyngier * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1600d01fd161SMarc Zyngier */ 1601d01fd161SMarc Zyngier .desc = "GICv3: Cavium erratum 38539", 1602d01fd161SMarc Zyngier .iidr = 0xa000034c, 1603d01fd161SMarc Zyngier .mask = 0xe8f00fff, 1604d01fd161SMarc Zyngier .init = gic_enable_quirk_cavium_38539, 1605d01fd161SMarc Zyngier }, 1606d01fd161SMarc Zyngier { 16077f2481b3SMarc Zyngier } 16087f2481b3SMarc Zyngier }; 16097f2481b3SMarc Zyngier 1610d98d0a99SJulien Thierry static void gic_enable_nmi_support(void) 1611d98d0a99SJulien Thierry { 1612101b35f7SJulien Thierry int i; 1613101b35f7SJulien Thierry 161481a43273SMarc Zyngier if (!gic_prio_masking_enabled()) 161581a43273SMarc Zyngier return; 161681a43273SMarc Zyngier 161781a43273SMarc Zyngier ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 161881a43273SMarc Zyngier if (!ppi_nmi_refs) 161981a43273SMarc Zyngier return; 162081a43273SMarc Zyngier 162181a43273SMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) 1622101b35f7SJulien Thierry refcount_set(&ppi_nmi_refs[i], 0); 1623101b35f7SJulien Thierry 1624f2266504SMarc Zyngier /* 1625f2266504SMarc Zyngier * Linux itself doesn't use 1:N distribution, so has no need to 1626f2266504SMarc Zyngier * set PMHE. The only reason to have it set is if EL3 requires it 1627f2266504SMarc Zyngier * (and we can't change it). 1628f2266504SMarc Zyngier */ 1629f2266504SMarc Zyngier if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) 1630f2266504SMarc Zyngier static_branch_enable(&gic_pmr_sync); 1631f2266504SMarc Zyngier 16324e594ad1SAlexandru Elisei pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 16334e594ad1SAlexandru Elisei static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed"); 1634f2266504SMarc Zyngier 163533678059SAlexandru Elisei /* 163633678059SAlexandru Elisei * How priority values are used by the GIC depends on two things: 163733678059SAlexandru Elisei * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 163833678059SAlexandru Elisei * and if Group 0 interrupts can be delivered to Linux in the non-secure 163933678059SAlexandru Elisei * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 164033678059SAlexandru Elisei * the ICC_PMR_EL1 register and the priority that software assigns to 164133678059SAlexandru Elisei * interrupts: 164233678059SAlexandru Elisei * 164333678059SAlexandru Elisei * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 164433678059SAlexandru Elisei * ----------------------------------------------------------- 164533678059SAlexandru Elisei * 1 | - | unchanged | unchanged 164633678059SAlexandru Elisei * ----------------------------------------------------------- 164733678059SAlexandru Elisei * 0 | 1 | non-secure | non-secure 164833678059SAlexandru Elisei * ----------------------------------------------------------- 164933678059SAlexandru Elisei * 0 | 0 | unchanged | non-secure 165033678059SAlexandru Elisei * 165133678059SAlexandru Elisei * where non-secure means that the value is right-shifted by one and the 165233678059SAlexandru Elisei * MSB bit set, to make it fit in the non-secure priority range. 165333678059SAlexandru Elisei * 165433678059SAlexandru Elisei * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 165533678059SAlexandru Elisei * are both either modified or unchanged, we can use the same set of 165633678059SAlexandru Elisei * priorities. 165733678059SAlexandru Elisei * 165833678059SAlexandru Elisei * In the last case, where only the interrupt priorities are modified to 165933678059SAlexandru Elisei * be in the non-secure range, we use a different PMR value to mask IRQs 166033678059SAlexandru Elisei * and the rest of the values that we use remain unchanged. 166133678059SAlexandru Elisei */ 166233678059SAlexandru Elisei if (gic_has_group0() && !gic_dist_security_disabled()) 166333678059SAlexandru Elisei static_branch_enable(&gic_nonsecure_priorities); 166433678059SAlexandru Elisei 1665d98d0a99SJulien Thierry static_branch_enable(&supports_pseudo_nmis); 1666101b35f7SJulien Thierry 1667101b35f7SJulien Thierry if (static_branch_likely(&supports_deactivate_key)) 1668101b35f7SJulien Thierry gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1669101b35f7SJulien Thierry else 1670101b35f7SJulien Thierry gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1671d98d0a99SJulien Thierry } 1672d98d0a99SJulien Thierry 1673db57d746STomasz Nowicki static int __init gic_init_bases(void __iomem *dist_base, 1674db57d746STomasz Nowicki struct redist_region *rdist_regs, 1675db57d746STomasz Nowicki u32 nr_redist_regions, 1676db57d746STomasz Nowicki u64 redist_stride, 1677db57d746STomasz Nowicki struct fwnode_handle *handle) 1678db57d746STomasz Nowicki { 1679db57d746STomasz Nowicki u32 typer; 1680db57d746STomasz Nowicki int err; 1681db57d746STomasz Nowicki 1682db57d746STomasz Nowicki if (!is_hyp_mode_available()) 1683d01d3274SDavidlohr Bueso static_branch_disable(&supports_deactivate_key); 1684db57d746STomasz Nowicki 1685d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 1686db57d746STomasz Nowicki pr_info("GIC: Using split EOI/Deactivate mode\n"); 1687db57d746STomasz Nowicki 1688e3825ba1SMarc Zyngier gic_data.fwnode = handle; 1689db57d746STomasz Nowicki gic_data.dist_base = dist_base; 1690db57d746STomasz Nowicki gic_data.redist_regions = rdist_regs; 1691db57d746STomasz Nowicki gic_data.nr_redist_regions = nr_redist_regions; 1692db57d746STomasz Nowicki gic_data.redist_stride = redist_stride; 1693db57d746STomasz Nowicki 1694db57d746STomasz Nowicki /* 1695db57d746STomasz Nowicki * Find out how many interrupts are supported. 1696db57d746STomasz Nowicki */ 1697db57d746STomasz Nowicki typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1698a4f9edb2SMarc Zyngier gic_data.rdists.gicd_typer = typer; 16997f2481b3SMarc Zyngier 17007f2481b3SMarc Zyngier gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 17017f2481b3SMarc Zyngier gic_quirks, &gic_data); 17027f2481b3SMarc Zyngier 1703211bddd2SMarc Zyngier pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 1704211bddd2SMarc Zyngier pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 1705f2d83409SMarc Zyngier 1706d01fd161SMarc Zyngier /* 1707d01fd161SMarc Zyngier * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 1708d01fd161SMarc Zyngier * architecture spec (which says that reserved registers are RES0). 1709d01fd161SMarc Zyngier */ 1710d01fd161SMarc Zyngier if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 1711f2d83409SMarc Zyngier gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 1712f2d83409SMarc Zyngier 1713db57d746STomasz Nowicki gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1714db57d746STomasz Nowicki &gic_data); 1715db57d746STomasz Nowicki gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 1716b25319d2SMarc Zyngier gic_data.rdists.has_rvpeid = true; 17170edc23eaSMarc Zyngier gic_data.rdists.has_vlpis = true; 17180edc23eaSMarc Zyngier gic_data.rdists.has_direct_lpi = true; 171996806229SMarc Zyngier gic_data.rdists.has_vpend_valid_dirty = true; 1720db57d746STomasz Nowicki 1721db57d746STomasz Nowicki if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1722db57d746STomasz Nowicki err = -ENOMEM; 1723db57d746STomasz Nowicki goto out_free; 1724db57d746STomasz Nowicki } 1725db57d746STomasz Nowicki 1726eeaa4b24Sluanshi irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1727eeaa4b24Sluanshi 1728eda0d04aSShanker Donthineni gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1729eda0d04aSShanker Donthineni pr_info("Distributor has %sRange Selector support\n", 1730eda0d04aSShanker Donthineni gic_data.has_rss ? "" : "no "); 1731eda0d04aSShanker Donthineni 173250528752SMarc Zyngier if (typer & GICD_TYPER_MBIS) { 173350528752SMarc Zyngier err = mbi_init(handle, gic_data.domain); 173450528752SMarc Zyngier if (err) 173550528752SMarc Zyngier pr_err("Failed to initialize MBIs\n"); 173650528752SMarc Zyngier } 173750528752SMarc Zyngier 1738db57d746STomasz Nowicki set_handle_irq(gic_handle_irq); 1739db57d746STomasz Nowicki 17401a60e1e6SMarc Zyngier gic_update_rdist_properties(); 17410edc23eaSMarc Zyngier 1742db57d746STomasz Nowicki gic_dist_init(); 1743db57d746STomasz Nowicki gic_cpu_init(); 174464b499d8SMarc Zyngier gic_smp_init(); 1745db57d746STomasz Nowicki gic_cpu_pm_init(); 1746db57d746STomasz Nowicki 1747d38a71c5SMarc Zyngier if (gic_dist_supports_lpis()) { 1748d38a71c5SMarc Zyngier its_init(handle, &gic_data.rdists, gic_data.domain); 1749d38a71c5SMarc Zyngier its_cpu_init(); 175090b4c555SZeev Zilberman } else { 175190b4c555SZeev Zilberman if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 175290b4c555SZeev Zilberman gicv2m_init(handle, gic_data.domain); 1753d38a71c5SMarc Zyngier } 1754d38a71c5SMarc Zyngier 1755d98d0a99SJulien Thierry gic_enable_nmi_support(); 1756d98d0a99SJulien Thierry 1757db57d746STomasz Nowicki return 0; 1758db57d746STomasz Nowicki 1759db57d746STomasz Nowicki out_free: 1760db57d746STomasz Nowicki if (gic_data.domain) 1761db57d746STomasz Nowicki irq_domain_remove(gic_data.domain); 1762db57d746STomasz Nowicki free_percpu(gic_data.rdists.rdist); 1763db57d746STomasz Nowicki return err; 1764db57d746STomasz Nowicki } 1765db57d746STomasz Nowicki 1766db57d746STomasz Nowicki static int __init gic_validate_dist_version(void __iomem *dist_base) 1767db57d746STomasz Nowicki { 1768db57d746STomasz Nowicki u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1769db57d746STomasz Nowicki 1770db57d746STomasz Nowicki if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 1771db57d746STomasz Nowicki return -ENODEV; 1772db57d746STomasz Nowicki 1773db57d746STomasz Nowicki return 0; 1774db57d746STomasz Nowicki } 1775db57d746STomasz Nowicki 1776e3825ba1SMarc Zyngier /* Create all possible partitions at boot time */ 17777beaa24bSLinus Torvalds static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 1778e3825ba1SMarc Zyngier { 1779e3825ba1SMarc Zyngier struct device_node *parts_node, *child_part; 1780e3825ba1SMarc Zyngier int part_idx = 0, i; 1781e3825ba1SMarc Zyngier int nr_parts; 1782e3825ba1SMarc Zyngier struct partition_affinity *parts; 1783e3825ba1SMarc Zyngier 178400ee9a1cSJohan Hovold parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 1785e3825ba1SMarc Zyngier if (!parts_node) 1786e3825ba1SMarc Zyngier return; 1787e3825ba1SMarc Zyngier 178852085d3fSMarc Zyngier gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 178952085d3fSMarc Zyngier if (!gic_data.ppi_descs) 179052085d3fSMarc Zyngier return; 179152085d3fSMarc Zyngier 1792e3825ba1SMarc Zyngier nr_parts = of_get_child_count(parts_node); 1793e3825ba1SMarc Zyngier 1794e3825ba1SMarc Zyngier if (!nr_parts) 179500ee9a1cSJohan Hovold goto out_put_node; 1796e3825ba1SMarc Zyngier 17976396bb22SKees Cook parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 1798e3825ba1SMarc Zyngier if (WARN_ON(!parts)) 179900ee9a1cSJohan Hovold goto out_put_node; 1800e3825ba1SMarc Zyngier 1801e3825ba1SMarc Zyngier for_each_child_of_node(parts_node, child_part) { 1802e3825ba1SMarc Zyngier struct partition_affinity *part; 1803e3825ba1SMarc Zyngier int n; 1804e3825ba1SMarc Zyngier 1805e3825ba1SMarc Zyngier part = &parts[part_idx]; 1806e3825ba1SMarc Zyngier 1807e3825ba1SMarc Zyngier part->partition_id = of_node_to_fwnode(child_part); 1808e3825ba1SMarc Zyngier 18092ef790dcSRob Herring pr_info("GIC: PPI partition %pOFn[%d] { ", 18102ef790dcSRob Herring child_part, part_idx); 1811e3825ba1SMarc Zyngier 1812e3825ba1SMarc Zyngier n = of_property_count_elems_of_size(child_part, "affinity", 1813e3825ba1SMarc Zyngier sizeof(u32)); 1814e3825ba1SMarc Zyngier WARN_ON(n <= 0); 1815e3825ba1SMarc Zyngier 1816e3825ba1SMarc Zyngier for (i = 0; i < n; i++) { 1817e3825ba1SMarc Zyngier int err, cpu; 1818e3825ba1SMarc Zyngier u32 cpu_phandle; 1819e3825ba1SMarc Zyngier struct device_node *cpu_node; 1820e3825ba1SMarc Zyngier 1821e3825ba1SMarc Zyngier err = of_property_read_u32_index(child_part, "affinity", 1822e3825ba1SMarc Zyngier i, &cpu_phandle); 1823e3825ba1SMarc Zyngier if (WARN_ON(err)) 1824e3825ba1SMarc Zyngier continue; 1825e3825ba1SMarc Zyngier 1826e3825ba1SMarc Zyngier cpu_node = of_find_node_by_phandle(cpu_phandle); 1827e3825ba1SMarc Zyngier if (WARN_ON(!cpu_node)) 1828e3825ba1SMarc Zyngier continue; 1829e3825ba1SMarc Zyngier 1830c08ec7daSSuzuki K Poulose cpu = of_cpu_node_to_id(cpu_node); 1831c08ec7daSSuzuki K Poulose if (WARN_ON(cpu < 0)) 1832e3825ba1SMarc Zyngier continue; 1833e3825ba1SMarc Zyngier 1834e81f54c6SRob Herring pr_cont("%pOF[%d] ", cpu_node, cpu); 1835e3825ba1SMarc Zyngier 1836e3825ba1SMarc Zyngier cpumask_set_cpu(cpu, &part->mask); 1837e3825ba1SMarc Zyngier } 1838e3825ba1SMarc Zyngier 1839e3825ba1SMarc Zyngier pr_cont("}\n"); 1840e3825ba1SMarc Zyngier part_idx++; 1841e3825ba1SMarc Zyngier } 1842e3825ba1SMarc Zyngier 184352085d3fSMarc Zyngier for (i = 0; i < gic_data.ppi_nr; i++) { 1844e3825ba1SMarc Zyngier unsigned int irq; 1845e3825ba1SMarc Zyngier struct partition_desc *desc; 1846e3825ba1SMarc Zyngier struct irq_fwspec ppi_fwspec = { 1847e3825ba1SMarc Zyngier .fwnode = gic_data.fwnode, 1848e3825ba1SMarc Zyngier .param_count = 3, 1849e3825ba1SMarc Zyngier .param = { 185065da7d19SMarc Zyngier [0] = GIC_IRQ_TYPE_PARTITION, 1851e3825ba1SMarc Zyngier [1] = i, 1852e3825ba1SMarc Zyngier [2] = IRQ_TYPE_NONE, 1853e3825ba1SMarc Zyngier }, 1854e3825ba1SMarc Zyngier }; 1855e3825ba1SMarc Zyngier 1856e3825ba1SMarc Zyngier irq = irq_create_fwspec_mapping(&ppi_fwspec); 1857e3825ba1SMarc Zyngier if (WARN_ON(!irq)) 1858e3825ba1SMarc Zyngier continue; 1859e3825ba1SMarc Zyngier desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 1860e3825ba1SMarc Zyngier irq, &partition_domain_ops); 1861e3825ba1SMarc Zyngier if (WARN_ON(!desc)) 1862e3825ba1SMarc Zyngier continue; 1863e3825ba1SMarc Zyngier 1864e3825ba1SMarc Zyngier gic_data.ppi_descs[i] = desc; 1865e3825ba1SMarc Zyngier } 186600ee9a1cSJohan Hovold 186700ee9a1cSJohan Hovold out_put_node: 186800ee9a1cSJohan Hovold of_node_put(parts_node); 1869e3825ba1SMarc Zyngier } 1870e3825ba1SMarc Zyngier 18711839e576SJulien Grall static void __init gic_of_setup_kvm_info(struct device_node *node) 18721839e576SJulien Grall { 18731839e576SJulien Grall int ret; 18741839e576SJulien Grall struct resource r; 18751839e576SJulien Grall u32 gicv_idx; 18761839e576SJulien Grall 18771839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 18781839e576SJulien Grall 18791839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 18801839e576SJulien Grall if (!gic_v3_kvm_info.maint_irq) 18811839e576SJulien Grall return; 18821839e576SJulien Grall 18831839e576SJulien Grall if (of_property_read_u32(node, "#redistributor-regions", 18841839e576SJulien Grall &gicv_idx)) 18851839e576SJulien Grall gicv_idx = 1; 18861839e576SJulien Grall 18871839e576SJulien Grall gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 18881839e576SJulien Grall ret = of_address_to_resource(node, gicv_idx, &r); 18891839e576SJulien Grall if (!ret) 18901839e576SJulien Grall gic_v3_kvm_info.vcpu = r; 18911839e576SJulien Grall 18924bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 18933c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 18940e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info); 18951839e576SJulien Grall } 18961839e576SJulien Grall 1897021f6537SMarc Zyngier static int __init gic_of_init(struct device_node *node, struct device_node *parent) 1898021f6537SMarc Zyngier { 1899021f6537SMarc Zyngier void __iomem *dist_base; 1900f5c1434cSMarc Zyngier struct redist_region *rdist_regs; 1901021f6537SMarc Zyngier u64 redist_stride; 1902f5c1434cSMarc Zyngier u32 nr_redist_regions; 1903db57d746STomasz Nowicki int err, i; 1904021f6537SMarc Zyngier 1905021f6537SMarc Zyngier dist_base = of_iomap(node, 0); 1906021f6537SMarc Zyngier if (!dist_base) { 1907e81f54c6SRob Herring pr_err("%pOF: unable to map gic dist registers\n", node); 1908021f6537SMarc Zyngier return -ENXIO; 1909021f6537SMarc Zyngier } 1910021f6537SMarc Zyngier 1911db57d746STomasz Nowicki err = gic_validate_dist_version(dist_base); 1912db57d746STomasz Nowicki if (err) { 1913e81f54c6SRob Herring pr_err("%pOF: no distributor detected, giving up\n", node); 1914021f6537SMarc Zyngier goto out_unmap_dist; 1915021f6537SMarc Zyngier } 1916021f6537SMarc Zyngier 1917f5c1434cSMarc Zyngier if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 1918f5c1434cSMarc Zyngier nr_redist_regions = 1; 1919021f6537SMarc Zyngier 19206396bb22SKees Cook rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 19216396bb22SKees Cook GFP_KERNEL); 1922f5c1434cSMarc Zyngier if (!rdist_regs) { 1923021f6537SMarc Zyngier err = -ENOMEM; 1924021f6537SMarc Zyngier goto out_unmap_dist; 1925021f6537SMarc Zyngier } 1926021f6537SMarc Zyngier 1927f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) { 1928f5c1434cSMarc Zyngier struct resource res; 1929f5c1434cSMarc Zyngier int ret; 1930f5c1434cSMarc Zyngier 1931f5c1434cSMarc Zyngier ret = of_address_to_resource(node, 1 + i, &res); 1932f5c1434cSMarc Zyngier rdist_regs[i].redist_base = of_iomap(node, 1 + i); 1933f5c1434cSMarc Zyngier if (ret || !rdist_regs[i].redist_base) { 1934e81f54c6SRob Herring pr_err("%pOF: couldn't map region %d\n", node, i); 1935021f6537SMarc Zyngier err = -ENODEV; 1936021f6537SMarc Zyngier goto out_unmap_rdist; 1937021f6537SMarc Zyngier } 1938f5c1434cSMarc Zyngier rdist_regs[i].phys_base = res.start; 1939021f6537SMarc Zyngier } 1940021f6537SMarc Zyngier 1941021f6537SMarc Zyngier if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 1942021f6537SMarc Zyngier redist_stride = 0; 1943021f6537SMarc Zyngier 1944f70fdb42SSrinivas Kandagatla gic_enable_of_quirks(node, gic_quirks, &gic_data); 1945f70fdb42SSrinivas Kandagatla 1946db57d746STomasz Nowicki err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions, 1947db57d746STomasz Nowicki redist_stride, &node->fwnode); 1948e3825ba1SMarc Zyngier if (err) 1949e3825ba1SMarc Zyngier goto out_unmap_rdist; 1950e3825ba1SMarc Zyngier 1951e3825ba1SMarc Zyngier gic_populate_ppi_partitions(node); 1952d33a3c8cSChristoffer Dall 1953d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 19541839e576SJulien Grall gic_of_setup_kvm_info(node); 1955021f6537SMarc Zyngier return 0; 1956021f6537SMarc Zyngier 1957021f6537SMarc Zyngier out_unmap_rdist: 1958f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) 1959f5c1434cSMarc Zyngier if (rdist_regs[i].redist_base) 1960f5c1434cSMarc Zyngier iounmap(rdist_regs[i].redist_base); 1961f5c1434cSMarc Zyngier kfree(rdist_regs); 1962021f6537SMarc Zyngier out_unmap_dist: 1963021f6537SMarc Zyngier iounmap(dist_base); 1964021f6537SMarc Zyngier return err; 1965021f6537SMarc Zyngier } 1966021f6537SMarc Zyngier 1967021f6537SMarc Zyngier IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 1968ffa7d616STomasz Nowicki 1969ffa7d616STomasz Nowicki #ifdef CONFIG_ACPI 1970611f039fSJulien Grall static struct 1971611f039fSJulien Grall { 1972611f039fSJulien Grall void __iomem *dist_base; 1973611f039fSJulien Grall struct redist_region *redist_regs; 1974611f039fSJulien Grall u32 nr_redist_regions; 1975611f039fSJulien Grall bool single_redist; 1976926b5dfaSMarc Zyngier int enabled_rdists; 19771839e576SJulien Grall u32 maint_irq; 19781839e576SJulien Grall int maint_irq_mode; 19791839e576SJulien Grall phys_addr_t vcpu_base; 1980611f039fSJulien Grall } acpi_data __initdata; 1981b70fb7afSTomasz Nowicki 1982b70fb7afSTomasz Nowicki static void __init 1983b70fb7afSTomasz Nowicki gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 1984b70fb7afSTomasz Nowicki { 1985b70fb7afSTomasz Nowicki static int count = 0; 1986b70fb7afSTomasz Nowicki 1987611f039fSJulien Grall acpi_data.redist_regs[count].phys_base = phys_base; 1988611f039fSJulien Grall acpi_data.redist_regs[count].redist_base = redist_base; 1989611f039fSJulien Grall acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 1990b70fb7afSTomasz Nowicki count++; 1991b70fb7afSTomasz Nowicki } 1992ffa7d616STomasz Nowicki 1993ffa7d616STomasz Nowicki static int __init 199460574d1eSKeith Busch gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 1995ffa7d616STomasz Nowicki const unsigned long end) 1996ffa7d616STomasz Nowicki { 1997ffa7d616STomasz Nowicki struct acpi_madt_generic_redistributor *redist = 1998ffa7d616STomasz Nowicki (struct acpi_madt_generic_redistributor *)header; 1999ffa7d616STomasz Nowicki void __iomem *redist_base; 2000ffa7d616STomasz Nowicki 2001ffa7d616STomasz Nowicki redist_base = ioremap(redist->base_address, redist->length); 2002ffa7d616STomasz Nowicki if (!redist_base) { 2003ffa7d616STomasz Nowicki pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2004ffa7d616STomasz Nowicki return -ENOMEM; 2005ffa7d616STomasz Nowicki } 2006ffa7d616STomasz Nowicki 2007b70fb7afSTomasz Nowicki gic_acpi_register_redist(redist->base_address, redist_base); 2008ffa7d616STomasz Nowicki return 0; 2009ffa7d616STomasz Nowicki } 2010ffa7d616STomasz Nowicki 2011b70fb7afSTomasz Nowicki static int __init 201260574d1eSKeith Busch gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2013b70fb7afSTomasz Nowicki const unsigned long end) 2014b70fb7afSTomasz Nowicki { 2015b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 2016b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 2017611f039fSJulien Grall u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2018b70fb7afSTomasz Nowicki u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2019b70fb7afSTomasz Nowicki void __iomem *redist_base; 2020b70fb7afSTomasz Nowicki 2021ebe2f871SShanker Donthineni /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 2022ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 2023ebe2f871SShanker Donthineni return 0; 2024ebe2f871SShanker Donthineni 2025b70fb7afSTomasz Nowicki redist_base = ioremap(gicc->gicr_base_address, size); 2026b70fb7afSTomasz Nowicki if (!redist_base) 2027b70fb7afSTomasz Nowicki return -ENOMEM; 2028b70fb7afSTomasz Nowicki 2029b70fb7afSTomasz Nowicki gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2030b70fb7afSTomasz Nowicki return 0; 2031b70fb7afSTomasz Nowicki } 2032b70fb7afSTomasz Nowicki 2033b70fb7afSTomasz Nowicki static int __init gic_acpi_collect_gicr_base(void) 2034b70fb7afSTomasz Nowicki { 2035b70fb7afSTomasz Nowicki acpi_tbl_entry_handler redist_parser; 2036b70fb7afSTomasz Nowicki enum acpi_madt_type type; 2037b70fb7afSTomasz Nowicki 2038611f039fSJulien Grall if (acpi_data.single_redist) { 2039b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2040b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_gicc; 2041b70fb7afSTomasz Nowicki } else { 2042b70fb7afSTomasz Nowicki type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2043b70fb7afSTomasz Nowicki redist_parser = gic_acpi_parse_madt_redist; 2044b70fb7afSTomasz Nowicki } 2045b70fb7afSTomasz Nowicki 2046b70fb7afSTomasz Nowicki /* Collect redistributor base addresses in GICR entries */ 2047b70fb7afSTomasz Nowicki if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2048b70fb7afSTomasz Nowicki return 0; 2049b70fb7afSTomasz Nowicki 2050b70fb7afSTomasz Nowicki pr_info("No valid GICR entries exist\n"); 2051b70fb7afSTomasz Nowicki return -ENODEV; 2052b70fb7afSTomasz Nowicki } 2053b70fb7afSTomasz Nowicki 205460574d1eSKeith Busch static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2055ffa7d616STomasz Nowicki const unsigned long end) 2056ffa7d616STomasz Nowicki { 2057ffa7d616STomasz Nowicki /* Subtable presence means that redist exists, that's it */ 2058ffa7d616STomasz Nowicki return 0; 2059ffa7d616STomasz Nowicki } 2060ffa7d616STomasz Nowicki 206160574d1eSKeith Busch static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2062b70fb7afSTomasz Nowicki const unsigned long end) 2063b70fb7afSTomasz Nowicki { 2064b70fb7afSTomasz Nowicki struct acpi_madt_generic_interrupt *gicc = 2065b70fb7afSTomasz Nowicki (struct acpi_madt_generic_interrupt *)header; 2066b70fb7afSTomasz Nowicki 2067b70fb7afSTomasz Nowicki /* 2068b70fb7afSTomasz Nowicki * If GICC is enabled and has valid gicr base address, then it means 2069b70fb7afSTomasz Nowicki * GICR base is presented via GICC 2070b70fb7afSTomasz Nowicki */ 2071926b5dfaSMarc Zyngier if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2072926b5dfaSMarc Zyngier acpi_data.enabled_rdists++; 2073b70fb7afSTomasz Nowicki return 0; 2074926b5dfaSMarc Zyngier } 2075b70fb7afSTomasz Nowicki 2076ebe2f871SShanker Donthineni /* 2077ebe2f871SShanker Donthineni * It's perfectly valid firmware can pass disabled GICC entry, driver 2078ebe2f871SShanker Donthineni * should not treat as errors, skip the entry instead of probe fail. 2079ebe2f871SShanker Donthineni */ 2080ebe2f871SShanker Donthineni if (!(gicc->flags & ACPI_MADT_ENABLED)) 2081ebe2f871SShanker Donthineni return 0; 2082ebe2f871SShanker Donthineni 2083b70fb7afSTomasz Nowicki return -ENODEV; 2084b70fb7afSTomasz Nowicki } 2085b70fb7afSTomasz Nowicki 2086b70fb7afSTomasz Nowicki static int __init gic_acpi_count_gicr_regions(void) 2087b70fb7afSTomasz Nowicki { 2088b70fb7afSTomasz Nowicki int count; 2089b70fb7afSTomasz Nowicki 2090b70fb7afSTomasz Nowicki /* 2091b70fb7afSTomasz Nowicki * Count how many redistributor regions we have. It is not allowed 2092b70fb7afSTomasz Nowicki * to mix redistributor description, GICR and GICC subtables have to be 2093b70fb7afSTomasz Nowicki * mutually exclusive. 2094b70fb7afSTomasz Nowicki */ 2095b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2096b70fb7afSTomasz Nowicki gic_acpi_match_gicr, 0); 2097b70fb7afSTomasz Nowicki if (count > 0) { 2098611f039fSJulien Grall acpi_data.single_redist = false; 2099b70fb7afSTomasz Nowicki return count; 2100b70fb7afSTomasz Nowicki } 2101b70fb7afSTomasz Nowicki 2102b70fb7afSTomasz Nowicki count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2103b70fb7afSTomasz Nowicki gic_acpi_match_gicc, 0); 2104926b5dfaSMarc Zyngier if (count > 0) { 2105611f039fSJulien Grall acpi_data.single_redist = true; 2106926b5dfaSMarc Zyngier count = acpi_data.enabled_rdists; 2107926b5dfaSMarc Zyngier } 2108b70fb7afSTomasz Nowicki 2109b70fb7afSTomasz Nowicki return count; 2110b70fb7afSTomasz Nowicki } 2111b70fb7afSTomasz Nowicki 2112ffa7d616STomasz Nowicki static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2113ffa7d616STomasz Nowicki struct acpi_probe_entry *ape) 2114ffa7d616STomasz Nowicki { 2115ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2116ffa7d616STomasz Nowicki int count; 2117ffa7d616STomasz Nowicki 2118ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2119ffa7d616STomasz Nowicki if (dist->version != ape->driver_data) 2120ffa7d616STomasz Nowicki return false; 2121ffa7d616STomasz Nowicki 2122ffa7d616STomasz Nowicki /* We need to do that exercise anyway, the sooner the better */ 2123b70fb7afSTomasz Nowicki count = gic_acpi_count_gicr_regions(); 2124ffa7d616STomasz Nowicki if (count <= 0) 2125ffa7d616STomasz Nowicki return false; 2126ffa7d616STomasz Nowicki 2127611f039fSJulien Grall acpi_data.nr_redist_regions = count; 2128ffa7d616STomasz Nowicki return true; 2129ffa7d616STomasz Nowicki } 2130ffa7d616STomasz Nowicki 213160574d1eSKeith Busch static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 21321839e576SJulien Grall const unsigned long end) 21331839e576SJulien Grall { 21341839e576SJulien Grall struct acpi_madt_generic_interrupt *gicc = 21351839e576SJulien Grall (struct acpi_madt_generic_interrupt *)header; 21361839e576SJulien Grall int maint_irq_mode; 21371839e576SJulien Grall static int first_madt = true; 21381839e576SJulien Grall 21391839e576SJulien Grall /* Skip unusable CPUs */ 21401839e576SJulien Grall if (!(gicc->flags & ACPI_MADT_ENABLED)) 21411839e576SJulien Grall return 0; 21421839e576SJulien Grall 21431839e576SJulien Grall maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 21441839e576SJulien Grall ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 21451839e576SJulien Grall 21461839e576SJulien Grall if (first_madt) { 21471839e576SJulien Grall first_madt = false; 21481839e576SJulien Grall 21491839e576SJulien Grall acpi_data.maint_irq = gicc->vgic_interrupt; 21501839e576SJulien Grall acpi_data.maint_irq_mode = maint_irq_mode; 21511839e576SJulien Grall acpi_data.vcpu_base = gicc->gicv_base_address; 21521839e576SJulien Grall 21531839e576SJulien Grall return 0; 21541839e576SJulien Grall } 21551839e576SJulien Grall 21561839e576SJulien Grall /* 21571839e576SJulien Grall * The maintenance interrupt and GICV should be the same for every CPU 21581839e576SJulien Grall */ 21591839e576SJulien Grall if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 21601839e576SJulien Grall (acpi_data.maint_irq_mode != maint_irq_mode) || 21611839e576SJulien Grall (acpi_data.vcpu_base != gicc->gicv_base_address)) 21621839e576SJulien Grall return -EINVAL; 21631839e576SJulien Grall 21641839e576SJulien Grall return 0; 21651839e576SJulien Grall } 21661839e576SJulien Grall 21671839e576SJulien Grall static bool __init gic_acpi_collect_virt_info(void) 21681839e576SJulien Grall { 21691839e576SJulien Grall int count; 21701839e576SJulien Grall 21711839e576SJulien Grall count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 21721839e576SJulien Grall gic_acpi_parse_virt_madt_gicc, 0); 21731839e576SJulien Grall 21741839e576SJulien Grall return (count > 0); 21751839e576SJulien Grall } 21761839e576SJulien Grall 2177ffa7d616STomasz Nowicki #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 21781839e576SJulien Grall #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 21791839e576SJulien Grall #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 21801839e576SJulien Grall 21811839e576SJulien Grall static void __init gic_acpi_setup_kvm_info(void) 21821839e576SJulien Grall { 21831839e576SJulien Grall int irq; 21841839e576SJulien Grall 21851839e576SJulien Grall if (!gic_acpi_collect_virt_info()) { 21861839e576SJulien Grall pr_warn("Unable to get hardware information used for virtualization\n"); 21871839e576SJulien Grall return; 21881839e576SJulien Grall } 21891839e576SJulien Grall 21901839e576SJulien Grall gic_v3_kvm_info.type = GIC_V3; 21911839e576SJulien Grall 21921839e576SJulien Grall irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 21931839e576SJulien Grall acpi_data.maint_irq_mode, 21941839e576SJulien Grall ACPI_ACTIVE_HIGH); 21951839e576SJulien Grall if (irq <= 0) 21961839e576SJulien Grall return; 21971839e576SJulien Grall 21981839e576SJulien Grall gic_v3_kvm_info.maint_irq = irq; 21991839e576SJulien Grall 22001839e576SJulien Grall if (acpi_data.vcpu_base) { 22011839e576SJulien Grall struct resource *vcpu = &gic_v3_kvm_info.vcpu; 22021839e576SJulien Grall 22031839e576SJulien Grall vcpu->flags = IORESOURCE_MEM; 22041839e576SJulien Grall vcpu->start = acpi_data.vcpu_base; 22051839e576SJulien Grall vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 22061839e576SJulien Grall } 22071839e576SJulien Grall 22084bdf5025SMarc Zyngier gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 22093c40706dSMarc Zyngier gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 22100e5cb777SMarc Zyngier vgic_set_kvm_info(&gic_v3_kvm_info); 22111839e576SJulien Grall } 2212ffa7d616STomasz Nowicki 2213ffa7d616STomasz Nowicki static int __init 2214aba3c7edSOscar Carter gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2215ffa7d616STomasz Nowicki { 2216ffa7d616STomasz Nowicki struct acpi_madt_generic_distributor *dist; 2217ffa7d616STomasz Nowicki struct fwnode_handle *domain_handle; 2218611f039fSJulien Grall size_t size; 2219b70fb7afSTomasz Nowicki int i, err; 2220ffa7d616STomasz Nowicki 2221ffa7d616STomasz Nowicki /* Get distributor base address */ 2222ffa7d616STomasz Nowicki dist = (struct acpi_madt_generic_distributor *)header; 2223611f039fSJulien Grall acpi_data.dist_base = ioremap(dist->base_address, 2224611f039fSJulien Grall ACPI_GICV3_DIST_MEM_SIZE); 2225611f039fSJulien Grall if (!acpi_data.dist_base) { 2226ffa7d616STomasz Nowicki pr_err("Unable to map GICD registers\n"); 2227ffa7d616STomasz Nowicki return -ENOMEM; 2228ffa7d616STomasz Nowicki } 2229ffa7d616STomasz Nowicki 2230611f039fSJulien Grall err = gic_validate_dist_version(acpi_data.dist_base); 2231ffa7d616STomasz Nowicki if (err) { 223271192a68SArvind Yadav pr_err("No distributor detected at @%p, giving up\n", 2233611f039fSJulien Grall acpi_data.dist_base); 2234ffa7d616STomasz Nowicki goto out_dist_unmap; 2235ffa7d616STomasz Nowicki } 2236ffa7d616STomasz Nowicki 2237611f039fSJulien Grall size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2238611f039fSJulien Grall acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2239611f039fSJulien Grall if (!acpi_data.redist_regs) { 2240ffa7d616STomasz Nowicki err = -ENOMEM; 2241ffa7d616STomasz Nowicki goto out_dist_unmap; 2242ffa7d616STomasz Nowicki } 2243ffa7d616STomasz Nowicki 2244b70fb7afSTomasz Nowicki err = gic_acpi_collect_gicr_base(); 2245b70fb7afSTomasz Nowicki if (err) 2246ffa7d616STomasz Nowicki goto out_redist_unmap; 2247ffa7d616STomasz Nowicki 2248eeee0d09SMarc Zyngier domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2249ffa7d616STomasz Nowicki if (!domain_handle) { 2250ffa7d616STomasz Nowicki err = -ENOMEM; 2251ffa7d616STomasz Nowicki goto out_redist_unmap; 2252ffa7d616STomasz Nowicki } 2253ffa7d616STomasz Nowicki 2254611f039fSJulien Grall err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs, 2255611f039fSJulien Grall acpi_data.nr_redist_regions, 0, domain_handle); 2256ffa7d616STomasz Nowicki if (err) 2257ffa7d616STomasz Nowicki goto out_fwhandle_free; 2258ffa7d616STomasz Nowicki 2259ffa7d616STomasz Nowicki acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle); 2260d33a3c8cSChristoffer Dall 2261d01d3274SDavidlohr Bueso if (static_branch_likely(&supports_deactivate_key)) 22621839e576SJulien Grall gic_acpi_setup_kvm_info(); 22631839e576SJulien Grall 2264ffa7d616STomasz Nowicki return 0; 2265ffa7d616STomasz Nowicki 2266ffa7d616STomasz Nowicki out_fwhandle_free: 2267ffa7d616STomasz Nowicki irq_domain_free_fwnode(domain_handle); 2268ffa7d616STomasz Nowicki out_redist_unmap: 2269611f039fSJulien Grall for (i = 0; i < acpi_data.nr_redist_regions; i++) 2270611f039fSJulien Grall if (acpi_data.redist_regs[i].redist_base) 2271611f039fSJulien Grall iounmap(acpi_data.redist_regs[i].redist_base); 2272611f039fSJulien Grall kfree(acpi_data.redist_regs); 2273ffa7d616STomasz Nowicki out_dist_unmap: 2274611f039fSJulien Grall iounmap(acpi_data.dist_base); 2275ffa7d616STomasz Nowicki return err; 2276ffa7d616STomasz Nowicki } 2277ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2278ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2279ffa7d616STomasz Nowicki gic_acpi_init); 2280ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2281ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2282ffa7d616STomasz Nowicki gic_acpi_init); 2283ffa7d616STomasz Nowicki IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2284ffa7d616STomasz Nowicki acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2285ffa7d616STomasz Nowicki gic_acpi_init); 2286ffa7d616STomasz Nowicki #endif 2287