1021f6537SMarc Zyngier /* 2021f6537SMarc Zyngier * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. 3021f6537SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com> 4021f6537SMarc Zyngier * 5021f6537SMarc Zyngier * This program is free software; you can redistribute it and/or modify 6021f6537SMarc Zyngier * it under the terms of the GNU General Public License version 2 as 7021f6537SMarc Zyngier * published by the Free Software Foundation. 8021f6537SMarc Zyngier * 9021f6537SMarc Zyngier * This program is distributed in the hope that it will be useful, 10021f6537SMarc Zyngier * but WITHOUT ANY WARRANTY; without even the implied warranty of 11021f6537SMarc Zyngier * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12021f6537SMarc Zyngier * GNU General Public License for more details. 13021f6537SMarc Zyngier * 14021f6537SMarc Zyngier * You should have received a copy of the GNU General Public License 15021f6537SMarc Zyngier * along with this program. If not, see <http://www.gnu.org/licenses/>. 16021f6537SMarc Zyngier */ 17021f6537SMarc Zyngier 18021f6537SMarc Zyngier #include <linux/cpu.h> 193708d52fSSudeep Holla #include <linux/cpu_pm.h> 20021f6537SMarc Zyngier #include <linux/delay.h> 21021f6537SMarc Zyngier #include <linux/interrupt.h> 22021f6537SMarc Zyngier #include <linux/of.h> 23021f6537SMarc Zyngier #include <linux/of_address.h> 24021f6537SMarc Zyngier #include <linux/of_irq.h> 25021f6537SMarc Zyngier #include <linux/percpu.h> 26021f6537SMarc Zyngier #include <linux/slab.h> 27021f6537SMarc Zyngier 2841a83e06SJoel Porquet #include <linux/irqchip.h> 29021f6537SMarc Zyngier #include <linux/irqchip/arm-gic-v3.h> 30021f6537SMarc Zyngier 31021f6537SMarc Zyngier #include <asm/cputype.h> 32021f6537SMarc Zyngier #include <asm/exception.h> 33021f6537SMarc Zyngier #include <asm/smp_plat.h> 340b6a3da9SMarc Zyngier #include <asm/virt.h> 35021f6537SMarc Zyngier 36021f6537SMarc Zyngier #include "irq-gic-common.h" 37021f6537SMarc Zyngier 38f5c1434cSMarc Zyngier struct redist_region { 39f5c1434cSMarc Zyngier void __iomem *redist_base; 40f5c1434cSMarc Zyngier phys_addr_t phys_base; 41f5c1434cSMarc Zyngier }; 42f5c1434cSMarc Zyngier 43021f6537SMarc Zyngier struct gic_chip_data { 44021f6537SMarc Zyngier void __iomem *dist_base; 45f5c1434cSMarc Zyngier struct redist_region *redist_regions; 46f5c1434cSMarc Zyngier struct rdists rdists; 47021f6537SMarc Zyngier struct irq_domain *domain; 48021f6537SMarc Zyngier u64 redist_stride; 49f5c1434cSMarc Zyngier u32 nr_redist_regions; 50021f6537SMarc Zyngier unsigned int irq_nr; 51021f6537SMarc Zyngier }; 52021f6537SMarc Zyngier 53021f6537SMarc Zyngier static struct gic_chip_data gic_data __read_mostly; 540b6a3da9SMarc Zyngier static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE; 55021f6537SMarc Zyngier 56f5c1434cSMarc Zyngier #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 57f5c1434cSMarc Zyngier #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 58021f6537SMarc Zyngier #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 59021f6537SMarc Zyngier 60021f6537SMarc Zyngier /* Our default, arbitrary priority value. Linux only uses one anyway. */ 61021f6537SMarc Zyngier #define DEFAULT_PMR_VALUE 0xf0 62021f6537SMarc Zyngier 63021f6537SMarc Zyngier static inline unsigned int gic_irq(struct irq_data *d) 64021f6537SMarc Zyngier { 65021f6537SMarc Zyngier return d->hwirq; 66021f6537SMarc Zyngier } 67021f6537SMarc Zyngier 68021f6537SMarc Zyngier static inline int gic_irq_in_rdist(struct irq_data *d) 69021f6537SMarc Zyngier { 70021f6537SMarc Zyngier return gic_irq(d) < 32; 71021f6537SMarc Zyngier } 72021f6537SMarc Zyngier 73021f6537SMarc Zyngier static inline void __iomem *gic_dist_base(struct irq_data *d) 74021f6537SMarc Zyngier { 75021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */ 76021f6537SMarc Zyngier return gic_data_rdist_sgi_base(); 77021f6537SMarc Zyngier 78021f6537SMarc Zyngier if (d->hwirq <= 1023) /* SPI -> dist_base */ 79021f6537SMarc Zyngier return gic_data.dist_base; 80021f6537SMarc Zyngier 81021f6537SMarc Zyngier return NULL; 82021f6537SMarc Zyngier } 83021f6537SMarc Zyngier 84021f6537SMarc Zyngier static void gic_do_wait_for_rwp(void __iomem *base) 85021f6537SMarc Zyngier { 86021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 87021f6537SMarc Zyngier 88021f6537SMarc Zyngier while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) { 89021f6537SMarc Zyngier count--; 90021f6537SMarc Zyngier if (!count) { 91021f6537SMarc Zyngier pr_err_ratelimited("RWP timeout, gone fishing\n"); 92021f6537SMarc Zyngier return; 93021f6537SMarc Zyngier } 94021f6537SMarc Zyngier cpu_relax(); 95021f6537SMarc Zyngier udelay(1); 96021f6537SMarc Zyngier }; 97021f6537SMarc Zyngier } 98021f6537SMarc Zyngier 99021f6537SMarc Zyngier /* Wait for completion of a distributor change */ 100021f6537SMarc Zyngier static void gic_dist_wait_for_rwp(void) 101021f6537SMarc Zyngier { 102021f6537SMarc Zyngier gic_do_wait_for_rwp(gic_data.dist_base); 103021f6537SMarc Zyngier } 104021f6537SMarc Zyngier 105021f6537SMarc Zyngier /* Wait for completion of a redistributor change */ 106021f6537SMarc Zyngier static void gic_redist_wait_for_rwp(void) 107021f6537SMarc Zyngier { 108021f6537SMarc Zyngier gic_do_wait_for_rwp(gic_data_rdist_rd_base()); 109021f6537SMarc Zyngier } 110021f6537SMarc Zyngier 111021f6537SMarc Zyngier /* Low level accessors */ 1126d4e11c5SRobert Richter static u64 gic_read_iar_common(void) 113021f6537SMarc Zyngier { 114021f6537SMarc Zyngier u64 irqstat; 115021f6537SMarc Zyngier 11672c58395SCatalin Marinas asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat)); 117021f6537SMarc Zyngier return irqstat; 118021f6537SMarc Zyngier } 119021f6537SMarc Zyngier 1206d4e11c5SRobert Richter /* 1216d4e11c5SRobert Richter * Cavium ThunderX erratum 23154 1226d4e11c5SRobert Richter * 1236d4e11c5SRobert Richter * The gicv3 of ThunderX requires a modified version for reading the 1246d4e11c5SRobert Richter * IAR status to ensure data synchronization (access to icc_iar1_el1 1256d4e11c5SRobert Richter * is not sync'ed before and after). 1266d4e11c5SRobert Richter */ 1276d4e11c5SRobert Richter static u64 gic_read_iar_cavium_thunderx(void) 1286d4e11c5SRobert Richter { 1296d4e11c5SRobert Richter u64 irqstat; 1306d4e11c5SRobert Richter 1316d4e11c5SRobert Richter asm volatile( 1326d4e11c5SRobert Richter "nop;nop;nop;nop\n\t" 1336d4e11c5SRobert Richter "nop;nop;nop;nop\n\t" 1346d4e11c5SRobert Richter "mrs_s %0, " __stringify(ICC_IAR1_EL1) "\n\t" 1356d4e11c5SRobert Richter "nop;nop;nop;nop" 1366d4e11c5SRobert Richter : "=r" (irqstat)); 1376d4e11c5SRobert Richter mb(); 1386d4e11c5SRobert Richter 1396d4e11c5SRobert Richter return irqstat; 1406d4e11c5SRobert Richter } 1416d4e11c5SRobert Richter 1426d4e11c5SRobert Richter static struct static_key is_cavium_thunderx = STATIC_KEY_INIT_FALSE; 1436d4e11c5SRobert Richter 1446d4e11c5SRobert Richter static u64 __maybe_unused gic_read_iar(void) 1456d4e11c5SRobert Richter { 1466d4e11c5SRobert Richter if (static_key_false(&is_cavium_thunderx)) 1476d4e11c5SRobert Richter return gic_read_iar_cavium_thunderx(); 1486d4e11c5SRobert Richter else 1496d4e11c5SRobert Richter return gic_read_iar_common(); 1506d4e11c5SRobert Richter } 1516d4e11c5SRobert Richter 152c44e9d77SMark Brown static void __maybe_unused gic_write_pmr(u64 val) 153021f6537SMarc Zyngier { 15472c58395SCatalin Marinas asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val)); 155021f6537SMarc Zyngier } 156021f6537SMarc Zyngier 157c44e9d77SMark Brown static void __maybe_unused gic_write_ctlr(u64 val) 158021f6537SMarc Zyngier { 15972c58395SCatalin Marinas asm volatile("msr_s " __stringify(ICC_CTLR_EL1) ", %0" : : "r" (val)); 160021f6537SMarc Zyngier isb(); 161021f6537SMarc Zyngier } 162021f6537SMarc Zyngier 163c44e9d77SMark Brown static void __maybe_unused gic_write_grpen1(u64 val) 164021f6537SMarc Zyngier { 16572c58395SCatalin Marinas asm volatile("msr_s " __stringify(ICC_GRPEN1_EL1) ", %0" : : "r" (val)); 166021f6537SMarc Zyngier isb(); 167021f6537SMarc Zyngier } 168021f6537SMarc Zyngier 169c44e9d77SMark Brown static void __maybe_unused gic_write_sgi1r(u64 val) 170021f6537SMarc Zyngier { 17172c58395SCatalin Marinas asm volatile("msr_s " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val)); 172021f6537SMarc Zyngier } 173021f6537SMarc Zyngier 174021f6537SMarc Zyngier static void gic_enable_sre(void) 175021f6537SMarc Zyngier { 176021f6537SMarc Zyngier u64 val; 177021f6537SMarc Zyngier 17872c58395SCatalin Marinas asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val)); 179021f6537SMarc Zyngier val |= ICC_SRE_EL1_SRE; 18072c58395SCatalin Marinas asm volatile("msr_s " __stringify(ICC_SRE_EL1) ", %0" : : "r" (val)); 181021f6537SMarc Zyngier isb(); 182021f6537SMarc Zyngier 183021f6537SMarc Zyngier /* 184021f6537SMarc Zyngier * Need to check that the SRE bit has actually been set. If 185021f6537SMarc Zyngier * not, it means that SRE is disabled at EL2. We're going to 186021f6537SMarc Zyngier * die painfully, and there is nothing we can do about it. 187021f6537SMarc Zyngier * 188021f6537SMarc Zyngier * Kindly inform the luser. 189021f6537SMarc Zyngier */ 19072c58395SCatalin Marinas asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val)); 191021f6537SMarc Zyngier if (!(val & ICC_SRE_EL1_SRE)) 192021f6537SMarc Zyngier pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 193021f6537SMarc Zyngier } 194021f6537SMarc Zyngier 195a2c22510SSudeep Holla static void gic_enable_redist(bool enable) 196021f6537SMarc Zyngier { 197021f6537SMarc Zyngier void __iomem *rbase; 198021f6537SMarc Zyngier u32 count = 1000000; /* 1s! */ 199021f6537SMarc Zyngier u32 val; 200021f6537SMarc Zyngier 201021f6537SMarc Zyngier rbase = gic_data_rdist_rd_base(); 202021f6537SMarc Zyngier 203021f6537SMarc Zyngier val = readl_relaxed(rbase + GICR_WAKER); 204a2c22510SSudeep Holla if (enable) 205a2c22510SSudeep Holla /* Wake up this CPU redistributor */ 206021f6537SMarc Zyngier val &= ~GICR_WAKER_ProcessorSleep; 207a2c22510SSudeep Holla else 208a2c22510SSudeep Holla val |= GICR_WAKER_ProcessorSleep; 209021f6537SMarc Zyngier writel_relaxed(val, rbase + GICR_WAKER); 210021f6537SMarc Zyngier 211a2c22510SSudeep Holla if (!enable) { /* Check that GICR_WAKER is writeable */ 212a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 213a2c22510SSudeep Holla if (!(val & GICR_WAKER_ProcessorSleep)) 214a2c22510SSudeep Holla return; /* No PM support in this redistributor */ 215021f6537SMarc Zyngier } 216a2c22510SSudeep Holla 217a2c22510SSudeep Holla while (count--) { 218a2c22510SSudeep Holla val = readl_relaxed(rbase + GICR_WAKER); 219a2c22510SSudeep Holla if (enable ^ (val & GICR_WAKER_ChildrenAsleep)) 220a2c22510SSudeep Holla break; 221021f6537SMarc Zyngier cpu_relax(); 222021f6537SMarc Zyngier udelay(1); 223021f6537SMarc Zyngier }; 224a2c22510SSudeep Holla if (!count) 225a2c22510SSudeep Holla pr_err_ratelimited("redistributor failed to %s...\n", 226a2c22510SSudeep Holla enable ? "wakeup" : "sleep"); 227021f6537SMarc Zyngier } 228021f6537SMarc Zyngier 229021f6537SMarc Zyngier /* 230021f6537SMarc Zyngier * Routines to disable, enable, EOI and route interrupts 231021f6537SMarc Zyngier */ 232b594c6e2SMarc Zyngier static int gic_peek_irq(struct irq_data *d, u32 offset) 233b594c6e2SMarc Zyngier { 234b594c6e2SMarc Zyngier u32 mask = 1 << (gic_irq(d) % 32); 235b594c6e2SMarc Zyngier void __iomem *base; 236b594c6e2SMarc Zyngier 237b594c6e2SMarc Zyngier if (gic_irq_in_rdist(d)) 238b594c6e2SMarc Zyngier base = gic_data_rdist_sgi_base(); 239b594c6e2SMarc Zyngier else 240b594c6e2SMarc Zyngier base = gic_data.dist_base; 241b594c6e2SMarc Zyngier 242b594c6e2SMarc Zyngier return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask); 243b594c6e2SMarc Zyngier } 244b594c6e2SMarc Zyngier 245021f6537SMarc Zyngier static void gic_poke_irq(struct irq_data *d, u32 offset) 246021f6537SMarc Zyngier { 247021f6537SMarc Zyngier u32 mask = 1 << (gic_irq(d) % 32); 248021f6537SMarc Zyngier void (*rwp_wait)(void); 249021f6537SMarc Zyngier void __iomem *base; 250021f6537SMarc Zyngier 251021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) { 252021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 253021f6537SMarc Zyngier rwp_wait = gic_redist_wait_for_rwp; 254021f6537SMarc Zyngier } else { 255021f6537SMarc Zyngier base = gic_data.dist_base; 256021f6537SMarc Zyngier rwp_wait = gic_dist_wait_for_rwp; 257021f6537SMarc Zyngier } 258021f6537SMarc Zyngier 259021f6537SMarc Zyngier writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4); 260021f6537SMarc Zyngier rwp_wait(); 261021f6537SMarc Zyngier } 262021f6537SMarc Zyngier 263021f6537SMarc Zyngier static void gic_mask_irq(struct irq_data *d) 264021f6537SMarc Zyngier { 265021f6537SMarc Zyngier gic_poke_irq(d, GICD_ICENABLER); 266021f6537SMarc Zyngier } 267021f6537SMarc Zyngier 2680b6a3da9SMarc Zyngier static void gic_eoimode1_mask_irq(struct irq_data *d) 2690b6a3da9SMarc Zyngier { 2700b6a3da9SMarc Zyngier gic_mask_irq(d); 271530bf353SMarc Zyngier /* 272530bf353SMarc Zyngier * When masking a forwarded interrupt, make sure it is 273530bf353SMarc Zyngier * deactivated as well. 274530bf353SMarc Zyngier * 275530bf353SMarc Zyngier * This ensures that an interrupt that is getting 276530bf353SMarc Zyngier * disabled/masked will not get "stuck", because there is 277530bf353SMarc Zyngier * noone to deactivate it (guest is being terminated). 278530bf353SMarc Zyngier */ 2794df7f54dSThomas Gleixner if (irqd_is_forwarded_to_vcpu(d)) 280530bf353SMarc Zyngier gic_poke_irq(d, GICD_ICACTIVER); 2810b6a3da9SMarc Zyngier } 2820b6a3da9SMarc Zyngier 283021f6537SMarc Zyngier static void gic_unmask_irq(struct irq_data *d) 284021f6537SMarc Zyngier { 285021f6537SMarc Zyngier gic_poke_irq(d, GICD_ISENABLER); 286021f6537SMarc Zyngier } 287021f6537SMarc Zyngier 288b594c6e2SMarc Zyngier static int gic_irq_set_irqchip_state(struct irq_data *d, 289b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool val) 290b594c6e2SMarc Zyngier { 291b594c6e2SMarc Zyngier u32 reg; 292b594c6e2SMarc Zyngier 293b594c6e2SMarc Zyngier if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */ 294b594c6e2SMarc Zyngier return -EINVAL; 295b594c6e2SMarc Zyngier 296b594c6e2SMarc Zyngier switch (which) { 297b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 298b594c6e2SMarc Zyngier reg = val ? GICD_ISPENDR : GICD_ICPENDR; 299b594c6e2SMarc Zyngier break; 300b594c6e2SMarc Zyngier 301b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 302b594c6e2SMarc Zyngier reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 303b594c6e2SMarc Zyngier break; 304b594c6e2SMarc Zyngier 305b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 306b594c6e2SMarc Zyngier reg = val ? GICD_ICENABLER : GICD_ISENABLER; 307b594c6e2SMarc Zyngier break; 308b594c6e2SMarc Zyngier 309b594c6e2SMarc Zyngier default: 310b594c6e2SMarc Zyngier return -EINVAL; 311b594c6e2SMarc Zyngier } 312b594c6e2SMarc Zyngier 313b594c6e2SMarc Zyngier gic_poke_irq(d, reg); 314b594c6e2SMarc Zyngier return 0; 315b594c6e2SMarc Zyngier } 316b594c6e2SMarc Zyngier 317b594c6e2SMarc Zyngier static int gic_irq_get_irqchip_state(struct irq_data *d, 318b594c6e2SMarc Zyngier enum irqchip_irq_state which, bool *val) 319b594c6e2SMarc Zyngier { 320b594c6e2SMarc Zyngier if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */ 321b594c6e2SMarc Zyngier return -EINVAL; 322b594c6e2SMarc Zyngier 323b594c6e2SMarc Zyngier switch (which) { 324b594c6e2SMarc Zyngier case IRQCHIP_STATE_PENDING: 325b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISPENDR); 326b594c6e2SMarc Zyngier break; 327b594c6e2SMarc Zyngier 328b594c6e2SMarc Zyngier case IRQCHIP_STATE_ACTIVE: 329b594c6e2SMarc Zyngier *val = gic_peek_irq(d, GICD_ISACTIVER); 330b594c6e2SMarc Zyngier break; 331b594c6e2SMarc Zyngier 332b594c6e2SMarc Zyngier case IRQCHIP_STATE_MASKED: 333b594c6e2SMarc Zyngier *val = !gic_peek_irq(d, GICD_ISENABLER); 334b594c6e2SMarc Zyngier break; 335b594c6e2SMarc Zyngier 336b594c6e2SMarc Zyngier default: 337b594c6e2SMarc Zyngier return -EINVAL; 338b594c6e2SMarc Zyngier } 339b594c6e2SMarc Zyngier 340b594c6e2SMarc Zyngier return 0; 341b594c6e2SMarc Zyngier } 342b594c6e2SMarc Zyngier 343021f6537SMarc Zyngier static void gic_eoi_irq(struct irq_data *d) 344021f6537SMarc Zyngier { 345021f6537SMarc Zyngier gic_write_eoir(gic_irq(d)); 346021f6537SMarc Zyngier } 347021f6537SMarc Zyngier 3480b6a3da9SMarc Zyngier static void gic_eoimode1_eoi_irq(struct irq_data *d) 3490b6a3da9SMarc Zyngier { 3500b6a3da9SMarc Zyngier /* 351530bf353SMarc Zyngier * No need to deactivate an LPI, or an interrupt that 352530bf353SMarc Zyngier * is is getting forwarded to a vcpu. 3530b6a3da9SMarc Zyngier */ 3544df7f54dSThomas Gleixner if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 3550b6a3da9SMarc Zyngier return; 3560b6a3da9SMarc Zyngier gic_write_dir(gic_irq(d)); 3570b6a3da9SMarc Zyngier } 3580b6a3da9SMarc Zyngier 359021f6537SMarc Zyngier static int gic_set_type(struct irq_data *d, unsigned int type) 360021f6537SMarc Zyngier { 361021f6537SMarc Zyngier unsigned int irq = gic_irq(d); 362021f6537SMarc Zyngier void (*rwp_wait)(void); 363021f6537SMarc Zyngier void __iomem *base; 364021f6537SMarc Zyngier 365021f6537SMarc Zyngier /* Interrupt configuration for SGIs can't be changed */ 366021f6537SMarc Zyngier if (irq < 16) 367021f6537SMarc Zyngier return -EINVAL; 368021f6537SMarc Zyngier 369fb7e7debSLiviu Dudau /* SPIs have restrictions on the supported types */ 370fb7e7debSLiviu Dudau if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && 371fb7e7debSLiviu Dudau type != IRQ_TYPE_EDGE_RISING) 372021f6537SMarc Zyngier return -EINVAL; 373021f6537SMarc Zyngier 374021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) { 375021f6537SMarc Zyngier base = gic_data_rdist_sgi_base(); 376021f6537SMarc Zyngier rwp_wait = gic_redist_wait_for_rwp; 377021f6537SMarc Zyngier } else { 378021f6537SMarc Zyngier base = gic_data.dist_base; 379021f6537SMarc Zyngier rwp_wait = gic_dist_wait_for_rwp; 380021f6537SMarc Zyngier } 381021f6537SMarc Zyngier 382fb7e7debSLiviu Dudau return gic_configure_irq(irq, type, base, rwp_wait); 383021f6537SMarc Zyngier } 384021f6537SMarc Zyngier 385530bf353SMarc Zyngier static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 386530bf353SMarc Zyngier { 3874df7f54dSThomas Gleixner if (vcpu) 3884df7f54dSThomas Gleixner irqd_set_forwarded_to_vcpu(d); 3894df7f54dSThomas Gleixner else 3904df7f54dSThomas Gleixner irqd_clr_forwarded_to_vcpu(d); 391530bf353SMarc Zyngier return 0; 392530bf353SMarc Zyngier } 393530bf353SMarc Zyngier 394021f6537SMarc Zyngier static u64 gic_mpidr_to_affinity(u64 mpidr) 395021f6537SMarc Zyngier { 396021f6537SMarc Zyngier u64 aff; 397021f6537SMarc Zyngier 398021f6537SMarc Zyngier aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 399021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 400021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 401021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 402021f6537SMarc Zyngier 403021f6537SMarc Zyngier return aff; 404021f6537SMarc Zyngier } 405021f6537SMarc Zyngier 406021f6537SMarc Zyngier static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 407021f6537SMarc Zyngier { 408021f6537SMarc Zyngier u64 irqnr; 409021f6537SMarc Zyngier 410021f6537SMarc Zyngier do { 411021f6537SMarc Zyngier irqnr = gic_read_iar(); 412021f6537SMarc Zyngier 413da33f31dSMarc Zyngier if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) { 414ebc6de00SMarc Zyngier int err; 4150b6a3da9SMarc Zyngier 4160b6a3da9SMarc Zyngier if (static_key_true(&supports_deactivate)) 4170b6a3da9SMarc Zyngier gic_write_eoir(irqnr); 4180b6a3da9SMarc Zyngier 419ebc6de00SMarc Zyngier err = handle_domain_irq(gic_data.domain, irqnr, regs); 420ebc6de00SMarc Zyngier if (err) { 421da33f31dSMarc Zyngier WARN_ONCE(true, "Unexpected interrupt received!\n"); 4220b6a3da9SMarc Zyngier if (static_key_true(&supports_deactivate)) { 4230b6a3da9SMarc Zyngier if (irqnr < 8192) 4240b6a3da9SMarc Zyngier gic_write_dir(irqnr); 4250b6a3da9SMarc Zyngier } else { 426021f6537SMarc Zyngier gic_write_eoir(irqnr); 427021f6537SMarc Zyngier } 4280b6a3da9SMarc Zyngier } 429ebc6de00SMarc Zyngier continue; 430ebc6de00SMarc Zyngier } 431021f6537SMarc Zyngier if (irqnr < 16) { 432021f6537SMarc Zyngier gic_write_eoir(irqnr); 4330b6a3da9SMarc Zyngier if (static_key_true(&supports_deactivate)) 4340b6a3da9SMarc Zyngier gic_write_dir(irqnr); 435021f6537SMarc Zyngier #ifdef CONFIG_SMP 436021f6537SMarc Zyngier handle_IPI(irqnr, regs); 437021f6537SMarc Zyngier #else 438021f6537SMarc Zyngier WARN_ONCE(true, "Unexpected SGI received!\n"); 439021f6537SMarc Zyngier #endif 440021f6537SMarc Zyngier continue; 441021f6537SMarc Zyngier } 442021f6537SMarc Zyngier } while (irqnr != ICC_IAR1_EL1_SPURIOUS); 443021f6537SMarc Zyngier } 444021f6537SMarc Zyngier 445021f6537SMarc Zyngier static void __init gic_dist_init(void) 446021f6537SMarc Zyngier { 447021f6537SMarc Zyngier unsigned int i; 448021f6537SMarc Zyngier u64 affinity; 449021f6537SMarc Zyngier void __iomem *base = gic_data.dist_base; 450021f6537SMarc Zyngier 451021f6537SMarc Zyngier /* Disable the distributor */ 452021f6537SMarc Zyngier writel_relaxed(0, base + GICD_CTLR); 453021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 454021f6537SMarc Zyngier 455021f6537SMarc Zyngier gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp); 456021f6537SMarc Zyngier 457021f6537SMarc Zyngier /* Enable distributor with ARE, Group1 */ 458021f6537SMarc Zyngier writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, 459021f6537SMarc Zyngier base + GICD_CTLR); 460021f6537SMarc Zyngier 461021f6537SMarc Zyngier /* 462021f6537SMarc Zyngier * Set all global interrupts to the boot CPU only. ARE must be 463021f6537SMarc Zyngier * enabled. 464021f6537SMarc Zyngier */ 465021f6537SMarc Zyngier affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 466021f6537SMarc Zyngier for (i = 32; i < gic_data.irq_nr; i++) 467021f6537SMarc Zyngier writeq_relaxed(affinity, base + GICD_IROUTER + i * 8); 468021f6537SMarc Zyngier } 469021f6537SMarc Zyngier 470021f6537SMarc Zyngier static int gic_populate_rdist(void) 471021f6537SMarc Zyngier { 472021f6537SMarc Zyngier u64 mpidr = cpu_logical_map(smp_processor_id()); 473021f6537SMarc Zyngier u64 typer; 474021f6537SMarc Zyngier u32 aff; 475021f6537SMarc Zyngier int i; 476021f6537SMarc Zyngier 477021f6537SMarc Zyngier /* 478021f6537SMarc Zyngier * Convert affinity to a 32bit value that can be matched to 479021f6537SMarc Zyngier * GICR_TYPER bits [63:32]. 480021f6537SMarc Zyngier */ 481021f6537SMarc Zyngier aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 482021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 483021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 484021f6537SMarc Zyngier MPIDR_AFFINITY_LEVEL(mpidr, 0)); 485021f6537SMarc Zyngier 486f5c1434cSMarc Zyngier for (i = 0; i < gic_data.nr_redist_regions; i++) { 487f5c1434cSMarc Zyngier void __iomem *ptr = gic_data.redist_regions[i].redist_base; 488021f6537SMarc Zyngier u32 reg; 489021f6537SMarc Zyngier 490021f6537SMarc Zyngier reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 491021f6537SMarc Zyngier if (reg != GIC_PIDR2_ARCH_GICv3 && 492021f6537SMarc Zyngier reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 493021f6537SMarc Zyngier pr_warn("No redistributor present @%p\n", ptr); 494021f6537SMarc Zyngier break; 495021f6537SMarc Zyngier } 496021f6537SMarc Zyngier 497021f6537SMarc Zyngier do { 498021f6537SMarc Zyngier typer = readq_relaxed(ptr + GICR_TYPER); 499021f6537SMarc Zyngier if ((typer >> 32) == aff) { 500f5c1434cSMarc Zyngier u64 offset = ptr - gic_data.redist_regions[i].redist_base; 501021f6537SMarc Zyngier gic_data_rdist_rd_base() = ptr; 502f5c1434cSMarc Zyngier gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset; 503f5c1434cSMarc Zyngier pr_info("CPU%d: found redistributor %llx region %d:%pa\n", 504021f6537SMarc Zyngier smp_processor_id(), 505f5c1434cSMarc Zyngier (unsigned long long)mpidr, 506f5c1434cSMarc Zyngier i, &gic_data_rdist()->phys_base); 507021f6537SMarc Zyngier return 0; 508021f6537SMarc Zyngier } 509021f6537SMarc Zyngier 510021f6537SMarc Zyngier if (gic_data.redist_stride) { 511021f6537SMarc Zyngier ptr += gic_data.redist_stride; 512021f6537SMarc Zyngier } else { 513021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 514021f6537SMarc Zyngier if (typer & GICR_TYPER_VLPIS) 515021f6537SMarc Zyngier ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 516021f6537SMarc Zyngier } 517021f6537SMarc Zyngier } while (!(typer & GICR_TYPER_LAST)); 518021f6537SMarc Zyngier } 519021f6537SMarc Zyngier 520021f6537SMarc Zyngier /* We couldn't even deal with ourselves... */ 521021f6537SMarc Zyngier WARN(true, "CPU%d: mpidr %llx has no re-distributor!\n", 522021f6537SMarc Zyngier smp_processor_id(), (unsigned long long)mpidr); 523021f6537SMarc Zyngier return -ENODEV; 524021f6537SMarc Zyngier } 525021f6537SMarc Zyngier 5263708d52fSSudeep Holla static void gic_cpu_sys_reg_init(void) 527021f6537SMarc Zyngier { 528021f6537SMarc Zyngier /* Enable system registers */ 529021f6537SMarc Zyngier gic_enable_sre(); 530021f6537SMarc Zyngier 531021f6537SMarc Zyngier /* Set priority mask register */ 532021f6537SMarc Zyngier gic_write_pmr(DEFAULT_PMR_VALUE); 533021f6537SMarc Zyngier 5340b6a3da9SMarc Zyngier if (static_key_true(&supports_deactivate)) { 5350b6a3da9SMarc Zyngier /* EOI drops priority only (mode 1) */ 5360b6a3da9SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 5370b6a3da9SMarc Zyngier } else { 538021f6537SMarc Zyngier /* EOI deactivates interrupt too (mode 0) */ 539021f6537SMarc Zyngier gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 5400b6a3da9SMarc Zyngier } 541021f6537SMarc Zyngier 542021f6537SMarc Zyngier /* ... and let's hit the road... */ 543021f6537SMarc Zyngier gic_write_grpen1(1); 544021f6537SMarc Zyngier } 545021f6537SMarc Zyngier 546da33f31dSMarc Zyngier static int gic_dist_supports_lpis(void) 547da33f31dSMarc Zyngier { 548da33f31dSMarc Zyngier return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS); 549da33f31dSMarc Zyngier } 550da33f31dSMarc Zyngier 551021f6537SMarc Zyngier static void gic_cpu_init(void) 552021f6537SMarc Zyngier { 553021f6537SMarc Zyngier void __iomem *rbase; 554021f6537SMarc Zyngier 555021f6537SMarc Zyngier /* Register ourselves with the rest of the world */ 556021f6537SMarc Zyngier if (gic_populate_rdist()) 557021f6537SMarc Zyngier return; 558021f6537SMarc Zyngier 559a2c22510SSudeep Holla gic_enable_redist(true); 560021f6537SMarc Zyngier 561021f6537SMarc Zyngier rbase = gic_data_rdist_sgi_base(); 562021f6537SMarc Zyngier 563021f6537SMarc Zyngier gic_cpu_config(rbase, gic_redist_wait_for_rwp); 564021f6537SMarc Zyngier 565da33f31dSMarc Zyngier /* Give LPIs a spin */ 566da33f31dSMarc Zyngier if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis()) 567da33f31dSMarc Zyngier its_cpu_init(); 568da33f31dSMarc Zyngier 5693708d52fSSudeep Holla /* initialise system registers */ 5703708d52fSSudeep Holla gic_cpu_sys_reg_init(); 571021f6537SMarc Zyngier } 572021f6537SMarc Zyngier 573021f6537SMarc Zyngier #ifdef CONFIG_SMP 574021f6537SMarc Zyngier static int gic_secondary_init(struct notifier_block *nfb, 575021f6537SMarc Zyngier unsigned long action, void *hcpu) 576021f6537SMarc Zyngier { 577021f6537SMarc Zyngier if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) 578021f6537SMarc Zyngier gic_cpu_init(); 579021f6537SMarc Zyngier return NOTIFY_OK; 580021f6537SMarc Zyngier } 581021f6537SMarc Zyngier 582021f6537SMarc Zyngier /* 583021f6537SMarc Zyngier * Notifier for enabling the GIC CPU interface. Set an arbitrarily high 584021f6537SMarc Zyngier * priority because the GIC needs to be up before the ARM generic timers. 585021f6537SMarc Zyngier */ 586021f6537SMarc Zyngier static struct notifier_block gic_cpu_notifier = { 587021f6537SMarc Zyngier .notifier_call = gic_secondary_init, 588021f6537SMarc Zyngier .priority = 100, 589021f6537SMarc Zyngier }; 590021f6537SMarc Zyngier 591021f6537SMarc Zyngier static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 592021f6537SMarc Zyngier u64 cluster_id) 593021f6537SMarc Zyngier { 594021f6537SMarc Zyngier int cpu = *base_cpu; 595021f6537SMarc Zyngier u64 mpidr = cpu_logical_map(cpu); 596021f6537SMarc Zyngier u16 tlist = 0; 597021f6537SMarc Zyngier 598021f6537SMarc Zyngier while (cpu < nr_cpu_ids) { 599021f6537SMarc Zyngier /* 600021f6537SMarc Zyngier * If we ever get a cluster of more than 16 CPUs, just 601021f6537SMarc Zyngier * scream and skip that CPU. 602021f6537SMarc Zyngier */ 603021f6537SMarc Zyngier if (WARN_ON((mpidr & 0xff) >= 16)) 604021f6537SMarc Zyngier goto out; 605021f6537SMarc Zyngier 606021f6537SMarc Zyngier tlist |= 1 << (mpidr & 0xf); 607021f6537SMarc Zyngier 608021f6537SMarc Zyngier cpu = cpumask_next(cpu, mask); 609614be385SVladimir Murzin if (cpu >= nr_cpu_ids) 610021f6537SMarc Zyngier goto out; 611021f6537SMarc Zyngier 612021f6537SMarc Zyngier mpidr = cpu_logical_map(cpu); 613021f6537SMarc Zyngier 614021f6537SMarc Zyngier if (cluster_id != (mpidr & ~0xffUL)) { 615021f6537SMarc Zyngier cpu--; 616021f6537SMarc Zyngier goto out; 617021f6537SMarc Zyngier } 618021f6537SMarc Zyngier } 619021f6537SMarc Zyngier out: 620021f6537SMarc Zyngier *base_cpu = cpu; 621021f6537SMarc Zyngier return tlist; 622021f6537SMarc Zyngier } 623021f6537SMarc Zyngier 6247e580278SAndre Przywara #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 6257e580278SAndre Przywara (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 6267e580278SAndre Przywara << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 6277e580278SAndre Przywara 628021f6537SMarc Zyngier static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 629021f6537SMarc Zyngier { 630021f6537SMarc Zyngier u64 val; 631021f6537SMarc Zyngier 6327e580278SAndre Przywara val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 6337e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 6347e580278SAndre Przywara irq << ICC_SGI1R_SGI_ID_SHIFT | 6357e580278SAndre Przywara MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 6367e580278SAndre Przywara tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 637021f6537SMarc Zyngier 638021f6537SMarc Zyngier pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 639021f6537SMarc Zyngier gic_write_sgi1r(val); 640021f6537SMarc Zyngier } 641021f6537SMarc Zyngier 642021f6537SMarc Zyngier static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) 643021f6537SMarc Zyngier { 644021f6537SMarc Zyngier int cpu; 645021f6537SMarc Zyngier 646021f6537SMarc Zyngier if (WARN_ON(irq >= 16)) 647021f6537SMarc Zyngier return; 648021f6537SMarc Zyngier 649021f6537SMarc Zyngier /* 650021f6537SMarc Zyngier * Ensure that stores to Normal memory are visible to the 651021f6537SMarc Zyngier * other CPUs before issuing the IPI. 652021f6537SMarc Zyngier */ 653021f6537SMarc Zyngier smp_wmb(); 654021f6537SMarc Zyngier 655f9b531feSRusty Russell for_each_cpu(cpu, mask) { 656021f6537SMarc Zyngier u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL; 657021f6537SMarc Zyngier u16 tlist; 658021f6537SMarc Zyngier 659021f6537SMarc Zyngier tlist = gic_compute_target_list(&cpu, mask, cluster_id); 660021f6537SMarc Zyngier gic_send_sgi(cluster_id, tlist, irq); 661021f6537SMarc Zyngier } 662021f6537SMarc Zyngier 663021f6537SMarc Zyngier /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 664021f6537SMarc Zyngier isb(); 665021f6537SMarc Zyngier } 666021f6537SMarc Zyngier 667021f6537SMarc Zyngier static void gic_smp_init(void) 668021f6537SMarc Zyngier { 669021f6537SMarc Zyngier set_smp_cross_call(gic_raise_softirq); 670021f6537SMarc Zyngier register_cpu_notifier(&gic_cpu_notifier); 671021f6537SMarc Zyngier } 672021f6537SMarc Zyngier 673021f6537SMarc Zyngier static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 674021f6537SMarc Zyngier bool force) 675021f6537SMarc Zyngier { 676021f6537SMarc Zyngier unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask); 677021f6537SMarc Zyngier void __iomem *reg; 678021f6537SMarc Zyngier int enabled; 679021f6537SMarc Zyngier u64 val; 680021f6537SMarc Zyngier 681021f6537SMarc Zyngier if (gic_irq_in_rdist(d)) 682021f6537SMarc Zyngier return -EINVAL; 683021f6537SMarc Zyngier 684021f6537SMarc Zyngier /* If interrupt was enabled, disable it first */ 685021f6537SMarc Zyngier enabled = gic_peek_irq(d, GICD_ISENABLER); 686021f6537SMarc Zyngier if (enabled) 687021f6537SMarc Zyngier gic_mask_irq(d); 688021f6537SMarc Zyngier 689021f6537SMarc Zyngier reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8); 690021f6537SMarc Zyngier val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 691021f6537SMarc Zyngier 692021f6537SMarc Zyngier writeq_relaxed(val, reg); 693021f6537SMarc Zyngier 694021f6537SMarc Zyngier /* 695021f6537SMarc Zyngier * If the interrupt was enabled, enabled it again. Otherwise, 696021f6537SMarc Zyngier * just wait for the distributor to have digested our changes. 697021f6537SMarc Zyngier */ 698021f6537SMarc Zyngier if (enabled) 699021f6537SMarc Zyngier gic_unmask_irq(d); 700021f6537SMarc Zyngier else 701021f6537SMarc Zyngier gic_dist_wait_for_rwp(); 702021f6537SMarc Zyngier 703021f6537SMarc Zyngier return IRQ_SET_MASK_OK; 704021f6537SMarc Zyngier } 705021f6537SMarc Zyngier #else 706021f6537SMarc Zyngier #define gic_set_affinity NULL 707021f6537SMarc Zyngier #define gic_smp_init() do { } while(0) 708021f6537SMarc Zyngier #endif 709021f6537SMarc Zyngier 7103708d52fSSudeep Holla #ifdef CONFIG_CPU_PM 7113708d52fSSudeep Holla static int gic_cpu_pm_notifier(struct notifier_block *self, 7123708d52fSSudeep Holla unsigned long cmd, void *v) 7133708d52fSSudeep Holla { 7143708d52fSSudeep Holla if (cmd == CPU_PM_EXIT) { 7153708d52fSSudeep Holla gic_enable_redist(true); 7163708d52fSSudeep Holla gic_cpu_sys_reg_init(); 7173708d52fSSudeep Holla } else if (cmd == CPU_PM_ENTER) { 7183708d52fSSudeep Holla gic_write_grpen1(0); 7193708d52fSSudeep Holla gic_enable_redist(false); 7203708d52fSSudeep Holla } 7213708d52fSSudeep Holla return NOTIFY_OK; 7223708d52fSSudeep Holla } 7233708d52fSSudeep Holla 7243708d52fSSudeep Holla static struct notifier_block gic_cpu_pm_notifier_block = { 7253708d52fSSudeep Holla .notifier_call = gic_cpu_pm_notifier, 7263708d52fSSudeep Holla }; 7273708d52fSSudeep Holla 7283708d52fSSudeep Holla static void gic_cpu_pm_init(void) 7293708d52fSSudeep Holla { 7303708d52fSSudeep Holla cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 7313708d52fSSudeep Holla } 7323708d52fSSudeep Holla 7333708d52fSSudeep Holla #else 7343708d52fSSudeep Holla static inline void gic_cpu_pm_init(void) { } 7353708d52fSSudeep Holla #endif /* CONFIG_CPU_PM */ 7363708d52fSSudeep Holla 737021f6537SMarc Zyngier static struct irq_chip gic_chip = { 738021f6537SMarc Zyngier .name = "GICv3", 739021f6537SMarc Zyngier .irq_mask = gic_mask_irq, 740021f6537SMarc Zyngier .irq_unmask = gic_unmask_irq, 741021f6537SMarc Zyngier .irq_eoi = gic_eoi_irq, 742021f6537SMarc Zyngier .irq_set_type = gic_set_type, 743021f6537SMarc Zyngier .irq_set_affinity = gic_set_affinity, 744b594c6e2SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 745b594c6e2SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 74655963c9fSSudeep Holla .flags = IRQCHIP_SET_TYPE_MASKED, 747021f6537SMarc Zyngier }; 748021f6537SMarc Zyngier 7490b6a3da9SMarc Zyngier static struct irq_chip gic_eoimode1_chip = { 7500b6a3da9SMarc Zyngier .name = "GICv3", 7510b6a3da9SMarc Zyngier .irq_mask = gic_eoimode1_mask_irq, 7520b6a3da9SMarc Zyngier .irq_unmask = gic_unmask_irq, 7530b6a3da9SMarc Zyngier .irq_eoi = gic_eoimode1_eoi_irq, 7540b6a3da9SMarc Zyngier .irq_set_type = gic_set_type, 7550b6a3da9SMarc Zyngier .irq_set_affinity = gic_set_affinity, 7560b6a3da9SMarc Zyngier .irq_get_irqchip_state = gic_irq_get_irqchip_state, 7570b6a3da9SMarc Zyngier .irq_set_irqchip_state = gic_irq_set_irqchip_state, 758530bf353SMarc Zyngier .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 7590b6a3da9SMarc Zyngier .flags = IRQCHIP_SET_TYPE_MASKED, 7600b6a3da9SMarc Zyngier }; 7610b6a3da9SMarc Zyngier 762da33f31dSMarc Zyngier #define GIC_ID_NR (1U << gic_data.rdists.id_bits) 763da33f31dSMarc Zyngier 764021f6537SMarc Zyngier static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 765021f6537SMarc Zyngier irq_hw_number_t hw) 766021f6537SMarc Zyngier { 7670b6a3da9SMarc Zyngier struct irq_chip *chip = &gic_chip; 7680b6a3da9SMarc Zyngier 7690b6a3da9SMarc Zyngier if (static_key_true(&supports_deactivate)) 7700b6a3da9SMarc Zyngier chip = &gic_eoimode1_chip; 7710b6a3da9SMarc Zyngier 772021f6537SMarc Zyngier /* SGIs are private to the core kernel */ 773021f6537SMarc Zyngier if (hw < 16) 774021f6537SMarc Zyngier return -EPERM; 775da33f31dSMarc Zyngier /* Nothing here */ 776da33f31dSMarc Zyngier if (hw >= gic_data.irq_nr && hw < 8192) 777da33f31dSMarc Zyngier return -EPERM; 778da33f31dSMarc Zyngier /* Off limits */ 779da33f31dSMarc Zyngier if (hw >= GIC_ID_NR) 780da33f31dSMarc Zyngier return -EPERM; 781da33f31dSMarc Zyngier 782021f6537SMarc Zyngier /* PPIs */ 783021f6537SMarc Zyngier if (hw < 32) { 784021f6537SMarc Zyngier irq_set_percpu_devid(irq); 7850b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 786443acc4fSMarc Zyngier handle_percpu_devid_irq, NULL, NULL); 787d17cab44SRob Herring irq_set_status_flags(irq, IRQ_NOAUTOEN); 788021f6537SMarc Zyngier } 789021f6537SMarc Zyngier /* SPIs */ 790021f6537SMarc Zyngier if (hw >= 32 && hw < gic_data.irq_nr) { 7910b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 792443acc4fSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 793d17cab44SRob Herring irq_set_probe(irq); 794021f6537SMarc Zyngier } 795da33f31dSMarc Zyngier /* LPIs */ 796da33f31dSMarc Zyngier if (hw >= 8192 && hw < GIC_ID_NR) { 797da33f31dSMarc Zyngier if (!gic_dist_supports_lpis()) 798da33f31dSMarc Zyngier return -EPERM; 7990b6a3da9SMarc Zyngier irq_domain_set_info(d, irq, hw, chip, d->host_data, 800da33f31dSMarc Zyngier handle_fasteoi_irq, NULL, NULL); 801da33f31dSMarc Zyngier } 802da33f31dSMarc Zyngier 803021f6537SMarc Zyngier return 0; 804021f6537SMarc Zyngier } 805021f6537SMarc Zyngier 806021f6537SMarc Zyngier static int gic_irq_domain_xlate(struct irq_domain *d, 807021f6537SMarc Zyngier struct device_node *controller, 808021f6537SMarc Zyngier const u32 *intspec, unsigned int intsize, 809021f6537SMarc Zyngier unsigned long *out_hwirq, unsigned int *out_type) 810021f6537SMarc Zyngier { 811021f6537SMarc Zyngier if (d->of_node != controller) 812021f6537SMarc Zyngier return -EINVAL; 813021f6537SMarc Zyngier if (intsize < 3) 814021f6537SMarc Zyngier return -EINVAL; 815021f6537SMarc Zyngier 816021f6537SMarc Zyngier switch(intspec[0]) { 817021f6537SMarc Zyngier case 0: /* SPI */ 818021f6537SMarc Zyngier *out_hwirq = intspec[1] + 32; 819021f6537SMarc Zyngier break; 820021f6537SMarc Zyngier case 1: /* PPI */ 821021f6537SMarc Zyngier *out_hwirq = intspec[1] + 16; 822021f6537SMarc Zyngier break; 823da33f31dSMarc Zyngier case GIC_IRQ_TYPE_LPI: /* LPI */ 824da33f31dSMarc Zyngier *out_hwirq = intspec[1]; 825da33f31dSMarc Zyngier break; 826021f6537SMarc Zyngier default: 827021f6537SMarc Zyngier return -EINVAL; 828021f6537SMarc Zyngier } 829021f6537SMarc Zyngier 830021f6537SMarc Zyngier *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; 831021f6537SMarc Zyngier return 0; 832021f6537SMarc Zyngier } 833021f6537SMarc Zyngier 834443acc4fSMarc Zyngier static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 835443acc4fSMarc Zyngier unsigned int nr_irqs, void *arg) 836443acc4fSMarc Zyngier { 837443acc4fSMarc Zyngier int i, ret; 838443acc4fSMarc Zyngier irq_hw_number_t hwirq; 839443acc4fSMarc Zyngier unsigned int type = IRQ_TYPE_NONE; 840443acc4fSMarc Zyngier struct of_phandle_args *irq_data = arg; 841443acc4fSMarc Zyngier 842443acc4fSMarc Zyngier ret = gic_irq_domain_xlate(domain, irq_data->np, irq_data->args, 843443acc4fSMarc Zyngier irq_data->args_count, &hwirq, &type); 844443acc4fSMarc Zyngier if (ret) 845443acc4fSMarc Zyngier return ret; 846443acc4fSMarc Zyngier 847443acc4fSMarc Zyngier for (i = 0; i < nr_irqs; i++) 848443acc4fSMarc Zyngier gic_irq_domain_map(domain, virq + i, hwirq + i); 849443acc4fSMarc Zyngier 850443acc4fSMarc Zyngier return 0; 851443acc4fSMarc Zyngier } 852443acc4fSMarc Zyngier 853443acc4fSMarc Zyngier static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 854443acc4fSMarc Zyngier unsigned int nr_irqs) 855443acc4fSMarc Zyngier { 856443acc4fSMarc Zyngier int i; 857443acc4fSMarc Zyngier 858443acc4fSMarc Zyngier for (i = 0; i < nr_irqs; i++) { 859443acc4fSMarc Zyngier struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 860443acc4fSMarc Zyngier irq_set_handler(virq + i, NULL); 861443acc4fSMarc Zyngier irq_domain_reset_irq_data(d); 862443acc4fSMarc Zyngier } 863443acc4fSMarc Zyngier } 864443acc4fSMarc Zyngier 865021f6537SMarc Zyngier static const struct irq_domain_ops gic_irq_domain_ops = { 866021f6537SMarc Zyngier .xlate = gic_irq_domain_xlate, 867443acc4fSMarc Zyngier .alloc = gic_irq_domain_alloc, 868443acc4fSMarc Zyngier .free = gic_irq_domain_free, 869021f6537SMarc Zyngier }; 870021f6537SMarc Zyngier 8716d4e11c5SRobert Richter static void gicv3_enable_quirks(void) 8726d4e11c5SRobert Richter { 8736d4e11c5SRobert Richter if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154)) 8746d4e11c5SRobert Richter static_key_slow_inc(&is_cavium_thunderx); 8756d4e11c5SRobert Richter } 8766d4e11c5SRobert Richter 877021f6537SMarc Zyngier static int __init gic_of_init(struct device_node *node, struct device_node *parent) 878021f6537SMarc Zyngier { 879021f6537SMarc Zyngier void __iomem *dist_base; 880f5c1434cSMarc Zyngier struct redist_region *rdist_regs; 881021f6537SMarc Zyngier u64 redist_stride; 882f5c1434cSMarc Zyngier u32 nr_redist_regions; 883f5c1434cSMarc Zyngier u32 typer; 884021f6537SMarc Zyngier u32 reg; 885021f6537SMarc Zyngier int gic_irqs; 886021f6537SMarc Zyngier int err; 887021f6537SMarc Zyngier int i; 888021f6537SMarc Zyngier 889021f6537SMarc Zyngier dist_base = of_iomap(node, 0); 890021f6537SMarc Zyngier if (!dist_base) { 891021f6537SMarc Zyngier pr_err("%s: unable to map gic dist registers\n", 892021f6537SMarc Zyngier node->full_name); 893021f6537SMarc Zyngier return -ENXIO; 894021f6537SMarc Zyngier } 895021f6537SMarc Zyngier 896021f6537SMarc Zyngier reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 897021f6537SMarc Zyngier if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) { 898021f6537SMarc Zyngier pr_err("%s: no distributor detected, giving up\n", 899021f6537SMarc Zyngier node->full_name); 900021f6537SMarc Zyngier err = -ENODEV; 901021f6537SMarc Zyngier goto out_unmap_dist; 902021f6537SMarc Zyngier } 903021f6537SMarc Zyngier 904f5c1434cSMarc Zyngier if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 905f5c1434cSMarc Zyngier nr_redist_regions = 1; 906021f6537SMarc Zyngier 907f5c1434cSMarc Zyngier rdist_regs = kzalloc(sizeof(*rdist_regs) * nr_redist_regions, GFP_KERNEL); 908f5c1434cSMarc Zyngier if (!rdist_regs) { 909021f6537SMarc Zyngier err = -ENOMEM; 910021f6537SMarc Zyngier goto out_unmap_dist; 911021f6537SMarc Zyngier } 912021f6537SMarc Zyngier 913f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) { 914f5c1434cSMarc Zyngier struct resource res; 915f5c1434cSMarc Zyngier int ret; 916f5c1434cSMarc Zyngier 917f5c1434cSMarc Zyngier ret = of_address_to_resource(node, 1 + i, &res); 918f5c1434cSMarc Zyngier rdist_regs[i].redist_base = of_iomap(node, 1 + i); 919f5c1434cSMarc Zyngier if (ret || !rdist_regs[i].redist_base) { 920021f6537SMarc Zyngier pr_err("%s: couldn't map region %d\n", 921021f6537SMarc Zyngier node->full_name, i); 922021f6537SMarc Zyngier err = -ENODEV; 923021f6537SMarc Zyngier goto out_unmap_rdist; 924021f6537SMarc Zyngier } 925f5c1434cSMarc Zyngier rdist_regs[i].phys_base = res.start; 926021f6537SMarc Zyngier } 927021f6537SMarc Zyngier 928021f6537SMarc Zyngier if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 929021f6537SMarc Zyngier redist_stride = 0; 930021f6537SMarc Zyngier 9310b6a3da9SMarc Zyngier if (!is_hyp_mode_available()) 9320b6a3da9SMarc Zyngier static_key_slow_dec(&supports_deactivate); 9330b6a3da9SMarc Zyngier 9340b6a3da9SMarc Zyngier if (static_key_true(&supports_deactivate)) 9350b6a3da9SMarc Zyngier pr_info("GIC: Using split EOI/Deactivate mode\n"); 9360b6a3da9SMarc Zyngier 937021f6537SMarc Zyngier gic_data.dist_base = dist_base; 938f5c1434cSMarc Zyngier gic_data.redist_regions = rdist_regs; 939f5c1434cSMarc Zyngier gic_data.nr_redist_regions = nr_redist_regions; 940021f6537SMarc Zyngier gic_data.redist_stride = redist_stride; 941021f6537SMarc Zyngier 9426d4e11c5SRobert Richter gicv3_enable_quirks(); 9436d4e11c5SRobert Richter 944021f6537SMarc Zyngier /* 945021f6537SMarc Zyngier * Find out how many interrupts are supported. 946021f6537SMarc Zyngier * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI) 947021f6537SMarc Zyngier */ 948f5c1434cSMarc Zyngier typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 949f5c1434cSMarc Zyngier gic_data.rdists.id_bits = GICD_TYPER_ID_BITS(typer); 950f5c1434cSMarc Zyngier gic_irqs = GICD_TYPER_IRQS(typer); 951021f6537SMarc Zyngier if (gic_irqs > 1020) 952021f6537SMarc Zyngier gic_irqs = 1020; 953021f6537SMarc Zyngier gic_data.irq_nr = gic_irqs; 954021f6537SMarc Zyngier 955021f6537SMarc Zyngier gic_data.domain = irq_domain_add_tree(node, &gic_irq_domain_ops, 956021f6537SMarc Zyngier &gic_data); 957f5c1434cSMarc Zyngier gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 958021f6537SMarc Zyngier 959f5c1434cSMarc Zyngier if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 960021f6537SMarc Zyngier err = -ENOMEM; 961021f6537SMarc Zyngier goto out_free; 962021f6537SMarc Zyngier } 963021f6537SMarc Zyngier 964021f6537SMarc Zyngier set_handle_irq(gic_handle_irq); 965021f6537SMarc Zyngier 966da33f31dSMarc Zyngier if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis()) 967da33f31dSMarc Zyngier its_init(node, &gic_data.rdists, gic_data.domain); 968da33f31dSMarc Zyngier 969021f6537SMarc Zyngier gic_smp_init(); 970021f6537SMarc Zyngier gic_dist_init(); 971021f6537SMarc Zyngier gic_cpu_init(); 9723708d52fSSudeep Holla gic_cpu_pm_init(); 973021f6537SMarc Zyngier 974021f6537SMarc Zyngier return 0; 975021f6537SMarc Zyngier 976021f6537SMarc Zyngier out_free: 977021f6537SMarc Zyngier if (gic_data.domain) 978021f6537SMarc Zyngier irq_domain_remove(gic_data.domain); 979f5c1434cSMarc Zyngier free_percpu(gic_data.rdists.rdist); 980021f6537SMarc Zyngier out_unmap_rdist: 981f5c1434cSMarc Zyngier for (i = 0; i < nr_redist_regions; i++) 982f5c1434cSMarc Zyngier if (rdist_regs[i].redist_base) 983f5c1434cSMarc Zyngier iounmap(rdist_regs[i].redist_base); 984f5c1434cSMarc Zyngier kfree(rdist_regs); 985021f6537SMarc Zyngier out_unmap_dist: 986021f6537SMarc Zyngier iounmap(dist_base); 987021f6537SMarc Zyngier return err; 988021f6537SMarc Zyngier } 989021f6537SMarc Zyngier 990021f6537SMarc Zyngier IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 991