1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #define pr_fmt(fmt) "GICv3: " fmt 8 9 #include <linux/acpi.h> 10 #include <linux/cpu.h> 11 #include <linux/cpu_pm.h> 12 #include <linux/delay.h> 13 #include <linux/interrupt.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kstrtox.h> 16 #include <linux/of.h> 17 #include <linux/of_address.h> 18 #include <linux/of_irq.h> 19 #include <linux/percpu.h> 20 #include <linux/refcount.h> 21 #include <linux/slab.h> 22 23 #include <linux/irqchip.h> 24 #include <linux/irqchip/arm-gic-common.h> 25 #include <linux/irqchip/arm-gic-v3.h> 26 #include <linux/irqchip/irq-partition-percpu.h> 27 #include <linux/bitfield.h> 28 #include <linux/bits.h> 29 #include <linux/arm-smccc.h> 30 31 #include <asm/cputype.h> 32 #include <asm/exception.h> 33 #include <asm/smp_plat.h> 34 #include <asm/virt.h> 35 36 #include "irq-gic-common.h" 37 38 #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 39 40 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 41 #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 42 #define FLAGS_WORKAROUND_MTK_GICR_SAVE (1ULL << 2) 43 #define FLAGS_WORKAROUND_ASR_ERRATUM_8601001 (1ULL << 3) 44 45 #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 46 47 struct redist_region { 48 void __iomem *redist_base; 49 phys_addr_t phys_base; 50 bool single_redist; 51 }; 52 53 struct gic_chip_data { 54 struct fwnode_handle *fwnode; 55 phys_addr_t dist_phys_base; 56 void __iomem *dist_base; 57 struct redist_region *redist_regions; 58 struct rdists rdists; 59 struct irq_domain *domain; 60 u64 redist_stride; 61 u32 nr_redist_regions; 62 u64 flags; 63 bool has_rss; 64 unsigned int ppi_nr; 65 struct partition_desc **ppi_descs; 66 }; 67 68 #define T241_CHIPS_MAX 4 69 static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly; 70 static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum); 71 72 static DEFINE_STATIC_KEY_FALSE(gic_arm64_2941627_erratum); 73 74 static struct gic_chip_data gic_data __read_mostly; 75 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 76 77 #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 78 #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 79 #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 80 81 /* 82 * The behaviours of RPR and PMR registers differ depending on the value of 83 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 84 * distributor and redistributors depends on whether security is enabled in the 85 * GIC. 86 * 87 * When security is enabled, non-secure priority values from the (re)distributor 88 * are presented to the GIC CPUIF as follow: 89 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 90 * 91 * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 92 * EL1 are subject to a similar operation thus matching the priorities presented 93 * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 94 * these values are unchanged by the GIC. 95 * 96 * see GICv3/GICv4 Architecture Specification (IHI0069D): 97 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 98 * priorities. 99 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 100 * interrupt. 101 */ 102 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 103 104 DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 105 EXPORT_SYMBOL(gic_nonsecure_priorities); 106 107 /* 108 * When the Non-secure world has access to group 0 interrupts (as a 109 * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will 110 * return the Distributor's view of the interrupt priority. 111 * 112 * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority 113 * written by software is moved to the Non-secure range by the Distributor. 114 * 115 * If both are true (which is when gic_nonsecure_priorities gets enabled), 116 * we need to shift down the priority programmed by software to match it 117 * against the value returned by ICC_RPR_EL1. 118 */ 119 #define GICD_INT_RPR_PRI(priority) \ 120 ({ \ 121 u32 __priority = (priority); \ 122 if (static_branch_unlikely(&gic_nonsecure_priorities)) \ 123 __priority = 0x80 | (__priority >> 1); \ 124 \ 125 __priority; \ 126 }) 127 128 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 129 static refcount_t *ppi_nmi_refs; 130 131 static struct gic_kvm_info gic_v3_kvm_info __initdata; 132 static DEFINE_PER_CPU(bool, has_rss); 133 134 #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 135 #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 136 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 137 #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 138 139 /* Our default, arbitrary priority value. Linux only uses one anyway. */ 140 #define DEFAULT_PMR_VALUE 0xf0 141 142 enum gic_intid_range { 143 SGI_RANGE, 144 PPI_RANGE, 145 SPI_RANGE, 146 EPPI_RANGE, 147 ESPI_RANGE, 148 LPI_RANGE, 149 __INVALID_RANGE__ 150 }; 151 152 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 153 { 154 switch (hwirq) { 155 case 0 ... 15: 156 return SGI_RANGE; 157 case 16 ... 31: 158 return PPI_RANGE; 159 case 32 ... 1019: 160 return SPI_RANGE; 161 case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 162 return EPPI_RANGE; 163 case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 164 return ESPI_RANGE; 165 case 8192 ... GENMASK(23, 0): 166 return LPI_RANGE; 167 default: 168 return __INVALID_RANGE__; 169 } 170 } 171 172 static enum gic_intid_range get_intid_range(struct irq_data *d) 173 { 174 return __get_intid_range(d->hwirq); 175 } 176 177 static inline unsigned int gic_irq(struct irq_data *d) 178 { 179 return d->hwirq; 180 } 181 182 static inline bool gic_irq_in_rdist(struct irq_data *d) 183 { 184 switch (get_intid_range(d)) { 185 case SGI_RANGE: 186 case PPI_RANGE: 187 case EPPI_RANGE: 188 return true; 189 default: 190 return false; 191 } 192 } 193 194 static inline void __iomem *gic_dist_base_alias(struct irq_data *d) 195 { 196 if (static_branch_unlikely(&gic_nvidia_t241_erratum)) { 197 irq_hw_number_t hwirq = irqd_to_hwirq(d); 198 u32 chip; 199 200 /* 201 * For the erratum T241-FABRIC-4, read accesses to GICD_In{E} 202 * registers are directed to the chip that owns the SPI. The 203 * the alias region can also be used for writes to the 204 * GICD_In{E} except GICD_ICENABLERn. Each chip has support 205 * for 320 {E}SPIs. Mappings for all 4 chips: 206 * Chip0 = 32-351 207 * Chip1 = 352-671 208 * Chip2 = 672-991 209 * Chip3 = 4096-4415 210 */ 211 switch (__get_intid_range(hwirq)) { 212 case SPI_RANGE: 213 chip = (hwirq - 32) / 320; 214 break; 215 case ESPI_RANGE: 216 chip = 3; 217 break; 218 default: 219 unreachable(); 220 } 221 return t241_dist_base_alias[chip]; 222 } 223 224 return gic_data.dist_base; 225 } 226 227 static inline void __iomem *gic_dist_base(struct irq_data *d) 228 { 229 switch (get_intid_range(d)) { 230 case SGI_RANGE: 231 case PPI_RANGE: 232 case EPPI_RANGE: 233 /* SGI+PPI -> SGI_base for this CPU */ 234 return gic_data_rdist_sgi_base(); 235 236 case SPI_RANGE: 237 case ESPI_RANGE: 238 /* SPI -> dist_base */ 239 return gic_data.dist_base; 240 241 default: 242 return NULL; 243 } 244 } 245 246 static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) 247 { 248 u32 count = 1000000; /* 1s! */ 249 250 while (readl_relaxed(base + GICD_CTLR) & bit) { 251 count--; 252 if (!count) { 253 pr_err_ratelimited("RWP timeout, gone fishing\n"); 254 return; 255 } 256 cpu_relax(); 257 udelay(1); 258 } 259 } 260 261 /* Wait for completion of a distributor change */ 262 static void gic_dist_wait_for_rwp(void) 263 { 264 gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP); 265 } 266 267 /* Wait for completion of a redistributor change */ 268 static void gic_redist_wait_for_rwp(void) 269 { 270 gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); 271 } 272 273 #ifdef CONFIG_ARM64 274 275 static u64 __maybe_unused gic_read_iar(void) 276 { 277 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 278 return gic_read_iar_cavium_thunderx(); 279 else 280 return gic_read_iar_common(); 281 } 282 #endif 283 284 static void gic_enable_redist(bool enable) 285 { 286 void __iomem *rbase; 287 u32 count = 1000000; /* 1s! */ 288 u32 val; 289 290 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 291 return; 292 293 rbase = gic_data_rdist_rd_base(); 294 295 val = readl_relaxed(rbase + GICR_WAKER); 296 if (enable) 297 /* Wake up this CPU redistributor */ 298 val &= ~GICR_WAKER_ProcessorSleep; 299 else 300 val |= GICR_WAKER_ProcessorSleep; 301 writel_relaxed(val, rbase + GICR_WAKER); 302 303 if (!enable) { /* Check that GICR_WAKER is writeable */ 304 val = readl_relaxed(rbase + GICR_WAKER); 305 if (!(val & GICR_WAKER_ProcessorSleep)) 306 return; /* No PM support in this redistributor */ 307 } 308 309 while (--count) { 310 val = readl_relaxed(rbase + GICR_WAKER); 311 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 312 break; 313 cpu_relax(); 314 udelay(1); 315 } 316 if (!count) 317 pr_err_ratelimited("redistributor failed to %s...\n", 318 enable ? "wakeup" : "sleep"); 319 } 320 321 /* 322 * Routines to disable, enable, EOI and route interrupts 323 */ 324 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 325 { 326 switch (get_intid_range(d)) { 327 case SGI_RANGE: 328 case PPI_RANGE: 329 case SPI_RANGE: 330 *index = d->hwirq; 331 return offset; 332 case EPPI_RANGE: 333 /* 334 * Contrary to the ESPI range, the EPPI range is contiguous 335 * to the PPI range in the registers, so let's adjust the 336 * displacement accordingly. Consistency is overrated. 337 */ 338 *index = d->hwirq - EPPI_BASE_INTID + 32; 339 return offset; 340 case ESPI_RANGE: 341 *index = d->hwirq - ESPI_BASE_INTID; 342 switch (offset) { 343 case GICD_ISENABLER: 344 return GICD_ISENABLERnE; 345 case GICD_ICENABLER: 346 return GICD_ICENABLERnE; 347 case GICD_ISPENDR: 348 return GICD_ISPENDRnE; 349 case GICD_ICPENDR: 350 return GICD_ICPENDRnE; 351 case GICD_ISACTIVER: 352 return GICD_ISACTIVERnE; 353 case GICD_ICACTIVER: 354 return GICD_ICACTIVERnE; 355 case GICD_IPRIORITYR: 356 return GICD_IPRIORITYRnE; 357 case GICD_ICFGR: 358 return GICD_ICFGRnE; 359 case GICD_IROUTER: 360 return GICD_IROUTERnE; 361 default: 362 break; 363 } 364 break; 365 default: 366 break; 367 } 368 369 WARN_ON(1); 370 *index = d->hwirq; 371 return offset; 372 } 373 374 static int gic_peek_irq(struct irq_data *d, u32 offset) 375 { 376 void __iomem *base; 377 u32 index, mask; 378 379 offset = convert_offset_index(d, offset, &index); 380 mask = 1 << (index % 32); 381 382 if (gic_irq_in_rdist(d)) 383 base = gic_data_rdist_sgi_base(); 384 else 385 base = gic_dist_base_alias(d); 386 387 return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 388 } 389 390 static void gic_poke_irq(struct irq_data *d, u32 offset) 391 { 392 void __iomem *base; 393 u32 index, mask; 394 395 offset = convert_offset_index(d, offset, &index); 396 mask = 1 << (index % 32); 397 398 if (gic_irq_in_rdist(d)) 399 base = gic_data_rdist_sgi_base(); 400 else 401 base = gic_data.dist_base; 402 403 writel_relaxed(mask, base + offset + (index / 32) * 4); 404 } 405 406 static void gic_mask_irq(struct irq_data *d) 407 { 408 gic_poke_irq(d, GICD_ICENABLER); 409 if (gic_irq_in_rdist(d)) 410 gic_redist_wait_for_rwp(); 411 else 412 gic_dist_wait_for_rwp(); 413 } 414 415 static void gic_eoimode1_mask_irq(struct irq_data *d) 416 { 417 gic_mask_irq(d); 418 /* 419 * When masking a forwarded interrupt, make sure it is 420 * deactivated as well. 421 * 422 * This ensures that an interrupt that is getting 423 * disabled/masked will not get "stuck", because there is 424 * noone to deactivate it (guest is being terminated). 425 */ 426 if (irqd_is_forwarded_to_vcpu(d)) 427 gic_poke_irq(d, GICD_ICACTIVER); 428 } 429 430 static void gic_unmask_irq(struct irq_data *d) 431 { 432 gic_poke_irq(d, GICD_ISENABLER); 433 } 434 435 static inline bool gic_supports_nmi(void) 436 { 437 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 438 static_branch_likely(&supports_pseudo_nmis); 439 } 440 441 static int gic_irq_set_irqchip_state(struct irq_data *d, 442 enum irqchip_irq_state which, bool val) 443 { 444 u32 reg; 445 446 if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 447 return -EINVAL; 448 449 switch (which) { 450 case IRQCHIP_STATE_PENDING: 451 reg = val ? GICD_ISPENDR : GICD_ICPENDR; 452 break; 453 454 case IRQCHIP_STATE_ACTIVE: 455 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 456 break; 457 458 case IRQCHIP_STATE_MASKED: 459 if (val) { 460 gic_mask_irq(d); 461 return 0; 462 } 463 reg = GICD_ISENABLER; 464 break; 465 466 default: 467 return -EINVAL; 468 } 469 470 gic_poke_irq(d, reg); 471 return 0; 472 } 473 474 static int gic_irq_get_irqchip_state(struct irq_data *d, 475 enum irqchip_irq_state which, bool *val) 476 { 477 if (d->hwirq >= 8192) /* PPI/SPI only */ 478 return -EINVAL; 479 480 switch (which) { 481 case IRQCHIP_STATE_PENDING: 482 *val = gic_peek_irq(d, GICD_ISPENDR); 483 break; 484 485 case IRQCHIP_STATE_ACTIVE: 486 *val = gic_peek_irq(d, GICD_ISACTIVER); 487 break; 488 489 case IRQCHIP_STATE_MASKED: 490 *val = !gic_peek_irq(d, GICD_ISENABLER); 491 break; 492 493 default: 494 return -EINVAL; 495 } 496 497 return 0; 498 } 499 500 static void gic_irq_set_prio(struct irq_data *d, u8 prio) 501 { 502 void __iomem *base = gic_dist_base(d); 503 u32 offset, index; 504 505 offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 506 507 writeb_relaxed(prio, base + offset + index); 508 } 509 510 static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 511 { 512 switch (__get_intid_range(hwirq)) { 513 case PPI_RANGE: 514 return hwirq - 16; 515 case EPPI_RANGE: 516 return hwirq - EPPI_BASE_INTID + 16; 517 default: 518 unreachable(); 519 } 520 } 521 522 static u32 gic_get_ppi_index(struct irq_data *d) 523 { 524 return __gic_get_ppi_index(d->hwirq); 525 } 526 527 static int gic_irq_nmi_setup(struct irq_data *d) 528 { 529 struct irq_desc *desc = irq_to_desc(d->irq); 530 531 if (!gic_supports_nmi()) 532 return -EINVAL; 533 534 if (gic_peek_irq(d, GICD_ISENABLER)) { 535 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 536 return -EINVAL; 537 } 538 539 /* 540 * A secondary irq_chip should be in charge of LPI request, 541 * it should not be possible to get there 542 */ 543 if (WARN_ON(gic_irq(d) >= 8192)) 544 return -EINVAL; 545 546 /* desc lock should already be held */ 547 if (gic_irq_in_rdist(d)) { 548 u32 idx = gic_get_ppi_index(d); 549 550 /* Setting up PPI as NMI, only switch handler for first NMI */ 551 if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 552 refcount_set(&ppi_nmi_refs[idx], 1); 553 desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 554 } 555 } else { 556 desc->handle_irq = handle_fasteoi_nmi; 557 } 558 559 gic_irq_set_prio(d, GICD_INT_NMI_PRI); 560 561 return 0; 562 } 563 564 static void gic_irq_nmi_teardown(struct irq_data *d) 565 { 566 struct irq_desc *desc = irq_to_desc(d->irq); 567 568 if (WARN_ON(!gic_supports_nmi())) 569 return; 570 571 if (gic_peek_irq(d, GICD_ISENABLER)) { 572 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 573 return; 574 } 575 576 /* 577 * A secondary irq_chip should be in charge of LPI request, 578 * it should not be possible to get there 579 */ 580 if (WARN_ON(gic_irq(d) >= 8192)) 581 return; 582 583 /* desc lock should already be held */ 584 if (gic_irq_in_rdist(d)) { 585 u32 idx = gic_get_ppi_index(d); 586 587 /* Tearing down NMI, only switch handler for last NMI */ 588 if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 589 desc->handle_irq = handle_percpu_devid_irq; 590 } else { 591 desc->handle_irq = handle_fasteoi_irq; 592 } 593 594 gic_irq_set_prio(d, GICD_INT_DEF_PRI); 595 } 596 597 static bool gic_arm64_erratum_2941627_needed(struct irq_data *d) 598 { 599 enum gic_intid_range range; 600 601 if (!static_branch_unlikely(&gic_arm64_2941627_erratum)) 602 return false; 603 604 range = get_intid_range(d); 605 606 /* 607 * The workaround is needed if the IRQ is an SPI and 608 * the target cpu is different from the one we are 609 * executing on. 610 */ 611 return (range == SPI_RANGE || range == ESPI_RANGE) && 612 !cpumask_test_cpu(raw_smp_processor_id(), 613 irq_data_get_effective_affinity_mask(d)); 614 } 615 616 static void gic_eoi_irq(struct irq_data *d) 617 { 618 write_gicreg(gic_irq(d), ICC_EOIR1_EL1); 619 isb(); 620 621 if (gic_arm64_erratum_2941627_needed(d)) { 622 /* 623 * Make sure the GIC stream deactivate packet 624 * issued by ICC_EOIR1_EL1 has completed before 625 * deactivating through GICD_IACTIVER. 626 */ 627 dsb(sy); 628 gic_poke_irq(d, GICD_ICACTIVER); 629 } 630 } 631 632 static void gic_eoimode1_eoi_irq(struct irq_data *d) 633 { 634 /* 635 * No need to deactivate an LPI, or an interrupt that 636 * is is getting forwarded to a vcpu. 637 */ 638 if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 639 return; 640 641 if (!gic_arm64_erratum_2941627_needed(d)) 642 gic_write_dir(gic_irq(d)); 643 else 644 gic_poke_irq(d, GICD_ICACTIVER); 645 } 646 647 static int gic_set_type(struct irq_data *d, unsigned int type) 648 { 649 enum gic_intid_range range; 650 unsigned int irq = gic_irq(d); 651 void __iomem *base; 652 u32 offset, index; 653 int ret; 654 655 range = get_intid_range(d); 656 657 /* Interrupt configuration for SGIs can't be changed */ 658 if (range == SGI_RANGE) 659 return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 660 661 /* SPIs have restrictions on the supported types */ 662 if ((range == SPI_RANGE || range == ESPI_RANGE) && 663 type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 664 return -EINVAL; 665 666 if (gic_irq_in_rdist(d)) 667 base = gic_data_rdist_sgi_base(); 668 else 669 base = gic_dist_base_alias(d); 670 671 offset = convert_offset_index(d, GICD_ICFGR, &index); 672 673 ret = gic_configure_irq(index, type, base + offset, NULL); 674 if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 675 /* Misconfigured PPIs are usually not fatal */ 676 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 677 ret = 0; 678 } 679 680 return ret; 681 } 682 683 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 684 { 685 if (get_intid_range(d) == SGI_RANGE) 686 return -EINVAL; 687 688 if (vcpu) 689 irqd_set_forwarded_to_vcpu(d); 690 else 691 irqd_clr_forwarded_to_vcpu(d); 692 return 0; 693 } 694 695 static u64 gic_cpu_to_affinity(int cpu) 696 { 697 u64 mpidr = cpu_logical_map(cpu); 698 u64 aff; 699 700 /* ASR8601 needs to have its affinities shifted down... */ 701 if (unlikely(gic_data.flags & FLAGS_WORKAROUND_ASR_ERRATUM_8601001)) 702 mpidr = (MPIDR_AFFINITY_LEVEL(mpidr, 1) | 703 (MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8)); 704 705 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 706 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 707 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 708 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 709 710 return aff; 711 } 712 713 static void gic_deactivate_unhandled(u32 irqnr) 714 { 715 if (static_branch_likely(&supports_deactivate_key)) { 716 if (irqnr < 8192) 717 gic_write_dir(irqnr); 718 } else { 719 write_gicreg(irqnr, ICC_EOIR1_EL1); 720 isb(); 721 } 722 } 723 724 /* 725 * Follow a read of the IAR with any HW maintenance that needs to happen prior 726 * to invoking the relevant IRQ handler. We must do two things: 727 * 728 * (1) Ensure instruction ordering between a read of IAR and subsequent 729 * instructions in the IRQ handler using an ISB. 730 * 731 * It is possible for the IAR to report an IRQ which was signalled *after* 732 * the CPU took an IRQ exception as multiple interrupts can race to be 733 * recognized by the GIC, earlier interrupts could be withdrawn, and/or 734 * later interrupts could be prioritized by the GIC. 735 * 736 * For devices which are tightly coupled to the CPU, such as PMUs, a 737 * context synchronization event is necessary to ensure that system 738 * register state is not stale, as these may have been indirectly written 739 * *after* exception entry. 740 * 741 * (2) Deactivate the interrupt when EOI mode 1 is in use. 742 */ 743 static inline void gic_complete_ack(u32 irqnr) 744 { 745 if (static_branch_likely(&supports_deactivate_key)) 746 write_gicreg(irqnr, ICC_EOIR1_EL1); 747 748 isb(); 749 } 750 751 static bool gic_rpr_is_nmi_prio(void) 752 { 753 if (!gic_supports_nmi()) 754 return false; 755 756 return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI)); 757 } 758 759 static bool gic_irqnr_is_special(u32 irqnr) 760 { 761 return irqnr >= 1020 && irqnr <= 1023; 762 } 763 764 static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs) 765 { 766 if (gic_irqnr_is_special(irqnr)) 767 return; 768 769 gic_complete_ack(irqnr); 770 771 if (generic_handle_domain_irq(gic_data.domain, irqnr)) { 772 WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr); 773 gic_deactivate_unhandled(irqnr); 774 } 775 } 776 777 static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 778 { 779 if (gic_irqnr_is_special(irqnr)) 780 return; 781 782 gic_complete_ack(irqnr); 783 784 if (generic_handle_domain_nmi(gic_data.domain, irqnr)) { 785 WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr); 786 gic_deactivate_unhandled(irqnr); 787 } 788 } 789 790 /* 791 * An exception has been taken from a context with IRQs enabled, and this could 792 * be an IRQ or an NMI. 793 * 794 * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear 795 * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning, 796 * after handling any NMI but before handling any IRQ. 797 * 798 * The entry code has performed IRQ entry, and if an NMI is detected we must 799 * perform NMI entry/exit around invoking the handler. 800 */ 801 static void __gic_handle_irq_from_irqson(struct pt_regs *regs) 802 { 803 bool is_nmi; 804 u32 irqnr; 805 806 irqnr = gic_read_iar(); 807 808 is_nmi = gic_rpr_is_nmi_prio(); 809 810 if (is_nmi) { 811 nmi_enter(); 812 __gic_handle_nmi(irqnr, regs); 813 nmi_exit(); 814 } 815 816 if (gic_prio_masking_enabled()) { 817 gic_pmr_mask_irqs(); 818 gic_arch_enable_irqs(); 819 } 820 821 if (!is_nmi) 822 __gic_handle_irq(irqnr, regs); 823 } 824 825 /* 826 * An exception has been taken from a context with IRQs disabled, which can only 827 * be an NMI. 828 * 829 * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave 830 * DAIF.IF (and ICC_PMR_EL1) unchanged. 831 * 832 * The entry code has performed NMI entry. 833 */ 834 static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) 835 { 836 u64 pmr; 837 u32 irqnr; 838 839 /* 840 * We were in a context with IRQs disabled. However, the 841 * entry code has set PMR to a value that allows any 842 * interrupt to be acknowledged, and not just NMIs. This can 843 * lead to surprising effects if the NMI has been retired in 844 * the meantime, and that there is an IRQ pending. The IRQ 845 * would then be taken in NMI context, something that nobody 846 * wants to debug twice. 847 * 848 * Until we sort this, drop PMR again to a level that will 849 * actually only allow NMIs before reading IAR, and then 850 * restore it to what it was. 851 */ 852 pmr = gic_read_pmr(); 853 gic_pmr_mask_irqs(); 854 isb(); 855 irqnr = gic_read_iar(); 856 gic_write_pmr(pmr); 857 858 __gic_handle_nmi(irqnr, regs); 859 } 860 861 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 862 { 863 if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) 864 __gic_handle_irq_from_irqsoff(regs); 865 else 866 __gic_handle_irq_from_irqson(regs); 867 } 868 869 static u32 gic_get_pribits(void) 870 { 871 u32 pribits; 872 873 pribits = gic_read_ctlr(); 874 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 875 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 876 pribits++; 877 878 return pribits; 879 } 880 881 static bool gic_has_group0(void) 882 { 883 u32 val; 884 u32 old_pmr; 885 886 old_pmr = gic_read_pmr(); 887 888 /* 889 * Let's find out if Group0 is under control of EL3 or not by 890 * setting the highest possible, non-zero priority in PMR. 891 * 892 * If SCR_EL3.FIQ is set, the priority gets shifted down in 893 * order for the CPU interface to set bit 7, and keep the 894 * actual priority in the non-secure range. In the process, it 895 * looses the least significant bit and the actual priority 896 * becomes 0x80. Reading it back returns 0, indicating that 897 * we're don't have access to Group0. 898 */ 899 gic_write_pmr(BIT(8 - gic_get_pribits())); 900 val = gic_read_pmr(); 901 902 gic_write_pmr(old_pmr); 903 904 return val != 0; 905 } 906 907 static void __init gic_dist_init(void) 908 { 909 unsigned int i; 910 u64 affinity; 911 void __iomem *base = gic_data.dist_base; 912 u32 val; 913 914 /* Disable the distributor */ 915 writel_relaxed(0, base + GICD_CTLR); 916 gic_dist_wait_for_rwp(); 917 918 /* 919 * Configure SPIs as non-secure Group-1. This will only matter 920 * if the GIC only has a single security state. This will not 921 * do the right thing if the kernel is running in secure mode, 922 * but that's not the intended use case anyway. 923 */ 924 for (i = 32; i < GIC_LINE_NR; i += 32) 925 writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 926 927 /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 928 for (i = 0; i < GIC_ESPI_NR; i += 32) { 929 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 930 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 931 } 932 933 for (i = 0; i < GIC_ESPI_NR; i += 32) 934 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 935 936 for (i = 0; i < GIC_ESPI_NR; i += 16) 937 writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 938 939 for (i = 0; i < GIC_ESPI_NR; i += 4) 940 writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 941 942 /* Now do the common stuff */ 943 gic_dist_config(base, GIC_LINE_NR, NULL); 944 945 val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 946 if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 947 pr_info("Enabling SGIs without active state\n"); 948 val |= GICD_CTLR_nASSGIreq; 949 } 950 951 /* Enable distributor with ARE, Group1, and wait for it to drain */ 952 writel_relaxed(val, base + GICD_CTLR); 953 gic_dist_wait_for_rwp(); 954 955 /* 956 * Set all global interrupts to the boot CPU only. ARE must be 957 * enabled. 958 */ 959 affinity = gic_cpu_to_affinity(smp_processor_id()); 960 for (i = 32; i < GIC_LINE_NR; i++) 961 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 962 963 for (i = 0; i < GIC_ESPI_NR; i++) 964 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 965 } 966 967 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 968 { 969 int ret = -ENODEV; 970 int i; 971 972 for (i = 0; i < gic_data.nr_redist_regions; i++) { 973 void __iomem *ptr = gic_data.redist_regions[i].redist_base; 974 u64 typer; 975 u32 reg; 976 977 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 978 if (reg != GIC_PIDR2_ARCH_GICv3 && 979 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 980 pr_warn("No redistributor present @%p\n", ptr); 981 break; 982 } 983 984 do { 985 typer = gic_read_typer(ptr + GICR_TYPER); 986 ret = fn(gic_data.redist_regions + i, ptr); 987 if (!ret) 988 return 0; 989 990 if (gic_data.redist_regions[i].single_redist) 991 break; 992 993 if (gic_data.redist_stride) { 994 ptr += gic_data.redist_stride; 995 } else { 996 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 997 if (typer & GICR_TYPER_VLPIS) 998 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 999 } 1000 } while (!(typer & GICR_TYPER_LAST)); 1001 } 1002 1003 return ret ? -ENODEV : 0; 1004 } 1005 1006 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 1007 { 1008 unsigned long mpidr; 1009 u64 typer; 1010 u32 aff; 1011 1012 /* 1013 * Convert affinity to a 32bit value that can be matched to 1014 * GICR_TYPER bits [63:32]. 1015 */ 1016 mpidr = gic_cpu_to_affinity(smp_processor_id()); 1017 1018 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 1019 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 1020 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 1021 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 1022 1023 typer = gic_read_typer(ptr + GICR_TYPER); 1024 if ((typer >> 32) == aff) { 1025 u64 offset = ptr - region->redist_base; 1026 raw_spin_lock_init(&gic_data_rdist()->rd_lock); 1027 gic_data_rdist_rd_base() = ptr; 1028 gic_data_rdist()->phys_base = region->phys_base + offset; 1029 1030 pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 1031 smp_processor_id(), mpidr, 1032 (int)(region - gic_data.redist_regions), 1033 &gic_data_rdist()->phys_base); 1034 return 0; 1035 } 1036 1037 /* Try next one */ 1038 return 1; 1039 } 1040 1041 static int gic_populate_rdist(void) 1042 { 1043 if (gic_iterate_rdists(__gic_populate_rdist) == 0) 1044 return 0; 1045 1046 /* We couldn't even deal with ourselves... */ 1047 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 1048 smp_processor_id(), 1049 (unsigned long)cpu_logical_map(smp_processor_id())); 1050 return -ENODEV; 1051 } 1052 1053 static int __gic_update_rdist_properties(struct redist_region *region, 1054 void __iomem *ptr) 1055 { 1056 u64 typer = gic_read_typer(ptr + GICR_TYPER); 1057 u32 ctlr = readl_relaxed(ptr + GICR_CTLR); 1058 1059 /* Boot-time cleanup */ 1060 if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { 1061 u64 val; 1062 1063 /* Deactivate any present vPE */ 1064 val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); 1065 if (val & GICR_VPENDBASER_Valid) 1066 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 1067 ptr + SZ_128K + GICR_VPENDBASER); 1068 1069 /* Mark the VPE table as invalid */ 1070 val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); 1071 val &= ~GICR_VPROPBASER_4_1_VALID; 1072 gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); 1073 } 1074 1075 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 1076 1077 /* 1078 * TYPER.RVPEID implies some form of DirectLPI, no matter what the 1079 * doc says... :-/ And CTLR.IR implies another subset of DirectLPI 1080 * that the ITS driver can make use of for LPIs (and not VLPIs). 1081 * 1082 * These are 3 different ways to express the same thing, depending 1083 * on the revision of the architecture and its relaxations over 1084 * time. Just group them under the 'direct_lpi' banner. 1085 */ 1086 gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 1087 gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 1088 !!(ctlr & GICR_CTLR_IR) | 1089 gic_data.rdists.has_rvpeid); 1090 gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 1091 1092 /* Detect non-sensical configurations */ 1093 if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 1094 gic_data.rdists.has_direct_lpi = false; 1095 gic_data.rdists.has_vlpis = false; 1096 gic_data.rdists.has_rvpeid = false; 1097 } 1098 1099 gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 1100 1101 return 1; 1102 } 1103 1104 static void gic_update_rdist_properties(void) 1105 { 1106 gic_data.ppi_nr = UINT_MAX; 1107 gic_iterate_rdists(__gic_update_rdist_properties); 1108 if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 1109 gic_data.ppi_nr = 0; 1110 pr_info("GICv3 features: %d PPIs%s%s\n", 1111 gic_data.ppi_nr, 1112 gic_data.has_rss ? ", RSS" : "", 1113 gic_data.rdists.has_direct_lpi ? ", DirectLPI" : ""); 1114 1115 if (gic_data.rdists.has_vlpis) 1116 pr_info("GICv4 features: %s%s%s\n", 1117 gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 1118 gic_data.rdists.has_rvpeid ? "RVPEID " : "", 1119 gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 1120 } 1121 1122 /* Check whether it's single security state view */ 1123 static inline bool gic_dist_security_disabled(void) 1124 { 1125 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 1126 } 1127 1128 static void gic_cpu_sys_reg_init(void) 1129 { 1130 int i, cpu = smp_processor_id(); 1131 u64 mpidr = gic_cpu_to_affinity(cpu); 1132 u64 need_rss = MPIDR_RS(mpidr); 1133 bool group0; 1134 u32 pribits; 1135 1136 /* 1137 * Need to check that the SRE bit has actually been set. If 1138 * not, it means that SRE is disabled at EL2. We're going to 1139 * die painfully, and there is nothing we can do about it. 1140 * 1141 * Kindly inform the luser. 1142 */ 1143 if (!gic_enable_sre()) 1144 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 1145 1146 pribits = gic_get_pribits(); 1147 1148 group0 = gic_has_group0(); 1149 1150 /* Set priority mask register */ 1151 if (!gic_prio_masking_enabled()) { 1152 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 1153 } else if (gic_supports_nmi()) { 1154 /* 1155 * Mismatch configuration with boot CPU, the system is likely 1156 * to die as interrupt masking will not work properly on all 1157 * CPUs 1158 * 1159 * The boot CPU calls this function before enabling NMI support, 1160 * and as a result we'll never see this warning in the boot path 1161 * for that CPU. 1162 */ 1163 if (static_branch_unlikely(&gic_nonsecure_priorities)) 1164 WARN_ON(!group0 || gic_dist_security_disabled()); 1165 else 1166 WARN_ON(group0 && !gic_dist_security_disabled()); 1167 } 1168 1169 /* 1170 * Some firmwares hand over to the kernel with the BPR changed from 1171 * its reset value (and with a value large enough to prevent 1172 * any pre-emptive interrupts from working at all). Writing a zero 1173 * to BPR restores is reset value. 1174 */ 1175 gic_write_bpr1(0); 1176 1177 if (static_branch_likely(&supports_deactivate_key)) { 1178 /* EOI drops priority only (mode 1) */ 1179 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 1180 } else { 1181 /* EOI deactivates interrupt too (mode 0) */ 1182 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 1183 } 1184 1185 /* Always whack Group0 before Group1 */ 1186 if (group0) { 1187 switch(pribits) { 1188 case 8: 1189 case 7: 1190 write_gicreg(0, ICC_AP0R3_EL1); 1191 write_gicreg(0, ICC_AP0R2_EL1); 1192 fallthrough; 1193 case 6: 1194 write_gicreg(0, ICC_AP0R1_EL1); 1195 fallthrough; 1196 case 5: 1197 case 4: 1198 write_gicreg(0, ICC_AP0R0_EL1); 1199 } 1200 1201 isb(); 1202 } 1203 1204 switch(pribits) { 1205 case 8: 1206 case 7: 1207 write_gicreg(0, ICC_AP1R3_EL1); 1208 write_gicreg(0, ICC_AP1R2_EL1); 1209 fallthrough; 1210 case 6: 1211 write_gicreg(0, ICC_AP1R1_EL1); 1212 fallthrough; 1213 case 5: 1214 case 4: 1215 write_gicreg(0, ICC_AP1R0_EL1); 1216 } 1217 1218 isb(); 1219 1220 /* ... and let's hit the road... */ 1221 gic_write_grpen1(1); 1222 1223 /* Keep the RSS capability status in per_cpu variable */ 1224 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1225 1226 /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1227 for_each_online_cpu(i) { 1228 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1229 1230 need_rss |= MPIDR_RS(gic_cpu_to_affinity(i)); 1231 if (need_rss && (!have_rss)) 1232 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1233 cpu, (unsigned long)mpidr, 1234 i, (unsigned long)gic_cpu_to_affinity(i)); 1235 } 1236 1237 /** 1238 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1239 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1240 * UNPREDICTABLE choice of : 1241 * - The write is ignored. 1242 * - The RS field is treated as 0. 1243 */ 1244 if (need_rss && (!gic_data.has_rss)) 1245 pr_crit_once("RSS is required but GICD doesn't support it\n"); 1246 } 1247 1248 static bool gicv3_nolpi; 1249 1250 static int __init gicv3_nolpi_cfg(char *buf) 1251 { 1252 return kstrtobool(buf, &gicv3_nolpi); 1253 } 1254 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1255 1256 static int gic_dist_supports_lpis(void) 1257 { 1258 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1259 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1260 !gicv3_nolpi); 1261 } 1262 1263 static void gic_cpu_init(void) 1264 { 1265 void __iomem *rbase; 1266 int i; 1267 1268 /* Register ourselves with the rest of the world */ 1269 if (gic_populate_rdist()) 1270 return; 1271 1272 gic_enable_redist(true); 1273 1274 WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1275 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1276 "Distributor has extended ranges, but CPU%d doesn't\n", 1277 smp_processor_id()); 1278 1279 rbase = gic_data_rdist_sgi_base(); 1280 1281 /* Configure SGIs/PPIs as non-secure Group-1 */ 1282 for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 1283 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 1284 1285 gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1286 1287 /* initialise system registers */ 1288 gic_cpu_sys_reg_init(); 1289 } 1290 1291 #ifdef CONFIG_SMP 1292 1293 #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1294 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1295 1296 static int gic_starting_cpu(unsigned int cpu) 1297 { 1298 gic_cpu_init(); 1299 1300 if (gic_dist_supports_lpis()) 1301 its_cpu_init(); 1302 1303 return 0; 1304 } 1305 1306 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1307 unsigned long cluster_id) 1308 { 1309 int next_cpu, cpu = *base_cpu; 1310 unsigned long mpidr; 1311 u16 tlist = 0; 1312 1313 mpidr = gic_cpu_to_affinity(cpu); 1314 1315 while (cpu < nr_cpu_ids) { 1316 tlist |= 1 << (mpidr & 0xf); 1317 1318 next_cpu = cpumask_next(cpu, mask); 1319 if (next_cpu >= nr_cpu_ids) 1320 goto out; 1321 cpu = next_cpu; 1322 1323 mpidr = gic_cpu_to_affinity(cpu); 1324 1325 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1326 cpu--; 1327 goto out; 1328 } 1329 } 1330 out: 1331 *base_cpu = cpu; 1332 return tlist; 1333 } 1334 1335 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 1336 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 1337 << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 1338 1339 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1340 { 1341 u64 val; 1342 1343 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 1344 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 1345 irq << ICC_SGI1R_SGI_ID_SHIFT | 1346 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1347 MPIDR_TO_SGI_RS(cluster_id) | 1348 tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1349 1350 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1351 gic_write_sgi1r(val); 1352 } 1353 1354 static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1355 { 1356 int cpu; 1357 1358 if (WARN_ON(d->hwirq >= 16)) 1359 return; 1360 1361 /* 1362 * Ensure that stores to Normal memory are visible to the 1363 * other CPUs before issuing the IPI. 1364 */ 1365 dsb(ishst); 1366 1367 for_each_cpu(cpu, mask) { 1368 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(gic_cpu_to_affinity(cpu)); 1369 u16 tlist; 1370 1371 tlist = gic_compute_target_list(&cpu, mask, cluster_id); 1372 gic_send_sgi(cluster_id, tlist, d->hwirq); 1373 } 1374 1375 /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1376 isb(); 1377 } 1378 1379 static void __init gic_smp_init(void) 1380 { 1381 struct irq_fwspec sgi_fwspec = { 1382 .fwnode = gic_data.fwnode, 1383 .param_count = 1, 1384 }; 1385 int base_sgi; 1386 1387 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 1388 "irqchip/arm/gicv3:starting", 1389 gic_starting_cpu, NULL); 1390 1391 /* Register all 8 non-secure SGIs */ 1392 base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec); 1393 if (WARN_ON(base_sgi <= 0)) 1394 return; 1395 1396 set_smp_ipi_range(base_sgi, 8); 1397 } 1398 1399 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1400 bool force) 1401 { 1402 unsigned int cpu; 1403 u32 offset, index; 1404 void __iomem *reg; 1405 int enabled; 1406 u64 val; 1407 1408 if (force) 1409 cpu = cpumask_first(mask_val); 1410 else 1411 cpu = cpumask_any_and(mask_val, cpu_online_mask); 1412 1413 if (cpu >= nr_cpu_ids) 1414 return -EINVAL; 1415 1416 if (gic_irq_in_rdist(d)) 1417 return -EINVAL; 1418 1419 /* If interrupt was enabled, disable it first */ 1420 enabled = gic_peek_irq(d, GICD_ISENABLER); 1421 if (enabled) 1422 gic_mask_irq(d); 1423 1424 offset = convert_offset_index(d, GICD_IROUTER, &index); 1425 reg = gic_dist_base(d) + offset + (index * 8); 1426 val = gic_cpu_to_affinity(cpu); 1427 1428 gic_write_irouter(val, reg); 1429 1430 /* 1431 * If the interrupt was enabled, enabled it again. Otherwise, 1432 * just wait for the distributor to have digested our changes. 1433 */ 1434 if (enabled) 1435 gic_unmask_irq(d); 1436 1437 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1438 1439 return IRQ_SET_MASK_OK_DONE; 1440 } 1441 #else 1442 #define gic_set_affinity NULL 1443 #define gic_ipi_send_mask NULL 1444 #define gic_smp_init() do { } while(0) 1445 #endif 1446 1447 static int gic_retrigger(struct irq_data *data) 1448 { 1449 return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 1450 } 1451 1452 #ifdef CONFIG_CPU_PM 1453 static int gic_cpu_pm_notifier(struct notifier_block *self, 1454 unsigned long cmd, void *v) 1455 { 1456 if (cmd == CPU_PM_EXIT) { 1457 if (gic_dist_security_disabled()) 1458 gic_enable_redist(true); 1459 gic_cpu_sys_reg_init(); 1460 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 1461 gic_write_grpen1(0); 1462 gic_enable_redist(false); 1463 } 1464 return NOTIFY_OK; 1465 } 1466 1467 static struct notifier_block gic_cpu_pm_notifier_block = { 1468 .notifier_call = gic_cpu_pm_notifier, 1469 }; 1470 1471 static void gic_cpu_pm_init(void) 1472 { 1473 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 1474 } 1475 1476 #else 1477 static inline void gic_cpu_pm_init(void) { } 1478 #endif /* CONFIG_CPU_PM */ 1479 1480 static struct irq_chip gic_chip = { 1481 .name = "GICv3", 1482 .irq_mask = gic_mask_irq, 1483 .irq_unmask = gic_unmask_irq, 1484 .irq_eoi = gic_eoi_irq, 1485 .irq_set_type = gic_set_type, 1486 .irq_set_affinity = gic_set_affinity, 1487 .irq_retrigger = gic_retrigger, 1488 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1489 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1490 .irq_nmi_setup = gic_irq_nmi_setup, 1491 .irq_nmi_teardown = gic_irq_nmi_teardown, 1492 .ipi_send_mask = gic_ipi_send_mask, 1493 .flags = IRQCHIP_SET_TYPE_MASKED | 1494 IRQCHIP_SKIP_SET_WAKE | 1495 IRQCHIP_MASK_ON_SUSPEND, 1496 }; 1497 1498 static struct irq_chip gic_eoimode1_chip = { 1499 .name = "GICv3", 1500 .irq_mask = gic_eoimode1_mask_irq, 1501 .irq_unmask = gic_unmask_irq, 1502 .irq_eoi = gic_eoimode1_eoi_irq, 1503 .irq_set_type = gic_set_type, 1504 .irq_set_affinity = gic_set_affinity, 1505 .irq_retrigger = gic_retrigger, 1506 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1507 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1508 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1509 .irq_nmi_setup = gic_irq_nmi_setup, 1510 .irq_nmi_teardown = gic_irq_nmi_teardown, 1511 .ipi_send_mask = gic_ipi_send_mask, 1512 .flags = IRQCHIP_SET_TYPE_MASKED | 1513 IRQCHIP_SKIP_SET_WAKE | 1514 IRQCHIP_MASK_ON_SUSPEND, 1515 }; 1516 1517 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1518 irq_hw_number_t hw) 1519 { 1520 struct irq_chip *chip = &gic_chip; 1521 struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 1522 1523 if (static_branch_likely(&supports_deactivate_key)) 1524 chip = &gic_eoimode1_chip; 1525 1526 switch (__get_intid_range(hw)) { 1527 case SGI_RANGE: 1528 case PPI_RANGE: 1529 case EPPI_RANGE: 1530 irq_set_percpu_devid(irq); 1531 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1532 handle_percpu_devid_irq, NULL, NULL); 1533 break; 1534 1535 case SPI_RANGE: 1536 case ESPI_RANGE: 1537 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1538 handle_fasteoi_irq, NULL, NULL); 1539 irq_set_probe(irq); 1540 irqd_set_single_target(irqd); 1541 break; 1542 1543 case LPI_RANGE: 1544 if (!gic_dist_supports_lpis()) 1545 return -EPERM; 1546 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1547 handle_fasteoi_irq, NULL, NULL); 1548 break; 1549 1550 default: 1551 return -EPERM; 1552 } 1553 1554 /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 1555 irqd_set_handle_enforce_irqctx(irqd); 1556 return 0; 1557 } 1558 1559 static int gic_irq_domain_translate(struct irq_domain *d, 1560 struct irq_fwspec *fwspec, 1561 unsigned long *hwirq, 1562 unsigned int *type) 1563 { 1564 if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 1565 *hwirq = fwspec->param[0]; 1566 *type = IRQ_TYPE_EDGE_RISING; 1567 return 0; 1568 } 1569 1570 if (is_of_node(fwspec->fwnode)) { 1571 if (fwspec->param_count < 3) 1572 return -EINVAL; 1573 1574 switch (fwspec->param[0]) { 1575 case 0: /* SPI */ 1576 *hwirq = fwspec->param[1] + 32; 1577 break; 1578 case 1: /* PPI */ 1579 *hwirq = fwspec->param[1] + 16; 1580 break; 1581 case 2: /* ESPI */ 1582 *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1583 break; 1584 case 3: /* EPPI */ 1585 *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 1586 break; 1587 case GIC_IRQ_TYPE_LPI: /* LPI */ 1588 *hwirq = fwspec->param[1]; 1589 break; 1590 case GIC_IRQ_TYPE_PARTITION: 1591 *hwirq = fwspec->param[1]; 1592 if (fwspec->param[1] >= 16) 1593 *hwirq += EPPI_BASE_INTID - 16; 1594 else 1595 *hwirq += 16; 1596 break; 1597 default: 1598 return -EINVAL; 1599 } 1600 1601 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1602 1603 /* 1604 * Make it clear that broken DTs are... broken. 1605 * Partitioned PPIs are an unfortunate exception. 1606 */ 1607 WARN_ON(*type == IRQ_TYPE_NONE && 1608 fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1609 return 0; 1610 } 1611 1612 if (is_fwnode_irqchip(fwspec->fwnode)) { 1613 if(fwspec->param_count != 2) 1614 return -EINVAL; 1615 1616 if (fwspec->param[0] < 16) { 1617 pr_err(FW_BUG "Illegal GSI%d translation request\n", 1618 fwspec->param[0]); 1619 return -EINVAL; 1620 } 1621 1622 *hwirq = fwspec->param[0]; 1623 *type = fwspec->param[1]; 1624 1625 WARN_ON(*type == IRQ_TYPE_NONE); 1626 return 0; 1627 } 1628 1629 return -EINVAL; 1630 } 1631 1632 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1633 unsigned int nr_irqs, void *arg) 1634 { 1635 int i, ret; 1636 irq_hw_number_t hwirq; 1637 unsigned int type = IRQ_TYPE_NONE; 1638 struct irq_fwspec *fwspec = arg; 1639 1640 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1641 if (ret) 1642 return ret; 1643 1644 for (i = 0; i < nr_irqs; i++) { 1645 ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 1646 if (ret) 1647 return ret; 1648 } 1649 1650 return 0; 1651 } 1652 1653 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1654 unsigned int nr_irqs) 1655 { 1656 int i; 1657 1658 for (i = 0; i < nr_irqs; i++) { 1659 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1660 irq_set_handler(virq + i, NULL); 1661 irq_domain_reset_irq_data(d); 1662 } 1663 } 1664 1665 static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, 1666 irq_hw_number_t hwirq) 1667 { 1668 enum gic_intid_range range; 1669 1670 if (!gic_data.ppi_descs) 1671 return false; 1672 1673 if (!is_of_node(fwspec->fwnode)) 1674 return false; 1675 1676 if (fwspec->param_count < 4 || !fwspec->param[3]) 1677 return false; 1678 1679 range = __get_intid_range(hwirq); 1680 if (range != PPI_RANGE && range != EPPI_RANGE) 1681 return false; 1682 1683 return true; 1684 } 1685 1686 static int gic_irq_domain_select(struct irq_domain *d, 1687 struct irq_fwspec *fwspec, 1688 enum irq_domain_bus_token bus_token) 1689 { 1690 unsigned int type, ret, ppi_idx; 1691 irq_hw_number_t hwirq; 1692 1693 /* Not for us */ 1694 if (fwspec->fwnode != d->fwnode) 1695 return 0; 1696 1697 /* If this is not DT, then we have a single domain */ 1698 if (!is_of_node(fwspec->fwnode)) 1699 return 1; 1700 1701 ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type); 1702 if (WARN_ON_ONCE(ret)) 1703 return 0; 1704 1705 if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) 1706 return d == gic_data.domain; 1707 1708 /* 1709 * If this is a PPI and we have a 4th (non-null) parameter, 1710 * then we need to match the partition domain. 1711 */ 1712 ppi_idx = __gic_get_ppi_index(hwirq); 1713 return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); 1714 } 1715 1716 static const struct irq_domain_ops gic_irq_domain_ops = { 1717 .translate = gic_irq_domain_translate, 1718 .alloc = gic_irq_domain_alloc, 1719 .free = gic_irq_domain_free, 1720 .select = gic_irq_domain_select, 1721 }; 1722 1723 static int partition_domain_translate(struct irq_domain *d, 1724 struct irq_fwspec *fwspec, 1725 unsigned long *hwirq, 1726 unsigned int *type) 1727 { 1728 unsigned long ppi_intid; 1729 struct device_node *np; 1730 unsigned int ppi_idx; 1731 int ret; 1732 1733 if (!gic_data.ppi_descs) 1734 return -ENOMEM; 1735 1736 np = of_find_node_by_phandle(fwspec->param[3]); 1737 if (WARN_ON(!np)) 1738 return -EINVAL; 1739 1740 ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); 1741 if (WARN_ON_ONCE(ret)) 1742 return 0; 1743 1744 ppi_idx = __gic_get_ppi_index(ppi_intid); 1745 ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], 1746 of_node_to_fwnode(np)); 1747 if (ret < 0) 1748 return ret; 1749 1750 *hwirq = ret; 1751 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1752 1753 return 0; 1754 } 1755 1756 static const struct irq_domain_ops partition_domain_ops = { 1757 .translate = partition_domain_translate, 1758 .select = gic_irq_domain_select, 1759 }; 1760 1761 static bool gic_enable_quirk_msm8996(void *data) 1762 { 1763 struct gic_chip_data *d = data; 1764 1765 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 1766 1767 return true; 1768 } 1769 1770 static bool gic_enable_quirk_mtk_gicr(void *data) 1771 { 1772 struct gic_chip_data *d = data; 1773 1774 d->flags |= FLAGS_WORKAROUND_MTK_GICR_SAVE; 1775 1776 return true; 1777 } 1778 1779 static bool gic_enable_quirk_cavium_38539(void *data) 1780 { 1781 struct gic_chip_data *d = data; 1782 1783 d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1784 1785 return true; 1786 } 1787 1788 static bool gic_enable_quirk_hip06_07(void *data) 1789 { 1790 struct gic_chip_data *d = data; 1791 1792 /* 1793 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 1794 * not being an actual ARM implementation). The saving grace is 1795 * that GIC-600 doesn't have ESPI, so nothing to do in that case. 1796 * HIP07 doesn't even have a proper IIDR, and still pretends to 1797 * have ESPI. In both cases, put them right. 1798 */ 1799 if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 1800 /* Zero both ESPI and the RES0 field next to it... */ 1801 d->rdists.gicd_typer &= ~GENMASK(9, 8); 1802 return true; 1803 } 1804 1805 return false; 1806 } 1807 1808 #define T241_CHIPN_MASK GENMASK_ULL(45, 44) 1809 #define T241_CHIP_GICDA_OFFSET 0x1580000 1810 #define SMCCC_SOC_ID_T241 0x036b0241 1811 1812 static bool gic_enable_quirk_nvidia_t241(void *data) 1813 { 1814 s32 soc_id = arm_smccc_get_soc_id_version(); 1815 unsigned long chip_bmask = 0; 1816 phys_addr_t phys; 1817 u32 i; 1818 1819 /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */ 1820 if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241)) 1821 return false; 1822 1823 /* Find the chips based on GICR regions PHYS addr */ 1824 for (i = 0; i < gic_data.nr_redist_regions; i++) { 1825 chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK, 1826 (u64)gic_data.redist_regions[i].phys_base)); 1827 } 1828 1829 if (hweight32(chip_bmask) < 3) 1830 return false; 1831 1832 /* Setup GICD alias regions */ 1833 for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) { 1834 if (chip_bmask & BIT(i)) { 1835 phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET; 1836 phys |= FIELD_PREP(T241_CHIPN_MASK, i); 1837 t241_dist_base_alias[i] = ioremap(phys, SZ_64K); 1838 WARN_ON_ONCE(!t241_dist_base_alias[i]); 1839 } 1840 } 1841 static_branch_enable(&gic_nvidia_t241_erratum); 1842 return true; 1843 } 1844 1845 static bool gic_enable_quirk_asr8601(void *data) 1846 { 1847 struct gic_chip_data *d = data; 1848 1849 d->flags |= FLAGS_WORKAROUND_ASR_ERRATUM_8601001; 1850 1851 return true; 1852 } 1853 1854 static bool gic_enable_quirk_arm64_2941627(void *data) 1855 { 1856 static_branch_enable(&gic_arm64_2941627_erratum); 1857 return true; 1858 } 1859 1860 static const struct gic_quirk gic_quirks[] = { 1861 { 1862 .desc = "GICv3: Qualcomm MSM8996 broken firmware", 1863 .compatible = "qcom,msm8996-gic-v3", 1864 .init = gic_enable_quirk_msm8996, 1865 }, 1866 { 1867 .desc = "GICv3: ASR erratum 8601001", 1868 .compatible = "asr,asr8601-gic-v3", 1869 .init = gic_enable_quirk_asr8601, 1870 }, 1871 { 1872 .desc = "GICv3: Mediatek Chromebook GICR save problem", 1873 .property = "mediatek,broken-save-restore-fw", 1874 .init = gic_enable_quirk_mtk_gicr, 1875 }, 1876 { 1877 .desc = "GICv3: HIP06 erratum 161010803", 1878 .iidr = 0x0204043b, 1879 .mask = 0xffffffff, 1880 .init = gic_enable_quirk_hip06_07, 1881 }, 1882 { 1883 .desc = "GICv3: HIP07 erratum 161010803", 1884 .iidr = 0x00000000, 1885 .mask = 0xffffffff, 1886 .init = gic_enable_quirk_hip06_07, 1887 }, 1888 { 1889 /* 1890 * Reserved register accesses generate a Synchronous 1891 * External Abort. This erratum applies to: 1892 * - ThunderX: CN88xx 1893 * - OCTEON TX: CN83xx, CN81xx 1894 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1895 */ 1896 .desc = "GICv3: Cavium erratum 38539", 1897 .iidr = 0xa000034c, 1898 .mask = 0xe8f00fff, 1899 .init = gic_enable_quirk_cavium_38539, 1900 }, 1901 { 1902 .desc = "GICv3: NVIDIA erratum T241-FABRIC-4", 1903 .iidr = 0x0402043b, 1904 .mask = 0xffffffff, 1905 .init = gic_enable_quirk_nvidia_t241, 1906 }, 1907 { 1908 /* 1909 * GIC-700: 2941627 workaround - IP variant [0,1] 1910 * 1911 */ 1912 .desc = "GICv3: ARM64 erratum 2941627", 1913 .iidr = 0x0400043b, 1914 .mask = 0xff0e0fff, 1915 .init = gic_enable_quirk_arm64_2941627, 1916 }, 1917 { 1918 /* 1919 * GIC-700: 2941627 workaround - IP variant [2] 1920 */ 1921 .desc = "GICv3: ARM64 erratum 2941627", 1922 .iidr = 0x0402043b, 1923 .mask = 0xff0f0fff, 1924 .init = gic_enable_quirk_arm64_2941627, 1925 }, 1926 { 1927 } 1928 }; 1929 1930 static void gic_enable_nmi_support(void) 1931 { 1932 int i; 1933 1934 if (!gic_prio_masking_enabled()) 1935 return; 1936 1937 if (gic_data.flags & FLAGS_WORKAROUND_MTK_GICR_SAVE) { 1938 pr_warn("Skipping NMI enable due to firmware issues\n"); 1939 return; 1940 } 1941 1942 ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 1943 if (!ppi_nmi_refs) 1944 return; 1945 1946 for (i = 0; i < gic_data.ppi_nr; i++) 1947 refcount_set(&ppi_nmi_refs[i], 0); 1948 1949 pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 1950 gic_has_relaxed_pmr_sync() ? "relaxed" : "forced"); 1951 1952 /* 1953 * How priority values are used by the GIC depends on two things: 1954 * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 1955 * and if Group 0 interrupts can be delivered to Linux in the non-secure 1956 * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 1957 * ICC_PMR_EL1 register and the priority that software assigns to 1958 * interrupts: 1959 * 1960 * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 1961 * ----------------------------------------------------------- 1962 * 1 | - | unchanged | unchanged 1963 * ----------------------------------------------------------- 1964 * 0 | 1 | non-secure | non-secure 1965 * ----------------------------------------------------------- 1966 * 0 | 0 | unchanged | non-secure 1967 * 1968 * where non-secure means that the value is right-shifted by one and the 1969 * MSB bit set, to make it fit in the non-secure priority range. 1970 * 1971 * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 1972 * are both either modified or unchanged, we can use the same set of 1973 * priorities. 1974 * 1975 * In the last case, where only the interrupt priorities are modified to 1976 * be in the non-secure range, we use a different PMR value to mask IRQs 1977 * and the rest of the values that we use remain unchanged. 1978 */ 1979 if (gic_has_group0() && !gic_dist_security_disabled()) 1980 static_branch_enable(&gic_nonsecure_priorities); 1981 1982 static_branch_enable(&supports_pseudo_nmis); 1983 1984 if (static_branch_likely(&supports_deactivate_key)) 1985 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1986 else 1987 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1988 } 1989 1990 static int __init gic_init_bases(phys_addr_t dist_phys_base, 1991 void __iomem *dist_base, 1992 struct redist_region *rdist_regs, 1993 u32 nr_redist_regions, 1994 u64 redist_stride, 1995 struct fwnode_handle *handle) 1996 { 1997 u32 typer; 1998 int err; 1999 2000 if (!is_hyp_mode_available()) 2001 static_branch_disable(&supports_deactivate_key); 2002 2003 if (static_branch_likely(&supports_deactivate_key)) 2004 pr_info("GIC: Using split EOI/Deactivate mode\n"); 2005 2006 gic_data.fwnode = handle; 2007 gic_data.dist_phys_base = dist_phys_base; 2008 gic_data.dist_base = dist_base; 2009 gic_data.redist_regions = rdist_regs; 2010 gic_data.nr_redist_regions = nr_redist_regions; 2011 gic_data.redist_stride = redist_stride; 2012 2013 /* 2014 * Find out how many interrupts are supported. 2015 */ 2016 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 2017 gic_data.rdists.gicd_typer = typer; 2018 2019 gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 2020 gic_quirks, &gic_data); 2021 2022 pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 2023 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 2024 2025 /* 2026 * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 2027 * architecture spec (which says that reserved registers are RES0). 2028 */ 2029 if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 2030 gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 2031 2032 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 2033 &gic_data); 2034 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 2035 if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) { 2036 /* Disable GICv4.x features for the erratum T241-FABRIC-4 */ 2037 gic_data.rdists.has_rvpeid = true; 2038 gic_data.rdists.has_vlpis = true; 2039 gic_data.rdists.has_direct_lpi = true; 2040 gic_data.rdists.has_vpend_valid_dirty = true; 2041 } 2042 2043 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 2044 err = -ENOMEM; 2045 goto out_free; 2046 } 2047 2048 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 2049 2050 gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 2051 2052 if (typer & GICD_TYPER_MBIS) { 2053 err = mbi_init(handle, gic_data.domain); 2054 if (err) 2055 pr_err("Failed to initialize MBIs\n"); 2056 } 2057 2058 set_handle_irq(gic_handle_irq); 2059 2060 gic_update_rdist_properties(); 2061 2062 gic_dist_init(); 2063 gic_cpu_init(); 2064 gic_smp_init(); 2065 gic_cpu_pm_init(); 2066 2067 if (gic_dist_supports_lpis()) { 2068 its_init(handle, &gic_data.rdists, gic_data.domain); 2069 its_cpu_init(); 2070 its_lpi_memreserve_init(); 2071 } else { 2072 if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 2073 gicv2m_init(handle, gic_data.domain); 2074 } 2075 2076 gic_enable_nmi_support(); 2077 2078 return 0; 2079 2080 out_free: 2081 if (gic_data.domain) 2082 irq_domain_remove(gic_data.domain); 2083 free_percpu(gic_data.rdists.rdist); 2084 return err; 2085 } 2086 2087 static int __init gic_validate_dist_version(void __iomem *dist_base) 2088 { 2089 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2090 2091 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 2092 return -ENODEV; 2093 2094 return 0; 2095 } 2096 2097 /* Create all possible partitions at boot time */ 2098 static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 2099 { 2100 struct device_node *parts_node, *child_part; 2101 int part_idx = 0, i; 2102 int nr_parts; 2103 struct partition_affinity *parts; 2104 2105 parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 2106 if (!parts_node) 2107 return; 2108 2109 gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 2110 if (!gic_data.ppi_descs) 2111 goto out_put_node; 2112 2113 nr_parts = of_get_child_count(parts_node); 2114 2115 if (!nr_parts) 2116 goto out_put_node; 2117 2118 parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 2119 if (WARN_ON(!parts)) 2120 goto out_put_node; 2121 2122 for_each_child_of_node(parts_node, child_part) { 2123 struct partition_affinity *part; 2124 int n; 2125 2126 part = &parts[part_idx]; 2127 2128 part->partition_id = of_node_to_fwnode(child_part); 2129 2130 pr_info("GIC: PPI partition %pOFn[%d] { ", 2131 child_part, part_idx); 2132 2133 n = of_property_count_elems_of_size(child_part, "affinity", 2134 sizeof(u32)); 2135 WARN_ON(n <= 0); 2136 2137 for (i = 0; i < n; i++) { 2138 int err, cpu; 2139 u32 cpu_phandle; 2140 struct device_node *cpu_node; 2141 2142 err = of_property_read_u32_index(child_part, "affinity", 2143 i, &cpu_phandle); 2144 if (WARN_ON(err)) 2145 continue; 2146 2147 cpu_node = of_find_node_by_phandle(cpu_phandle); 2148 if (WARN_ON(!cpu_node)) 2149 continue; 2150 2151 cpu = of_cpu_node_to_id(cpu_node); 2152 if (WARN_ON(cpu < 0)) { 2153 of_node_put(cpu_node); 2154 continue; 2155 } 2156 2157 pr_cont("%pOF[%d] ", cpu_node, cpu); 2158 2159 cpumask_set_cpu(cpu, &part->mask); 2160 of_node_put(cpu_node); 2161 } 2162 2163 pr_cont("}\n"); 2164 part_idx++; 2165 } 2166 2167 for (i = 0; i < gic_data.ppi_nr; i++) { 2168 unsigned int irq; 2169 struct partition_desc *desc; 2170 struct irq_fwspec ppi_fwspec = { 2171 .fwnode = gic_data.fwnode, 2172 .param_count = 3, 2173 .param = { 2174 [0] = GIC_IRQ_TYPE_PARTITION, 2175 [1] = i, 2176 [2] = IRQ_TYPE_NONE, 2177 }, 2178 }; 2179 2180 irq = irq_create_fwspec_mapping(&ppi_fwspec); 2181 if (WARN_ON(!irq)) 2182 continue; 2183 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 2184 irq, &partition_domain_ops); 2185 if (WARN_ON(!desc)) 2186 continue; 2187 2188 gic_data.ppi_descs[i] = desc; 2189 } 2190 2191 out_put_node: 2192 of_node_put(parts_node); 2193 } 2194 2195 static void __init gic_of_setup_kvm_info(struct device_node *node) 2196 { 2197 int ret; 2198 struct resource r; 2199 u32 gicv_idx; 2200 2201 gic_v3_kvm_info.type = GIC_V3; 2202 2203 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 2204 if (!gic_v3_kvm_info.maint_irq) 2205 return; 2206 2207 if (of_property_read_u32(node, "#redistributor-regions", 2208 &gicv_idx)) 2209 gicv_idx = 1; 2210 2211 gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 2212 ret = of_address_to_resource(node, gicv_idx, &r); 2213 if (!ret) 2214 gic_v3_kvm_info.vcpu = r; 2215 2216 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2217 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2218 vgic_set_kvm_info(&gic_v3_kvm_info); 2219 } 2220 2221 static void gic_request_region(resource_size_t base, resource_size_t size, 2222 const char *name) 2223 { 2224 if (!request_mem_region(base, size, name)) 2225 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", 2226 name, &base); 2227 } 2228 2229 static void __iomem *gic_of_iomap(struct device_node *node, int idx, 2230 const char *name, struct resource *res) 2231 { 2232 void __iomem *base; 2233 int ret; 2234 2235 ret = of_address_to_resource(node, idx, res); 2236 if (ret) 2237 return IOMEM_ERR_PTR(ret); 2238 2239 gic_request_region(res->start, resource_size(res), name); 2240 base = of_iomap(node, idx); 2241 2242 return base ?: IOMEM_ERR_PTR(-ENOMEM); 2243 } 2244 2245 static int __init gic_of_init(struct device_node *node, struct device_node *parent) 2246 { 2247 phys_addr_t dist_phys_base; 2248 void __iomem *dist_base; 2249 struct redist_region *rdist_regs; 2250 struct resource res; 2251 u64 redist_stride; 2252 u32 nr_redist_regions; 2253 int err, i; 2254 2255 dist_base = gic_of_iomap(node, 0, "GICD", &res); 2256 if (IS_ERR(dist_base)) { 2257 pr_err("%pOF: unable to map gic dist registers\n", node); 2258 return PTR_ERR(dist_base); 2259 } 2260 2261 dist_phys_base = res.start; 2262 2263 err = gic_validate_dist_version(dist_base); 2264 if (err) { 2265 pr_err("%pOF: no distributor detected, giving up\n", node); 2266 goto out_unmap_dist; 2267 } 2268 2269 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 2270 nr_redist_regions = 1; 2271 2272 rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 2273 GFP_KERNEL); 2274 if (!rdist_regs) { 2275 err = -ENOMEM; 2276 goto out_unmap_dist; 2277 } 2278 2279 for (i = 0; i < nr_redist_regions; i++) { 2280 rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res); 2281 if (IS_ERR(rdist_regs[i].redist_base)) { 2282 pr_err("%pOF: couldn't map region %d\n", node, i); 2283 err = -ENODEV; 2284 goto out_unmap_rdist; 2285 } 2286 rdist_regs[i].phys_base = res.start; 2287 } 2288 2289 if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 2290 redist_stride = 0; 2291 2292 gic_enable_of_quirks(node, gic_quirks, &gic_data); 2293 2294 err = gic_init_bases(dist_phys_base, dist_base, rdist_regs, 2295 nr_redist_regions, redist_stride, &node->fwnode); 2296 if (err) 2297 goto out_unmap_rdist; 2298 2299 gic_populate_ppi_partitions(node); 2300 2301 if (static_branch_likely(&supports_deactivate_key)) 2302 gic_of_setup_kvm_info(node); 2303 return 0; 2304 2305 out_unmap_rdist: 2306 for (i = 0; i < nr_redist_regions; i++) 2307 if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base)) 2308 iounmap(rdist_regs[i].redist_base); 2309 kfree(rdist_regs); 2310 out_unmap_dist: 2311 iounmap(dist_base); 2312 return err; 2313 } 2314 2315 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 2316 2317 #ifdef CONFIG_ACPI 2318 static struct 2319 { 2320 void __iomem *dist_base; 2321 struct redist_region *redist_regs; 2322 u32 nr_redist_regions; 2323 bool single_redist; 2324 int enabled_rdists; 2325 u32 maint_irq; 2326 int maint_irq_mode; 2327 phys_addr_t vcpu_base; 2328 } acpi_data __initdata; 2329 2330 static void __init 2331 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 2332 { 2333 static int count = 0; 2334 2335 acpi_data.redist_regs[count].phys_base = phys_base; 2336 acpi_data.redist_regs[count].redist_base = redist_base; 2337 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 2338 count++; 2339 } 2340 2341 static int __init 2342 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 2343 const unsigned long end) 2344 { 2345 struct acpi_madt_generic_redistributor *redist = 2346 (struct acpi_madt_generic_redistributor *)header; 2347 void __iomem *redist_base; 2348 2349 redist_base = ioremap(redist->base_address, redist->length); 2350 if (!redist_base) { 2351 pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2352 return -ENOMEM; 2353 } 2354 gic_request_region(redist->base_address, redist->length, "GICR"); 2355 2356 gic_acpi_register_redist(redist->base_address, redist_base); 2357 return 0; 2358 } 2359 2360 static int __init 2361 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2362 const unsigned long end) 2363 { 2364 struct acpi_madt_generic_interrupt *gicc = 2365 (struct acpi_madt_generic_interrupt *)header; 2366 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2367 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2368 void __iomem *redist_base; 2369 2370 /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 2371 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2372 return 0; 2373 2374 redist_base = ioremap(gicc->gicr_base_address, size); 2375 if (!redist_base) 2376 return -ENOMEM; 2377 gic_request_region(gicc->gicr_base_address, size, "GICR"); 2378 2379 gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2380 return 0; 2381 } 2382 2383 static int __init gic_acpi_collect_gicr_base(void) 2384 { 2385 acpi_tbl_entry_handler redist_parser; 2386 enum acpi_madt_type type; 2387 2388 if (acpi_data.single_redist) { 2389 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2390 redist_parser = gic_acpi_parse_madt_gicc; 2391 } else { 2392 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2393 redist_parser = gic_acpi_parse_madt_redist; 2394 } 2395 2396 /* Collect redistributor base addresses in GICR entries */ 2397 if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2398 return 0; 2399 2400 pr_info("No valid GICR entries exist\n"); 2401 return -ENODEV; 2402 } 2403 2404 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2405 const unsigned long end) 2406 { 2407 /* Subtable presence means that redist exists, that's it */ 2408 return 0; 2409 } 2410 2411 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2412 const unsigned long end) 2413 { 2414 struct acpi_madt_generic_interrupt *gicc = 2415 (struct acpi_madt_generic_interrupt *)header; 2416 2417 /* 2418 * If GICC is enabled and has valid gicr base address, then it means 2419 * GICR base is presented via GICC 2420 */ 2421 if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2422 acpi_data.enabled_rdists++; 2423 return 0; 2424 } 2425 2426 /* 2427 * It's perfectly valid firmware can pass disabled GICC entry, driver 2428 * should not treat as errors, skip the entry instead of probe fail. 2429 */ 2430 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2431 return 0; 2432 2433 return -ENODEV; 2434 } 2435 2436 static int __init gic_acpi_count_gicr_regions(void) 2437 { 2438 int count; 2439 2440 /* 2441 * Count how many redistributor regions we have. It is not allowed 2442 * to mix redistributor description, GICR and GICC subtables have to be 2443 * mutually exclusive. 2444 */ 2445 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2446 gic_acpi_match_gicr, 0); 2447 if (count > 0) { 2448 acpi_data.single_redist = false; 2449 return count; 2450 } 2451 2452 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2453 gic_acpi_match_gicc, 0); 2454 if (count > 0) { 2455 acpi_data.single_redist = true; 2456 count = acpi_data.enabled_rdists; 2457 } 2458 2459 return count; 2460 } 2461 2462 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2463 struct acpi_probe_entry *ape) 2464 { 2465 struct acpi_madt_generic_distributor *dist; 2466 int count; 2467 2468 dist = (struct acpi_madt_generic_distributor *)header; 2469 if (dist->version != ape->driver_data) 2470 return false; 2471 2472 /* We need to do that exercise anyway, the sooner the better */ 2473 count = gic_acpi_count_gicr_regions(); 2474 if (count <= 0) 2475 return false; 2476 2477 acpi_data.nr_redist_regions = count; 2478 return true; 2479 } 2480 2481 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 2482 const unsigned long end) 2483 { 2484 struct acpi_madt_generic_interrupt *gicc = 2485 (struct acpi_madt_generic_interrupt *)header; 2486 int maint_irq_mode; 2487 static int first_madt = true; 2488 2489 /* Skip unusable CPUs */ 2490 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2491 return 0; 2492 2493 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 2494 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 2495 2496 if (first_madt) { 2497 first_madt = false; 2498 2499 acpi_data.maint_irq = gicc->vgic_interrupt; 2500 acpi_data.maint_irq_mode = maint_irq_mode; 2501 acpi_data.vcpu_base = gicc->gicv_base_address; 2502 2503 return 0; 2504 } 2505 2506 /* 2507 * The maintenance interrupt and GICV should be the same for every CPU 2508 */ 2509 if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 2510 (acpi_data.maint_irq_mode != maint_irq_mode) || 2511 (acpi_data.vcpu_base != gicc->gicv_base_address)) 2512 return -EINVAL; 2513 2514 return 0; 2515 } 2516 2517 static bool __init gic_acpi_collect_virt_info(void) 2518 { 2519 int count; 2520 2521 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2522 gic_acpi_parse_virt_madt_gicc, 0); 2523 2524 return (count > 0); 2525 } 2526 2527 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 2528 #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 2529 #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 2530 2531 static void __init gic_acpi_setup_kvm_info(void) 2532 { 2533 int irq; 2534 2535 if (!gic_acpi_collect_virt_info()) { 2536 pr_warn("Unable to get hardware information used for virtualization\n"); 2537 return; 2538 } 2539 2540 gic_v3_kvm_info.type = GIC_V3; 2541 2542 irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 2543 acpi_data.maint_irq_mode, 2544 ACPI_ACTIVE_HIGH); 2545 if (irq <= 0) 2546 return; 2547 2548 gic_v3_kvm_info.maint_irq = irq; 2549 2550 if (acpi_data.vcpu_base) { 2551 struct resource *vcpu = &gic_v3_kvm_info.vcpu; 2552 2553 vcpu->flags = IORESOURCE_MEM; 2554 vcpu->start = acpi_data.vcpu_base; 2555 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 2556 } 2557 2558 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2559 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2560 vgic_set_kvm_info(&gic_v3_kvm_info); 2561 } 2562 2563 static struct fwnode_handle *gsi_domain_handle; 2564 2565 static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi) 2566 { 2567 return gsi_domain_handle; 2568 } 2569 2570 static int __init 2571 gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2572 { 2573 struct acpi_madt_generic_distributor *dist; 2574 size_t size; 2575 int i, err; 2576 2577 /* Get distributor base address */ 2578 dist = (struct acpi_madt_generic_distributor *)header; 2579 acpi_data.dist_base = ioremap(dist->base_address, 2580 ACPI_GICV3_DIST_MEM_SIZE); 2581 if (!acpi_data.dist_base) { 2582 pr_err("Unable to map GICD registers\n"); 2583 return -ENOMEM; 2584 } 2585 gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD"); 2586 2587 err = gic_validate_dist_version(acpi_data.dist_base); 2588 if (err) { 2589 pr_err("No distributor detected at @%p, giving up\n", 2590 acpi_data.dist_base); 2591 goto out_dist_unmap; 2592 } 2593 2594 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2595 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2596 if (!acpi_data.redist_regs) { 2597 err = -ENOMEM; 2598 goto out_dist_unmap; 2599 } 2600 2601 err = gic_acpi_collect_gicr_base(); 2602 if (err) 2603 goto out_redist_unmap; 2604 2605 gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2606 if (!gsi_domain_handle) { 2607 err = -ENOMEM; 2608 goto out_redist_unmap; 2609 } 2610 2611 err = gic_init_bases(dist->base_address, acpi_data.dist_base, 2612 acpi_data.redist_regs, acpi_data.nr_redist_regions, 2613 0, gsi_domain_handle); 2614 if (err) 2615 goto out_fwhandle_free; 2616 2617 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id); 2618 2619 if (static_branch_likely(&supports_deactivate_key)) 2620 gic_acpi_setup_kvm_info(); 2621 2622 return 0; 2623 2624 out_fwhandle_free: 2625 irq_domain_free_fwnode(gsi_domain_handle); 2626 out_redist_unmap: 2627 for (i = 0; i < acpi_data.nr_redist_regions; i++) 2628 if (acpi_data.redist_regs[i].redist_base) 2629 iounmap(acpi_data.redist_regs[i].redist_base); 2630 kfree(acpi_data.redist_regs); 2631 out_dist_unmap: 2632 iounmap(acpi_data.dist_base); 2633 return err; 2634 } 2635 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2636 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2637 gic_acpi_init); 2638 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2639 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2640 gic_acpi_init); 2641 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2642 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2643 gic_acpi_init); 2644 #endif 2645