1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #define pr_fmt(fmt) "GICv3: " fmt 8 9 #include <linux/acpi.h> 10 #include <linux/cpu.h> 11 #include <linux/cpu_pm.h> 12 #include <linux/delay.h> 13 #include <linux/interrupt.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kstrtox.h> 16 #include <linux/of.h> 17 #include <linux/of_address.h> 18 #include <linux/of_irq.h> 19 #include <linux/percpu.h> 20 #include <linux/refcount.h> 21 #include <linux/slab.h> 22 23 #include <linux/irqchip.h> 24 #include <linux/irqchip/arm-gic-common.h> 25 #include <linux/irqchip/arm-gic-v3.h> 26 #include <linux/irqchip/irq-partition-percpu.h> 27 #include <linux/bitfield.h> 28 #include <linux/bits.h> 29 #include <linux/arm-smccc.h> 30 31 #include <asm/cputype.h> 32 #include <asm/exception.h> 33 #include <asm/smp_plat.h> 34 #include <asm/virt.h> 35 36 #include "irq-gic-common.h" 37 38 #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 39 40 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 41 #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 42 #define FLAGS_WORKAROUND_MTK_GICR_SAVE (1ULL << 2) 43 44 #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 45 46 struct redist_region { 47 void __iomem *redist_base; 48 phys_addr_t phys_base; 49 bool single_redist; 50 }; 51 52 struct gic_chip_data { 53 struct fwnode_handle *fwnode; 54 phys_addr_t dist_phys_base; 55 void __iomem *dist_base; 56 struct redist_region *redist_regions; 57 struct rdists rdists; 58 struct irq_domain *domain; 59 u64 redist_stride; 60 u32 nr_redist_regions; 61 u64 flags; 62 bool has_rss; 63 unsigned int ppi_nr; 64 struct partition_desc **ppi_descs; 65 }; 66 67 #define T241_CHIPS_MAX 4 68 static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly; 69 static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum); 70 71 static struct gic_chip_data gic_data __read_mostly; 72 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 73 74 #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 75 #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 76 #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 77 78 /* 79 * The behaviours of RPR and PMR registers differ depending on the value of 80 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 81 * distributor and redistributors depends on whether security is enabled in the 82 * GIC. 83 * 84 * When security is enabled, non-secure priority values from the (re)distributor 85 * are presented to the GIC CPUIF as follow: 86 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 87 * 88 * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 89 * EL1 are subject to a similar operation thus matching the priorities presented 90 * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 91 * these values are unchanged by the GIC. 92 * 93 * see GICv3/GICv4 Architecture Specification (IHI0069D): 94 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 95 * priorities. 96 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 97 * interrupt. 98 */ 99 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 100 101 DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 102 EXPORT_SYMBOL(gic_nonsecure_priorities); 103 104 /* 105 * When the Non-secure world has access to group 0 interrupts (as a 106 * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will 107 * return the Distributor's view of the interrupt priority. 108 * 109 * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority 110 * written by software is moved to the Non-secure range by the Distributor. 111 * 112 * If both are true (which is when gic_nonsecure_priorities gets enabled), 113 * we need to shift down the priority programmed by software to match it 114 * against the value returned by ICC_RPR_EL1. 115 */ 116 #define GICD_INT_RPR_PRI(priority) \ 117 ({ \ 118 u32 __priority = (priority); \ 119 if (static_branch_unlikely(&gic_nonsecure_priorities)) \ 120 __priority = 0x80 | (__priority >> 1); \ 121 \ 122 __priority; \ 123 }) 124 125 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 126 static refcount_t *ppi_nmi_refs; 127 128 static struct gic_kvm_info gic_v3_kvm_info __initdata; 129 static DEFINE_PER_CPU(bool, has_rss); 130 131 #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 132 #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 133 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 134 #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 135 136 /* Our default, arbitrary priority value. Linux only uses one anyway. */ 137 #define DEFAULT_PMR_VALUE 0xf0 138 139 enum gic_intid_range { 140 SGI_RANGE, 141 PPI_RANGE, 142 SPI_RANGE, 143 EPPI_RANGE, 144 ESPI_RANGE, 145 LPI_RANGE, 146 __INVALID_RANGE__ 147 }; 148 149 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 150 { 151 switch (hwirq) { 152 case 0 ... 15: 153 return SGI_RANGE; 154 case 16 ... 31: 155 return PPI_RANGE; 156 case 32 ... 1019: 157 return SPI_RANGE; 158 case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 159 return EPPI_RANGE; 160 case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 161 return ESPI_RANGE; 162 case 8192 ... GENMASK(23, 0): 163 return LPI_RANGE; 164 default: 165 return __INVALID_RANGE__; 166 } 167 } 168 169 static enum gic_intid_range get_intid_range(struct irq_data *d) 170 { 171 return __get_intid_range(d->hwirq); 172 } 173 174 static inline unsigned int gic_irq(struct irq_data *d) 175 { 176 return d->hwirq; 177 } 178 179 static inline bool gic_irq_in_rdist(struct irq_data *d) 180 { 181 switch (get_intid_range(d)) { 182 case SGI_RANGE: 183 case PPI_RANGE: 184 case EPPI_RANGE: 185 return true; 186 default: 187 return false; 188 } 189 } 190 191 static inline void __iomem *gic_dist_base_alias(struct irq_data *d) 192 { 193 if (static_branch_unlikely(&gic_nvidia_t241_erratum)) { 194 irq_hw_number_t hwirq = irqd_to_hwirq(d); 195 u32 chip; 196 197 /* 198 * For the erratum T241-FABRIC-4, read accesses to GICD_In{E} 199 * registers are directed to the chip that owns the SPI. The 200 * the alias region can also be used for writes to the 201 * GICD_In{E} except GICD_ICENABLERn. Each chip has support 202 * for 320 {E}SPIs. Mappings for all 4 chips: 203 * Chip0 = 32-351 204 * Chip1 = 352-671 205 * Chip2 = 672-991 206 * Chip3 = 4096-4415 207 */ 208 switch (__get_intid_range(hwirq)) { 209 case SPI_RANGE: 210 chip = (hwirq - 32) / 320; 211 break; 212 case ESPI_RANGE: 213 chip = 3; 214 break; 215 default: 216 unreachable(); 217 } 218 return t241_dist_base_alias[chip]; 219 } 220 221 return gic_data.dist_base; 222 } 223 224 static inline void __iomem *gic_dist_base(struct irq_data *d) 225 { 226 switch (get_intid_range(d)) { 227 case SGI_RANGE: 228 case PPI_RANGE: 229 case EPPI_RANGE: 230 /* SGI+PPI -> SGI_base for this CPU */ 231 return gic_data_rdist_sgi_base(); 232 233 case SPI_RANGE: 234 case ESPI_RANGE: 235 /* SPI -> dist_base */ 236 return gic_data.dist_base; 237 238 default: 239 return NULL; 240 } 241 } 242 243 static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) 244 { 245 u32 count = 1000000; /* 1s! */ 246 247 while (readl_relaxed(base + GICD_CTLR) & bit) { 248 count--; 249 if (!count) { 250 pr_err_ratelimited("RWP timeout, gone fishing\n"); 251 return; 252 } 253 cpu_relax(); 254 udelay(1); 255 } 256 } 257 258 /* Wait for completion of a distributor change */ 259 static void gic_dist_wait_for_rwp(void) 260 { 261 gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP); 262 } 263 264 /* Wait for completion of a redistributor change */ 265 static void gic_redist_wait_for_rwp(void) 266 { 267 gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); 268 } 269 270 #ifdef CONFIG_ARM64 271 272 static u64 __maybe_unused gic_read_iar(void) 273 { 274 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 275 return gic_read_iar_cavium_thunderx(); 276 else 277 return gic_read_iar_common(); 278 } 279 #endif 280 281 static void gic_enable_redist(bool enable) 282 { 283 void __iomem *rbase; 284 u32 count = 1000000; /* 1s! */ 285 u32 val; 286 287 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 288 return; 289 290 rbase = gic_data_rdist_rd_base(); 291 292 val = readl_relaxed(rbase + GICR_WAKER); 293 if (enable) 294 /* Wake up this CPU redistributor */ 295 val &= ~GICR_WAKER_ProcessorSleep; 296 else 297 val |= GICR_WAKER_ProcessorSleep; 298 writel_relaxed(val, rbase + GICR_WAKER); 299 300 if (!enable) { /* Check that GICR_WAKER is writeable */ 301 val = readl_relaxed(rbase + GICR_WAKER); 302 if (!(val & GICR_WAKER_ProcessorSleep)) 303 return; /* No PM support in this redistributor */ 304 } 305 306 while (--count) { 307 val = readl_relaxed(rbase + GICR_WAKER); 308 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 309 break; 310 cpu_relax(); 311 udelay(1); 312 } 313 if (!count) 314 pr_err_ratelimited("redistributor failed to %s...\n", 315 enable ? "wakeup" : "sleep"); 316 } 317 318 /* 319 * Routines to disable, enable, EOI and route interrupts 320 */ 321 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 322 { 323 switch (get_intid_range(d)) { 324 case SGI_RANGE: 325 case PPI_RANGE: 326 case SPI_RANGE: 327 *index = d->hwirq; 328 return offset; 329 case EPPI_RANGE: 330 /* 331 * Contrary to the ESPI range, the EPPI range is contiguous 332 * to the PPI range in the registers, so let's adjust the 333 * displacement accordingly. Consistency is overrated. 334 */ 335 *index = d->hwirq - EPPI_BASE_INTID + 32; 336 return offset; 337 case ESPI_RANGE: 338 *index = d->hwirq - ESPI_BASE_INTID; 339 switch (offset) { 340 case GICD_ISENABLER: 341 return GICD_ISENABLERnE; 342 case GICD_ICENABLER: 343 return GICD_ICENABLERnE; 344 case GICD_ISPENDR: 345 return GICD_ISPENDRnE; 346 case GICD_ICPENDR: 347 return GICD_ICPENDRnE; 348 case GICD_ISACTIVER: 349 return GICD_ISACTIVERnE; 350 case GICD_ICACTIVER: 351 return GICD_ICACTIVERnE; 352 case GICD_IPRIORITYR: 353 return GICD_IPRIORITYRnE; 354 case GICD_ICFGR: 355 return GICD_ICFGRnE; 356 case GICD_IROUTER: 357 return GICD_IROUTERnE; 358 default: 359 break; 360 } 361 break; 362 default: 363 break; 364 } 365 366 WARN_ON(1); 367 *index = d->hwirq; 368 return offset; 369 } 370 371 static int gic_peek_irq(struct irq_data *d, u32 offset) 372 { 373 void __iomem *base; 374 u32 index, mask; 375 376 offset = convert_offset_index(d, offset, &index); 377 mask = 1 << (index % 32); 378 379 if (gic_irq_in_rdist(d)) 380 base = gic_data_rdist_sgi_base(); 381 else 382 base = gic_dist_base_alias(d); 383 384 return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 385 } 386 387 static void gic_poke_irq(struct irq_data *d, u32 offset) 388 { 389 void __iomem *base; 390 u32 index, mask; 391 392 offset = convert_offset_index(d, offset, &index); 393 mask = 1 << (index % 32); 394 395 if (gic_irq_in_rdist(d)) 396 base = gic_data_rdist_sgi_base(); 397 else 398 base = gic_data.dist_base; 399 400 writel_relaxed(mask, base + offset + (index / 32) * 4); 401 } 402 403 static void gic_mask_irq(struct irq_data *d) 404 { 405 gic_poke_irq(d, GICD_ICENABLER); 406 if (gic_irq_in_rdist(d)) 407 gic_redist_wait_for_rwp(); 408 else 409 gic_dist_wait_for_rwp(); 410 } 411 412 static void gic_eoimode1_mask_irq(struct irq_data *d) 413 { 414 gic_mask_irq(d); 415 /* 416 * When masking a forwarded interrupt, make sure it is 417 * deactivated as well. 418 * 419 * This ensures that an interrupt that is getting 420 * disabled/masked will not get "stuck", because there is 421 * noone to deactivate it (guest is being terminated). 422 */ 423 if (irqd_is_forwarded_to_vcpu(d)) 424 gic_poke_irq(d, GICD_ICACTIVER); 425 } 426 427 static void gic_unmask_irq(struct irq_data *d) 428 { 429 gic_poke_irq(d, GICD_ISENABLER); 430 } 431 432 static inline bool gic_supports_nmi(void) 433 { 434 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 435 static_branch_likely(&supports_pseudo_nmis); 436 } 437 438 static int gic_irq_set_irqchip_state(struct irq_data *d, 439 enum irqchip_irq_state which, bool val) 440 { 441 u32 reg; 442 443 if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 444 return -EINVAL; 445 446 switch (which) { 447 case IRQCHIP_STATE_PENDING: 448 reg = val ? GICD_ISPENDR : GICD_ICPENDR; 449 break; 450 451 case IRQCHIP_STATE_ACTIVE: 452 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 453 break; 454 455 case IRQCHIP_STATE_MASKED: 456 if (val) { 457 gic_mask_irq(d); 458 return 0; 459 } 460 reg = GICD_ISENABLER; 461 break; 462 463 default: 464 return -EINVAL; 465 } 466 467 gic_poke_irq(d, reg); 468 return 0; 469 } 470 471 static int gic_irq_get_irqchip_state(struct irq_data *d, 472 enum irqchip_irq_state which, bool *val) 473 { 474 if (d->hwirq >= 8192) /* PPI/SPI only */ 475 return -EINVAL; 476 477 switch (which) { 478 case IRQCHIP_STATE_PENDING: 479 *val = gic_peek_irq(d, GICD_ISPENDR); 480 break; 481 482 case IRQCHIP_STATE_ACTIVE: 483 *val = gic_peek_irq(d, GICD_ISACTIVER); 484 break; 485 486 case IRQCHIP_STATE_MASKED: 487 *val = !gic_peek_irq(d, GICD_ISENABLER); 488 break; 489 490 default: 491 return -EINVAL; 492 } 493 494 return 0; 495 } 496 497 static void gic_irq_set_prio(struct irq_data *d, u8 prio) 498 { 499 void __iomem *base = gic_dist_base(d); 500 u32 offset, index; 501 502 offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 503 504 writeb_relaxed(prio, base + offset + index); 505 } 506 507 static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 508 { 509 switch (__get_intid_range(hwirq)) { 510 case PPI_RANGE: 511 return hwirq - 16; 512 case EPPI_RANGE: 513 return hwirq - EPPI_BASE_INTID + 16; 514 default: 515 unreachable(); 516 } 517 } 518 519 static u32 gic_get_ppi_index(struct irq_data *d) 520 { 521 return __gic_get_ppi_index(d->hwirq); 522 } 523 524 static int gic_irq_nmi_setup(struct irq_data *d) 525 { 526 struct irq_desc *desc = irq_to_desc(d->irq); 527 528 if (!gic_supports_nmi()) 529 return -EINVAL; 530 531 if (gic_peek_irq(d, GICD_ISENABLER)) { 532 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 533 return -EINVAL; 534 } 535 536 /* 537 * A secondary irq_chip should be in charge of LPI request, 538 * it should not be possible to get there 539 */ 540 if (WARN_ON(gic_irq(d) >= 8192)) 541 return -EINVAL; 542 543 /* desc lock should already be held */ 544 if (gic_irq_in_rdist(d)) { 545 u32 idx = gic_get_ppi_index(d); 546 547 /* Setting up PPI as NMI, only switch handler for first NMI */ 548 if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 549 refcount_set(&ppi_nmi_refs[idx], 1); 550 desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 551 } 552 } else { 553 desc->handle_irq = handle_fasteoi_nmi; 554 } 555 556 gic_irq_set_prio(d, GICD_INT_NMI_PRI); 557 558 return 0; 559 } 560 561 static void gic_irq_nmi_teardown(struct irq_data *d) 562 { 563 struct irq_desc *desc = irq_to_desc(d->irq); 564 565 if (WARN_ON(!gic_supports_nmi())) 566 return; 567 568 if (gic_peek_irq(d, GICD_ISENABLER)) { 569 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 570 return; 571 } 572 573 /* 574 * A secondary irq_chip should be in charge of LPI request, 575 * it should not be possible to get there 576 */ 577 if (WARN_ON(gic_irq(d) >= 8192)) 578 return; 579 580 /* desc lock should already be held */ 581 if (gic_irq_in_rdist(d)) { 582 u32 idx = gic_get_ppi_index(d); 583 584 /* Tearing down NMI, only switch handler for last NMI */ 585 if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 586 desc->handle_irq = handle_percpu_devid_irq; 587 } else { 588 desc->handle_irq = handle_fasteoi_irq; 589 } 590 591 gic_irq_set_prio(d, GICD_INT_DEF_PRI); 592 } 593 594 static void gic_eoi_irq(struct irq_data *d) 595 { 596 write_gicreg(gic_irq(d), ICC_EOIR1_EL1); 597 isb(); 598 } 599 600 static void gic_eoimode1_eoi_irq(struct irq_data *d) 601 { 602 /* 603 * No need to deactivate an LPI, or an interrupt that 604 * is is getting forwarded to a vcpu. 605 */ 606 if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 607 return; 608 gic_write_dir(gic_irq(d)); 609 } 610 611 static int gic_set_type(struct irq_data *d, unsigned int type) 612 { 613 enum gic_intid_range range; 614 unsigned int irq = gic_irq(d); 615 void __iomem *base; 616 u32 offset, index; 617 int ret; 618 619 range = get_intid_range(d); 620 621 /* Interrupt configuration for SGIs can't be changed */ 622 if (range == SGI_RANGE) 623 return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 624 625 /* SPIs have restrictions on the supported types */ 626 if ((range == SPI_RANGE || range == ESPI_RANGE) && 627 type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 628 return -EINVAL; 629 630 if (gic_irq_in_rdist(d)) 631 base = gic_data_rdist_sgi_base(); 632 else 633 base = gic_dist_base_alias(d); 634 635 offset = convert_offset_index(d, GICD_ICFGR, &index); 636 637 ret = gic_configure_irq(index, type, base + offset, NULL); 638 if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 639 /* Misconfigured PPIs are usually not fatal */ 640 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 641 ret = 0; 642 } 643 644 return ret; 645 } 646 647 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 648 { 649 if (get_intid_range(d) == SGI_RANGE) 650 return -EINVAL; 651 652 if (vcpu) 653 irqd_set_forwarded_to_vcpu(d); 654 else 655 irqd_clr_forwarded_to_vcpu(d); 656 return 0; 657 } 658 659 static u64 gic_mpidr_to_affinity(unsigned long mpidr) 660 { 661 u64 aff; 662 663 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 664 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 665 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 666 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 667 668 return aff; 669 } 670 671 static void gic_deactivate_unhandled(u32 irqnr) 672 { 673 if (static_branch_likely(&supports_deactivate_key)) { 674 if (irqnr < 8192) 675 gic_write_dir(irqnr); 676 } else { 677 write_gicreg(irqnr, ICC_EOIR1_EL1); 678 isb(); 679 } 680 } 681 682 /* 683 * Follow a read of the IAR with any HW maintenance that needs to happen prior 684 * to invoking the relevant IRQ handler. We must do two things: 685 * 686 * (1) Ensure instruction ordering between a read of IAR and subsequent 687 * instructions in the IRQ handler using an ISB. 688 * 689 * It is possible for the IAR to report an IRQ which was signalled *after* 690 * the CPU took an IRQ exception as multiple interrupts can race to be 691 * recognized by the GIC, earlier interrupts could be withdrawn, and/or 692 * later interrupts could be prioritized by the GIC. 693 * 694 * For devices which are tightly coupled to the CPU, such as PMUs, a 695 * context synchronization event is necessary to ensure that system 696 * register state is not stale, as these may have been indirectly written 697 * *after* exception entry. 698 * 699 * (2) Deactivate the interrupt when EOI mode 1 is in use. 700 */ 701 static inline void gic_complete_ack(u32 irqnr) 702 { 703 if (static_branch_likely(&supports_deactivate_key)) 704 write_gicreg(irqnr, ICC_EOIR1_EL1); 705 706 isb(); 707 } 708 709 static bool gic_rpr_is_nmi_prio(void) 710 { 711 if (!gic_supports_nmi()) 712 return false; 713 714 return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI)); 715 } 716 717 static bool gic_irqnr_is_special(u32 irqnr) 718 { 719 return irqnr >= 1020 && irqnr <= 1023; 720 } 721 722 static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs) 723 { 724 if (gic_irqnr_is_special(irqnr)) 725 return; 726 727 gic_complete_ack(irqnr); 728 729 if (generic_handle_domain_irq(gic_data.domain, irqnr)) { 730 WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr); 731 gic_deactivate_unhandled(irqnr); 732 } 733 } 734 735 static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 736 { 737 if (gic_irqnr_is_special(irqnr)) 738 return; 739 740 gic_complete_ack(irqnr); 741 742 if (generic_handle_domain_nmi(gic_data.domain, irqnr)) { 743 WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr); 744 gic_deactivate_unhandled(irqnr); 745 } 746 } 747 748 /* 749 * An exception has been taken from a context with IRQs enabled, and this could 750 * be an IRQ or an NMI. 751 * 752 * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear 753 * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning, 754 * after handling any NMI but before handling any IRQ. 755 * 756 * The entry code has performed IRQ entry, and if an NMI is detected we must 757 * perform NMI entry/exit around invoking the handler. 758 */ 759 static void __gic_handle_irq_from_irqson(struct pt_regs *regs) 760 { 761 bool is_nmi; 762 u32 irqnr; 763 764 irqnr = gic_read_iar(); 765 766 is_nmi = gic_rpr_is_nmi_prio(); 767 768 if (is_nmi) { 769 nmi_enter(); 770 __gic_handle_nmi(irqnr, regs); 771 nmi_exit(); 772 } 773 774 if (gic_prio_masking_enabled()) { 775 gic_pmr_mask_irqs(); 776 gic_arch_enable_irqs(); 777 } 778 779 if (!is_nmi) 780 __gic_handle_irq(irqnr, regs); 781 } 782 783 /* 784 * An exception has been taken from a context with IRQs disabled, which can only 785 * be an NMI. 786 * 787 * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave 788 * DAIF.IF (and ICC_PMR_EL1) unchanged. 789 * 790 * The entry code has performed NMI entry. 791 */ 792 static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) 793 { 794 u64 pmr; 795 u32 irqnr; 796 797 /* 798 * We were in a context with IRQs disabled. However, the 799 * entry code has set PMR to a value that allows any 800 * interrupt to be acknowledged, and not just NMIs. This can 801 * lead to surprising effects if the NMI has been retired in 802 * the meantime, and that there is an IRQ pending. The IRQ 803 * would then be taken in NMI context, something that nobody 804 * wants to debug twice. 805 * 806 * Until we sort this, drop PMR again to a level that will 807 * actually only allow NMIs before reading IAR, and then 808 * restore it to what it was. 809 */ 810 pmr = gic_read_pmr(); 811 gic_pmr_mask_irqs(); 812 isb(); 813 irqnr = gic_read_iar(); 814 gic_write_pmr(pmr); 815 816 __gic_handle_nmi(irqnr, regs); 817 } 818 819 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 820 { 821 if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) 822 __gic_handle_irq_from_irqsoff(regs); 823 else 824 __gic_handle_irq_from_irqson(regs); 825 } 826 827 static u32 gic_get_pribits(void) 828 { 829 u32 pribits; 830 831 pribits = gic_read_ctlr(); 832 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 833 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 834 pribits++; 835 836 return pribits; 837 } 838 839 static bool gic_has_group0(void) 840 { 841 u32 val; 842 u32 old_pmr; 843 844 old_pmr = gic_read_pmr(); 845 846 /* 847 * Let's find out if Group0 is under control of EL3 or not by 848 * setting the highest possible, non-zero priority in PMR. 849 * 850 * If SCR_EL3.FIQ is set, the priority gets shifted down in 851 * order for the CPU interface to set bit 7, and keep the 852 * actual priority in the non-secure range. In the process, it 853 * looses the least significant bit and the actual priority 854 * becomes 0x80. Reading it back returns 0, indicating that 855 * we're don't have access to Group0. 856 */ 857 gic_write_pmr(BIT(8 - gic_get_pribits())); 858 val = gic_read_pmr(); 859 860 gic_write_pmr(old_pmr); 861 862 return val != 0; 863 } 864 865 static void __init gic_dist_init(void) 866 { 867 unsigned int i; 868 u64 affinity; 869 void __iomem *base = gic_data.dist_base; 870 u32 val; 871 872 /* Disable the distributor */ 873 writel_relaxed(0, base + GICD_CTLR); 874 gic_dist_wait_for_rwp(); 875 876 /* 877 * Configure SPIs as non-secure Group-1. This will only matter 878 * if the GIC only has a single security state. This will not 879 * do the right thing if the kernel is running in secure mode, 880 * but that's not the intended use case anyway. 881 */ 882 for (i = 32; i < GIC_LINE_NR; i += 32) 883 writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 884 885 /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 886 for (i = 0; i < GIC_ESPI_NR; i += 32) { 887 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 888 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 889 } 890 891 for (i = 0; i < GIC_ESPI_NR; i += 32) 892 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 893 894 for (i = 0; i < GIC_ESPI_NR; i += 16) 895 writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 896 897 for (i = 0; i < GIC_ESPI_NR; i += 4) 898 writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 899 900 /* Now do the common stuff */ 901 gic_dist_config(base, GIC_LINE_NR, NULL); 902 903 val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 904 if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 905 pr_info("Enabling SGIs without active state\n"); 906 val |= GICD_CTLR_nASSGIreq; 907 } 908 909 /* Enable distributor with ARE, Group1, and wait for it to drain */ 910 writel_relaxed(val, base + GICD_CTLR); 911 gic_dist_wait_for_rwp(); 912 913 /* 914 * Set all global interrupts to the boot CPU only. ARE must be 915 * enabled. 916 */ 917 affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 918 for (i = 32; i < GIC_LINE_NR; i++) 919 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 920 921 for (i = 0; i < GIC_ESPI_NR; i++) 922 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 923 } 924 925 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 926 { 927 int ret = -ENODEV; 928 int i; 929 930 for (i = 0; i < gic_data.nr_redist_regions; i++) { 931 void __iomem *ptr = gic_data.redist_regions[i].redist_base; 932 u64 typer; 933 u32 reg; 934 935 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 936 if (reg != GIC_PIDR2_ARCH_GICv3 && 937 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 938 pr_warn("No redistributor present @%p\n", ptr); 939 break; 940 } 941 942 do { 943 typer = gic_read_typer(ptr + GICR_TYPER); 944 ret = fn(gic_data.redist_regions + i, ptr); 945 if (!ret) 946 return 0; 947 948 if (gic_data.redist_regions[i].single_redist) 949 break; 950 951 if (gic_data.redist_stride) { 952 ptr += gic_data.redist_stride; 953 } else { 954 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 955 if (typer & GICR_TYPER_VLPIS) 956 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 957 } 958 } while (!(typer & GICR_TYPER_LAST)); 959 } 960 961 return ret ? -ENODEV : 0; 962 } 963 964 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 965 { 966 unsigned long mpidr = cpu_logical_map(smp_processor_id()); 967 u64 typer; 968 u32 aff; 969 970 /* 971 * Convert affinity to a 32bit value that can be matched to 972 * GICR_TYPER bits [63:32]. 973 */ 974 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 975 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 976 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 977 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 978 979 typer = gic_read_typer(ptr + GICR_TYPER); 980 if ((typer >> 32) == aff) { 981 u64 offset = ptr - region->redist_base; 982 raw_spin_lock_init(&gic_data_rdist()->rd_lock); 983 gic_data_rdist_rd_base() = ptr; 984 gic_data_rdist()->phys_base = region->phys_base + offset; 985 986 pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 987 smp_processor_id(), mpidr, 988 (int)(region - gic_data.redist_regions), 989 &gic_data_rdist()->phys_base); 990 return 0; 991 } 992 993 /* Try next one */ 994 return 1; 995 } 996 997 static int gic_populate_rdist(void) 998 { 999 if (gic_iterate_rdists(__gic_populate_rdist) == 0) 1000 return 0; 1001 1002 /* We couldn't even deal with ourselves... */ 1003 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 1004 smp_processor_id(), 1005 (unsigned long)cpu_logical_map(smp_processor_id())); 1006 return -ENODEV; 1007 } 1008 1009 static int __gic_update_rdist_properties(struct redist_region *region, 1010 void __iomem *ptr) 1011 { 1012 u64 typer = gic_read_typer(ptr + GICR_TYPER); 1013 u32 ctlr = readl_relaxed(ptr + GICR_CTLR); 1014 1015 /* Boot-time cleanup */ 1016 if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { 1017 u64 val; 1018 1019 /* Deactivate any present vPE */ 1020 val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); 1021 if (val & GICR_VPENDBASER_Valid) 1022 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 1023 ptr + SZ_128K + GICR_VPENDBASER); 1024 1025 /* Mark the VPE table as invalid */ 1026 val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); 1027 val &= ~GICR_VPROPBASER_4_1_VALID; 1028 gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); 1029 } 1030 1031 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 1032 1033 /* 1034 * TYPER.RVPEID implies some form of DirectLPI, no matter what the 1035 * doc says... :-/ And CTLR.IR implies another subset of DirectLPI 1036 * that the ITS driver can make use of for LPIs (and not VLPIs). 1037 * 1038 * These are 3 different ways to express the same thing, depending 1039 * on the revision of the architecture and its relaxations over 1040 * time. Just group them under the 'direct_lpi' banner. 1041 */ 1042 gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 1043 gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 1044 !!(ctlr & GICR_CTLR_IR) | 1045 gic_data.rdists.has_rvpeid); 1046 gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 1047 1048 /* Detect non-sensical configurations */ 1049 if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 1050 gic_data.rdists.has_direct_lpi = false; 1051 gic_data.rdists.has_vlpis = false; 1052 gic_data.rdists.has_rvpeid = false; 1053 } 1054 1055 gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 1056 1057 return 1; 1058 } 1059 1060 static void gic_update_rdist_properties(void) 1061 { 1062 gic_data.ppi_nr = UINT_MAX; 1063 gic_iterate_rdists(__gic_update_rdist_properties); 1064 if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 1065 gic_data.ppi_nr = 0; 1066 pr_info("GICv3 features: %d PPIs%s%s\n", 1067 gic_data.ppi_nr, 1068 gic_data.has_rss ? ", RSS" : "", 1069 gic_data.rdists.has_direct_lpi ? ", DirectLPI" : ""); 1070 1071 if (gic_data.rdists.has_vlpis) 1072 pr_info("GICv4 features: %s%s%s\n", 1073 gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 1074 gic_data.rdists.has_rvpeid ? "RVPEID " : "", 1075 gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 1076 } 1077 1078 /* Check whether it's single security state view */ 1079 static inline bool gic_dist_security_disabled(void) 1080 { 1081 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 1082 } 1083 1084 static void gic_cpu_sys_reg_init(void) 1085 { 1086 int i, cpu = smp_processor_id(); 1087 u64 mpidr = cpu_logical_map(cpu); 1088 u64 need_rss = MPIDR_RS(mpidr); 1089 bool group0; 1090 u32 pribits; 1091 1092 /* 1093 * Need to check that the SRE bit has actually been set. If 1094 * not, it means that SRE is disabled at EL2. We're going to 1095 * die painfully, and there is nothing we can do about it. 1096 * 1097 * Kindly inform the luser. 1098 */ 1099 if (!gic_enable_sre()) 1100 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 1101 1102 pribits = gic_get_pribits(); 1103 1104 group0 = gic_has_group0(); 1105 1106 /* Set priority mask register */ 1107 if (!gic_prio_masking_enabled()) { 1108 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 1109 } else if (gic_supports_nmi()) { 1110 /* 1111 * Mismatch configuration with boot CPU, the system is likely 1112 * to die as interrupt masking will not work properly on all 1113 * CPUs 1114 * 1115 * The boot CPU calls this function before enabling NMI support, 1116 * and as a result we'll never see this warning in the boot path 1117 * for that CPU. 1118 */ 1119 if (static_branch_unlikely(&gic_nonsecure_priorities)) 1120 WARN_ON(!group0 || gic_dist_security_disabled()); 1121 else 1122 WARN_ON(group0 && !gic_dist_security_disabled()); 1123 } 1124 1125 /* 1126 * Some firmwares hand over to the kernel with the BPR changed from 1127 * its reset value (and with a value large enough to prevent 1128 * any pre-emptive interrupts from working at all). Writing a zero 1129 * to BPR restores is reset value. 1130 */ 1131 gic_write_bpr1(0); 1132 1133 if (static_branch_likely(&supports_deactivate_key)) { 1134 /* EOI drops priority only (mode 1) */ 1135 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 1136 } else { 1137 /* EOI deactivates interrupt too (mode 0) */ 1138 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 1139 } 1140 1141 /* Always whack Group0 before Group1 */ 1142 if (group0) { 1143 switch(pribits) { 1144 case 8: 1145 case 7: 1146 write_gicreg(0, ICC_AP0R3_EL1); 1147 write_gicreg(0, ICC_AP0R2_EL1); 1148 fallthrough; 1149 case 6: 1150 write_gicreg(0, ICC_AP0R1_EL1); 1151 fallthrough; 1152 case 5: 1153 case 4: 1154 write_gicreg(0, ICC_AP0R0_EL1); 1155 } 1156 1157 isb(); 1158 } 1159 1160 switch(pribits) { 1161 case 8: 1162 case 7: 1163 write_gicreg(0, ICC_AP1R3_EL1); 1164 write_gicreg(0, ICC_AP1R2_EL1); 1165 fallthrough; 1166 case 6: 1167 write_gicreg(0, ICC_AP1R1_EL1); 1168 fallthrough; 1169 case 5: 1170 case 4: 1171 write_gicreg(0, ICC_AP1R0_EL1); 1172 } 1173 1174 isb(); 1175 1176 /* ... and let's hit the road... */ 1177 gic_write_grpen1(1); 1178 1179 /* Keep the RSS capability status in per_cpu variable */ 1180 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1181 1182 /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1183 for_each_online_cpu(i) { 1184 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1185 1186 need_rss |= MPIDR_RS(cpu_logical_map(i)); 1187 if (need_rss && (!have_rss)) 1188 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1189 cpu, (unsigned long)mpidr, 1190 i, (unsigned long)cpu_logical_map(i)); 1191 } 1192 1193 /** 1194 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1195 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1196 * UNPREDICTABLE choice of : 1197 * - The write is ignored. 1198 * - The RS field is treated as 0. 1199 */ 1200 if (need_rss && (!gic_data.has_rss)) 1201 pr_crit_once("RSS is required but GICD doesn't support it\n"); 1202 } 1203 1204 static bool gicv3_nolpi; 1205 1206 static int __init gicv3_nolpi_cfg(char *buf) 1207 { 1208 return kstrtobool(buf, &gicv3_nolpi); 1209 } 1210 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1211 1212 static int gic_dist_supports_lpis(void) 1213 { 1214 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1215 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1216 !gicv3_nolpi); 1217 } 1218 1219 static void gic_cpu_init(void) 1220 { 1221 void __iomem *rbase; 1222 int i; 1223 1224 /* Register ourselves with the rest of the world */ 1225 if (gic_populate_rdist()) 1226 return; 1227 1228 gic_enable_redist(true); 1229 1230 WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1231 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1232 "Distributor has extended ranges, but CPU%d doesn't\n", 1233 smp_processor_id()); 1234 1235 rbase = gic_data_rdist_sgi_base(); 1236 1237 /* Configure SGIs/PPIs as non-secure Group-1 */ 1238 for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 1239 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 1240 1241 gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1242 1243 /* initialise system registers */ 1244 gic_cpu_sys_reg_init(); 1245 } 1246 1247 #ifdef CONFIG_SMP 1248 1249 #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1250 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1251 1252 static int gic_starting_cpu(unsigned int cpu) 1253 { 1254 gic_cpu_init(); 1255 1256 if (gic_dist_supports_lpis()) 1257 its_cpu_init(); 1258 1259 return 0; 1260 } 1261 1262 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1263 unsigned long cluster_id) 1264 { 1265 int next_cpu, cpu = *base_cpu; 1266 unsigned long mpidr = cpu_logical_map(cpu); 1267 u16 tlist = 0; 1268 1269 while (cpu < nr_cpu_ids) { 1270 tlist |= 1 << (mpidr & 0xf); 1271 1272 next_cpu = cpumask_next(cpu, mask); 1273 if (next_cpu >= nr_cpu_ids) 1274 goto out; 1275 cpu = next_cpu; 1276 1277 mpidr = cpu_logical_map(cpu); 1278 1279 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1280 cpu--; 1281 goto out; 1282 } 1283 } 1284 out: 1285 *base_cpu = cpu; 1286 return tlist; 1287 } 1288 1289 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 1290 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 1291 << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 1292 1293 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1294 { 1295 u64 val; 1296 1297 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 1298 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 1299 irq << ICC_SGI1R_SGI_ID_SHIFT | 1300 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1301 MPIDR_TO_SGI_RS(cluster_id) | 1302 tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1303 1304 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1305 gic_write_sgi1r(val); 1306 } 1307 1308 static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1309 { 1310 int cpu; 1311 1312 if (WARN_ON(d->hwirq >= 16)) 1313 return; 1314 1315 /* 1316 * Ensure that stores to Normal memory are visible to the 1317 * other CPUs before issuing the IPI. 1318 */ 1319 dsb(ishst); 1320 1321 for_each_cpu(cpu, mask) { 1322 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1323 u16 tlist; 1324 1325 tlist = gic_compute_target_list(&cpu, mask, cluster_id); 1326 gic_send_sgi(cluster_id, tlist, d->hwirq); 1327 } 1328 1329 /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1330 isb(); 1331 } 1332 1333 static void __init gic_smp_init(void) 1334 { 1335 struct irq_fwspec sgi_fwspec = { 1336 .fwnode = gic_data.fwnode, 1337 .param_count = 1, 1338 }; 1339 int base_sgi; 1340 1341 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 1342 "irqchip/arm/gicv3:starting", 1343 gic_starting_cpu, NULL); 1344 1345 /* Register all 8 non-secure SGIs */ 1346 base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec); 1347 if (WARN_ON(base_sgi <= 0)) 1348 return; 1349 1350 set_smp_ipi_range(base_sgi, 8); 1351 } 1352 1353 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1354 bool force) 1355 { 1356 unsigned int cpu; 1357 u32 offset, index; 1358 void __iomem *reg; 1359 int enabled; 1360 u64 val; 1361 1362 if (force) 1363 cpu = cpumask_first(mask_val); 1364 else 1365 cpu = cpumask_any_and(mask_val, cpu_online_mask); 1366 1367 if (cpu >= nr_cpu_ids) 1368 return -EINVAL; 1369 1370 if (gic_irq_in_rdist(d)) 1371 return -EINVAL; 1372 1373 /* If interrupt was enabled, disable it first */ 1374 enabled = gic_peek_irq(d, GICD_ISENABLER); 1375 if (enabled) 1376 gic_mask_irq(d); 1377 1378 offset = convert_offset_index(d, GICD_IROUTER, &index); 1379 reg = gic_dist_base(d) + offset + (index * 8); 1380 val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 1381 1382 gic_write_irouter(val, reg); 1383 1384 /* 1385 * If the interrupt was enabled, enabled it again. Otherwise, 1386 * just wait for the distributor to have digested our changes. 1387 */ 1388 if (enabled) 1389 gic_unmask_irq(d); 1390 1391 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1392 1393 return IRQ_SET_MASK_OK_DONE; 1394 } 1395 #else 1396 #define gic_set_affinity NULL 1397 #define gic_ipi_send_mask NULL 1398 #define gic_smp_init() do { } while(0) 1399 #endif 1400 1401 static int gic_retrigger(struct irq_data *data) 1402 { 1403 return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 1404 } 1405 1406 #ifdef CONFIG_CPU_PM 1407 static int gic_cpu_pm_notifier(struct notifier_block *self, 1408 unsigned long cmd, void *v) 1409 { 1410 if (cmd == CPU_PM_EXIT) { 1411 if (gic_dist_security_disabled()) 1412 gic_enable_redist(true); 1413 gic_cpu_sys_reg_init(); 1414 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 1415 gic_write_grpen1(0); 1416 gic_enable_redist(false); 1417 } 1418 return NOTIFY_OK; 1419 } 1420 1421 static struct notifier_block gic_cpu_pm_notifier_block = { 1422 .notifier_call = gic_cpu_pm_notifier, 1423 }; 1424 1425 static void gic_cpu_pm_init(void) 1426 { 1427 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 1428 } 1429 1430 #else 1431 static inline void gic_cpu_pm_init(void) { } 1432 #endif /* CONFIG_CPU_PM */ 1433 1434 static struct irq_chip gic_chip = { 1435 .name = "GICv3", 1436 .irq_mask = gic_mask_irq, 1437 .irq_unmask = gic_unmask_irq, 1438 .irq_eoi = gic_eoi_irq, 1439 .irq_set_type = gic_set_type, 1440 .irq_set_affinity = gic_set_affinity, 1441 .irq_retrigger = gic_retrigger, 1442 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1443 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1444 .irq_nmi_setup = gic_irq_nmi_setup, 1445 .irq_nmi_teardown = gic_irq_nmi_teardown, 1446 .ipi_send_mask = gic_ipi_send_mask, 1447 .flags = IRQCHIP_SET_TYPE_MASKED | 1448 IRQCHIP_SKIP_SET_WAKE | 1449 IRQCHIP_MASK_ON_SUSPEND, 1450 }; 1451 1452 static struct irq_chip gic_eoimode1_chip = { 1453 .name = "GICv3", 1454 .irq_mask = gic_eoimode1_mask_irq, 1455 .irq_unmask = gic_unmask_irq, 1456 .irq_eoi = gic_eoimode1_eoi_irq, 1457 .irq_set_type = gic_set_type, 1458 .irq_set_affinity = gic_set_affinity, 1459 .irq_retrigger = gic_retrigger, 1460 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1461 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1462 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1463 .irq_nmi_setup = gic_irq_nmi_setup, 1464 .irq_nmi_teardown = gic_irq_nmi_teardown, 1465 .ipi_send_mask = gic_ipi_send_mask, 1466 .flags = IRQCHIP_SET_TYPE_MASKED | 1467 IRQCHIP_SKIP_SET_WAKE | 1468 IRQCHIP_MASK_ON_SUSPEND, 1469 }; 1470 1471 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1472 irq_hw_number_t hw) 1473 { 1474 struct irq_chip *chip = &gic_chip; 1475 struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 1476 1477 if (static_branch_likely(&supports_deactivate_key)) 1478 chip = &gic_eoimode1_chip; 1479 1480 switch (__get_intid_range(hw)) { 1481 case SGI_RANGE: 1482 case PPI_RANGE: 1483 case EPPI_RANGE: 1484 irq_set_percpu_devid(irq); 1485 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1486 handle_percpu_devid_irq, NULL, NULL); 1487 break; 1488 1489 case SPI_RANGE: 1490 case ESPI_RANGE: 1491 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1492 handle_fasteoi_irq, NULL, NULL); 1493 irq_set_probe(irq); 1494 irqd_set_single_target(irqd); 1495 break; 1496 1497 case LPI_RANGE: 1498 if (!gic_dist_supports_lpis()) 1499 return -EPERM; 1500 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1501 handle_fasteoi_irq, NULL, NULL); 1502 break; 1503 1504 default: 1505 return -EPERM; 1506 } 1507 1508 /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 1509 irqd_set_handle_enforce_irqctx(irqd); 1510 return 0; 1511 } 1512 1513 static int gic_irq_domain_translate(struct irq_domain *d, 1514 struct irq_fwspec *fwspec, 1515 unsigned long *hwirq, 1516 unsigned int *type) 1517 { 1518 if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 1519 *hwirq = fwspec->param[0]; 1520 *type = IRQ_TYPE_EDGE_RISING; 1521 return 0; 1522 } 1523 1524 if (is_of_node(fwspec->fwnode)) { 1525 if (fwspec->param_count < 3) 1526 return -EINVAL; 1527 1528 switch (fwspec->param[0]) { 1529 case 0: /* SPI */ 1530 *hwirq = fwspec->param[1] + 32; 1531 break; 1532 case 1: /* PPI */ 1533 *hwirq = fwspec->param[1] + 16; 1534 break; 1535 case 2: /* ESPI */ 1536 *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1537 break; 1538 case 3: /* EPPI */ 1539 *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 1540 break; 1541 case GIC_IRQ_TYPE_LPI: /* LPI */ 1542 *hwirq = fwspec->param[1]; 1543 break; 1544 case GIC_IRQ_TYPE_PARTITION: 1545 *hwirq = fwspec->param[1]; 1546 if (fwspec->param[1] >= 16) 1547 *hwirq += EPPI_BASE_INTID - 16; 1548 else 1549 *hwirq += 16; 1550 break; 1551 default: 1552 return -EINVAL; 1553 } 1554 1555 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1556 1557 /* 1558 * Make it clear that broken DTs are... broken. 1559 * Partitioned PPIs are an unfortunate exception. 1560 */ 1561 WARN_ON(*type == IRQ_TYPE_NONE && 1562 fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1563 return 0; 1564 } 1565 1566 if (is_fwnode_irqchip(fwspec->fwnode)) { 1567 if(fwspec->param_count != 2) 1568 return -EINVAL; 1569 1570 if (fwspec->param[0] < 16) { 1571 pr_err(FW_BUG "Illegal GSI%d translation request\n", 1572 fwspec->param[0]); 1573 return -EINVAL; 1574 } 1575 1576 *hwirq = fwspec->param[0]; 1577 *type = fwspec->param[1]; 1578 1579 WARN_ON(*type == IRQ_TYPE_NONE); 1580 return 0; 1581 } 1582 1583 return -EINVAL; 1584 } 1585 1586 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1587 unsigned int nr_irqs, void *arg) 1588 { 1589 int i, ret; 1590 irq_hw_number_t hwirq; 1591 unsigned int type = IRQ_TYPE_NONE; 1592 struct irq_fwspec *fwspec = arg; 1593 1594 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1595 if (ret) 1596 return ret; 1597 1598 for (i = 0; i < nr_irqs; i++) { 1599 ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 1600 if (ret) 1601 return ret; 1602 } 1603 1604 return 0; 1605 } 1606 1607 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1608 unsigned int nr_irqs) 1609 { 1610 int i; 1611 1612 for (i = 0; i < nr_irqs; i++) { 1613 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1614 irq_set_handler(virq + i, NULL); 1615 irq_domain_reset_irq_data(d); 1616 } 1617 } 1618 1619 static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, 1620 irq_hw_number_t hwirq) 1621 { 1622 enum gic_intid_range range; 1623 1624 if (!gic_data.ppi_descs) 1625 return false; 1626 1627 if (!is_of_node(fwspec->fwnode)) 1628 return false; 1629 1630 if (fwspec->param_count < 4 || !fwspec->param[3]) 1631 return false; 1632 1633 range = __get_intid_range(hwirq); 1634 if (range != PPI_RANGE && range != EPPI_RANGE) 1635 return false; 1636 1637 return true; 1638 } 1639 1640 static int gic_irq_domain_select(struct irq_domain *d, 1641 struct irq_fwspec *fwspec, 1642 enum irq_domain_bus_token bus_token) 1643 { 1644 unsigned int type, ret, ppi_idx; 1645 irq_hw_number_t hwirq; 1646 1647 /* Not for us */ 1648 if (fwspec->fwnode != d->fwnode) 1649 return 0; 1650 1651 /* If this is not DT, then we have a single domain */ 1652 if (!is_of_node(fwspec->fwnode)) 1653 return 1; 1654 1655 ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type); 1656 if (WARN_ON_ONCE(ret)) 1657 return 0; 1658 1659 if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) 1660 return d == gic_data.domain; 1661 1662 /* 1663 * If this is a PPI and we have a 4th (non-null) parameter, 1664 * then we need to match the partition domain. 1665 */ 1666 ppi_idx = __gic_get_ppi_index(hwirq); 1667 return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); 1668 } 1669 1670 static const struct irq_domain_ops gic_irq_domain_ops = { 1671 .translate = gic_irq_domain_translate, 1672 .alloc = gic_irq_domain_alloc, 1673 .free = gic_irq_domain_free, 1674 .select = gic_irq_domain_select, 1675 }; 1676 1677 static int partition_domain_translate(struct irq_domain *d, 1678 struct irq_fwspec *fwspec, 1679 unsigned long *hwirq, 1680 unsigned int *type) 1681 { 1682 unsigned long ppi_intid; 1683 struct device_node *np; 1684 unsigned int ppi_idx; 1685 int ret; 1686 1687 if (!gic_data.ppi_descs) 1688 return -ENOMEM; 1689 1690 np = of_find_node_by_phandle(fwspec->param[3]); 1691 if (WARN_ON(!np)) 1692 return -EINVAL; 1693 1694 ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); 1695 if (WARN_ON_ONCE(ret)) 1696 return 0; 1697 1698 ppi_idx = __gic_get_ppi_index(ppi_intid); 1699 ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], 1700 of_node_to_fwnode(np)); 1701 if (ret < 0) 1702 return ret; 1703 1704 *hwirq = ret; 1705 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1706 1707 return 0; 1708 } 1709 1710 static const struct irq_domain_ops partition_domain_ops = { 1711 .translate = partition_domain_translate, 1712 .select = gic_irq_domain_select, 1713 }; 1714 1715 static bool gic_enable_quirk_msm8996(void *data) 1716 { 1717 struct gic_chip_data *d = data; 1718 1719 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 1720 1721 return true; 1722 } 1723 1724 static bool gic_enable_quirk_mtk_gicr(void *data) 1725 { 1726 struct gic_chip_data *d = data; 1727 1728 d->flags |= FLAGS_WORKAROUND_MTK_GICR_SAVE; 1729 1730 return true; 1731 } 1732 1733 static bool gic_enable_quirk_cavium_38539(void *data) 1734 { 1735 struct gic_chip_data *d = data; 1736 1737 d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1738 1739 return true; 1740 } 1741 1742 static bool gic_enable_quirk_hip06_07(void *data) 1743 { 1744 struct gic_chip_data *d = data; 1745 1746 /* 1747 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 1748 * not being an actual ARM implementation). The saving grace is 1749 * that GIC-600 doesn't have ESPI, so nothing to do in that case. 1750 * HIP07 doesn't even have a proper IIDR, and still pretends to 1751 * have ESPI. In both cases, put them right. 1752 */ 1753 if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 1754 /* Zero both ESPI and the RES0 field next to it... */ 1755 d->rdists.gicd_typer &= ~GENMASK(9, 8); 1756 return true; 1757 } 1758 1759 return false; 1760 } 1761 1762 #define T241_CHIPN_MASK GENMASK_ULL(45, 44) 1763 #define T241_CHIP_GICDA_OFFSET 0x1580000 1764 #define SMCCC_SOC_ID_T241 0x036b0241 1765 1766 static bool gic_enable_quirk_nvidia_t241(void *data) 1767 { 1768 s32 soc_id = arm_smccc_get_soc_id_version(); 1769 unsigned long chip_bmask = 0; 1770 phys_addr_t phys; 1771 u32 i; 1772 1773 /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */ 1774 if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241)) 1775 return false; 1776 1777 /* Find the chips based on GICR regions PHYS addr */ 1778 for (i = 0; i < gic_data.nr_redist_regions; i++) { 1779 chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK, 1780 (u64)gic_data.redist_regions[i].phys_base)); 1781 } 1782 1783 if (hweight32(chip_bmask) < 3) 1784 return false; 1785 1786 /* Setup GICD alias regions */ 1787 for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) { 1788 if (chip_bmask & BIT(i)) { 1789 phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET; 1790 phys |= FIELD_PREP(T241_CHIPN_MASK, i); 1791 t241_dist_base_alias[i] = ioremap(phys, SZ_64K); 1792 WARN_ON_ONCE(!t241_dist_base_alias[i]); 1793 } 1794 } 1795 static_branch_enable(&gic_nvidia_t241_erratum); 1796 return true; 1797 } 1798 1799 static const struct gic_quirk gic_quirks[] = { 1800 { 1801 .desc = "GICv3: Qualcomm MSM8996 broken firmware", 1802 .compatible = "qcom,msm8996-gic-v3", 1803 .init = gic_enable_quirk_msm8996, 1804 }, 1805 { 1806 .desc = "GICv3: Mediatek Chromebook GICR save problem", 1807 .property = "mediatek,broken-save-restore-fw", 1808 .init = gic_enable_quirk_mtk_gicr, 1809 }, 1810 { 1811 .desc = "GICv3: HIP06 erratum 161010803", 1812 .iidr = 0x0204043b, 1813 .mask = 0xffffffff, 1814 .init = gic_enable_quirk_hip06_07, 1815 }, 1816 { 1817 .desc = "GICv3: HIP07 erratum 161010803", 1818 .iidr = 0x00000000, 1819 .mask = 0xffffffff, 1820 .init = gic_enable_quirk_hip06_07, 1821 }, 1822 { 1823 /* 1824 * Reserved register accesses generate a Synchronous 1825 * External Abort. This erratum applies to: 1826 * - ThunderX: CN88xx 1827 * - OCTEON TX: CN83xx, CN81xx 1828 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1829 */ 1830 .desc = "GICv3: Cavium erratum 38539", 1831 .iidr = 0xa000034c, 1832 .mask = 0xe8f00fff, 1833 .init = gic_enable_quirk_cavium_38539, 1834 }, 1835 { 1836 .desc = "GICv3: NVIDIA erratum T241-FABRIC-4", 1837 .iidr = 0x0402043b, 1838 .mask = 0xffffffff, 1839 .init = gic_enable_quirk_nvidia_t241, 1840 }, 1841 { 1842 } 1843 }; 1844 1845 static void gic_enable_nmi_support(void) 1846 { 1847 int i; 1848 1849 if (!gic_prio_masking_enabled()) 1850 return; 1851 1852 if (gic_data.flags & FLAGS_WORKAROUND_MTK_GICR_SAVE) { 1853 pr_warn("Skipping NMI enable due to firmware issues\n"); 1854 return; 1855 } 1856 1857 ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 1858 if (!ppi_nmi_refs) 1859 return; 1860 1861 for (i = 0; i < gic_data.ppi_nr; i++) 1862 refcount_set(&ppi_nmi_refs[i], 0); 1863 1864 pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 1865 gic_has_relaxed_pmr_sync() ? "relaxed" : "forced"); 1866 1867 /* 1868 * How priority values are used by the GIC depends on two things: 1869 * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 1870 * and if Group 0 interrupts can be delivered to Linux in the non-secure 1871 * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 1872 * ICC_PMR_EL1 register and the priority that software assigns to 1873 * interrupts: 1874 * 1875 * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 1876 * ----------------------------------------------------------- 1877 * 1 | - | unchanged | unchanged 1878 * ----------------------------------------------------------- 1879 * 0 | 1 | non-secure | non-secure 1880 * ----------------------------------------------------------- 1881 * 0 | 0 | unchanged | non-secure 1882 * 1883 * where non-secure means that the value is right-shifted by one and the 1884 * MSB bit set, to make it fit in the non-secure priority range. 1885 * 1886 * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 1887 * are both either modified or unchanged, we can use the same set of 1888 * priorities. 1889 * 1890 * In the last case, where only the interrupt priorities are modified to 1891 * be in the non-secure range, we use a different PMR value to mask IRQs 1892 * and the rest of the values that we use remain unchanged. 1893 */ 1894 if (gic_has_group0() && !gic_dist_security_disabled()) 1895 static_branch_enable(&gic_nonsecure_priorities); 1896 1897 static_branch_enable(&supports_pseudo_nmis); 1898 1899 if (static_branch_likely(&supports_deactivate_key)) 1900 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1901 else 1902 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1903 } 1904 1905 static int __init gic_init_bases(phys_addr_t dist_phys_base, 1906 void __iomem *dist_base, 1907 struct redist_region *rdist_regs, 1908 u32 nr_redist_regions, 1909 u64 redist_stride, 1910 struct fwnode_handle *handle) 1911 { 1912 u32 typer; 1913 int err; 1914 1915 if (!is_hyp_mode_available()) 1916 static_branch_disable(&supports_deactivate_key); 1917 1918 if (static_branch_likely(&supports_deactivate_key)) 1919 pr_info("GIC: Using split EOI/Deactivate mode\n"); 1920 1921 gic_data.fwnode = handle; 1922 gic_data.dist_phys_base = dist_phys_base; 1923 gic_data.dist_base = dist_base; 1924 gic_data.redist_regions = rdist_regs; 1925 gic_data.nr_redist_regions = nr_redist_regions; 1926 gic_data.redist_stride = redist_stride; 1927 1928 /* 1929 * Find out how many interrupts are supported. 1930 */ 1931 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1932 gic_data.rdists.gicd_typer = typer; 1933 1934 gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 1935 gic_quirks, &gic_data); 1936 1937 pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 1938 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 1939 1940 /* 1941 * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 1942 * architecture spec (which says that reserved registers are RES0). 1943 */ 1944 if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 1945 gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 1946 1947 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1948 &gic_data); 1949 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 1950 if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) { 1951 /* Disable GICv4.x features for the erratum T241-FABRIC-4 */ 1952 gic_data.rdists.has_rvpeid = true; 1953 gic_data.rdists.has_vlpis = true; 1954 gic_data.rdists.has_direct_lpi = true; 1955 gic_data.rdists.has_vpend_valid_dirty = true; 1956 } 1957 1958 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1959 err = -ENOMEM; 1960 goto out_free; 1961 } 1962 1963 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1964 1965 gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1966 1967 if (typer & GICD_TYPER_MBIS) { 1968 err = mbi_init(handle, gic_data.domain); 1969 if (err) 1970 pr_err("Failed to initialize MBIs\n"); 1971 } 1972 1973 set_handle_irq(gic_handle_irq); 1974 1975 gic_update_rdist_properties(); 1976 1977 gic_dist_init(); 1978 gic_cpu_init(); 1979 gic_smp_init(); 1980 gic_cpu_pm_init(); 1981 1982 if (gic_dist_supports_lpis()) { 1983 its_init(handle, &gic_data.rdists, gic_data.domain); 1984 its_cpu_init(); 1985 its_lpi_memreserve_init(); 1986 } else { 1987 if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 1988 gicv2m_init(handle, gic_data.domain); 1989 } 1990 1991 gic_enable_nmi_support(); 1992 1993 return 0; 1994 1995 out_free: 1996 if (gic_data.domain) 1997 irq_domain_remove(gic_data.domain); 1998 free_percpu(gic_data.rdists.rdist); 1999 return err; 2000 } 2001 2002 static int __init gic_validate_dist_version(void __iomem *dist_base) 2003 { 2004 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2005 2006 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 2007 return -ENODEV; 2008 2009 return 0; 2010 } 2011 2012 /* Create all possible partitions at boot time */ 2013 static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 2014 { 2015 struct device_node *parts_node, *child_part; 2016 int part_idx = 0, i; 2017 int nr_parts; 2018 struct partition_affinity *parts; 2019 2020 parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 2021 if (!parts_node) 2022 return; 2023 2024 gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 2025 if (!gic_data.ppi_descs) 2026 goto out_put_node; 2027 2028 nr_parts = of_get_child_count(parts_node); 2029 2030 if (!nr_parts) 2031 goto out_put_node; 2032 2033 parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 2034 if (WARN_ON(!parts)) 2035 goto out_put_node; 2036 2037 for_each_child_of_node(parts_node, child_part) { 2038 struct partition_affinity *part; 2039 int n; 2040 2041 part = &parts[part_idx]; 2042 2043 part->partition_id = of_node_to_fwnode(child_part); 2044 2045 pr_info("GIC: PPI partition %pOFn[%d] { ", 2046 child_part, part_idx); 2047 2048 n = of_property_count_elems_of_size(child_part, "affinity", 2049 sizeof(u32)); 2050 WARN_ON(n <= 0); 2051 2052 for (i = 0; i < n; i++) { 2053 int err, cpu; 2054 u32 cpu_phandle; 2055 struct device_node *cpu_node; 2056 2057 err = of_property_read_u32_index(child_part, "affinity", 2058 i, &cpu_phandle); 2059 if (WARN_ON(err)) 2060 continue; 2061 2062 cpu_node = of_find_node_by_phandle(cpu_phandle); 2063 if (WARN_ON(!cpu_node)) 2064 continue; 2065 2066 cpu = of_cpu_node_to_id(cpu_node); 2067 if (WARN_ON(cpu < 0)) { 2068 of_node_put(cpu_node); 2069 continue; 2070 } 2071 2072 pr_cont("%pOF[%d] ", cpu_node, cpu); 2073 2074 cpumask_set_cpu(cpu, &part->mask); 2075 of_node_put(cpu_node); 2076 } 2077 2078 pr_cont("}\n"); 2079 part_idx++; 2080 } 2081 2082 for (i = 0; i < gic_data.ppi_nr; i++) { 2083 unsigned int irq; 2084 struct partition_desc *desc; 2085 struct irq_fwspec ppi_fwspec = { 2086 .fwnode = gic_data.fwnode, 2087 .param_count = 3, 2088 .param = { 2089 [0] = GIC_IRQ_TYPE_PARTITION, 2090 [1] = i, 2091 [2] = IRQ_TYPE_NONE, 2092 }, 2093 }; 2094 2095 irq = irq_create_fwspec_mapping(&ppi_fwspec); 2096 if (WARN_ON(!irq)) 2097 continue; 2098 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 2099 irq, &partition_domain_ops); 2100 if (WARN_ON(!desc)) 2101 continue; 2102 2103 gic_data.ppi_descs[i] = desc; 2104 } 2105 2106 out_put_node: 2107 of_node_put(parts_node); 2108 } 2109 2110 static void __init gic_of_setup_kvm_info(struct device_node *node) 2111 { 2112 int ret; 2113 struct resource r; 2114 u32 gicv_idx; 2115 2116 gic_v3_kvm_info.type = GIC_V3; 2117 2118 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 2119 if (!gic_v3_kvm_info.maint_irq) 2120 return; 2121 2122 if (of_property_read_u32(node, "#redistributor-regions", 2123 &gicv_idx)) 2124 gicv_idx = 1; 2125 2126 gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 2127 ret = of_address_to_resource(node, gicv_idx, &r); 2128 if (!ret) 2129 gic_v3_kvm_info.vcpu = r; 2130 2131 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2132 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2133 vgic_set_kvm_info(&gic_v3_kvm_info); 2134 } 2135 2136 static void gic_request_region(resource_size_t base, resource_size_t size, 2137 const char *name) 2138 { 2139 if (!request_mem_region(base, size, name)) 2140 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", 2141 name, &base); 2142 } 2143 2144 static void __iomem *gic_of_iomap(struct device_node *node, int idx, 2145 const char *name, struct resource *res) 2146 { 2147 void __iomem *base; 2148 int ret; 2149 2150 ret = of_address_to_resource(node, idx, res); 2151 if (ret) 2152 return IOMEM_ERR_PTR(ret); 2153 2154 gic_request_region(res->start, resource_size(res), name); 2155 base = of_iomap(node, idx); 2156 2157 return base ?: IOMEM_ERR_PTR(-ENOMEM); 2158 } 2159 2160 static int __init gic_of_init(struct device_node *node, struct device_node *parent) 2161 { 2162 phys_addr_t dist_phys_base; 2163 void __iomem *dist_base; 2164 struct redist_region *rdist_regs; 2165 struct resource res; 2166 u64 redist_stride; 2167 u32 nr_redist_regions; 2168 int err, i; 2169 2170 dist_base = gic_of_iomap(node, 0, "GICD", &res); 2171 if (IS_ERR(dist_base)) { 2172 pr_err("%pOF: unable to map gic dist registers\n", node); 2173 return PTR_ERR(dist_base); 2174 } 2175 2176 dist_phys_base = res.start; 2177 2178 err = gic_validate_dist_version(dist_base); 2179 if (err) { 2180 pr_err("%pOF: no distributor detected, giving up\n", node); 2181 goto out_unmap_dist; 2182 } 2183 2184 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 2185 nr_redist_regions = 1; 2186 2187 rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 2188 GFP_KERNEL); 2189 if (!rdist_regs) { 2190 err = -ENOMEM; 2191 goto out_unmap_dist; 2192 } 2193 2194 for (i = 0; i < nr_redist_regions; i++) { 2195 rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res); 2196 if (IS_ERR(rdist_regs[i].redist_base)) { 2197 pr_err("%pOF: couldn't map region %d\n", node, i); 2198 err = -ENODEV; 2199 goto out_unmap_rdist; 2200 } 2201 rdist_regs[i].phys_base = res.start; 2202 } 2203 2204 if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 2205 redist_stride = 0; 2206 2207 gic_enable_of_quirks(node, gic_quirks, &gic_data); 2208 2209 err = gic_init_bases(dist_phys_base, dist_base, rdist_regs, 2210 nr_redist_regions, redist_stride, &node->fwnode); 2211 if (err) 2212 goto out_unmap_rdist; 2213 2214 gic_populate_ppi_partitions(node); 2215 2216 if (static_branch_likely(&supports_deactivate_key)) 2217 gic_of_setup_kvm_info(node); 2218 return 0; 2219 2220 out_unmap_rdist: 2221 for (i = 0; i < nr_redist_regions; i++) 2222 if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base)) 2223 iounmap(rdist_regs[i].redist_base); 2224 kfree(rdist_regs); 2225 out_unmap_dist: 2226 iounmap(dist_base); 2227 return err; 2228 } 2229 2230 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 2231 2232 #ifdef CONFIG_ACPI 2233 static struct 2234 { 2235 void __iomem *dist_base; 2236 struct redist_region *redist_regs; 2237 u32 nr_redist_regions; 2238 bool single_redist; 2239 int enabled_rdists; 2240 u32 maint_irq; 2241 int maint_irq_mode; 2242 phys_addr_t vcpu_base; 2243 } acpi_data __initdata; 2244 2245 static void __init 2246 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 2247 { 2248 static int count = 0; 2249 2250 acpi_data.redist_regs[count].phys_base = phys_base; 2251 acpi_data.redist_regs[count].redist_base = redist_base; 2252 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 2253 count++; 2254 } 2255 2256 static int __init 2257 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 2258 const unsigned long end) 2259 { 2260 struct acpi_madt_generic_redistributor *redist = 2261 (struct acpi_madt_generic_redistributor *)header; 2262 void __iomem *redist_base; 2263 2264 redist_base = ioremap(redist->base_address, redist->length); 2265 if (!redist_base) { 2266 pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2267 return -ENOMEM; 2268 } 2269 gic_request_region(redist->base_address, redist->length, "GICR"); 2270 2271 gic_acpi_register_redist(redist->base_address, redist_base); 2272 return 0; 2273 } 2274 2275 static int __init 2276 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2277 const unsigned long end) 2278 { 2279 struct acpi_madt_generic_interrupt *gicc = 2280 (struct acpi_madt_generic_interrupt *)header; 2281 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2282 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2283 void __iomem *redist_base; 2284 2285 /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 2286 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2287 return 0; 2288 2289 redist_base = ioremap(gicc->gicr_base_address, size); 2290 if (!redist_base) 2291 return -ENOMEM; 2292 gic_request_region(gicc->gicr_base_address, size, "GICR"); 2293 2294 gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2295 return 0; 2296 } 2297 2298 static int __init gic_acpi_collect_gicr_base(void) 2299 { 2300 acpi_tbl_entry_handler redist_parser; 2301 enum acpi_madt_type type; 2302 2303 if (acpi_data.single_redist) { 2304 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2305 redist_parser = gic_acpi_parse_madt_gicc; 2306 } else { 2307 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2308 redist_parser = gic_acpi_parse_madt_redist; 2309 } 2310 2311 /* Collect redistributor base addresses in GICR entries */ 2312 if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2313 return 0; 2314 2315 pr_info("No valid GICR entries exist\n"); 2316 return -ENODEV; 2317 } 2318 2319 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2320 const unsigned long end) 2321 { 2322 /* Subtable presence means that redist exists, that's it */ 2323 return 0; 2324 } 2325 2326 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2327 const unsigned long end) 2328 { 2329 struct acpi_madt_generic_interrupt *gicc = 2330 (struct acpi_madt_generic_interrupt *)header; 2331 2332 /* 2333 * If GICC is enabled and has valid gicr base address, then it means 2334 * GICR base is presented via GICC 2335 */ 2336 if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2337 acpi_data.enabled_rdists++; 2338 return 0; 2339 } 2340 2341 /* 2342 * It's perfectly valid firmware can pass disabled GICC entry, driver 2343 * should not treat as errors, skip the entry instead of probe fail. 2344 */ 2345 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2346 return 0; 2347 2348 return -ENODEV; 2349 } 2350 2351 static int __init gic_acpi_count_gicr_regions(void) 2352 { 2353 int count; 2354 2355 /* 2356 * Count how many redistributor regions we have. It is not allowed 2357 * to mix redistributor description, GICR and GICC subtables have to be 2358 * mutually exclusive. 2359 */ 2360 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2361 gic_acpi_match_gicr, 0); 2362 if (count > 0) { 2363 acpi_data.single_redist = false; 2364 return count; 2365 } 2366 2367 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2368 gic_acpi_match_gicc, 0); 2369 if (count > 0) { 2370 acpi_data.single_redist = true; 2371 count = acpi_data.enabled_rdists; 2372 } 2373 2374 return count; 2375 } 2376 2377 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2378 struct acpi_probe_entry *ape) 2379 { 2380 struct acpi_madt_generic_distributor *dist; 2381 int count; 2382 2383 dist = (struct acpi_madt_generic_distributor *)header; 2384 if (dist->version != ape->driver_data) 2385 return false; 2386 2387 /* We need to do that exercise anyway, the sooner the better */ 2388 count = gic_acpi_count_gicr_regions(); 2389 if (count <= 0) 2390 return false; 2391 2392 acpi_data.nr_redist_regions = count; 2393 return true; 2394 } 2395 2396 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 2397 const unsigned long end) 2398 { 2399 struct acpi_madt_generic_interrupt *gicc = 2400 (struct acpi_madt_generic_interrupt *)header; 2401 int maint_irq_mode; 2402 static int first_madt = true; 2403 2404 /* Skip unusable CPUs */ 2405 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2406 return 0; 2407 2408 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 2409 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 2410 2411 if (first_madt) { 2412 first_madt = false; 2413 2414 acpi_data.maint_irq = gicc->vgic_interrupt; 2415 acpi_data.maint_irq_mode = maint_irq_mode; 2416 acpi_data.vcpu_base = gicc->gicv_base_address; 2417 2418 return 0; 2419 } 2420 2421 /* 2422 * The maintenance interrupt and GICV should be the same for every CPU 2423 */ 2424 if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 2425 (acpi_data.maint_irq_mode != maint_irq_mode) || 2426 (acpi_data.vcpu_base != gicc->gicv_base_address)) 2427 return -EINVAL; 2428 2429 return 0; 2430 } 2431 2432 static bool __init gic_acpi_collect_virt_info(void) 2433 { 2434 int count; 2435 2436 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2437 gic_acpi_parse_virt_madt_gicc, 0); 2438 2439 return (count > 0); 2440 } 2441 2442 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 2443 #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 2444 #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 2445 2446 static void __init gic_acpi_setup_kvm_info(void) 2447 { 2448 int irq; 2449 2450 if (!gic_acpi_collect_virt_info()) { 2451 pr_warn("Unable to get hardware information used for virtualization\n"); 2452 return; 2453 } 2454 2455 gic_v3_kvm_info.type = GIC_V3; 2456 2457 irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 2458 acpi_data.maint_irq_mode, 2459 ACPI_ACTIVE_HIGH); 2460 if (irq <= 0) 2461 return; 2462 2463 gic_v3_kvm_info.maint_irq = irq; 2464 2465 if (acpi_data.vcpu_base) { 2466 struct resource *vcpu = &gic_v3_kvm_info.vcpu; 2467 2468 vcpu->flags = IORESOURCE_MEM; 2469 vcpu->start = acpi_data.vcpu_base; 2470 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 2471 } 2472 2473 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2474 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2475 vgic_set_kvm_info(&gic_v3_kvm_info); 2476 } 2477 2478 static struct fwnode_handle *gsi_domain_handle; 2479 2480 static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi) 2481 { 2482 return gsi_domain_handle; 2483 } 2484 2485 static int __init 2486 gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2487 { 2488 struct acpi_madt_generic_distributor *dist; 2489 size_t size; 2490 int i, err; 2491 2492 /* Get distributor base address */ 2493 dist = (struct acpi_madt_generic_distributor *)header; 2494 acpi_data.dist_base = ioremap(dist->base_address, 2495 ACPI_GICV3_DIST_MEM_SIZE); 2496 if (!acpi_data.dist_base) { 2497 pr_err("Unable to map GICD registers\n"); 2498 return -ENOMEM; 2499 } 2500 gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD"); 2501 2502 err = gic_validate_dist_version(acpi_data.dist_base); 2503 if (err) { 2504 pr_err("No distributor detected at @%p, giving up\n", 2505 acpi_data.dist_base); 2506 goto out_dist_unmap; 2507 } 2508 2509 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2510 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2511 if (!acpi_data.redist_regs) { 2512 err = -ENOMEM; 2513 goto out_dist_unmap; 2514 } 2515 2516 err = gic_acpi_collect_gicr_base(); 2517 if (err) 2518 goto out_redist_unmap; 2519 2520 gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2521 if (!gsi_domain_handle) { 2522 err = -ENOMEM; 2523 goto out_redist_unmap; 2524 } 2525 2526 err = gic_init_bases(dist->base_address, acpi_data.dist_base, 2527 acpi_data.redist_regs, acpi_data.nr_redist_regions, 2528 0, gsi_domain_handle); 2529 if (err) 2530 goto out_fwhandle_free; 2531 2532 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id); 2533 2534 if (static_branch_likely(&supports_deactivate_key)) 2535 gic_acpi_setup_kvm_info(); 2536 2537 return 0; 2538 2539 out_fwhandle_free: 2540 irq_domain_free_fwnode(gsi_domain_handle); 2541 out_redist_unmap: 2542 for (i = 0; i < acpi_data.nr_redist_regions; i++) 2543 if (acpi_data.redist_regs[i].redist_base) 2544 iounmap(acpi_data.redist_regs[i].redist_base); 2545 kfree(acpi_data.redist_regs); 2546 out_dist_unmap: 2547 iounmap(acpi_data.dist_base); 2548 return err; 2549 } 2550 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2551 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2552 gic_acpi_init); 2553 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2554 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2555 gic_acpi_init); 2556 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2557 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2558 gic_acpi_init); 2559 #endif 2560