1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #define pr_fmt(fmt) "GICv3: " fmt 8 9 #include <linux/acpi.h> 10 #include <linux/cpu.h> 11 #include <linux/cpu_pm.h> 12 #include <linux/delay.h> 13 #include <linux/interrupt.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kstrtox.h> 16 #include <linux/of.h> 17 #include <linux/of_address.h> 18 #include <linux/of_irq.h> 19 #include <linux/percpu.h> 20 #include <linux/refcount.h> 21 #include <linux/slab.h> 22 23 #include <linux/irqchip.h> 24 #include <linux/irqchip/arm-gic-common.h> 25 #include <linux/irqchip/arm-gic-v3.h> 26 #include <linux/irqchip/irq-partition-percpu.h> 27 28 #include <asm/cputype.h> 29 #include <asm/exception.h> 30 #include <asm/smp_plat.h> 31 #include <asm/virt.h> 32 33 #include "irq-gic-common.h" 34 35 #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 36 37 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 38 #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 39 40 #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 41 42 struct redist_region { 43 void __iomem *redist_base; 44 phys_addr_t phys_base; 45 bool single_redist; 46 }; 47 48 struct gic_chip_data { 49 struct fwnode_handle *fwnode; 50 void __iomem *dist_base; 51 struct redist_region *redist_regions; 52 struct rdists rdists; 53 struct irq_domain *domain; 54 u64 redist_stride; 55 u32 nr_redist_regions; 56 u64 flags; 57 bool has_rss; 58 unsigned int ppi_nr; 59 struct partition_desc **ppi_descs; 60 }; 61 62 static struct gic_chip_data gic_data __read_mostly; 63 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 64 65 #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 66 #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 67 #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 68 69 /* 70 * The behaviours of RPR and PMR registers differ depending on the value of 71 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 72 * distributor and redistributors depends on whether security is enabled in the 73 * GIC. 74 * 75 * When security is enabled, non-secure priority values from the (re)distributor 76 * are presented to the GIC CPUIF as follow: 77 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 78 * 79 * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 80 * EL1 are subject to a similar operation thus matching the priorities presented 81 * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 82 * these values are unchanged by the GIC. 83 * 84 * see GICv3/GICv4 Architecture Specification (IHI0069D): 85 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 86 * priorities. 87 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 88 * interrupt. 89 */ 90 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 91 92 /* 93 * Global static key controlling whether an update to PMR allowing more 94 * interrupts requires to be propagated to the redistributor (DSB SY). 95 * And this needs to be exported for modules to be able to enable 96 * interrupts... 97 */ 98 DEFINE_STATIC_KEY_FALSE(gic_pmr_sync); 99 EXPORT_SYMBOL(gic_pmr_sync); 100 101 DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 102 EXPORT_SYMBOL(gic_nonsecure_priorities); 103 104 /* 105 * When the Non-secure world has access to group 0 interrupts (as a 106 * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will 107 * return the Distributor's view of the interrupt priority. 108 * 109 * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority 110 * written by software is moved to the Non-secure range by the Distributor. 111 * 112 * If both are true (which is when gic_nonsecure_priorities gets enabled), 113 * we need to shift down the priority programmed by software to match it 114 * against the value returned by ICC_RPR_EL1. 115 */ 116 #define GICD_INT_RPR_PRI(priority) \ 117 ({ \ 118 u32 __priority = (priority); \ 119 if (static_branch_unlikely(&gic_nonsecure_priorities)) \ 120 __priority = 0x80 | (__priority >> 1); \ 121 \ 122 __priority; \ 123 }) 124 125 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 126 static refcount_t *ppi_nmi_refs; 127 128 static struct gic_kvm_info gic_v3_kvm_info __initdata; 129 static DEFINE_PER_CPU(bool, has_rss); 130 131 #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 132 #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 133 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 134 #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 135 136 /* Our default, arbitrary priority value. Linux only uses one anyway. */ 137 #define DEFAULT_PMR_VALUE 0xf0 138 139 enum gic_intid_range { 140 SGI_RANGE, 141 PPI_RANGE, 142 SPI_RANGE, 143 EPPI_RANGE, 144 ESPI_RANGE, 145 LPI_RANGE, 146 __INVALID_RANGE__ 147 }; 148 149 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 150 { 151 switch (hwirq) { 152 case 0 ... 15: 153 return SGI_RANGE; 154 case 16 ... 31: 155 return PPI_RANGE; 156 case 32 ... 1019: 157 return SPI_RANGE; 158 case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 159 return EPPI_RANGE; 160 case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 161 return ESPI_RANGE; 162 case 8192 ... GENMASK(23, 0): 163 return LPI_RANGE; 164 default: 165 return __INVALID_RANGE__; 166 } 167 } 168 169 static enum gic_intid_range get_intid_range(struct irq_data *d) 170 { 171 return __get_intid_range(d->hwirq); 172 } 173 174 static inline unsigned int gic_irq(struct irq_data *d) 175 { 176 return d->hwirq; 177 } 178 179 static inline bool gic_irq_in_rdist(struct irq_data *d) 180 { 181 switch (get_intid_range(d)) { 182 case SGI_RANGE: 183 case PPI_RANGE: 184 case EPPI_RANGE: 185 return true; 186 default: 187 return false; 188 } 189 } 190 191 static inline void __iomem *gic_dist_base(struct irq_data *d) 192 { 193 switch (get_intid_range(d)) { 194 case SGI_RANGE: 195 case PPI_RANGE: 196 case EPPI_RANGE: 197 /* SGI+PPI -> SGI_base for this CPU */ 198 return gic_data_rdist_sgi_base(); 199 200 case SPI_RANGE: 201 case ESPI_RANGE: 202 /* SPI -> dist_base */ 203 return gic_data.dist_base; 204 205 default: 206 return NULL; 207 } 208 } 209 210 static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) 211 { 212 u32 count = 1000000; /* 1s! */ 213 214 while (readl_relaxed(base + GICD_CTLR) & bit) { 215 count--; 216 if (!count) { 217 pr_err_ratelimited("RWP timeout, gone fishing\n"); 218 return; 219 } 220 cpu_relax(); 221 udelay(1); 222 } 223 } 224 225 /* Wait for completion of a distributor change */ 226 static void gic_dist_wait_for_rwp(void) 227 { 228 gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP); 229 } 230 231 /* Wait for completion of a redistributor change */ 232 static void gic_redist_wait_for_rwp(void) 233 { 234 gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); 235 } 236 237 #ifdef CONFIG_ARM64 238 239 static u64 __maybe_unused gic_read_iar(void) 240 { 241 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 242 return gic_read_iar_cavium_thunderx(); 243 else 244 return gic_read_iar_common(); 245 } 246 #endif 247 248 static void gic_enable_redist(bool enable) 249 { 250 void __iomem *rbase; 251 u32 count = 1000000; /* 1s! */ 252 u32 val; 253 254 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 255 return; 256 257 rbase = gic_data_rdist_rd_base(); 258 259 val = readl_relaxed(rbase + GICR_WAKER); 260 if (enable) 261 /* Wake up this CPU redistributor */ 262 val &= ~GICR_WAKER_ProcessorSleep; 263 else 264 val |= GICR_WAKER_ProcessorSleep; 265 writel_relaxed(val, rbase + GICR_WAKER); 266 267 if (!enable) { /* Check that GICR_WAKER is writeable */ 268 val = readl_relaxed(rbase + GICR_WAKER); 269 if (!(val & GICR_WAKER_ProcessorSleep)) 270 return; /* No PM support in this redistributor */ 271 } 272 273 while (--count) { 274 val = readl_relaxed(rbase + GICR_WAKER); 275 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 276 break; 277 cpu_relax(); 278 udelay(1); 279 } 280 if (!count) 281 pr_err_ratelimited("redistributor failed to %s...\n", 282 enable ? "wakeup" : "sleep"); 283 } 284 285 /* 286 * Routines to disable, enable, EOI and route interrupts 287 */ 288 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 289 { 290 switch (get_intid_range(d)) { 291 case SGI_RANGE: 292 case PPI_RANGE: 293 case SPI_RANGE: 294 *index = d->hwirq; 295 return offset; 296 case EPPI_RANGE: 297 /* 298 * Contrary to the ESPI range, the EPPI range is contiguous 299 * to the PPI range in the registers, so let's adjust the 300 * displacement accordingly. Consistency is overrated. 301 */ 302 *index = d->hwirq - EPPI_BASE_INTID + 32; 303 return offset; 304 case ESPI_RANGE: 305 *index = d->hwirq - ESPI_BASE_INTID; 306 switch (offset) { 307 case GICD_ISENABLER: 308 return GICD_ISENABLERnE; 309 case GICD_ICENABLER: 310 return GICD_ICENABLERnE; 311 case GICD_ISPENDR: 312 return GICD_ISPENDRnE; 313 case GICD_ICPENDR: 314 return GICD_ICPENDRnE; 315 case GICD_ISACTIVER: 316 return GICD_ISACTIVERnE; 317 case GICD_ICACTIVER: 318 return GICD_ICACTIVERnE; 319 case GICD_IPRIORITYR: 320 return GICD_IPRIORITYRnE; 321 case GICD_ICFGR: 322 return GICD_ICFGRnE; 323 case GICD_IROUTER: 324 return GICD_IROUTERnE; 325 default: 326 break; 327 } 328 break; 329 default: 330 break; 331 } 332 333 WARN_ON(1); 334 *index = d->hwirq; 335 return offset; 336 } 337 338 static int gic_peek_irq(struct irq_data *d, u32 offset) 339 { 340 void __iomem *base; 341 u32 index, mask; 342 343 offset = convert_offset_index(d, offset, &index); 344 mask = 1 << (index % 32); 345 346 if (gic_irq_in_rdist(d)) 347 base = gic_data_rdist_sgi_base(); 348 else 349 base = gic_data.dist_base; 350 351 return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 352 } 353 354 static void gic_poke_irq(struct irq_data *d, u32 offset) 355 { 356 void __iomem *base; 357 u32 index, mask; 358 359 offset = convert_offset_index(d, offset, &index); 360 mask = 1 << (index % 32); 361 362 if (gic_irq_in_rdist(d)) 363 base = gic_data_rdist_sgi_base(); 364 else 365 base = gic_data.dist_base; 366 367 writel_relaxed(mask, base + offset + (index / 32) * 4); 368 } 369 370 static void gic_mask_irq(struct irq_data *d) 371 { 372 gic_poke_irq(d, GICD_ICENABLER); 373 if (gic_irq_in_rdist(d)) 374 gic_redist_wait_for_rwp(); 375 else 376 gic_dist_wait_for_rwp(); 377 } 378 379 static void gic_eoimode1_mask_irq(struct irq_data *d) 380 { 381 gic_mask_irq(d); 382 /* 383 * When masking a forwarded interrupt, make sure it is 384 * deactivated as well. 385 * 386 * This ensures that an interrupt that is getting 387 * disabled/masked will not get "stuck", because there is 388 * noone to deactivate it (guest is being terminated). 389 */ 390 if (irqd_is_forwarded_to_vcpu(d)) 391 gic_poke_irq(d, GICD_ICACTIVER); 392 } 393 394 static void gic_unmask_irq(struct irq_data *d) 395 { 396 gic_poke_irq(d, GICD_ISENABLER); 397 } 398 399 static inline bool gic_supports_nmi(void) 400 { 401 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 402 static_branch_likely(&supports_pseudo_nmis); 403 } 404 405 static int gic_irq_set_irqchip_state(struct irq_data *d, 406 enum irqchip_irq_state which, bool val) 407 { 408 u32 reg; 409 410 if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 411 return -EINVAL; 412 413 switch (which) { 414 case IRQCHIP_STATE_PENDING: 415 reg = val ? GICD_ISPENDR : GICD_ICPENDR; 416 break; 417 418 case IRQCHIP_STATE_ACTIVE: 419 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 420 break; 421 422 case IRQCHIP_STATE_MASKED: 423 if (val) { 424 gic_mask_irq(d); 425 return 0; 426 } 427 reg = GICD_ISENABLER; 428 break; 429 430 default: 431 return -EINVAL; 432 } 433 434 gic_poke_irq(d, reg); 435 return 0; 436 } 437 438 static int gic_irq_get_irqchip_state(struct irq_data *d, 439 enum irqchip_irq_state which, bool *val) 440 { 441 if (d->hwirq >= 8192) /* PPI/SPI only */ 442 return -EINVAL; 443 444 switch (which) { 445 case IRQCHIP_STATE_PENDING: 446 *val = gic_peek_irq(d, GICD_ISPENDR); 447 break; 448 449 case IRQCHIP_STATE_ACTIVE: 450 *val = gic_peek_irq(d, GICD_ISACTIVER); 451 break; 452 453 case IRQCHIP_STATE_MASKED: 454 *val = !gic_peek_irq(d, GICD_ISENABLER); 455 break; 456 457 default: 458 return -EINVAL; 459 } 460 461 return 0; 462 } 463 464 static void gic_irq_set_prio(struct irq_data *d, u8 prio) 465 { 466 void __iomem *base = gic_dist_base(d); 467 u32 offset, index; 468 469 offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 470 471 writeb_relaxed(prio, base + offset + index); 472 } 473 474 static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 475 { 476 switch (__get_intid_range(hwirq)) { 477 case PPI_RANGE: 478 return hwirq - 16; 479 case EPPI_RANGE: 480 return hwirq - EPPI_BASE_INTID + 16; 481 default: 482 unreachable(); 483 } 484 } 485 486 static u32 gic_get_ppi_index(struct irq_data *d) 487 { 488 return __gic_get_ppi_index(d->hwirq); 489 } 490 491 static int gic_irq_nmi_setup(struct irq_data *d) 492 { 493 struct irq_desc *desc = irq_to_desc(d->irq); 494 495 if (!gic_supports_nmi()) 496 return -EINVAL; 497 498 if (gic_peek_irq(d, GICD_ISENABLER)) { 499 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 500 return -EINVAL; 501 } 502 503 /* 504 * A secondary irq_chip should be in charge of LPI request, 505 * it should not be possible to get there 506 */ 507 if (WARN_ON(gic_irq(d) >= 8192)) 508 return -EINVAL; 509 510 /* desc lock should already be held */ 511 if (gic_irq_in_rdist(d)) { 512 u32 idx = gic_get_ppi_index(d); 513 514 /* Setting up PPI as NMI, only switch handler for first NMI */ 515 if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 516 refcount_set(&ppi_nmi_refs[idx], 1); 517 desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 518 } 519 } else { 520 desc->handle_irq = handle_fasteoi_nmi; 521 } 522 523 gic_irq_set_prio(d, GICD_INT_NMI_PRI); 524 525 return 0; 526 } 527 528 static void gic_irq_nmi_teardown(struct irq_data *d) 529 { 530 struct irq_desc *desc = irq_to_desc(d->irq); 531 532 if (WARN_ON(!gic_supports_nmi())) 533 return; 534 535 if (gic_peek_irq(d, GICD_ISENABLER)) { 536 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 537 return; 538 } 539 540 /* 541 * A secondary irq_chip should be in charge of LPI request, 542 * it should not be possible to get there 543 */ 544 if (WARN_ON(gic_irq(d) >= 8192)) 545 return; 546 547 /* desc lock should already be held */ 548 if (gic_irq_in_rdist(d)) { 549 u32 idx = gic_get_ppi_index(d); 550 551 /* Tearing down NMI, only switch handler for last NMI */ 552 if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 553 desc->handle_irq = handle_percpu_devid_irq; 554 } else { 555 desc->handle_irq = handle_fasteoi_irq; 556 } 557 558 gic_irq_set_prio(d, GICD_INT_DEF_PRI); 559 } 560 561 static void gic_eoi_irq(struct irq_data *d) 562 { 563 write_gicreg(gic_irq(d), ICC_EOIR1_EL1); 564 isb(); 565 } 566 567 static void gic_eoimode1_eoi_irq(struct irq_data *d) 568 { 569 /* 570 * No need to deactivate an LPI, or an interrupt that 571 * is is getting forwarded to a vcpu. 572 */ 573 if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 574 return; 575 gic_write_dir(gic_irq(d)); 576 } 577 578 static int gic_set_type(struct irq_data *d, unsigned int type) 579 { 580 enum gic_intid_range range; 581 unsigned int irq = gic_irq(d); 582 void __iomem *base; 583 u32 offset, index; 584 int ret; 585 586 range = get_intid_range(d); 587 588 /* Interrupt configuration for SGIs can't be changed */ 589 if (range == SGI_RANGE) 590 return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 591 592 /* SPIs have restrictions on the supported types */ 593 if ((range == SPI_RANGE || range == ESPI_RANGE) && 594 type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 595 return -EINVAL; 596 597 if (gic_irq_in_rdist(d)) 598 base = gic_data_rdist_sgi_base(); 599 else 600 base = gic_data.dist_base; 601 602 offset = convert_offset_index(d, GICD_ICFGR, &index); 603 604 ret = gic_configure_irq(index, type, base + offset, NULL); 605 if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 606 /* Misconfigured PPIs are usually not fatal */ 607 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 608 ret = 0; 609 } 610 611 return ret; 612 } 613 614 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 615 { 616 if (get_intid_range(d) == SGI_RANGE) 617 return -EINVAL; 618 619 if (vcpu) 620 irqd_set_forwarded_to_vcpu(d); 621 else 622 irqd_clr_forwarded_to_vcpu(d); 623 return 0; 624 } 625 626 static u64 gic_mpidr_to_affinity(unsigned long mpidr) 627 { 628 u64 aff; 629 630 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 631 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 632 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 633 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 634 635 return aff; 636 } 637 638 static void gic_deactivate_unhandled(u32 irqnr) 639 { 640 if (static_branch_likely(&supports_deactivate_key)) { 641 if (irqnr < 8192) 642 gic_write_dir(irqnr); 643 } else { 644 write_gicreg(irqnr, ICC_EOIR1_EL1); 645 isb(); 646 } 647 } 648 649 /* 650 * Follow a read of the IAR with any HW maintenance that needs to happen prior 651 * to invoking the relevant IRQ handler. We must do two things: 652 * 653 * (1) Ensure instruction ordering between a read of IAR and subsequent 654 * instructions in the IRQ handler using an ISB. 655 * 656 * It is possible for the IAR to report an IRQ which was signalled *after* 657 * the CPU took an IRQ exception as multiple interrupts can race to be 658 * recognized by the GIC, earlier interrupts could be withdrawn, and/or 659 * later interrupts could be prioritized by the GIC. 660 * 661 * For devices which are tightly coupled to the CPU, such as PMUs, a 662 * context synchronization event is necessary to ensure that system 663 * register state is not stale, as these may have been indirectly written 664 * *after* exception entry. 665 * 666 * (2) Deactivate the interrupt when EOI mode 1 is in use. 667 */ 668 static inline void gic_complete_ack(u32 irqnr) 669 { 670 if (static_branch_likely(&supports_deactivate_key)) 671 write_gicreg(irqnr, ICC_EOIR1_EL1); 672 673 isb(); 674 } 675 676 static bool gic_rpr_is_nmi_prio(void) 677 { 678 if (!gic_supports_nmi()) 679 return false; 680 681 return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI)); 682 } 683 684 static bool gic_irqnr_is_special(u32 irqnr) 685 { 686 return irqnr >= 1020 && irqnr <= 1023; 687 } 688 689 static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs) 690 { 691 if (gic_irqnr_is_special(irqnr)) 692 return; 693 694 gic_complete_ack(irqnr); 695 696 if (generic_handle_domain_irq(gic_data.domain, irqnr)) { 697 WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr); 698 gic_deactivate_unhandled(irqnr); 699 } 700 } 701 702 static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 703 { 704 if (gic_irqnr_is_special(irqnr)) 705 return; 706 707 gic_complete_ack(irqnr); 708 709 if (generic_handle_domain_nmi(gic_data.domain, irqnr)) { 710 WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr); 711 gic_deactivate_unhandled(irqnr); 712 } 713 } 714 715 /* 716 * An exception has been taken from a context with IRQs enabled, and this could 717 * be an IRQ or an NMI. 718 * 719 * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear 720 * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning, 721 * after handling any NMI but before handling any IRQ. 722 * 723 * The entry code has performed IRQ entry, and if an NMI is detected we must 724 * perform NMI entry/exit around invoking the handler. 725 */ 726 static void __gic_handle_irq_from_irqson(struct pt_regs *regs) 727 { 728 bool is_nmi; 729 u32 irqnr; 730 731 irqnr = gic_read_iar(); 732 733 is_nmi = gic_rpr_is_nmi_prio(); 734 735 if (is_nmi) { 736 nmi_enter(); 737 __gic_handle_nmi(irqnr, regs); 738 nmi_exit(); 739 } 740 741 if (gic_prio_masking_enabled()) { 742 gic_pmr_mask_irqs(); 743 gic_arch_enable_irqs(); 744 } 745 746 if (!is_nmi) 747 __gic_handle_irq(irqnr, regs); 748 } 749 750 /* 751 * An exception has been taken from a context with IRQs disabled, which can only 752 * be an NMI. 753 * 754 * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave 755 * DAIF.IF (and ICC_PMR_EL1) unchanged. 756 * 757 * The entry code has performed NMI entry. 758 */ 759 static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) 760 { 761 u64 pmr; 762 u32 irqnr; 763 764 /* 765 * We were in a context with IRQs disabled. However, the 766 * entry code has set PMR to a value that allows any 767 * interrupt to be acknowledged, and not just NMIs. This can 768 * lead to surprising effects if the NMI has been retired in 769 * the meantime, and that there is an IRQ pending. The IRQ 770 * would then be taken in NMI context, something that nobody 771 * wants to debug twice. 772 * 773 * Until we sort this, drop PMR again to a level that will 774 * actually only allow NMIs before reading IAR, and then 775 * restore it to what it was. 776 */ 777 pmr = gic_read_pmr(); 778 gic_pmr_mask_irqs(); 779 isb(); 780 irqnr = gic_read_iar(); 781 gic_write_pmr(pmr); 782 783 __gic_handle_nmi(irqnr, regs); 784 } 785 786 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 787 { 788 if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) 789 __gic_handle_irq_from_irqsoff(regs); 790 else 791 __gic_handle_irq_from_irqson(regs); 792 } 793 794 static u32 gic_get_pribits(void) 795 { 796 u32 pribits; 797 798 pribits = gic_read_ctlr(); 799 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 800 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 801 pribits++; 802 803 return pribits; 804 } 805 806 static bool gic_has_group0(void) 807 { 808 u32 val; 809 u32 old_pmr; 810 811 old_pmr = gic_read_pmr(); 812 813 /* 814 * Let's find out if Group0 is under control of EL3 or not by 815 * setting the highest possible, non-zero priority in PMR. 816 * 817 * If SCR_EL3.FIQ is set, the priority gets shifted down in 818 * order for the CPU interface to set bit 7, and keep the 819 * actual priority in the non-secure range. In the process, it 820 * looses the least significant bit and the actual priority 821 * becomes 0x80. Reading it back returns 0, indicating that 822 * we're don't have access to Group0. 823 */ 824 gic_write_pmr(BIT(8 - gic_get_pribits())); 825 val = gic_read_pmr(); 826 827 gic_write_pmr(old_pmr); 828 829 return val != 0; 830 } 831 832 static void __init gic_dist_init(void) 833 { 834 unsigned int i; 835 u64 affinity; 836 void __iomem *base = gic_data.dist_base; 837 u32 val; 838 839 /* Disable the distributor */ 840 writel_relaxed(0, base + GICD_CTLR); 841 gic_dist_wait_for_rwp(); 842 843 /* 844 * Configure SPIs as non-secure Group-1. This will only matter 845 * if the GIC only has a single security state. This will not 846 * do the right thing if the kernel is running in secure mode, 847 * but that's not the intended use case anyway. 848 */ 849 for (i = 32; i < GIC_LINE_NR; i += 32) 850 writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 851 852 /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 853 for (i = 0; i < GIC_ESPI_NR; i += 32) { 854 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 855 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 856 } 857 858 for (i = 0; i < GIC_ESPI_NR; i += 32) 859 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 860 861 for (i = 0; i < GIC_ESPI_NR; i += 16) 862 writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 863 864 for (i = 0; i < GIC_ESPI_NR; i += 4) 865 writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 866 867 /* Now do the common stuff */ 868 gic_dist_config(base, GIC_LINE_NR, NULL); 869 870 val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 871 if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 872 pr_info("Enabling SGIs without active state\n"); 873 val |= GICD_CTLR_nASSGIreq; 874 } 875 876 /* Enable distributor with ARE, Group1, and wait for it to drain */ 877 writel_relaxed(val, base + GICD_CTLR); 878 gic_dist_wait_for_rwp(); 879 880 /* 881 * Set all global interrupts to the boot CPU only. ARE must be 882 * enabled. 883 */ 884 affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 885 for (i = 32; i < GIC_LINE_NR; i++) 886 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 887 888 for (i = 0; i < GIC_ESPI_NR; i++) 889 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 890 } 891 892 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 893 { 894 int ret = -ENODEV; 895 int i; 896 897 for (i = 0; i < gic_data.nr_redist_regions; i++) { 898 void __iomem *ptr = gic_data.redist_regions[i].redist_base; 899 u64 typer; 900 u32 reg; 901 902 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 903 if (reg != GIC_PIDR2_ARCH_GICv3 && 904 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 905 pr_warn("No redistributor present @%p\n", ptr); 906 break; 907 } 908 909 do { 910 typer = gic_read_typer(ptr + GICR_TYPER); 911 ret = fn(gic_data.redist_regions + i, ptr); 912 if (!ret) 913 return 0; 914 915 if (gic_data.redist_regions[i].single_redist) 916 break; 917 918 if (gic_data.redist_stride) { 919 ptr += gic_data.redist_stride; 920 } else { 921 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 922 if (typer & GICR_TYPER_VLPIS) 923 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 924 } 925 } while (!(typer & GICR_TYPER_LAST)); 926 } 927 928 return ret ? -ENODEV : 0; 929 } 930 931 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 932 { 933 unsigned long mpidr = cpu_logical_map(smp_processor_id()); 934 u64 typer; 935 u32 aff; 936 937 /* 938 * Convert affinity to a 32bit value that can be matched to 939 * GICR_TYPER bits [63:32]. 940 */ 941 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 942 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 943 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 944 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 945 946 typer = gic_read_typer(ptr + GICR_TYPER); 947 if ((typer >> 32) == aff) { 948 u64 offset = ptr - region->redist_base; 949 raw_spin_lock_init(&gic_data_rdist()->rd_lock); 950 gic_data_rdist_rd_base() = ptr; 951 gic_data_rdist()->phys_base = region->phys_base + offset; 952 953 pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 954 smp_processor_id(), mpidr, 955 (int)(region - gic_data.redist_regions), 956 &gic_data_rdist()->phys_base); 957 return 0; 958 } 959 960 /* Try next one */ 961 return 1; 962 } 963 964 static int gic_populate_rdist(void) 965 { 966 if (gic_iterate_rdists(__gic_populate_rdist) == 0) 967 return 0; 968 969 /* We couldn't even deal with ourselves... */ 970 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 971 smp_processor_id(), 972 (unsigned long)cpu_logical_map(smp_processor_id())); 973 return -ENODEV; 974 } 975 976 static int __gic_update_rdist_properties(struct redist_region *region, 977 void __iomem *ptr) 978 { 979 u64 typer = gic_read_typer(ptr + GICR_TYPER); 980 u32 ctlr = readl_relaxed(ptr + GICR_CTLR); 981 982 /* Boot-time cleanup */ 983 if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { 984 u64 val; 985 986 /* Deactivate any present vPE */ 987 val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); 988 if (val & GICR_VPENDBASER_Valid) 989 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 990 ptr + SZ_128K + GICR_VPENDBASER); 991 992 /* Mark the VPE table as invalid */ 993 val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); 994 val &= ~GICR_VPROPBASER_4_1_VALID; 995 gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); 996 } 997 998 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 999 1000 /* 1001 * TYPER.RVPEID implies some form of DirectLPI, no matter what the 1002 * doc says... :-/ And CTLR.IR implies another subset of DirectLPI 1003 * that the ITS driver can make use of for LPIs (and not VLPIs). 1004 * 1005 * These are 3 different ways to express the same thing, depending 1006 * on the revision of the architecture and its relaxations over 1007 * time. Just group them under the 'direct_lpi' banner. 1008 */ 1009 gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 1010 gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 1011 !!(ctlr & GICR_CTLR_IR) | 1012 gic_data.rdists.has_rvpeid); 1013 gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 1014 1015 /* Detect non-sensical configurations */ 1016 if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 1017 gic_data.rdists.has_direct_lpi = false; 1018 gic_data.rdists.has_vlpis = false; 1019 gic_data.rdists.has_rvpeid = false; 1020 } 1021 1022 gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 1023 1024 return 1; 1025 } 1026 1027 static void gic_update_rdist_properties(void) 1028 { 1029 gic_data.ppi_nr = UINT_MAX; 1030 gic_iterate_rdists(__gic_update_rdist_properties); 1031 if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 1032 gic_data.ppi_nr = 0; 1033 pr_info("GICv3 features: %d PPIs%s%s\n", 1034 gic_data.ppi_nr, 1035 gic_data.has_rss ? ", RSS" : "", 1036 gic_data.rdists.has_direct_lpi ? ", DirectLPI" : ""); 1037 1038 if (gic_data.rdists.has_vlpis) 1039 pr_info("GICv4 features: %s%s%s\n", 1040 gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 1041 gic_data.rdists.has_rvpeid ? "RVPEID " : "", 1042 gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 1043 } 1044 1045 /* Check whether it's single security state view */ 1046 static inline bool gic_dist_security_disabled(void) 1047 { 1048 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 1049 } 1050 1051 static void gic_cpu_sys_reg_init(void) 1052 { 1053 int i, cpu = smp_processor_id(); 1054 u64 mpidr = cpu_logical_map(cpu); 1055 u64 need_rss = MPIDR_RS(mpidr); 1056 bool group0; 1057 u32 pribits; 1058 1059 /* 1060 * Need to check that the SRE bit has actually been set. If 1061 * not, it means that SRE is disabled at EL2. We're going to 1062 * die painfully, and there is nothing we can do about it. 1063 * 1064 * Kindly inform the luser. 1065 */ 1066 if (!gic_enable_sre()) 1067 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 1068 1069 pribits = gic_get_pribits(); 1070 1071 group0 = gic_has_group0(); 1072 1073 /* Set priority mask register */ 1074 if (!gic_prio_masking_enabled()) { 1075 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 1076 } else if (gic_supports_nmi()) { 1077 /* 1078 * Mismatch configuration with boot CPU, the system is likely 1079 * to die as interrupt masking will not work properly on all 1080 * CPUs 1081 * 1082 * The boot CPU calls this function before enabling NMI support, 1083 * and as a result we'll never see this warning in the boot path 1084 * for that CPU. 1085 */ 1086 if (static_branch_unlikely(&gic_nonsecure_priorities)) 1087 WARN_ON(!group0 || gic_dist_security_disabled()); 1088 else 1089 WARN_ON(group0 && !gic_dist_security_disabled()); 1090 } 1091 1092 /* 1093 * Some firmwares hand over to the kernel with the BPR changed from 1094 * its reset value (and with a value large enough to prevent 1095 * any pre-emptive interrupts from working at all). Writing a zero 1096 * to BPR restores is reset value. 1097 */ 1098 gic_write_bpr1(0); 1099 1100 if (static_branch_likely(&supports_deactivate_key)) { 1101 /* EOI drops priority only (mode 1) */ 1102 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 1103 } else { 1104 /* EOI deactivates interrupt too (mode 0) */ 1105 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 1106 } 1107 1108 /* Always whack Group0 before Group1 */ 1109 if (group0) { 1110 switch(pribits) { 1111 case 8: 1112 case 7: 1113 write_gicreg(0, ICC_AP0R3_EL1); 1114 write_gicreg(0, ICC_AP0R2_EL1); 1115 fallthrough; 1116 case 6: 1117 write_gicreg(0, ICC_AP0R1_EL1); 1118 fallthrough; 1119 case 5: 1120 case 4: 1121 write_gicreg(0, ICC_AP0R0_EL1); 1122 } 1123 1124 isb(); 1125 } 1126 1127 switch(pribits) { 1128 case 8: 1129 case 7: 1130 write_gicreg(0, ICC_AP1R3_EL1); 1131 write_gicreg(0, ICC_AP1R2_EL1); 1132 fallthrough; 1133 case 6: 1134 write_gicreg(0, ICC_AP1R1_EL1); 1135 fallthrough; 1136 case 5: 1137 case 4: 1138 write_gicreg(0, ICC_AP1R0_EL1); 1139 } 1140 1141 isb(); 1142 1143 /* ... and let's hit the road... */ 1144 gic_write_grpen1(1); 1145 1146 /* Keep the RSS capability status in per_cpu variable */ 1147 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1148 1149 /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1150 for_each_online_cpu(i) { 1151 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1152 1153 need_rss |= MPIDR_RS(cpu_logical_map(i)); 1154 if (need_rss && (!have_rss)) 1155 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1156 cpu, (unsigned long)mpidr, 1157 i, (unsigned long)cpu_logical_map(i)); 1158 } 1159 1160 /** 1161 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1162 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1163 * UNPREDICTABLE choice of : 1164 * - The write is ignored. 1165 * - The RS field is treated as 0. 1166 */ 1167 if (need_rss && (!gic_data.has_rss)) 1168 pr_crit_once("RSS is required but GICD doesn't support it\n"); 1169 } 1170 1171 static bool gicv3_nolpi; 1172 1173 static int __init gicv3_nolpi_cfg(char *buf) 1174 { 1175 return kstrtobool(buf, &gicv3_nolpi); 1176 } 1177 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1178 1179 static int gic_dist_supports_lpis(void) 1180 { 1181 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1182 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1183 !gicv3_nolpi); 1184 } 1185 1186 static void gic_cpu_init(void) 1187 { 1188 void __iomem *rbase; 1189 int i; 1190 1191 /* Register ourselves with the rest of the world */ 1192 if (gic_populate_rdist()) 1193 return; 1194 1195 gic_enable_redist(true); 1196 1197 WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1198 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1199 "Distributor has extended ranges, but CPU%d doesn't\n", 1200 smp_processor_id()); 1201 1202 rbase = gic_data_rdist_sgi_base(); 1203 1204 /* Configure SGIs/PPIs as non-secure Group-1 */ 1205 for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 1206 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 1207 1208 gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1209 1210 /* initialise system registers */ 1211 gic_cpu_sys_reg_init(); 1212 } 1213 1214 #ifdef CONFIG_SMP 1215 1216 #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1217 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1218 1219 static int gic_starting_cpu(unsigned int cpu) 1220 { 1221 gic_cpu_init(); 1222 1223 if (gic_dist_supports_lpis()) 1224 its_cpu_init(); 1225 1226 return 0; 1227 } 1228 1229 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1230 unsigned long cluster_id) 1231 { 1232 int next_cpu, cpu = *base_cpu; 1233 unsigned long mpidr = cpu_logical_map(cpu); 1234 u16 tlist = 0; 1235 1236 while (cpu < nr_cpu_ids) { 1237 tlist |= 1 << (mpidr & 0xf); 1238 1239 next_cpu = cpumask_next(cpu, mask); 1240 if (next_cpu >= nr_cpu_ids) 1241 goto out; 1242 cpu = next_cpu; 1243 1244 mpidr = cpu_logical_map(cpu); 1245 1246 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1247 cpu--; 1248 goto out; 1249 } 1250 } 1251 out: 1252 *base_cpu = cpu; 1253 return tlist; 1254 } 1255 1256 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 1257 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 1258 << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 1259 1260 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1261 { 1262 u64 val; 1263 1264 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 1265 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 1266 irq << ICC_SGI1R_SGI_ID_SHIFT | 1267 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1268 MPIDR_TO_SGI_RS(cluster_id) | 1269 tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1270 1271 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1272 gic_write_sgi1r(val); 1273 } 1274 1275 static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1276 { 1277 int cpu; 1278 1279 if (WARN_ON(d->hwirq >= 16)) 1280 return; 1281 1282 /* 1283 * Ensure that stores to Normal memory are visible to the 1284 * other CPUs before issuing the IPI. 1285 */ 1286 dsb(ishst); 1287 1288 for_each_cpu(cpu, mask) { 1289 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1290 u16 tlist; 1291 1292 tlist = gic_compute_target_list(&cpu, mask, cluster_id); 1293 gic_send_sgi(cluster_id, tlist, d->hwirq); 1294 } 1295 1296 /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1297 isb(); 1298 } 1299 1300 static void __init gic_smp_init(void) 1301 { 1302 struct irq_fwspec sgi_fwspec = { 1303 .fwnode = gic_data.fwnode, 1304 .param_count = 1, 1305 }; 1306 int base_sgi; 1307 1308 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 1309 "irqchip/arm/gicv3:starting", 1310 gic_starting_cpu, NULL); 1311 1312 /* Register all 8 non-secure SGIs */ 1313 base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8, 1314 NUMA_NO_NODE, &sgi_fwspec, 1315 false, NULL); 1316 if (WARN_ON(base_sgi <= 0)) 1317 return; 1318 1319 set_smp_ipi_range(base_sgi, 8); 1320 } 1321 1322 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1323 bool force) 1324 { 1325 unsigned int cpu; 1326 u32 offset, index; 1327 void __iomem *reg; 1328 int enabled; 1329 u64 val; 1330 1331 if (force) 1332 cpu = cpumask_first(mask_val); 1333 else 1334 cpu = cpumask_any_and(mask_val, cpu_online_mask); 1335 1336 if (cpu >= nr_cpu_ids) 1337 return -EINVAL; 1338 1339 if (gic_irq_in_rdist(d)) 1340 return -EINVAL; 1341 1342 /* If interrupt was enabled, disable it first */ 1343 enabled = gic_peek_irq(d, GICD_ISENABLER); 1344 if (enabled) 1345 gic_mask_irq(d); 1346 1347 offset = convert_offset_index(d, GICD_IROUTER, &index); 1348 reg = gic_dist_base(d) + offset + (index * 8); 1349 val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 1350 1351 gic_write_irouter(val, reg); 1352 1353 /* 1354 * If the interrupt was enabled, enabled it again. Otherwise, 1355 * just wait for the distributor to have digested our changes. 1356 */ 1357 if (enabled) 1358 gic_unmask_irq(d); 1359 1360 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1361 1362 return IRQ_SET_MASK_OK_DONE; 1363 } 1364 #else 1365 #define gic_set_affinity NULL 1366 #define gic_ipi_send_mask NULL 1367 #define gic_smp_init() do { } while(0) 1368 #endif 1369 1370 static int gic_retrigger(struct irq_data *data) 1371 { 1372 return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 1373 } 1374 1375 #ifdef CONFIG_CPU_PM 1376 static int gic_cpu_pm_notifier(struct notifier_block *self, 1377 unsigned long cmd, void *v) 1378 { 1379 if (cmd == CPU_PM_EXIT) { 1380 if (gic_dist_security_disabled()) 1381 gic_enable_redist(true); 1382 gic_cpu_sys_reg_init(); 1383 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 1384 gic_write_grpen1(0); 1385 gic_enable_redist(false); 1386 } 1387 return NOTIFY_OK; 1388 } 1389 1390 static struct notifier_block gic_cpu_pm_notifier_block = { 1391 .notifier_call = gic_cpu_pm_notifier, 1392 }; 1393 1394 static void gic_cpu_pm_init(void) 1395 { 1396 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 1397 } 1398 1399 #else 1400 static inline void gic_cpu_pm_init(void) { } 1401 #endif /* CONFIG_CPU_PM */ 1402 1403 static struct irq_chip gic_chip = { 1404 .name = "GICv3", 1405 .irq_mask = gic_mask_irq, 1406 .irq_unmask = gic_unmask_irq, 1407 .irq_eoi = gic_eoi_irq, 1408 .irq_set_type = gic_set_type, 1409 .irq_set_affinity = gic_set_affinity, 1410 .irq_retrigger = gic_retrigger, 1411 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1412 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1413 .irq_nmi_setup = gic_irq_nmi_setup, 1414 .irq_nmi_teardown = gic_irq_nmi_teardown, 1415 .ipi_send_mask = gic_ipi_send_mask, 1416 .flags = IRQCHIP_SET_TYPE_MASKED | 1417 IRQCHIP_SKIP_SET_WAKE | 1418 IRQCHIP_MASK_ON_SUSPEND, 1419 }; 1420 1421 static struct irq_chip gic_eoimode1_chip = { 1422 .name = "GICv3", 1423 .irq_mask = gic_eoimode1_mask_irq, 1424 .irq_unmask = gic_unmask_irq, 1425 .irq_eoi = gic_eoimode1_eoi_irq, 1426 .irq_set_type = gic_set_type, 1427 .irq_set_affinity = gic_set_affinity, 1428 .irq_retrigger = gic_retrigger, 1429 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1430 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1431 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1432 .irq_nmi_setup = gic_irq_nmi_setup, 1433 .irq_nmi_teardown = gic_irq_nmi_teardown, 1434 .ipi_send_mask = gic_ipi_send_mask, 1435 .flags = IRQCHIP_SET_TYPE_MASKED | 1436 IRQCHIP_SKIP_SET_WAKE | 1437 IRQCHIP_MASK_ON_SUSPEND, 1438 }; 1439 1440 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1441 irq_hw_number_t hw) 1442 { 1443 struct irq_chip *chip = &gic_chip; 1444 struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 1445 1446 if (static_branch_likely(&supports_deactivate_key)) 1447 chip = &gic_eoimode1_chip; 1448 1449 switch (__get_intid_range(hw)) { 1450 case SGI_RANGE: 1451 case PPI_RANGE: 1452 case EPPI_RANGE: 1453 irq_set_percpu_devid(irq); 1454 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1455 handle_percpu_devid_irq, NULL, NULL); 1456 break; 1457 1458 case SPI_RANGE: 1459 case ESPI_RANGE: 1460 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1461 handle_fasteoi_irq, NULL, NULL); 1462 irq_set_probe(irq); 1463 irqd_set_single_target(irqd); 1464 break; 1465 1466 case LPI_RANGE: 1467 if (!gic_dist_supports_lpis()) 1468 return -EPERM; 1469 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1470 handle_fasteoi_irq, NULL, NULL); 1471 break; 1472 1473 default: 1474 return -EPERM; 1475 } 1476 1477 /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 1478 irqd_set_handle_enforce_irqctx(irqd); 1479 return 0; 1480 } 1481 1482 static int gic_irq_domain_translate(struct irq_domain *d, 1483 struct irq_fwspec *fwspec, 1484 unsigned long *hwirq, 1485 unsigned int *type) 1486 { 1487 if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 1488 *hwirq = fwspec->param[0]; 1489 *type = IRQ_TYPE_EDGE_RISING; 1490 return 0; 1491 } 1492 1493 if (is_of_node(fwspec->fwnode)) { 1494 if (fwspec->param_count < 3) 1495 return -EINVAL; 1496 1497 switch (fwspec->param[0]) { 1498 case 0: /* SPI */ 1499 *hwirq = fwspec->param[1] + 32; 1500 break; 1501 case 1: /* PPI */ 1502 *hwirq = fwspec->param[1] + 16; 1503 break; 1504 case 2: /* ESPI */ 1505 *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1506 break; 1507 case 3: /* EPPI */ 1508 *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 1509 break; 1510 case GIC_IRQ_TYPE_LPI: /* LPI */ 1511 *hwirq = fwspec->param[1]; 1512 break; 1513 case GIC_IRQ_TYPE_PARTITION: 1514 *hwirq = fwspec->param[1]; 1515 if (fwspec->param[1] >= 16) 1516 *hwirq += EPPI_BASE_INTID - 16; 1517 else 1518 *hwirq += 16; 1519 break; 1520 default: 1521 return -EINVAL; 1522 } 1523 1524 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1525 1526 /* 1527 * Make it clear that broken DTs are... broken. 1528 * Partitioned PPIs are an unfortunate exception. 1529 */ 1530 WARN_ON(*type == IRQ_TYPE_NONE && 1531 fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1532 return 0; 1533 } 1534 1535 if (is_fwnode_irqchip(fwspec->fwnode)) { 1536 if(fwspec->param_count != 2) 1537 return -EINVAL; 1538 1539 if (fwspec->param[0] < 16) { 1540 pr_err(FW_BUG "Illegal GSI%d translation request\n", 1541 fwspec->param[0]); 1542 return -EINVAL; 1543 } 1544 1545 *hwirq = fwspec->param[0]; 1546 *type = fwspec->param[1]; 1547 1548 WARN_ON(*type == IRQ_TYPE_NONE); 1549 return 0; 1550 } 1551 1552 return -EINVAL; 1553 } 1554 1555 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1556 unsigned int nr_irqs, void *arg) 1557 { 1558 int i, ret; 1559 irq_hw_number_t hwirq; 1560 unsigned int type = IRQ_TYPE_NONE; 1561 struct irq_fwspec *fwspec = arg; 1562 1563 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1564 if (ret) 1565 return ret; 1566 1567 for (i = 0; i < nr_irqs; i++) { 1568 ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 1569 if (ret) 1570 return ret; 1571 } 1572 1573 return 0; 1574 } 1575 1576 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1577 unsigned int nr_irqs) 1578 { 1579 int i; 1580 1581 for (i = 0; i < nr_irqs; i++) { 1582 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1583 irq_set_handler(virq + i, NULL); 1584 irq_domain_reset_irq_data(d); 1585 } 1586 } 1587 1588 static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, 1589 irq_hw_number_t hwirq) 1590 { 1591 enum gic_intid_range range; 1592 1593 if (!gic_data.ppi_descs) 1594 return false; 1595 1596 if (!is_of_node(fwspec->fwnode)) 1597 return false; 1598 1599 if (fwspec->param_count < 4 || !fwspec->param[3]) 1600 return false; 1601 1602 range = __get_intid_range(hwirq); 1603 if (range != PPI_RANGE && range != EPPI_RANGE) 1604 return false; 1605 1606 return true; 1607 } 1608 1609 static int gic_irq_domain_select(struct irq_domain *d, 1610 struct irq_fwspec *fwspec, 1611 enum irq_domain_bus_token bus_token) 1612 { 1613 unsigned int type, ret, ppi_idx; 1614 irq_hw_number_t hwirq; 1615 1616 /* Not for us */ 1617 if (fwspec->fwnode != d->fwnode) 1618 return 0; 1619 1620 /* If this is not DT, then we have a single domain */ 1621 if (!is_of_node(fwspec->fwnode)) 1622 return 1; 1623 1624 ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type); 1625 if (WARN_ON_ONCE(ret)) 1626 return 0; 1627 1628 if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) 1629 return d == gic_data.domain; 1630 1631 /* 1632 * If this is a PPI and we have a 4th (non-null) parameter, 1633 * then we need to match the partition domain. 1634 */ 1635 ppi_idx = __gic_get_ppi_index(hwirq); 1636 return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); 1637 } 1638 1639 static const struct irq_domain_ops gic_irq_domain_ops = { 1640 .translate = gic_irq_domain_translate, 1641 .alloc = gic_irq_domain_alloc, 1642 .free = gic_irq_domain_free, 1643 .select = gic_irq_domain_select, 1644 }; 1645 1646 static int partition_domain_translate(struct irq_domain *d, 1647 struct irq_fwspec *fwspec, 1648 unsigned long *hwirq, 1649 unsigned int *type) 1650 { 1651 unsigned long ppi_intid; 1652 struct device_node *np; 1653 unsigned int ppi_idx; 1654 int ret; 1655 1656 if (!gic_data.ppi_descs) 1657 return -ENOMEM; 1658 1659 np = of_find_node_by_phandle(fwspec->param[3]); 1660 if (WARN_ON(!np)) 1661 return -EINVAL; 1662 1663 ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); 1664 if (WARN_ON_ONCE(ret)) 1665 return 0; 1666 1667 ppi_idx = __gic_get_ppi_index(ppi_intid); 1668 ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], 1669 of_node_to_fwnode(np)); 1670 if (ret < 0) 1671 return ret; 1672 1673 *hwirq = ret; 1674 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1675 1676 return 0; 1677 } 1678 1679 static const struct irq_domain_ops partition_domain_ops = { 1680 .translate = partition_domain_translate, 1681 .select = gic_irq_domain_select, 1682 }; 1683 1684 static bool gic_enable_quirk_msm8996(void *data) 1685 { 1686 struct gic_chip_data *d = data; 1687 1688 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 1689 1690 return true; 1691 } 1692 1693 static bool gic_enable_quirk_cavium_38539(void *data) 1694 { 1695 struct gic_chip_data *d = data; 1696 1697 d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1698 1699 return true; 1700 } 1701 1702 static bool gic_enable_quirk_hip06_07(void *data) 1703 { 1704 struct gic_chip_data *d = data; 1705 1706 /* 1707 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 1708 * not being an actual ARM implementation). The saving grace is 1709 * that GIC-600 doesn't have ESPI, so nothing to do in that case. 1710 * HIP07 doesn't even have a proper IIDR, and still pretends to 1711 * have ESPI. In both cases, put them right. 1712 */ 1713 if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 1714 /* Zero both ESPI and the RES0 field next to it... */ 1715 d->rdists.gicd_typer &= ~GENMASK(9, 8); 1716 return true; 1717 } 1718 1719 return false; 1720 } 1721 1722 static const struct gic_quirk gic_quirks[] = { 1723 { 1724 .desc = "GICv3: Qualcomm MSM8996 broken firmware", 1725 .compatible = "qcom,msm8996-gic-v3", 1726 .init = gic_enable_quirk_msm8996, 1727 }, 1728 { 1729 .desc = "GICv3: HIP06 erratum 161010803", 1730 .iidr = 0x0204043b, 1731 .mask = 0xffffffff, 1732 .init = gic_enable_quirk_hip06_07, 1733 }, 1734 { 1735 .desc = "GICv3: HIP07 erratum 161010803", 1736 .iidr = 0x00000000, 1737 .mask = 0xffffffff, 1738 .init = gic_enable_quirk_hip06_07, 1739 }, 1740 { 1741 /* 1742 * Reserved register accesses generate a Synchronous 1743 * External Abort. This erratum applies to: 1744 * - ThunderX: CN88xx 1745 * - OCTEON TX: CN83xx, CN81xx 1746 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1747 */ 1748 .desc = "GICv3: Cavium erratum 38539", 1749 .iidr = 0xa000034c, 1750 .mask = 0xe8f00fff, 1751 .init = gic_enable_quirk_cavium_38539, 1752 }, 1753 { 1754 } 1755 }; 1756 1757 static void gic_enable_nmi_support(void) 1758 { 1759 int i; 1760 1761 if (!gic_prio_masking_enabled()) 1762 return; 1763 1764 ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 1765 if (!ppi_nmi_refs) 1766 return; 1767 1768 for (i = 0; i < gic_data.ppi_nr; i++) 1769 refcount_set(&ppi_nmi_refs[i], 0); 1770 1771 /* 1772 * Linux itself doesn't use 1:N distribution, so has no need to 1773 * set PMHE. The only reason to have it set is if EL3 requires it 1774 * (and we can't change it). 1775 */ 1776 if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) 1777 static_branch_enable(&gic_pmr_sync); 1778 1779 pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 1780 static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed"); 1781 1782 /* 1783 * How priority values are used by the GIC depends on two things: 1784 * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 1785 * and if Group 0 interrupts can be delivered to Linux in the non-secure 1786 * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 1787 * ICC_PMR_EL1 register and the priority that software assigns to 1788 * interrupts: 1789 * 1790 * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 1791 * ----------------------------------------------------------- 1792 * 1 | - | unchanged | unchanged 1793 * ----------------------------------------------------------- 1794 * 0 | 1 | non-secure | non-secure 1795 * ----------------------------------------------------------- 1796 * 0 | 0 | unchanged | non-secure 1797 * 1798 * where non-secure means that the value is right-shifted by one and the 1799 * MSB bit set, to make it fit in the non-secure priority range. 1800 * 1801 * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 1802 * are both either modified or unchanged, we can use the same set of 1803 * priorities. 1804 * 1805 * In the last case, where only the interrupt priorities are modified to 1806 * be in the non-secure range, we use a different PMR value to mask IRQs 1807 * and the rest of the values that we use remain unchanged. 1808 */ 1809 if (gic_has_group0() && !gic_dist_security_disabled()) 1810 static_branch_enable(&gic_nonsecure_priorities); 1811 1812 static_branch_enable(&supports_pseudo_nmis); 1813 1814 if (static_branch_likely(&supports_deactivate_key)) 1815 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1816 else 1817 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1818 } 1819 1820 static int __init gic_init_bases(void __iomem *dist_base, 1821 struct redist_region *rdist_regs, 1822 u32 nr_redist_regions, 1823 u64 redist_stride, 1824 struct fwnode_handle *handle) 1825 { 1826 u32 typer; 1827 int err; 1828 1829 if (!is_hyp_mode_available()) 1830 static_branch_disable(&supports_deactivate_key); 1831 1832 if (static_branch_likely(&supports_deactivate_key)) 1833 pr_info("GIC: Using split EOI/Deactivate mode\n"); 1834 1835 gic_data.fwnode = handle; 1836 gic_data.dist_base = dist_base; 1837 gic_data.redist_regions = rdist_regs; 1838 gic_data.nr_redist_regions = nr_redist_regions; 1839 gic_data.redist_stride = redist_stride; 1840 1841 /* 1842 * Find out how many interrupts are supported. 1843 */ 1844 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1845 gic_data.rdists.gicd_typer = typer; 1846 1847 gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 1848 gic_quirks, &gic_data); 1849 1850 pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 1851 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 1852 1853 /* 1854 * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 1855 * architecture spec (which says that reserved registers are RES0). 1856 */ 1857 if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 1858 gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 1859 1860 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1861 &gic_data); 1862 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 1863 gic_data.rdists.has_rvpeid = true; 1864 gic_data.rdists.has_vlpis = true; 1865 gic_data.rdists.has_direct_lpi = true; 1866 gic_data.rdists.has_vpend_valid_dirty = true; 1867 1868 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1869 err = -ENOMEM; 1870 goto out_free; 1871 } 1872 1873 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1874 1875 gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1876 1877 if (typer & GICD_TYPER_MBIS) { 1878 err = mbi_init(handle, gic_data.domain); 1879 if (err) 1880 pr_err("Failed to initialize MBIs\n"); 1881 } 1882 1883 set_handle_irq(gic_handle_irq); 1884 1885 gic_update_rdist_properties(); 1886 1887 gic_dist_init(); 1888 gic_cpu_init(); 1889 gic_smp_init(); 1890 gic_cpu_pm_init(); 1891 1892 if (gic_dist_supports_lpis()) { 1893 its_init(handle, &gic_data.rdists, gic_data.domain); 1894 its_cpu_init(); 1895 its_lpi_memreserve_init(); 1896 } else { 1897 if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 1898 gicv2m_init(handle, gic_data.domain); 1899 } 1900 1901 gic_enable_nmi_support(); 1902 1903 return 0; 1904 1905 out_free: 1906 if (gic_data.domain) 1907 irq_domain_remove(gic_data.domain); 1908 free_percpu(gic_data.rdists.rdist); 1909 return err; 1910 } 1911 1912 static int __init gic_validate_dist_version(void __iomem *dist_base) 1913 { 1914 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1915 1916 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 1917 return -ENODEV; 1918 1919 return 0; 1920 } 1921 1922 /* Create all possible partitions at boot time */ 1923 static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 1924 { 1925 struct device_node *parts_node, *child_part; 1926 int part_idx = 0, i; 1927 int nr_parts; 1928 struct partition_affinity *parts; 1929 1930 parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 1931 if (!parts_node) 1932 return; 1933 1934 gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 1935 if (!gic_data.ppi_descs) 1936 goto out_put_node; 1937 1938 nr_parts = of_get_child_count(parts_node); 1939 1940 if (!nr_parts) 1941 goto out_put_node; 1942 1943 parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 1944 if (WARN_ON(!parts)) 1945 goto out_put_node; 1946 1947 for_each_child_of_node(parts_node, child_part) { 1948 struct partition_affinity *part; 1949 int n; 1950 1951 part = &parts[part_idx]; 1952 1953 part->partition_id = of_node_to_fwnode(child_part); 1954 1955 pr_info("GIC: PPI partition %pOFn[%d] { ", 1956 child_part, part_idx); 1957 1958 n = of_property_count_elems_of_size(child_part, "affinity", 1959 sizeof(u32)); 1960 WARN_ON(n <= 0); 1961 1962 for (i = 0; i < n; i++) { 1963 int err, cpu; 1964 u32 cpu_phandle; 1965 struct device_node *cpu_node; 1966 1967 err = of_property_read_u32_index(child_part, "affinity", 1968 i, &cpu_phandle); 1969 if (WARN_ON(err)) 1970 continue; 1971 1972 cpu_node = of_find_node_by_phandle(cpu_phandle); 1973 if (WARN_ON(!cpu_node)) 1974 continue; 1975 1976 cpu = of_cpu_node_to_id(cpu_node); 1977 if (WARN_ON(cpu < 0)) { 1978 of_node_put(cpu_node); 1979 continue; 1980 } 1981 1982 pr_cont("%pOF[%d] ", cpu_node, cpu); 1983 1984 cpumask_set_cpu(cpu, &part->mask); 1985 of_node_put(cpu_node); 1986 } 1987 1988 pr_cont("}\n"); 1989 part_idx++; 1990 } 1991 1992 for (i = 0; i < gic_data.ppi_nr; i++) { 1993 unsigned int irq; 1994 struct partition_desc *desc; 1995 struct irq_fwspec ppi_fwspec = { 1996 .fwnode = gic_data.fwnode, 1997 .param_count = 3, 1998 .param = { 1999 [0] = GIC_IRQ_TYPE_PARTITION, 2000 [1] = i, 2001 [2] = IRQ_TYPE_NONE, 2002 }, 2003 }; 2004 2005 irq = irq_create_fwspec_mapping(&ppi_fwspec); 2006 if (WARN_ON(!irq)) 2007 continue; 2008 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 2009 irq, &partition_domain_ops); 2010 if (WARN_ON(!desc)) 2011 continue; 2012 2013 gic_data.ppi_descs[i] = desc; 2014 } 2015 2016 out_put_node: 2017 of_node_put(parts_node); 2018 } 2019 2020 static void __init gic_of_setup_kvm_info(struct device_node *node) 2021 { 2022 int ret; 2023 struct resource r; 2024 u32 gicv_idx; 2025 2026 gic_v3_kvm_info.type = GIC_V3; 2027 2028 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 2029 if (!gic_v3_kvm_info.maint_irq) 2030 return; 2031 2032 if (of_property_read_u32(node, "#redistributor-regions", 2033 &gicv_idx)) 2034 gicv_idx = 1; 2035 2036 gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 2037 ret = of_address_to_resource(node, gicv_idx, &r); 2038 if (!ret) 2039 gic_v3_kvm_info.vcpu = r; 2040 2041 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2042 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2043 vgic_set_kvm_info(&gic_v3_kvm_info); 2044 } 2045 2046 static void gic_request_region(resource_size_t base, resource_size_t size, 2047 const char *name) 2048 { 2049 if (!request_mem_region(base, size, name)) 2050 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", 2051 name, &base); 2052 } 2053 2054 static void __iomem *gic_of_iomap(struct device_node *node, int idx, 2055 const char *name, struct resource *res) 2056 { 2057 void __iomem *base; 2058 int ret; 2059 2060 ret = of_address_to_resource(node, idx, res); 2061 if (ret) 2062 return IOMEM_ERR_PTR(ret); 2063 2064 gic_request_region(res->start, resource_size(res), name); 2065 base = of_iomap(node, idx); 2066 2067 return base ?: IOMEM_ERR_PTR(-ENOMEM); 2068 } 2069 2070 static int __init gic_of_init(struct device_node *node, struct device_node *parent) 2071 { 2072 void __iomem *dist_base; 2073 struct redist_region *rdist_regs; 2074 struct resource res; 2075 u64 redist_stride; 2076 u32 nr_redist_regions; 2077 int err, i; 2078 2079 dist_base = gic_of_iomap(node, 0, "GICD", &res); 2080 if (IS_ERR(dist_base)) { 2081 pr_err("%pOF: unable to map gic dist registers\n", node); 2082 return PTR_ERR(dist_base); 2083 } 2084 2085 err = gic_validate_dist_version(dist_base); 2086 if (err) { 2087 pr_err("%pOF: no distributor detected, giving up\n", node); 2088 goto out_unmap_dist; 2089 } 2090 2091 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 2092 nr_redist_regions = 1; 2093 2094 rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 2095 GFP_KERNEL); 2096 if (!rdist_regs) { 2097 err = -ENOMEM; 2098 goto out_unmap_dist; 2099 } 2100 2101 for (i = 0; i < nr_redist_regions; i++) { 2102 rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res); 2103 if (IS_ERR(rdist_regs[i].redist_base)) { 2104 pr_err("%pOF: couldn't map region %d\n", node, i); 2105 err = -ENODEV; 2106 goto out_unmap_rdist; 2107 } 2108 rdist_regs[i].phys_base = res.start; 2109 } 2110 2111 if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 2112 redist_stride = 0; 2113 2114 gic_enable_of_quirks(node, gic_quirks, &gic_data); 2115 2116 err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions, 2117 redist_stride, &node->fwnode); 2118 if (err) 2119 goto out_unmap_rdist; 2120 2121 gic_populate_ppi_partitions(node); 2122 2123 if (static_branch_likely(&supports_deactivate_key)) 2124 gic_of_setup_kvm_info(node); 2125 return 0; 2126 2127 out_unmap_rdist: 2128 for (i = 0; i < nr_redist_regions; i++) 2129 if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base)) 2130 iounmap(rdist_regs[i].redist_base); 2131 kfree(rdist_regs); 2132 out_unmap_dist: 2133 iounmap(dist_base); 2134 return err; 2135 } 2136 2137 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 2138 2139 #ifdef CONFIG_ACPI 2140 static struct 2141 { 2142 void __iomem *dist_base; 2143 struct redist_region *redist_regs; 2144 u32 nr_redist_regions; 2145 bool single_redist; 2146 int enabled_rdists; 2147 u32 maint_irq; 2148 int maint_irq_mode; 2149 phys_addr_t vcpu_base; 2150 } acpi_data __initdata; 2151 2152 static void __init 2153 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 2154 { 2155 static int count = 0; 2156 2157 acpi_data.redist_regs[count].phys_base = phys_base; 2158 acpi_data.redist_regs[count].redist_base = redist_base; 2159 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 2160 count++; 2161 } 2162 2163 static int __init 2164 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 2165 const unsigned long end) 2166 { 2167 struct acpi_madt_generic_redistributor *redist = 2168 (struct acpi_madt_generic_redistributor *)header; 2169 void __iomem *redist_base; 2170 2171 redist_base = ioremap(redist->base_address, redist->length); 2172 if (!redist_base) { 2173 pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2174 return -ENOMEM; 2175 } 2176 gic_request_region(redist->base_address, redist->length, "GICR"); 2177 2178 gic_acpi_register_redist(redist->base_address, redist_base); 2179 return 0; 2180 } 2181 2182 static int __init 2183 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2184 const unsigned long end) 2185 { 2186 struct acpi_madt_generic_interrupt *gicc = 2187 (struct acpi_madt_generic_interrupt *)header; 2188 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2189 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2190 void __iomem *redist_base; 2191 2192 /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 2193 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2194 return 0; 2195 2196 redist_base = ioremap(gicc->gicr_base_address, size); 2197 if (!redist_base) 2198 return -ENOMEM; 2199 gic_request_region(gicc->gicr_base_address, size, "GICR"); 2200 2201 gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2202 return 0; 2203 } 2204 2205 static int __init gic_acpi_collect_gicr_base(void) 2206 { 2207 acpi_tbl_entry_handler redist_parser; 2208 enum acpi_madt_type type; 2209 2210 if (acpi_data.single_redist) { 2211 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2212 redist_parser = gic_acpi_parse_madt_gicc; 2213 } else { 2214 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2215 redist_parser = gic_acpi_parse_madt_redist; 2216 } 2217 2218 /* Collect redistributor base addresses in GICR entries */ 2219 if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2220 return 0; 2221 2222 pr_info("No valid GICR entries exist\n"); 2223 return -ENODEV; 2224 } 2225 2226 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2227 const unsigned long end) 2228 { 2229 /* Subtable presence means that redist exists, that's it */ 2230 return 0; 2231 } 2232 2233 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2234 const unsigned long end) 2235 { 2236 struct acpi_madt_generic_interrupt *gicc = 2237 (struct acpi_madt_generic_interrupt *)header; 2238 2239 /* 2240 * If GICC is enabled and has valid gicr base address, then it means 2241 * GICR base is presented via GICC 2242 */ 2243 if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2244 acpi_data.enabled_rdists++; 2245 return 0; 2246 } 2247 2248 /* 2249 * It's perfectly valid firmware can pass disabled GICC entry, driver 2250 * should not treat as errors, skip the entry instead of probe fail. 2251 */ 2252 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2253 return 0; 2254 2255 return -ENODEV; 2256 } 2257 2258 static int __init gic_acpi_count_gicr_regions(void) 2259 { 2260 int count; 2261 2262 /* 2263 * Count how many redistributor regions we have. It is not allowed 2264 * to mix redistributor description, GICR and GICC subtables have to be 2265 * mutually exclusive. 2266 */ 2267 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2268 gic_acpi_match_gicr, 0); 2269 if (count > 0) { 2270 acpi_data.single_redist = false; 2271 return count; 2272 } 2273 2274 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2275 gic_acpi_match_gicc, 0); 2276 if (count > 0) { 2277 acpi_data.single_redist = true; 2278 count = acpi_data.enabled_rdists; 2279 } 2280 2281 return count; 2282 } 2283 2284 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2285 struct acpi_probe_entry *ape) 2286 { 2287 struct acpi_madt_generic_distributor *dist; 2288 int count; 2289 2290 dist = (struct acpi_madt_generic_distributor *)header; 2291 if (dist->version != ape->driver_data) 2292 return false; 2293 2294 /* We need to do that exercise anyway, the sooner the better */ 2295 count = gic_acpi_count_gicr_regions(); 2296 if (count <= 0) 2297 return false; 2298 2299 acpi_data.nr_redist_regions = count; 2300 return true; 2301 } 2302 2303 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 2304 const unsigned long end) 2305 { 2306 struct acpi_madt_generic_interrupt *gicc = 2307 (struct acpi_madt_generic_interrupt *)header; 2308 int maint_irq_mode; 2309 static int first_madt = true; 2310 2311 /* Skip unusable CPUs */ 2312 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2313 return 0; 2314 2315 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 2316 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 2317 2318 if (first_madt) { 2319 first_madt = false; 2320 2321 acpi_data.maint_irq = gicc->vgic_interrupt; 2322 acpi_data.maint_irq_mode = maint_irq_mode; 2323 acpi_data.vcpu_base = gicc->gicv_base_address; 2324 2325 return 0; 2326 } 2327 2328 /* 2329 * The maintenance interrupt and GICV should be the same for every CPU 2330 */ 2331 if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 2332 (acpi_data.maint_irq_mode != maint_irq_mode) || 2333 (acpi_data.vcpu_base != gicc->gicv_base_address)) 2334 return -EINVAL; 2335 2336 return 0; 2337 } 2338 2339 static bool __init gic_acpi_collect_virt_info(void) 2340 { 2341 int count; 2342 2343 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2344 gic_acpi_parse_virt_madt_gicc, 0); 2345 2346 return (count > 0); 2347 } 2348 2349 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 2350 #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 2351 #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 2352 2353 static void __init gic_acpi_setup_kvm_info(void) 2354 { 2355 int irq; 2356 2357 if (!gic_acpi_collect_virt_info()) { 2358 pr_warn("Unable to get hardware information used for virtualization\n"); 2359 return; 2360 } 2361 2362 gic_v3_kvm_info.type = GIC_V3; 2363 2364 irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 2365 acpi_data.maint_irq_mode, 2366 ACPI_ACTIVE_HIGH); 2367 if (irq <= 0) 2368 return; 2369 2370 gic_v3_kvm_info.maint_irq = irq; 2371 2372 if (acpi_data.vcpu_base) { 2373 struct resource *vcpu = &gic_v3_kvm_info.vcpu; 2374 2375 vcpu->flags = IORESOURCE_MEM; 2376 vcpu->start = acpi_data.vcpu_base; 2377 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 2378 } 2379 2380 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2381 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2382 vgic_set_kvm_info(&gic_v3_kvm_info); 2383 } 2384 2385 static struct fwnode_handle *gsi_domain_handle; 2386 2387 static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi) 2388 { 2389 return gsi_domain_handle; 2390 } 2391 2392 static int __init 2393 gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2394 { 2395 struct acpi_madt_generic_distributor *dist; 2396 size_t size; 2397 int i, err; 2398 2399 /* Get distributor base address */ 2400 dist = (struct acpi_madt_generic_distributor *)header; 2401 acpi_data.dist_base = ioremap(dist->base_address, 2402 ACPI_GICV3_DIST_MEM_SIZE); 2403 if (!acpi_data.dist_base) { 2404 pr_err("Unable to map GICD registers\n"); 2405 return -ENOMEM; 2406 } 2407 gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD"); 2408 2409 err = gic_validate_dist_version(acpi_data.dist_base); 2410 if (err) { 2411 pr_err("No distributor detected at @%p, giving up\n", 2412 acpi_data.dist_base); 2413 goto out_dist_unmap; 2414 } 2415 2416 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2417 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2418 if (!acpi_data.redist_regs) { 2419 err = -ENOMEM; 2420 goto out_dist_unmap; 2421 } 2422 2423 err = gic_acpi_collect_gicr_base(); 2424 if (err) 2425 goto out_redist_unmap; 2426 2427 gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2428 if (!gsi_domain_handle) { 2429 err = -ENOMEM; 2430 goto out_redist_unmap; 2431 } 2432 2433 err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs, 2434 acpi_data.nr_redist_regions, 0, gsi_domain_handle); 2435 if (err) 2436 goto out_fwhandle_free; 2437 2438 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id); 2439 2440 if (static_branch_likely(&supports_deactivate_key)) 2441 gic_acpi_setup_kvm_info(); 2442 2443 return 0; 2444 2445 out_fwhandle_free: 2446 irq_domain_free_fwnode(gsi_domain_handle); 2447 out_redist_unmap: 2448 for (i = 0; i < acpi_data.nr_redist_regions; i++) 2449 if (acpi_data.redist_regs[i].redist_base) 2450 iounmap(acpi_data.redist_regs[i].redist_base); 2451 kfree(acpi_data.redist_regs); 2452 out_dist_unmap: 2453 iounmap(acpi_data.dist_base); 2454 return err; 2455 } 2456 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2457 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2458 gic_acpi_init); 2459 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2460 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2461 gic_acpi_init); 2462 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2463 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2464 gic_acpi_init); 2465 #endif 2466