1 /* 2 * Copyright (C) 2002 ARM Limited, All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * Interrupt architecture for the GIC: 9 * 10 * o There is one Interrupt Distributor, which receives interrupts 11 * from system devices and sends them to the Interrupt Controllers. 12 * 13 * o There is one CPU Interface per CPU, which sends interrupts sent 14 * by the Distributor, and interrupts generated locally, to the 15 * associated CPU. The base address of the CPU interface is usually 16 * aliased so that the same address points to different chips depending 17 * on the CPU it is accessed from. 18 * 19 * Note that IRQs 0-31 are special - they are local to each CPU. 20 * As such, the enable set/clear, pending set/clear and active bit 21 * registers are banked per-cpu for these sources. 22 */ 23 #include <linux/init.h> 24 #include <linux/kernel.h> 25 #include <linux/err.h> 26 #include <linux/module.h> 27 #include <linux/list.h> 28 #include <linux/smp.h> 29 #include <linux/cpu.h> 30 #include <linux/cpu_pm.h> 31 #include <linux/cpumask.h> 32 #include <linux/io.h> 33 #include <linux/of.h> 34 #include <linux/of_address.h> 35 #include <linux/of_irq.h> 36 #include <linux/acpi.h> 37 #include <linux/irqdomain.h> 38 #include <linux/interrupt.h> 39 #include <linux/percpu.h> 40 #include <linux/slab.h> 41 #include <linux/irqchip.h> 42 #include <linux/irqchip/chained_irq.h> 43 #include <linux/irqchip/arm-gic.h> 44 #include <linux/irqchip/arm-gic-acpi.h> 45 46 #include <asm/cputype.h> 47 #include <asm/irq.h> 48 #include <asm/exception.h> 49 #include <asm/smp_plat.h> 50 #include <asm/virt.h> 51 52 #include "irq-gic-common.h" 53 54 union gic_base { 55 void __iomem *common_base; 56 void __percpu * __iomem *percpu_base; 57 }; 58 59 struct gic_chip_data { 60 union gic_base dist_base; 61 union gic_base cpu_base; 62 #ifdef CONFIG_CPU_PM 63 u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)]; 64 u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)]; 65 u32 saved_spi_target[DIV_ROUND_UP(1020, 4)]; 66 u32 __percpu *saved_ppi_enable; 67 u32 __percpu *saved_ppi_conf; 68 #endif 69 struct irq_domain *domain; 70 unsigned int gic_irqs; 71 #ifdef CONFIG_GIC_NON_BANKED 72 void __iomem *(*get_base)(union gic_base *); 73 #endif 74 }; 75 76 static DEFINE_RAW_SPINLOCK(irq_controller_lock); 77 78 /* 79 * The GIC mapping of CPU interfaces does not necessarily match 80 * the logical CPU numbering. Let's use a mapping as returned 81 * by the GIC itself. 82 */ 83 #define NR_GIC_CPU_IF 8 84 static u8 gic_cpu_map[NR_GIC_CPU_IF] __read_mostly; 85 86 static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE; 87 88 #ifndef MAX_GIC_NR 89 #define MAX_GIC_NR 1 90 #endif 91 92 static struct gic_chip_data gic_data[MAX_GIC_NR] __read_mostly; 93 94 #ifdef CONFIG_GIC_NON_BANKED 95 static void __iomem *gic_get_percpu_base(union gic_base *base) 96 { 97 return raw_cpu_read(*base->percpu_base); 98 } 99 100 static void __iomem *gic_get_common_base(union gic_base *base) 101 { 102 return base->common_base; 103 } 104 105 static inline void __iomem *gic_data_dist_base(struct gic_chip_data *data) 106 { 107 return data->get_base(&data->dist_base); 108 } 109 110 static inline void __iomem *gic_data_cpu_base(struct gic_chip_data *data) 111 { 112 return data->get_base(&data->cpu_base); 113 } 114 115 static inline void gic_set_base_accessor(struct gic_chip_data *data, 116 void __iomem *(*f)(union gic_base *)) 117 { 118 data->get_base = f; 119 } 120 #else 121 #define gic_data_dist_base(d) ((d)->dist_base.common_base) 122 #define gic_data_cpu_base(d) ((d)->cpu_base.common_base) 123 #define gic_set_base_accessor(d, f) 124 #endif 125 126 static inline void __iomem *gic_dist_base(struct irq_data *d) 127 { 128 struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d); 129 return gic_data_dist_base(gic_data); 130 } 131 132 static inline void __iomem *gic_cpu_base(struct irq_data *d) 133 { 134 struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d); 135 return gic_data_cpu_base(gic_data); 136 } 137 138 static inline unsigned int gic_irq(struct irq_data *d) 139 { 140 return d->hwirq; 141 } 142 143 static inline bool cascading_gic_irq(struct irq_data *d) 144 { 145 void *data = irq_data_get_irq_handler_data(d); 146 147 /* 148 * If handler_data pointing to one of the secondary GICs, then 149 * this is a cascading interrupt, and it cannot possibly be 150 * forwarded. 151 */ 152 if (data >= (void *)(gic_data + 1) && 153 data < (void *)(gic_data + MAX_GIC_NR)) 154 return true; 155 156 return false; 157 } 158 159 static inline bool forwarded_irq(struct irq_data *d) 160 { 161 /* 162 * A forwarded interrupt: 163 * - is on the primary GIC 164 * - has its handler_data set to a value 165 * - that isn't a secondary GIC 166 */ 167 if (d->handler_data && !cascading_gic_irq(d)) 168 return true; 169 170 return false; 171 } 172 173 /* 174 * Routines to acknowledge, disable and enable interrupts 175 */ 176 static void gic_poke_irq(struct irq_data *d, u32 offset) 177 { 178 u32 mask = 1 << (gic_irq(d) % 32); 179 writel_relaxed(mask, gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4); 180 } 181 182 static int gic_peek_irq(struct irq_data *d, u32 offset) 183 { 184 u32 mask = 1 << (gic_irq(d) % 32); 185 return !!(readl_relaxed(gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4) & mask); 186 } 187 188 static void gic_mask_irq(struct irq_data *d) 189 { 190 gic_poke_irq(d, GIC_DIST_ENABLE_CLEAR); 191 } 192 193 static void gic_eoimode1_mask_irq(struct irq_data *d) 194 { 195 gic_mask_irq(d); 196 /* 197 * When masking a forwarded interrupt, make sure it is 198 * deactivated as well. 199 * 200 * This ensures that an interrupt that is getting 201 * disabled/masked will not get "stuck", because there is 202 * noone to deactivate it (guest is being terminated). 203 */ 204 if (forwarded_irq(d)) 205 gic_poke_irq(d, GIC_DIST_ACTIVE_CLEAR); 206 } 207 208 static void gic_unmask_irq(struct irq_data *d) 209 { 210 gic_poke_irq(d, GIC_DIST_ENABLE_SET); 211 } 212 213 static void gic_eoi_irq(struct irq_data *d) 214 { 215 writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI); 216 } 217 218 static void gic_eoimode1_eoi_irq(struct irq_data *d) 219 { 220 /* Do not deactivate an IRQ forwarded to a vcpu. */ 221 if (forwarded_irq(d)) 222 return; 223 224 writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_DEACTIVATE); 225 } 226 227 static int gic_irq_set_irqchip_state(struct irq_data *d, 228 enum irqchip_irq_state which, bool val) 229 { 230 u32 reg; 231 232 switch (which) { 233 case IRQCHIP_STATE_PENDING: 234 reg = val ? GIC_DIST_PENDING_SET : GIC_DIST_PENDING_CLEAR; 235 break; 236 237 case IRQCHIP_STATE_ACTIVE: 238 reg = val ? GIC_DIST_ACTIVE_SET : GIC_DIST_ACTIVE_CLEAR; 239 break; 240 241 case IRQCHIP_STATE_MASKED: 242 reg = val ? GIC_DIST_ENABLE_CLEAR : GIC_DIST_ENABLE_SET; 243 break; 244 245 default: 246 return -EINVAL; 247 } 248 249 gic_poke_irq(d, reg); 250 return 0; 251 } 252 253 static int gic_irq_get_irqchip_state(struct irq_data *d, 254 enum irqchip_irq_state which, bool *val) 255 { 256 switch (which) { 257 case IRQCHIP_STATE_PENDING: 258 *val = gic_peek_irq(d, GIC_DIST_PENDING_SET); 259 break; 260 261 case IRQCHIP_STATE_ACTIVE: 262 *val = gic_peek_irq(d, GIC_DIST_ACTIVE_SET); 263 break; 264 265 case IRQCHIP_STATE_MASKED: 266 *val = !gic_peek_irq(d, GIC_DIST_ENABLE_SET); 267 break; 268 269 default: 270 return -EINVAL; 271 } 272 273 return 0; 274 } 275 276 static int gic_set_type(struct irq_data *d, unsigned int type) 277 { 278 void __iomem *base = gic_dist_base(d); 279 unsigned int gicirq = gic_irq(d); 280 281 /* Interrupt configuration for SGIs can't be changed */ 282 if (gicirq < 16) 283 return -EINVAL; 284 285 /* SPIs have restrictions on the supported types */ 286 if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && 287 type != IRQ_TYPE_EDGE_RISING) 288 return -EINVAL; 289 290 return gic_configure_irq(gicirq, type, base, NULL); 291 } 292 293 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 294 { 295 /* Only interrupts on the primary GIC can be forwarded to a vcpu. */ 296 if (cascading_gic_irq(d)) 297 return -EINVAL; 298 299 d->handler_data = vcpu; 300 return 0; 301 } 302 303 #ifdef CONFIG_SMP 304 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 305 bool force) 306 { 307 void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + (gic_irq(d) & ~3); 308 unsigned int cpu, shift = (gic_irq(d) % 4) * 8; 309 u32 val, mask, bit; 310 unsigned long flags; 311 312 if (!force) 313 cpu = cpumask_any_and(mask_val, cpu_online_mask); 314 else 315 cpu = cpumask_first(mask_val); 316 317 if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids) 318 return -EINVAL; 319 320 raw_spin_lock_irqsave(&irq_controller_lock, flags); 321 mask = 0xff << shift; 322 bit = gic_cpu_map[cpu] << shift; 323 val = readl_relaxed(reg) & ~mask; 324 writel_relaxed(val | bit, reg); 325 raw_spin_unlock_irqrestore(&irq_controller_lock, flags); 326 327 return IRQ_SET_MASK_OK; 328 } 329 #endif 330 331 static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 332 { 333 u32 irqstat, irqnr; 334 struct gic_chip_data *gic = &gic_data[0]; 335 void __iomem *cpu_base = gic_data_cpu_base(gic); 336 337 do { 338 irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK); 339 irqnr = irqstat & GICC_IAR_INT_ID_MASK; 340 341 if (likely(irqnr > 15 && irqnr < 1021)) { 342 if (static_key_true(&supports_deactivate)) 343 writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); 344 handle_domain_irq(gic->domain, irqnr, regs); 345 continue; 346 } 347 if (irqnr < 16) { 348 writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); 349 if (static_key_true(&supports_deactivate)) 350 writel_relaxed(irqstat, cpu_base + GIC_CPU_DEACTIVATE); 351 #ifdef CONFIG_SMP 352 handle_IPI(irqnr, regs); 353 #endif 354 continue; 355 } 356 break; 357 } while (1); 358 } 359 360 static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc) 361 { 362 struct gic_chip_data *chip_data = irq_desc_get_handler_data(desc); 363 struct irq_chip *chip = irq_desc_get_chip(desc); 364 unsigned int cascade_irq, gic_irq; 365 unsigned long status; 366 367 chained_irq_enter(chip, desc); 368 369 raw_spin_lock(&irq_controller_lock); 370 status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK); 371 raw_spin_unlock(&irq_controller_lock); 372 373 gic_irq = (status & GICC_IAR_INT_ID_MASK); 374 if (gic_irq == GICC_INT_SPURIOUS) 375 goto out; 376 377 cascade_irq = irq_find_mapping(chip_data->domain, gic_irq); 378 if (unlikely(gic_irq < 32 || gic_irq > 1020)) 379 handle_bad_irq(cascade_irq, desc); 380 else 381 generic_handle_irq(cascade_irq); 382 383 out: 384 chained_irq_exit(chip, desc); 385 } 386 387 static struct irq_chip gic_chip = { 388 .name = "GIC", 389 .irq_mask = gic_mask_irq, 390 .irq_unmask = gic_unmask_irq, 391 .irq_eoi = gic_eoi_irq, 392 .irq_set_type = gic_set_type, 393 #ifdef CONFIG_SMP 394 .irq_set_affinity = gic_set_affinity, 395 #endif 396 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 397 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 398 .flags = IRQCHIP_SET_TYPE_MASKED | 399 IRQCHIP_SKIP_SET_WAKE | 400 IRQCHIP_MASK_ON_SUSPEND, 401 }; 402 403 static struct irq_chip gic_eoimode1_chip = { 404 .name = "GICv2", 405 .irq_mask = gic_eoimode1_mask_irq, 406 .irq_unmask = gic_unmask_irq, 407 .irq_eoi = gic_eoimode1_eoi_irq, 408 .irq_set_type = gic_set_type, 409 #ifdef CONFIG_SMP 410 .irq_set_affinity = gic_set_affinity, 411 #endif 412 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 413 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 414 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 415 .flags = IRQCHIP_SET_TYPE_MASKED | 416 IRQCHIP_SKIP_SET_WAKE | 417 IRQCHIP_MASK_ON_SUSPEND, 418 }; 419 420 void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq) 421 { 422 if (gic_nr >= MAX_GIC_NR) 423 BUG(); 424 irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq, 425 &gic_data[gic_nr]); 426 } 427 428 static u8 gic_get_cpumask(struct gic_chip_data *gic) 429 { 430 void __iomem *base = gic_data_dist_base(gic); 431 u32 mask, i; 432 433 for (i = mask = 0; i < 32; i += 4) { 434 mask = readl_relaxed(base + GIC_DIST_TARGET + i); 435 mask |= mask >> 16; 436 mask |= mask >> 8; 437 if (mask) 438 break; 439 } 440 441 if (!mask && num_possible_cpus() > 1) 442 pr_crit("GIC CPU mask not found - kernel will fail to boot.\n"); 443 444 return mask; 445 } 446 447 static void gic_cpu_if_up(struct gic_chip_data *gic) 448 { 449 void __iomem *cpu_base = gic_data_cpu_base(gic); 450 u32 bypass = 0; 451 u32 mode = 0; 452 453 if (static_key_true(&supports_deactivate)) 454 mode = GIC_CPU_CTRL_EOImodeNS; 455 456 /* 457 * Preserve bypass disable bits to be written back later 458 */ 459 bypass = readl(cpu_base + GIC_CPU_CTRL); 460 bypass &= GICC_DIS_BYPASS_MASK; 461 462 writel_relaxed(bypass | mode | GICC_ENABLE, cpu_base + GIC_CPU_CTRL); 463 } 464 465 466 static void __init gic_dist_init(struct gic_chip_data *gic) 467 { 468 unsigned int i; 469 u32 cpumask; 470 unsigned int gic_irqs = gic->gic_irqs; 471 void __iomem *base = gic_data_dist_base(gic); 472 473 writel_relaxed(GICD_DISABLE, base + GIC_DIST_CTRL); 474 475 /* 476 * Set all global interrupts to this CPU only. 477 */ 478 cpumask = gic_get_cpumask(gic); 479 cpumask |= cpumask << 8; 480 cpumask |= cpumask << 16; 481 for (i = 32; i < gic_irqs; i += 4) 482 writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4); 483 484 gic_dist_config(base, gic_irqs, NULL); 485 486 writel_relaxed(GICD_ENABLE, base + GIC_DIST_CTRL); 487 } 488 489 static void gic_cpu_init(struct gic_chip_data *gic) 490 { 491 void __iomem *dist_base = gic_data_dist_base(gic); 492 void __iomem *base = gic_data_cpu_base(gic); 493 unsigned int cpu_mask, cpu = smp_processor_id(); 494 int i; 495 496 /* 497 * Setting up the CPU map is only relevant for the primary GIC 498 * because any nested/secondary GICs do not directly interface 499 * with the CPU(s). 500 */ 501 if (gic == &gic_data[0]) { 502 /* 503 * Get what the GIC says our CPU mask is. 504 */ 505 BUG_ON(cpu >= NR_GIC_CPU_IF); 506 cpu_mask = gic_get_cpumask(gic); 507 gic_cpu_map[cpu] = cpu_mask; 508 509 /* 510 * Clear our mask from the other map entries in case they're 511 * still undefined. 512 */ 513 for (i = 0; i < NR_GIC_CPU_IF; i++) 514 if (i != cpu) 515 gic_cpu_map[i] &= ~cpu_mask; 516 } 517 518 gic_cpu_config(dist_base, NULL); 519 520 writel_relaxed(GICC_INT_PRI_THRESHOLD, base + GIC_CPU_PRIMASK); 521 gic_cpu_if_up(gic); 522 } 523 524 int gic_cpu_if_down(unsigned int gic_nr) 525 { 526 void __iomem *cpu_base; 527 u32 val = 0; 528 529 if (gic_nr >= MAX_GIC_NR) 530 return -EINVAL; 531 532 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]); 533 val = readl(cpu_base + GIC_CPU_CTRL); 534 val &= ~GICC_ENABLE; 535 writel_relaxed(val, cpu_base + GIC_CPU_CTRL); 536 537 return 0; 538 } 539 540 #ifdef CONFIG_CPU_PM 541 /* 542 * Saves the GIC distributor registers during suspend or idle. Must be called 543 * with interrupts disabled but before powering down the GIC. After calling 544 * this function, no interrupts will be delivered by the GIC, and another 545 * platform-specific wakeup source must be enabled. 546 */ 547 static void gic_dist_save(unsigned int gic_nr) 548 { 549 unsigned int gic_irqs; 550 void __iomem *dist_base; 551 int i; 552 553 if (gic_nr >= MAX_GIC_NR) 554 BUG(); 555 556 gic_irqs = gic_data[gic_nr].gic_irqs; 557 dist_base = gic_data_dist_base(&gic_data[gic_nr]); 558 559 if (!dist_base) 560 return; 561 562 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++) 563 gic_data[gic_nr].saved_spi_conf[i] = 564 readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4); 565 566 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) 567 gic_data[gic_nr].saved_spi_target[i] = 568 readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4); 569 570 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) 571 gic_data[gic_nr].saved_spi_enable[i] = 572 readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4); 573 } 574 575 /* 576 * Restores the GIC distributor registers during resume or when coming out of 577 * idle. Must be called before enabling interrupts. If a level interrupt 578 * that occured while the GIC was suspended is still present, it will be 579 * handled normally, but any edge interrupts that occured will not be seen by 580 * the GIC and need to be handled by the platform-specific wakeup source. 581 */ 582 static void gic_dist_restore(unsigned int gic_nr) 583 { 584 unsigned int gic_irqs; 585 unsigned int i; 586 void __iomem *dist_base; 587 588 if (gic_nr >= MAX_GIC_NR) 589 BUG(); 590 591 gic_irqs = gic_data[gic_nr].gic_irqs; 592 dist_base = gic_data_dist_base(&gic_data[gic_nr]); 593 594 if (!dist_base) 595 return; 596 597 writel_relaxed(GICD_DISABLE, dist_base + GIC_DIST_CTRL); 598 599 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++) 600 writel_relaxed(gic_data[gic_nr].saved_spi_conf[i], 601 dist_base + GIC_DIST_CONFIG + i * 4); 602 603 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) 604 writel_relaxed(GICD_INT_DEF_PRI_X4, 605 dist_base + GIC_DIST_PRI + i * 4); 606 607 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) 608 writel_relaxed(gic_data[gic_nr].saved_spi_target[i], 609 dist_base + GIC_DIST_TARGET + i * 4); 610 611 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) 612 writel_relaxed(gic_data[gic_nr].saved_spi_enable[i], 613 dist_base + GIC_DIST_ENABLE_SET + i * 4); 614 615 writel_relaxed(GICD_ENABLE, dist_base + GIC_DIST_CTRL); 616 } 617 618 static void gic_cpu_save(unsigned int gic_nr) 619 { 620 int i; 621 u32 *ptr; 622 void __iomem *dist_base; 623 void __iomem *cpu_base; 624 625 if (gic_nr >= MAX_GIC_NR) 626 BUG(); 627 628 dist_base = gic_data_dist_base(&gic_data[gic_nr]); 629 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]); 630 631 if (!dist_base || !cpu_base) 632 return; 633 634 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_enable); 635 for (i = 0; i < DIV_ROUND_UP(32, 32); i++) 636 ptr[i] = readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4); 637 638 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_conf); 639 for (i = 0; i < DIV_ROUND_UP(32, 16); i++) 640 ptr[i] = readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4); 641 642 } 643 644 static void gic_cpu_restore(unsigned int gic_nr) 645 { 646 int i; 647 u32 *ptr; 648 void __iomem *dist_base; 649 void __iomem *cpu_base; 650 651 if (gic_nr >= MAX_GIC_NR) 652 BUG(); 653 654 dist_base = gic_data_dist_base(&gic_data[gic_nr]); 655 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]); 656 657 if (!dist_base || !cpu_base) 658 return; 659 660 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_enable); 661 for (i = 0; i < DIV_ROUND_UP(32, 32); i++) 662 writel_relaxed(ptr[i], dist_base + GIC_DIST_ENABLE_SET + i * 4); 663 664 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_conf); 665 for (i = 0; i < DIV_ROUND_UP(32, 16); i++) 666 writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4); 667 668 for (i = 0; i < DIV_ROUND_UP(32, 4); i++) 669 writel_relaxed(GICD_INT_DEF_PRI_X4, 670 dist_base + GIC_DIST_PRI + i * 4); 671 672 writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK); 673 gic_cpu_if_up(&gic_data[gic_nr]); 674 } 675 676 static int gic_notifier(struct notifier_block *self, unsigned long cmd, void *v) 677 { 678 int i; 679 680 for (i = 0; i < MAX_GIC_NR; i++) { 681 #ifdef CONFIG_GIC_NON_BANKED 682 /* Skip over unused GICs */ 683 if (!gic_data[i].get_base) 684 continue; 685 #endif 686 switch (cmd) { 687 case CPU_PM_ENTER: 688 gic_cpu_save(i); 689 break; 690 case CPU_PM_ENTER_FAILED: 691 case CPU_PM_EXIT: 692 gic_cpu_restore(i); 693 break; 694 case CPU_CLUSTER_PM_ENTER: 695 gic_dist_save(i); 696 break; 697 case CPU_CLUSTER_PM_ENTER_FAILED: 698 case CPU_CLUSTER_PM_EXIT: 699 gic_dist_restore(i); 700 break; 701 } 702 } 703 704 return NOTIFY_OK; 705 } 706 707 static struct notifier_block gic_notifier_block = { 708 .notifier_call = gic_notifier, 709 }; 710 711 static void __init gic_pm_init(struct gic_chip_data *gic) 712 { 713 gic->saved_ppi_enable = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4, 714 sizeof(u32)); 715 BUG_ON(!gic->saved_ppi_enable); 716 717 gic->saved_ppi_conf = __alloc_percpu(DIV_ROUND_UP(32, 16) * 4, 718 sizeof(u32)); 719 BUG_ON(!gic->saved_ppi_conf); 720 721 if (gic == &gic_data[0]) 722 cpu_pm_register_notifier(&gic_notifier_block); 723 } 724 #else 725 static void __init gic_pm_init(struct gic_chip_data *gic) 726 { 727 } 728 #endif 729 730 #ifdef CONFIG_SMP 731 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) 732 { 733 int cpu; 734 unsigned long flags, map = 0; 735 736 raw_spin_lock_irqsave(&irq_controller_lock, flags); 737 738 /* Convert our logical CPU mask into a physical one. */ 739 for_each_cpu(cpu, mask) 740 map |= gic_cpu_map[cpu]; 741 742 /* 743 * Ensure that stores to Normal memory are visible to the 744 * other CPUs before they observe us issuing the IPI. 745 */ 746 dmb(ishst); 747 748 /* this always happens on GIC0 */ 749 writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); 750 751 raw_spin_unlock_irqrestore(&irq_controller_lock, flags); 752 } 753 #endif 754 755 #ifdef CONFIG_BL_SWITCHER 756 /* 757 * gic_send_sgi - send a SGI directly to given CPU interface number 758 * 759 * cpu_id: the ID for the destination CPU interface 760 * irq: the IPI number to send a SGI for 761 */ 762 void gic_send_sgi(unsigned int cpu_id, unsigned int irq) 763 { 764 BUG_ON(cpu_id >= NR_GIC_CPU_IF); 765 cpu_id = 1 << cpu_id; 766 /* this always happens on GIC0 */ 767 writel_relaxed((cpu_id << 16) | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); 768 } 769 770 /* 771 * gic_get_cpu_id - get the CPU interface ID for the specified CPU 772 * 773 * @cpu: the logical CPU number to get the GIC ID for. 774 * 775 * Return the CPU interface ID for the given logical CPU number, 776 * or -1 if the CPU number is too large or the interface ID is 777 * unknown (more than one bit set). 778 */ 779 int gic_get_cpu_id(unsigned int cpu) 780 { 781 unsigned int cpu_bit; 782 783 if (cpu >= NR_GIC_CPU_IF) 784 return -1; 785 cpu_bit = gic_cpu_map[cpu]; 786 if (cpu_bit & (cpu_bit - 1)) 787 return -1; 788 return __ffs(cpu_bit); 789 } 790 791 /* 792 * gic_migrate_target - migrate IRQs to another CPU interface 793 * 794 * @new_cpu_id: the CPU target ID to migrate IRQs to 795 * 796 * Migrate all peripheral interrupts with a target matching the current CPU 797 * to the interface corresponding to @new_cpu_id. The CPU interface mapping 798 * is also updated. Targets to other CPU interfaces are unchanged. 799 * This must be called with IRQs locally disabled. 800 */ 801 void gic_migrate_target(unsigned int new_cpu_id) 802 { 803 unsigned int cur_cpu_id, gic_irqs, gic_nr = 0; 804 void __iomem *dist_base; 805 int i, ror_val, cpu = smp_processor_id(); 806 u32 val, cur_target_mask, active_mask; 807 808 if (gic_nr >= MAX_GIC_NR) 809 BUG(); 810 811 dist_base = gic_data_dist_base(&gic_data[gic_nr]); 812 if (!dist_base) 813 return; 814 gic_irqs = gic_data[gic_nr].gic_irqs; 815 816 cur_cpu_id = __ffs(gic_cpu_map[cpu]); 817 cur_target_mask = 0x01010101 << cur_cpu_id; 818 ror_val = (cur_cpu_id - new_cpu_id) & 31; 819 820 raw_spin_lock(&irq_controller_lock); 821 822 /* Update the target interface for this logical CPU */ 823 gic_cpu_map[cpu] = 1 << new_cpu_id; 824 825 /* 826 * Find all the peripheral interrupts targetting the current 827 * CPU interface and migrate them to the new CPU interface. 828 * We skip DIST_TARGET 0 to 7 as they are read-only. 829 */ 830 for (i = 8; i < DIV_ROUND_UP(gic_irqs, 4); i++) { 831 val = readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4); 832 active_mask = val & cur_target_mask; 833 if (active_mask) { 834 val &= ~active_mask; 835 val |= ror32(active_mask, ror_val); 836 writel_relaxed(val, dist_base + GIC_DIST_TARGET + i*4); 837 } 838 } 839 840 raw_spin_unlock(&irq_controller_lock); 841 842 /* 843 * Now let's migrate and clear any potential SGIs that might be 844 * pending for us (cur_cpu_id). Since GIC_DIST_SGI_PENDING_SET 845 * is a banked register, we can only forward the SGI using 846 * GIC_DIST_SOFTINT. The original SGI source is lost but Linux 847 * doesn't use that information anyway. 848 * 849 * For the same reason we do not adjust SGI source information 850 * for previously sent SGIs by us to other CPUs either. 851 */ 852 for (i = 0; i < 16; i += 4) { 853 int j; 854 val = readl_relaxed(dist_base + GIC_DIST_SGI_PENDING_SET + i); 855 if (!val) 856 continue; 857 writel_relaxed(val, dist_base + GIC_DIST_SGI_PENDING_CLEAR + i); 858 for (j = i; j < i + 4; j++) { 859 if (val & 0xff) 860 writel_relaxed((1 << (new_cpu_id + 16)) | j, 861 dist_base + GIC_DIST_SOFTINT); 862 val >>= 8; 863 } 864 } 865 } 866 867 /* 868 * gic_get_sgir_physaddr - get the physical address for the SGI register 869 * 870 * REturn the physical address of the SGI register to be used 871 * by some early assembly code when the kernel is not yet available. 872 */ 873 static unsigned long gic_dist_physaddr; 874 875 unsigned long gic_get_sgir_physaddr(void) 876 { 877 if (!gic_dist_physaddr) 878 return 0; 879 return gic_dist_physaddr + GIC_DIST_SOFTINT; 880 } 881 882 void __init gic_init_physaddr(struct device_node *node) 883 { 884 struct resource res; 885 if (of_address_to_resource(node, 0, &res) == 0) { 886 gic_dist_physaddr = res.start; 887 pr_info("GIC physical location is %#lx\n", gic_dist_physaddr); 888 } 889 } 890 891 #else 892 #define gic_init_physaddr(node) do { } while (0) 893 #endif 894 895 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 896 irq_hw_number_t hw) 897 { 898 struct irq_chip *chip = &gic_chip; 899 900 if (static_key_true(&supports_deactivate)) { 901 if (d->host_data == (void *)&gic_data[0]) 902 chip = &gic_eoimode1_chip; 903 } 904 905 if (hw < 32) { 906 irq_set_percpu_devid(irq); 907 irq_domain_set_info(d, irq, hw, chip, d->host_data, 908 handle_percpu_devid_irq, NULL, NULL); 909 set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN); 910 } else { 911 irq_domain_set_info(d, irq, hw, chip, d->host_data, 912 handle_fasteoi_irq, NULL, NULL); 913 set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); 914 } 915 return 0; 916 } 917 918 static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq) 919 { 920 } 921 922 static int gic_irq_domain_xlate(struct irq_domain *d, 923 struct device_node *controller, 924 const u32 *intspec, unsigned int intsize, 925 unsigned long *out_hwirq, unsigned int *out_type) 926 { 927 unsigned long ret = 0; 928 929 if (d->of_node != controller) 930 return -EINVAL; 931 if (intsize < 3) 932 return -EINVAL; 933 934 /* Get the interrupt number and add 16 to skip over SGIs */ 935 *out_hwirq = intspec[1] + 16; 936 937 /* For SPIs, we need to add 16 more to get the GIC irq ID number */ 938 if (!intspec[0]) 939 *out_hwirq += 16; 940 941 *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; 942 943 return ret; 944 } 945 946 #ifdef CONFIG_SMP 947 static int gic_secondary_init(struct notifier_block *nfb, unsigned long action, 948 void *hcpu) 949 { 950 if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) 951 gic_cpu_init(&gic_data[0]); 952 return NOTIFY_OK; 953 } 954 955 /* 956 * Notifier for enabling the GIC CPU interface. Set an arbitrarily high 957 * priority because the GIC needs to be up before the ARM generic timers. 958 */ 959 static struct notifier_block gic_cpu_notifier = { 960 .notifier_call = gic_secondary_init, 961 .priority = 100, 962 }; 963 #endif 964 965 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 966 unsigned int nr_irqs, void *arg) 967 { 968 int i, ret; 969 irq_hw_number_t hwirq; 970 unsigned int type = IRQ_TYPE_NONE; 971 struct of_phandle_args *irq_data = arg; 972 973 ret = gic_irq_domain_xlate(domain, irq_data->np, irq_data->args, 974 irq_data->args_count, &hwirq, &type); 975 if (ret) 976 return ret; 977 978 for (i = 0; i < nr_irqs; i++) 979 gic_irq_domain_map(domain, virq + i, hwirq + i); 980 981 return 0; 982 } 983 984 static const struct irq_domain_ops gic_irq_domain_hierarchy_ops = { 985 .xlate = gic_irq_domain_xlate, 986 .alloc = gic_irq_domain_alloc, 987 .free = irq_domain_free_irqs_top, 988 }; 989 990 static const struct irq_domain_ops gic_irq_domain_ops = { 991 .map = gic_irq_domain_map, 992 .unmap = gic_irq_domain_unmap, 993 .xlate = gic_irq_domain_xlate, 994 }; 995 996 static void __init __gic_init_bases(unsigned int gic_nr, int irq_start, 997 void __iomem *dist_base, void __iomem *cpu_base, 998 u32 percpu_offset, struct device_node *node) 999 { 1000 irq_hw_number_t hwirq_base; 1001 struct gic_chip_data *gic; 1002 int gic_irqs, irq_base, i; 1003 1004 BUG_ON(gic_nr >= MAX_GIC_NR); 1005 1006 gic = &gic_data[gic_nr]; 1007 #ifdef CONFIG_GIC_NON_BANKED 1008 if (percpu_offset) { /* Frankein-GIC without banked registers... */ 1009 unsigned int cpu; 1010 1011 gic->dist_base.percpu_base = alloc_percpu(void __iomem *); 1012 gic->cpu_base.percpu_base = alloc_percpu(void __iomem *); 1013 if (WARN_ON(!gic->dist_base.percpu_base || 1014 !gic->cpu_base.percpu_base)) { 1015 free_percpu(gic->dist_base.percpu_base); 1016 free_percpu(gic->cpu_base.percpu_base); 1017 return; 1018 } 1019 1020 for_each_possible_cpu(cpu) { 1021 u32 mpidr = cpu_logical_map(cpu); 1022 u32 core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); 1023 unsigned long offset = percpu_offset * core_id; 1024 *per_cpu_ptr(gic->dist_base.percpu_base, cpu) = dist_base + offset; 1025 *per_cpu_ptr(gic->cpu_base.percpu_base, cpu) = cpu_base + offset; 1026 } 1027 1028 gic_set_base_accessor(gic, gic_get_percpu_base); 1029 } else 1030 #endif 1031 { /* Normal, sane GIC... */ 1032 WARN(percpu_offset, 1033 "GIC_NON_BANKED not enabled, ignoring %08x offset!", 1034 percpu_offset); 1035 gic->dist_base.common_base = dist_base; 1036 gic->cpu_base.common_base = cpu_base; 1037 gic_set_base_accessor(gic, gic_get_common_base); 1038 } 1039 1040 /* 1041 * Find out how many interrupts are supported. 1042 * The GIC only supports up to 1020 interrupt sources. 1043 */ 1044 gic_irqs = readl_relaxed(gic_data_dist_base(gic) + GIC_DIST_CTR) & 0x1f; 1045 gic_irqs = (gic_irqs + 1) * 32; 1046 if (gic_irqs > 1020) 1047 gic_irqs = 1020; 1048 gic->gic_irqs = gic_irqs; 1049 1050 if (node) { /* DT case */ 1051 gic->domain = irq_domain_add_linear(node, gic_irqs, 1052 &gic_irq_domain_hierarchy_ops, 1053 gic); 1054 } else { /* Non-DT case */ 1055 /* 1056 * For primary GICs, skip over SGIs. 1057 * For secondary GICs, skip over PPIs, too. 1058 */ 1059 if (gic_nr == 0 && (irq_start & 31) > 0) { 1060 hwirq_base = 16; 1061 if (irq_start != -1) 1062 irq_start = (irq_start & ~31) + 16; 1063 } else { 1064 hwirq_base = 32; 1065 } 1066 1067 gic_irqs -= hwirq_base; /* calculate # of irqs to allocate */ 1068 1069 irq_base = irq_alloc_descs(irq_start, 16, gic_irqs, 1070 numa_node_id()); 1071 if (IS_ERR_VALUE(irq_base)) { 1072 WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n", 1073 irq_start); 1074 irq_base = irq_start; 1075 } 1076 1077 gic->domain = irq_domain_add_legacy(node, gic_irqs, irq_base, 1078 hwirq_base, &gic_irq_domain_ops, gic); 1079 } 1080 1081 if (WARN_ON(!gic->domain)) 1082 return; 1083 1084 if (gic_nr == 0) { 1085 /* 1086 * Initialize the CPU interface map to all CPUs. 1087 * It will be refined as each CPU probes its ID. 1088 * This is only necessary for the primary GIC. 1089 */ 1090 for (i = 0; i < NR_GIC_CPU_IF; i++) 1091 gic_cpu_map[i] = 0xff; 1092 #ifdef CONFIG_SMP 1093 set_smp_cross_call(gic_raise_softirq); 1094 register_cpu_notifier(&gic_cpu_notifier); 1095 #endif 1096 set_handle_irq(gic_handle_irq); 1097 if (static_key_true(&supports_deactivate)) 1098 pr_info("GIC: Using split EOI/Deactivate mode\n"); 1099 } 1100 1101 gic_dist_init(gic); 1102 gic_cpu_init(gic); 1103 gic_pm_init(gic); 1104 } 1105 1106 void __init gic_init_bases(unsigned int gic_nr, int irq_start, 1107 void __iomem *dist_base, void __iomem *cpu_base, 1108 u32 percpu_offset, struct device_node *node) 1109 { 1110 /* 1111 * Non-DT/ACPI systems won't run a hypervisor, so let's not 1112 * bother with these... 1113 */ 1114 static_key_slow_dec(&supports_deactivate); 1115 __gic_init_bases(gic_nr, irq_start, dist_base, cpu_base, 1116 percpu_offset, node); 1117 } 1118 1119 #ifdef CONFIG_OF 1120 static int gic_cnt __initdata; 1121 1122 static int __init 1123 gic_of_init(struct device_node *node, struct device_node *parent) 1124 { 1125 void __iomem *cpu_base; 1126 void __iomem *dist_base; 1127 struct resource cpu_res; 1128 u32 percpu_offset; 1129 int irq; 1130 1131 if (WARN_ON(!node)) 1132 return -ENODEV; 1133 1134 dist_base = of_iomap(node, 0); 1135 WARN(!dist_base, "unable to map gic dist registers\n"); 1136 1137 cpu_base = of_iomap(node, 1); 1138 WARN(!cpu_base, "unable to map gic cpu registers\n"); 1139 1140 of_address_to_resource(node, 1, &cpu_res); 1141 1142 /* 1143 * Disable split EOI/Deactivate if either HYP is not available 1144 * or the CPU interface is too small. 1145 */ 1146 if (gic_cnt == 0 && (!is_hyp_mode_available() || 1147 resource_size(&cpu_res) < SZ_8K)) 1148 static_key_slow_dec(&supports_deactivate); 1149 1150 if (of_property_read_u32(node, "cpu-offset", &percpu_offset)) 1151 percpu_offset = 0; 1152 1153 __gic_init_bases(gic_cnt, -1, dist_base, cpu_base, percpu_offset, node); 1154 if (!gic_cnt) 1155 gic_init_physaddr(node); 1156 1157 if (parent) { 1158 irq = irq_of_parse_and_map(node, 0); 1159 gic_cascade_irq(gic_cnt, irq); 1160 } 1161 1162 if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 1163 gicv2m_of_init(node, gic_data[gic_cnt].domain); 1164 1165 gic_cnt++; 1166 return 0; 1167 } 1168 IRQCHIP_DECLARE(gic_400, "arm,gic-400", gic_of_init); 1169 IRQCHIP_DECLARE(arm11mp_gic, "arm,arm11mp-gic", gic_of_init); 1170 IRQCHIP_DECLARE(arm1176jzf_dc_gic, "arm,arm1176jzf-devchip-gic", gic_of_init); 1171 IRQCHIP_DECLARE(cortex_a15_gic, "arm,cortex-a15-gic", gic_of_init); 1172 IRQCHIP_DECLARE(cortex_a9_gic, "arm,cortex-a9-gic", gic_of_init); 1173 IRQCHIP_DECLARE(cortex_a7_gic, "arm,cortex-a7-gic", gic_of_init); 1174 IRQCHIP_DECLARE(msm_8660_qgic, "qcom,msm-8660-qgic", gic_of_init); 1175 IRQCHIP_DECLARE(msm_qgic2, "qcom,msm-qgic2", gic_of_init); 1176 1177 #endif 1178 1179 #ifdef CONFIG_ACPI 1180 static phys_addr_t dist_phy_base, cpu_phy_base __initdata; 1181 1182 static int __init 1183 gic_acpi_parse_madt_cpu(struct acpi_subtable_header *header, 1184 const unsigned long end) 1185 { 1186 struct acpi_madt_generic_interrupt *processor; 1187 phys_addr_t gic_cpu_base; 1188 static int cpu_base_assigned; 1189 1190 processor = (struct acpi_madt_generic_interrupt *)header; 1191 1192 if (BAD_MADT_GICC_ENTRY(processor, end)) 1193 return -EINVAL; 1194 1195 /* 1196 * There is no support for non-banked GICv1/2 register in ACPI spec. 1197 * All CPU interface addresses have to be the same. 1198 */ 1199 gic_cpu_base = processor->base_address; 1200 if (cpu_base_assigned && gic_cpu_base != cpu_phy_base) 1201 return -EINVAL; 1202 1203 cpu_phy_base = gic_cpu_base; 1204 cpu_base_assigned = 1; 1205 return 0; 1206 } 1207 1208 static int __init 1209 gic_acpi_parse_madt_distributor(struct acpi_subtable_header *header, 1210 const unsigned long end) 1211 { 1212 struct acpi_madt_generic_distributor *dist; 1213 1214 dist = (struct acpi_madt_generic_distributor *)header; 1215 1216 if (BAD_MADT_ENTRY(dist, end)) 1217 return -EINVAL; 1218 1219 dist_phy_base = dist->base_address; 1220 return 0; 1221 } 1222 1223 int __init 1224 gic_v2_acpi_init(struct acpi_table_header *table) 1225 { 1226 void __iomem *cpu_base, *dist_base; 1227 int count; 1228 1229 /* Collect CPU base addresses */ 1230 count = acpi_parse_entries(ACPI_SIG_MADT, 1231 sizeof(struct acpi_table_madt), 1232 gic_acpi_parse_madt_cpu, table, 1233 ACPI_MADT_TYPE_GENERIC_INTERRUPT, 0); 1234 if (count <= 0) { 1235 pr_err("No valid GICC entries exist\n"); 1236 return -EINVAL; 1237 } 1238 1239 /* 1240 * Find distributor base address. We expect one distributor entry since 1241 * ACPI 5.1 spec neither support multi-GIC instances nor GIC cascade. 1242 */ 1243 count = acpi_parse_entries(ACPI_SIG_MADT, 1244 sizeof(struct acpi_table_madt), 1245 gic_acpi_parse_madt_distributor, table, 1246 ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 0); 1247 if (count <= 0) { 1248 pr_err("No valid GICD entries exist\n"); 1249 return -EINVAL; 1250 } else if (count > 1) { 1251 pr_err("More than one GICD entry detected\n"); 1252 return -EINVAL; 1253 } 1254 1255 cpu_base = ioremap(cpu_phy_base, ACPI_GIC_CPU_IF_MEM_SIZE); 1256 if (!cpu_base) { 1257 pr_err("Unable to map GICC registers\n"); 1258 return -ENOMEM; 1259 } 1260 1261 dist_base = ioremap(dist_phy_base, ACPI_GICV2_DIST_MEM_SIZE); 1262 if (!dist_base) { 1263 pr_err("Unable to map GICD registers\n"); 1264 iounmap(cpu_base); 1265 return -ENOMEM; 1266 } 1267 1268 /* 1269 * Disable split EOI/Deactivate if HYP is not available. ACPI 1270 * guarantees that we'll always have a GICv2, so the CPU 1271 * interface will always be the right size. 1272 */ 1273 if (!is_hyp_mode_available()) 1274 static_key_slow_dec(&supports_deactivate); 1275 1276 /* 1277 * Initialize zero GIC instance (no multi-GIC support). Also, set GIC 1278 * as default IRQ domain to allow for GSI registration and GSI to IRQ 1279 * number translation (see acpi_register_gsi() and acpi_gsi_to_irq()). 1280 */ 1281 __gic_init_bases(0, -1, dist_base, cpu_base, 0, NULL); 1282 irq_set_default_host(gic_data[0].domain); 1283 1284 acpi_irq_model = ACPI_IRQ_MODEL_GIC; 1285 return 0; 1286 } 1287 #endif 1288