1 /* 2 * Marvell Armada 370 and Armada XP SoC IRQ handling 3 * 4 * Copyright (C) 2012 Marvell 5 * 6 * Lior Amsalem <alior@marvell.com> 7 * Gregory CLEMENT <gregory.clement@free-electrons.com> 8 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 9 * Ben Dooks <ben.dooks@codethink.co.uk> 10 * 11 * This file is licensed under the terms of the GNU General Public 12 * License version 2. This program is licensed "as is" without any 13 * warranty of any kind, whether express or implied. 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/init.h> 19 #include <linux/irq.h> 20 #include <linux/interrupt.h> 21 #include <linux/irqchip/chained_irq.h> 22 #include <linux/cpu.h> 23 #include <linux/io.h> 24 #include <linux/of_address.h> 25 #include <linux/of_irq.h> 26 #include <linux/of_pci.h> 27 #include <linux/irqdomain.h> 28 #include <linux/slab.h> 29 #include <linux/msi.h> 30 #include <asm/mach/arch.h> 31 #include <asm/exception.h> 32 #include <asm/smp_plat.h> 33 #include <asm/mach/irq.h> 34 35 #include "irqchip.h" 36 37 /* Interrupt Controller Registers Map */ 38 #define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) 39 #define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) 40 41 #define ARMADA_370_XP_INT_CONTROL (0x00) 42 #define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) 43 #define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) 44 #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) 45 #define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF 46 #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) 47 48 #define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) 49 #define ARMADA_375_PPI_CAUSE (0x10) 50 51 #define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x4) 52 #define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0xc) 53 #define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x8) 54 55 #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) 56 57 #define ARMADA_370_XP_TIMER0_PER_CPU_IRQ (5) 58 59 #define IPI_DOORBELL_START (0) 60 #define IPI_DOORBELL_END (8) 61 #define IPI_DOORBELL_MASK 0xFF 62 #define PCI_MSI_DOORBELL_START (16) 63 #define PCI_MSI_DOORBELL_NR (16) 64 #define PCI_MSI_DOORBELL_END (32) 65 #define PCI_MSI_DOORBELL_MASK 0xFFFF0000 66 67 static void __iomem *per_cpu_int_base; 68 static void __iomem *main_int_base; 69 static struct irq_domain *armada_370_xp_mpic_domain; 70 #ifdef CONFIG_PCI_MSI 71 static struct irq_domain *armada_370_xp_msi_domain; 72 static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR); 73 static DEFINE_MUTEX(msi_used_lock); 74 static phys_addr_t msi_doorbell_addr; 75 #endif 76 77 /* 78 * In SMP mode: 79 * For shared global interrupts, mask/unmask global enable bit 80 * For CPU interrupts, mask/unmask the calling CPU's bit 81 */ 82 static void armada_370_xp_irq_mask(struct irq_data *d) 83 { 84 irq_hw_number_t hwirq = irqd_to_hwirq(d); 85 86 if (hwirq != ARMADA_370_XP_TIMER0_PER_CPU_IRQ) 87 writel(hwirq, main_int_base + 88 ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); 89 else 90 writel(hwirq, per_cpu_int_base + 91 ARMADA_370_XP_INT_SET_MASK_OFFS); 92 } 93 94 static void armada_370_xp_irq_unmask(struct irq_data *d) 95 { 96 irq_hw_number_t hwirq = irqd_to_hwirq(d); 97 98 if (hwirq != ARMADA_370_XP_TIMER0_PER_CPU_IRQ) 99 writel(hwirq, main_int_base + 100 ARMADA_370_XP_INT_SET_ENABLE_OFFS); 101 else 102 writel(hwirq, per_cpu_int_base + 103 ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 104 } 105 106 #ifdef CONFIG_PCI_MSI 107 108 static int armada_370_xp_alloc_msi(void) 109 { 110 int hwirq; 111 112 mutex_lock(&msi_used_lock); 113 hwirq = find_first_zero_bit(&msi_used, PCI_MSI_DOORBELL_NR); 114 if (hwirq >= PCI_MSI_DOORBELL_NR) 115 hwirq = -ENOSPC; 116 else 117 set_bit(hwirq, msi_used); 118 mutex_unlock(&msi_used_lock); 119 120 return hwirq; 121 } 122 123 static void armada_370_xp_free_msi(int hwirq) 124 { 125 mutex_lock(&msi_used_lock); 126 if (!test_bit(hwirq, msi_used)) 127 pr_err("trying to free unused MSI#%d\n", hwirq); 128 else 129 clear_bit(hwirq, msi_used); 130 mutex_unlock(&msi_used_lock); 131 } 132 133 static int armada_370_xp_setup_msi_irq(struct msi_chip *chip, 134 struct pci_dev *pdev, 135 struct msi_desc *desc) 136 { 137 struct msi_msg msg; 138 int virq, hwirq; 139 140 /* We support MSI, but not MSI-X */ 141 if (desc->msi_attrib.is_msix) 142 return -EINVAL; 143 144 hwirq = armada_370_xp_alloc_msi(); 145 if (hwirq < 0) 146 return hwirq; 147 148 virq = irq_create_mapping(armada_370_xp_msi_domain, hwirq); 149 if (!virq) { 150 armada_370_xp_free_msi(hwirq); 151 return -EINVAL; 152 } 153 154 irq_set_msi_desc(virq, desc); 155 156 msg.address_lo = msi_doorbell_addr; 157 msg.address_hi = 0; 158 msg.data = 0xf00 | (hwirq + 16); 159 160 write_msi_msg(virq, &msg); 161 return 0; 162 } 163 164 static void armada_370_xp_teardown_msi_irq(struct msi_chip *chip, 165 unsigned int irq) 166 { 167 struct irq_data *d = irq_get_irq_data(irq); 168 unsigned long hwirq = d->hwirq; 169 170 irq_dispose_mapping(irq); 171 armada_370_xp_free_msi(hwirq); 172 } 173 174 static struct irq_chip armada_370_xp_msi_irq_chip = { 175 .name = "armada_370_xp_msi_irq", 176 .irq_enable = unmask_msi_irq, 177 .irq_disable = mask_msi_irq, 178 .irq_mask = mask_msi_irq, 179 .irq_unmask = unmask_msi_irq, 180 }; 181 182 static int armada_370_xp_msi_map(struct irq_domain *domain, unsigned int virq, 183 irq_hw_number_t hw) 184 { 185 irq_set_chip_and_handler(virq, &armada_370_xp_msi_irq_chip, 186 handle_simple_irq); 187 set_irq_flags(virq, IRQF_VALID); 188 189 return 0; 190 } 191 192 static const struct irq_domain_ops armada_370_xp_msi_irq_ops = { 193 .map = armada_370_xp_msi_map, 194 }; 195 196 static int armada_370_xp_msi_init(struct device_node *node, 197 phys_addr_t main_int_phys_base) 198 { 199 struct msi_chip *msi_chip; 200 u32 reg; 201 int ret; 202 203 msi_doorbell_addr = main_int_phys_base + 204 ARMADA_370_XP_SW_TRIG_INT_OFFS; 205 206 msi_chip = kzalloc(sizeof(*msi_chip), GFP_KERNEL); 207 if (!msi_chip) 208 return -ENOMEM; 209 210 msi_chip->setup_irq = armada_370_xp_setup_msi_irq; 211 msi_chip->teardown_irq = armada_370_xp_teardown_msi_irq; 212 msi_chip->of_node = node; 213 214 armada_370_xp_msi_domain = 215 irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR, 216 &armada_370_xp_msi_irq_ops, 217 NULL); 218 if (!armada_370_xp_msi_domain) { 219 kfree(msi_chip); 220 return -ENOMEM; 221 } 222 223 ret = of_pci_msi_chip_add(msi_chip); 224 if (ret < 0) { 225 irq_domain_remove(armada_370_xp_msi_domain); 226 kfree(msi_chip); 227 return ret; 228 } 229 230 reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS) 231 | PCI_MSI_DOORBELL_MASK; 232 233 writel(reg, per_cpu_int_base + 234 ARMADA_370_XP_IN_DRBEL_MSK_OFFS); 235 236 /* Unmask IPI interrupt */ 237 writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 238 239 return 0; 240 } 241 #else 242 static inline int armada_370_xp_msi_init(struct device_node *node, 243 phys_addr_t main_int_phys_base) 244 { 245 return 0; 246 } 247 #endif 248 249 #ifdef CONFIG_SMP 250 static DEFINE_RAW_SPINLOCK(irq_controller_lock); 251 252 static int armada_xp_set_affinity(struct irq_data *d, 253 const struct cpumask *mask_val, bool force) 254 { 255 irq_hw_number_t hwirq = irqd_to_hwirq(d); 256 unsigned long reg, mask; 257 int cpu; 258 259 /* Select a single core from the affinity mask which is online */ 260 cpu = cpumask_any_and(mask_val, cpu_online_mask); 261 mask = 1UL << cpu_logical_map(cpu); 262 263 raw_spin_lock(&irq_controller_lock); 264 reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); 265 reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask; 266 writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); 267 raw_spin_unlock(&irq_controller_lock); 268 269 return 0; 270 } 271 #endif 272 273 static struct irq_chip armada_370_xp_irq_chip = { 274 .name = "armada_370_xp_irq", 275 .irq_mask = armada_370_xp_irq_mask, 276 .irq_mask_ack = armada_370_xp_irq_mask, 277 .irq_unmask = armada_370_xp_irq_unmask, 278 #ifdef CONFIG_SMP 279 .irq_set_affinity = armada_xp_set_affinity, 280 #endif 281 }; 282 283 static int armada_370_xp_mpic_irq_map(struct irq_domain *h, 284 unsigned int virq, irq_hw_number_t hw) 285 { 286 armada_370_xp_irq_mask(irq_get_irq_data(virq)); 287 if (hw != ARMADA_370_XP_TIMER0_PER_CPU_IRQ) 288 writel(hw, per_cpu_int_base + 289 ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 290 else 291 writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS); 292 irq_set_status_flags(virq, IRQ_LEVEL); 293 294 if (hw == ARMADA_370_XP_TIMER0_PER_CPU_IRQ) { 295 irq_set_percpu_devid(virq); 296 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, 297 handle_percpu_devid_irq); 298 299 } else { 300 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, 301 handle_level_irq); 302 } 303 set_irq_flags(virq, IRQF_VALID | IRQF_PROBE); 304 305 return 0; 306 } 307 308 #ifdef CONFIG_SMP 309 static void armada_mpic_send_doorbell(const struct cpumask *mask, 310 unsigned int irq) 311 { 312 int cpu; 313 unsigned long map = 0; 314 315 /* Convert our logical CPU mask into a physical one. */ 316 for_each_cpu(cpu, mask) 317 map |= 1 << cpu_logical_map(cpu); 318 319 /* 320 * Ensure that stores to Normal memory are visible to the 321 * other CPUs before issuing the IPI. 322 */ 323 dsb(); 324 325 /* submit softirq */ 326 writel((map << 8) | irq, main_int_base + 327 ARMADA_370_XP_SW_TRIG_INT_OFFS); 328 } 329 330 static void armada_xp_mpic_smp_cpu_init(void) 331 { 332 u32 control; 333 int nr_irqs, i; 334 335 control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); 336 nr_irqs = (control >> 2) & 0x3ff; 337 338 for (i = 0; i < nr_irqs; i++) 339 writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); 340 341 /* Clear pending IPIs */ 342 writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); 343 344 /* Enable first 8 IPIs */ 345 writel(IPI_DOORBELL_MASK, per_cpu_int_base + 346 ARMADA_370_XP_IN_DRBEL_MSK_OFFS); 347 348 /* Unmask IPI interrupt */ 349 writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 350 } 351 352 static int armada_xp_mpic_secondary_init(struct notifier_block *nfb, 353 unsigned long action, void *hcpu) 354 { 355 if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) 356 armada_xp_mpic_smp_cpu_init(); 357 return NOTIFY_OK; 358 } 359 360 static struct notifier_block armada_370_xp_mpic_cpu_notifier = { 361 .notifier_call = armada_xp_mpic_secondary_init, 362 .priority = 100, 363 }; 364 365 #endif /* CONFIG_SMP */ 366 367 static struct irq_domain_ops armada_370_xp_mpic_irq_ops = { 368 .map = armada_370_xp_mpic_irq_map, 369 .xlate = irq_domain_xlate_onecell, 370 }; 371 372 #ifdef CONFIG_PCI_MSI 373 static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) 374 { 375 u32 msimask, msinr; 376 377 msimask = readl_relaxed(per_cpu_int_base + 378 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) 379 & PCI_MSI_DOORBELL_MASK; 380 381 writel(~msimask, per_cpu_int_base + 382 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); 383 384 for (msinr = PCI_MSI_DOORBELL_START; 385 msinr < PCI_MSI_DOORBELL_END; msinr++) { 386 int irq; 387 388 if (!(msimask & BIT(msinr))) 389 continue; 390 391 if (is_chained) { 392 irq = irq_find_mapping(armada_370_xp_msi_domain, 393 msinr - 16); 394 generic_handle_irq(irq); 395 } else { 396 irq = msinr - 16; 397 handle_domain_irq(armada_370_xp_msi_domain, 398 irq, regs); 399 } 400 } 401 } 402 #else 403 static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {} 404 #endif 405 406 static void armada_370_xp_mpic_handle_cascade_irq(unsigned int irq, 407 struct irq_desc *desc) 408 { 409 struct irq_chip *chip = irq_get_chip(irq); 410 unsigned long irqmap, irqn, irqsrc, cpuid; 411 unsigned int cascade_irq; 412 413 chained_irq_enter(chip, desc); 414 415 irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE); 416 cpuid = cpu_logical_map(smp_processor_id()); 417 418 for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { 419 irqsrc = readl_relaxed(main_int_base + 420 ARMADA_370_XP_INT_SOURCE_CTL(irqn)); 421 422 /* Check if the interrupt is not masked on current CPU. 423 * Test IRQ (0-1) and FIQ (8-9) mask bits. 424 */ 425 if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid))) 426 continue; 427 428 if (irqn == 1) { 429 armada_370_xp_handle_msi_irq(NULL, true); 430 continue; 431 } 432 433 cascade_irq = irq_find_mapping(armada_370_xp_mpic_domain, irqn); 434 generic_handle_irq(cascade_irq); 435 } 436 437 chained_irq_exit(chip, desc); 438 } 439 440 static void __exception_irq_entry 441 armada_370_xp_handle_irq(struct pt_regs *regs) 442 { 443 u32 irqstat, irqnr; 444 445 do { 446 irqstat = readl_relaxed(per_cpu_int_base + 447 ARMADA_370_XP_CPU_INTACK_OFFS); 448 irqnr = irqstat & 0x3FF; 449 450 if (irqnr > 1022) 451 break; 452 453 if (irqnr > 1) { 454 handle_domain_irq(armada_370_xp_mpic_domain, 455 irqnr, regs); 456 continue; 457 } 458 459 /* MSI handling */ 460 if (irqnr == 1) 461 armada_370_xp_handle_msi_irq(regs, false); 462 463 #ifdef CONFIG_SMP 464 /* IPI Handling */ 465 if (irqnr == 0) { 466 u32 ipimask, ipinr; 467 468 ipimask = readl_relaxed(per_cpu_int_base + 469 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) 470 & IPI_DOORBELL_MASK; 471 472 writel(~ipimask, per_cpu_int_base + 473 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); 474 475 /* Handle all pending doorbells */ 476 for (ipinr = IPI_DOORBELL_START; 477 ipinr < IPI_DOORBELL_END; ipinr++) { 478 if (ipimask & (0x1 << ipinr)) 479 handle_IPI(ipinr, regs); 480 } 481 continue; 482 } 483 #endif 484 485 } while (1); 486 } 487 488 static int __init armada_370_xp_mpic_of_init(struct device_node *node, 489 struct device_node *parent) 490 { 491 struct resource main_int_res, per_cpu_int_res; 492 int parent_irq, nr_irqs, i; 493 u32 control; 494 495 BUG_ON(of_address_to_resource(node, 0, &main_int_res)); 496 BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); 497 498 BUG_ON(!request_mem_region(main_int_res.start, 499 resource_size(&main_int_res), 500 node->full_name)); 501 BUG_ON(!request_mem_region(per_cpu_int_res.start, 502 resource_size(&per_cpu_int_res), 503 node->full_name)); 504 505 main_int_base = ioremap(main_int_res.start, 506 resource_size(&main_int_res)); 507 BUG_ON(!main_int_base); 508 509 per_cpu_int_base = ioremap(per_cpu_int_res.start, 510 resource_size(&per_cpu_int_res)); 511 BUG_ON(!per_cpu_int_base); 512 513 control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); 514 nr_irqs = (control >> 2) & 0x3ff; 515 516 for (i = 0; i < nr_irqs; i++) 517 writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); 518 519 armada_370_xp_mpic_domain = 520 irq_domain_add_linear(node, nr_irqs, 521 &armada_370_xp_mpic_irq_ops, NULL); 522 523 BUG_ON(!armada_370_xp_mpic_domain); 524 525 #ifdef CONFIG_SMP 526 armada_xp_mpic_smp_cpu_init(); 527 #endif 528 529 armada_370_xp_msi_init(node, main_int_res.start); 530 531 parent_irq = irq_of_parse_and_map(node, 0); 532 if (parent_irq <= 0) { 533 irq_set_default_host(armada_370_xp_mpic_domain); 534 set_handle_irq(armada_370_xp_handle_irq); 535 #ifdef CONFIG_SMP 536 set_smp_cross_call(armada_mpic_send_doorbell); 537 register_cpu_notifier(&armada_370_xp_mpic_cpu_notifier); 538 #endif 539 } else { 540 irq_set_chained_handler(parent_irq, 541 armada_370_xp_mpic_handle_cascade_irq); 542 } 543 544 return 0; 545 } 546 547 IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init); 548