1 /* 2 * Marvell Armada 370 and Armada XP SoC IRQ handling 3 * 4 * Copyright (C) 2012 Marvell 5 * 6 * Lior Amsalem <alior@marvell.com> 7 * Gregory CLEMENT <gregory.clement@free-electrons.com> 8 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 9 * Ben Dooks <ben.dooks@codethink.co.uk> 10 * 11 * This file is licensed under the terms of the GNU General Public 12 * License version 2. This program is licensed "as is" without any 13 * warranty of any kind, whether express or implied. 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/init.h> 19 #include <linux/irq.h> 20 #include <linux/interrupt.h> 21 #include <linux/irqchip/chained_irq.h> 22 #include <linux/io.h> 23 #include <linux/of_address.h> 24 #include <linux/of_irq.h> 25 #include <linux/of_pci.h> 26 #include <linux/irqdomain.h> 27 #include <linux/slab.h> 28 #include <linux/msi.h> 29 #include <asm/mach/arch.h> 30 #include <asm/exception.h> 31 #include <asm/smp_plat.h> 32 #include <asm/mach/irq.h> 33 34 #include "irqchip.h" 35 36 /* Interrupt Controller Registers Map */ 37 #define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) 38 #define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) 39 40 #define ARMADA_370_XP_INT_CONTROL (0x00) 41 #define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) 42 #define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) 43 #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) 44 45 #define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) 46 #define ARMADA_375_PPI_CAUSE (0x10) 47 48 #define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x4) 49 #define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0xc) 50 #define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x8) 51 52 #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) 53 54 #define ARMADA_370_XP_TIMER0_PER_CPU_IRQ (5) 55 56 #define IPI_DOORBELL_START (0) 57 #define IPI_DOORBELL_END (8) 58 #define IPI_DOORBELL_MASK 0xFF 59 #define PCI_MSI_DOORBELL_START (16) 60 #define PCI_MSI_DOORBELL_NR (16) 61 #define PCI_MSI_DOORBELL_END (32) 62 #define PCI_MSI_DOORBELL_MASK 0xFFFF0000 63 64 static void __iomem *per_cpu_int_base; 65 static void __iomem *main_int_base; 66 static struct irq_domain *armada_370_xp_mpic_domain; 67 #ifdef CONFIG_PCI_MSI 68 static struct irq_domain *armada_370_xp_msi_domain; 69 static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR); 70 static DEFINE_MUTEX(msi_used_lock); 71 static phys_addr_t msi_doorbell_addr; 72 #endif 73 74 /* 75 * In SMP mode: 76 * For shared global interrupts, mask/unmask global enable bit 77 * For CPU interrupts, mask/unmask the calling CPU's bit 78 */ 79 static void armada_370_xp_irq_mask(struct irq_data *d) 80 { 81 irq_hw_number_t hwirq = irqd_to_hwirq(d); 82 83 if (hwirq != ARMADA_370_XP_TIMER0_PER_CPU_IRQ) 84 writel(hwirq, main_int_base + 85 ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); 86 else 87 writel(hwirq, per_cpu_int_base + 88 ARMADA_370_XP_INT_SET_MASK_OFFS); 89 } 90 91 static void armada_370_xp_irq_unmask(struct irq_data *d) 92 { 93 irq_hw_number_t hwirq = irqd_to_hwirq(d); 94 95 if (hwirq != ARMADA_370_XP_TIMER0_PER_CPU_IRQ) 96 writel(hwirq, main_int_base + 97 ARMADA_370_XP_INT_SET_ENABLE_OFFS); 98 else 99 writel(hwirq, per_cpu_int_base + 100 ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 101 } 102 103 #ifdef CONFIG_PCI_MSI 104 105 static int armada_370_xp_alloc_msi(void) 106 { 107 int hwirq; 108 109 mutex_lock(&msi_used_lock); 110 hwirq = find_first_zero_bit(&msi_used, PCI_MSI_DOORBELL_NR); 111 if (hwirq >= PCI_MSI_DOORBELL_NR) 112 hwirq = -ENOSPC; 113 else 114 set_bit(hwirq, msi_used); 115 mutex_unlock(&msi_used_lock); 116 117 return hwirq; 118 } 119 120 static void armada_370_xp_free_msi(int hwirq) 121 { 122 mutex_lock(&msi_used_lock); 123 if (!test_bit(hwirq, msi_used)) 124 pr_err("trying to free unused MSI#%d\n", hwirq); 125 else 126 clear_bit(hwirq, msi_used); 127 mutex_unlock(&msi_used_lock); 128 } 129 130 static int armada_370_xp_setup_msi_irq(struct msi_chip *chip, 131 struct pci_dev *pdev, 132 struct msi_desc *desc) 133 { 134 struct msi_msg msg; 135 irq_hw_number_t hwirq; 136 int virq; 137 138 hwirq = armada_370_xp_alloc_msi(); 139 if (hwirq < 0) 140 return hwirq; 141 142 virq = irq_create_mapping(armada_370_xp_msi_domain, hwirq); 143 if (!virq) { 144 armada_370_xp_free_msi(hwirq); 145 return -EINVAL; 146 } 147 148 irq_set_msi_desc(virq, desc); 149 150 msg.address_lo = msi_doorbell_addr; 151 msg.address_hi = 0; 152 msg.data = 0xf00 | (hwirq + 16); 153 154 write_msi_msg(virq, &msg); 155 return 0; 156 } 157 158 static void armada_370_xp_teardown_msi_irq(struct msi_chip *chip, 159 unsigned int irq) 160 { 161 struct irq_data *d = irq_get_irq_data(irq); 162 irq_dispose_mapping(irq); 163 armada_370_xp_free_msi(d->hwirq); 164 } 165 166 static struct irq_chip armada_370_xp_msi_irq_chip = { 167 .name = "armada_370_xp_msi_irq", 168 .irq_enable = unmask_msi_irq, 169 .irq_disable = mask_msi_irq, 170 .irq_mask = mask_msi_irq, 171 .irq_unmask = unmask_msi_irq, 172 }; 173 174 static int armada_370_xp_msi_map(struct irq_domain *domain, unsigned int virq, 175 irq_hw_number_t hw) 176 { 177 irq_set_chip_and_handler(virq, &armada_370_xp_msi_irq_chip, 178 handle_simple_irq); 179 set_irq_flags(virq, IRQF_VALID); 180 181 return 0; 182 } 183 184 static const struct irq_domain_ops armada_370_xp_msi_irq_ops = { 185 .map = armada_370_xp_msi_map, 186 }; 187 188 static int armada_370_xp_msi_init(struct device_node *node, 189 phys_addr_t main_int_phys_base) 190 { 191 struct msi_chip *msi_chip; 192 u32 reg; 193 int ret; 194 195 msi_doorbell_addr = main_int_phys_base + 196 ARMADA_370_XP_SW_TRIG_INT_OFFS; 197 198 msi_chip = kzalloc(sizeof(*msi_chip), GFP_KERNEL); 199 if (!msi_chip) 200 return -ENOMEM; 201 202 msi_chip->setup_irq = armada_370_xp_setup_msi_irq; 203 msi_chip->teardown_irq = armada_370_xp_teardown_msi_irq; 204 msi_chip->of_node = node; 205 206 armada_370_xp_msi_domain = 207 irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR, 208 &armada_370_xp_msi_irq_ops, 209 NULL); 210 if (!armada_370_xp_msi_domain) { 211 kfree(msi_chip); 212 return -ENOMEM; 213 } 214 215 ret = of_pci_msi_chip_add(msi_chip); 216 if (ret < 0) { 217 irq_domain_remove(armada_370_xp_msi_domain); 218 kfree(msi_chip); 219 return ret; 220 } 221 222 reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS) 223 | PCI_MSI_DOORBELL_MASK; 224 225 writel(reg, per_cpu_int_base + 226 ARMADA_370_XP_IN_DRBEL_MSK_OFFS); 227 228 /* Unmask IPI interrupt */ 229 writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 230 231 return 0; 232 } 233 #else 234 static inline int armada_370_xp_msi_init(struct device_node *node, 235 phys_addr_t main_int_phys_base) 236 { 237 return 0; 238 } 239 #endif 240 241 #ifdef CONFIG_SMP 242 static DEFINE_RAW_SPINLOCK(irq_controller_lock); 243 244 static int armada_xp_set_affinity(struct irq_data *d, 245 const struct cpumask *mask_val, bool force) 246 { 247 unsigned long reg; 248 unsigned long new_mask = 0; 249 unsigned long online_mask = 0; 250 unsigned long count = 0; 251 irq_hw_number_t hwirq = irqd_to_hwirq(d); 252 int cpu; 253 254 for_each_cpu(cpu, mask_val) { 255 new_mask |= 1 << cpu_logical_map(cpu); 256 count++; 257 } 258 259 /* 260 * Forbid mutlicore interrupt affinity 261 * This is required since the MPIC HW doesn't limit 262 * several CPUs from acknowledging the same interrupt. 263 */ 264 if (count > 1) 265 return -EINVAL; 266 267 for_each_cpu(cpu, cpu_online_mask) 268 online_mask |= 1 << cpu_logical_map(cpu); 269 270 raw_spin_lock(&irq_controller_lock); 271 272 reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); 273 reg = (reg & (~online_mask)) | new_mask; 274 writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); 275 276 raw_spin_unlock(&irq_controller_lock); 277 278 return 0; 279 } 280 #endif 281 282 static struct irq_chip armada_370_xp_irq_chip = { 283 .name = "armada_370_xp_irq", 284 .irq_mask = armada_370_xp_irq_mask, 285 .irq_mask_ack = armada_370_xp_irq_mask, 286 .irq_unmask = armada_370_xp_irq_unmask, 287 #ifdef CONFIG_SMP 288 .irq_set_affinity = armada_xp_set_affinity, 289 #endif 290 }; 291 292 static int armada_370_xp_mpic_irq_map(struct irq_domain *h, 293 unsigned int virq, irq_hw_number_t hw) 294 { 295 armada_370_xp_irq_mask(irq_get_irq_data(virq)); 296 if (hw != ARMADA_370_XP_TIMER0_PER_CPU_IRQ) 297 writel(hw, per_cpu_int_base + 298 ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 299 else 300 writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS); 301 irq_set_status_flags(virq, IRQ_LEVEL); 302 303 if (hw == ARMADA_370_XP_TIMER0_PER_CPU_IRQ) { 304 irq_set_percpu_devid(virq); 305 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, 306 handle_percpu_devid_irq); 307 308 } else { 309 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, 310 handle_level_irq); 311 } 312 set_irq_flags(virq, IRQF_VALID | IRQF_PROBE); 313 314 return 0; 315 } 316 317 #ifdef CONFIG_SMP 318 void armada_mpic_send_doorbell(const struct cpumask *mask, unsigned int irq) 319 { 320 int cpu; 321 unsigned long map = 0; 322 323 /* Convert our logical CPU mask into a physical one. */ 324 for_each_cpu(cpu, mask) 325 map |= 1 << cpu_logical_map(cpu); 326 327 /* 328 * Ensure that stores to Normal memory are visible to the 329 * other CPUs before issuing the IPI. 330 */ 331 dsb(); 332 333 /* submit softirq */ 334 writel((map << 8) | irq, main_int_base + 335 ARMADA_370_XP_SW_TRIG_INT_OFFS); 336 } 337 338 void armada_xp_mpic_smp_cpu_init(void) 339 { 340 /* Clear pending IPIs */ 341 writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); 342 343 /* Enable first 8 IPIs */ 344 writel(IPI_DOORBELL_MASK, per_cpu_int_base + 345 ARMADA_370_XP_IN_DRBEL_MSK_OFFS); 346 347 /* Unmask IPI interrupt */ 348 writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 349 } 350 #endif /* CONFIG_SMP */ 351 352 static struct irq_domain_ops armada_370_xp_mpic_irq_ops = { 353 .map = armada_370_xp_mpic_irq_map, 354 .xlate = irq_domain_xlate_onecell, 355 }; 356 357 #ifdef CONFIG_PCI_MSI 358 static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) 359 { 360 u32 msimask, msinr; 361 362 msimask = readl_relaxed(per_cpu_int_base + 363 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) 364 & PCI_MSI_DOORBELL_MASK; 365 366 writel(~msimask, per_cpu_int_base + 367 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); 368 369 for (msinr = PCI_MSI_DOORBELL_START; 370 msinr < PCI_MSI_DOORBELL_END; msinr++) { 371 int irq; 372 373 if (!(msimask & BIT(msinr))) 374 continue; 375 376 irq = irq_find_mapping(armada_370_xp_msi_domain, 377 msinr - 16); 378 379 if (is_chained) 380 generic_handle_irq(irq); 381 else 382 handle_IRQ(irq, regs); 383 } 384 } 385 #else 386 static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {} 387 #endif 388 389 static void armada_370_xp_mpic_handle_cascade_irq(unsigned int irq, 390 struct irq_desc *desc) 391 { 392 struct irq_chip *chip = irq_get_chip(irq); 393 unsigned long irqmap, irqn; 394 unsigned int cascade_irq; 395 396 chained_irq_enter(chip, desc); 397 398 irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE); 399 400 if (irqmap & BIT(0)) { 401 armada_370_xp_handle_msi_irq(NULL, true); 402 irqmap &= ~BIT(0); 403 } 404 405 for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { 406 cascade_irq = irq_find_mapping(armada_370_xp_mpic_domain, irqn); 407 generic_handle_irq(cascade_irq); 408 } 409 410 chained_irq_exit(chip, desc); 411 } 412 413 static void __exception_irq_entry 414 armada_370_xp_handle_irq(struct pt_regs *regs) 415 { 416 u32 irqstat, irqnr; 417 418 do { 419 irqstat = readl_relaxed(per_cpu_int_base + 420 ARMADA_370_XP_CPU_INTACK_OFFS); 421 irqnr = irqstat & 0x3FF; 422 423 if (irqnr > 1022) 424 break; 425 426 if (irqnr > 1) { 427 irqnr = irq_find_mapping(armada_370_xp_mpic_domain, 428 irqnr); 429 handle_IRQ(irqnr, regs); 430 continue; 431 } 432 433 /* MSI handling */ 434 if (irqnr == 1) 435 armada_370_xp_handle_msi_irq(regs, false); 436 437 #ifdef CONFIG_SMP 438 /* IPI Handling */ 439 if (irqnr == 0) { 440 u32 ipimask, ipinr; 441 442 ipimask = readl_relaxed(per_cpu_int_base + 443 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) 444 & IPI_DOORBELL_MASK; 445 446 writel(~ipimask, per_cpu_int_base + 447 ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); 448 449 /* Handle all pending doorbells */ 450 for (ipinr = IPI_DOORBELL_START; 451 ipinr < IPI_DOORBELL_END; ipinr++) { 452 if (ipimask & (0x1 << ipinr)) 453 handle_IPI(ipinr, regs); 454 } 455 continue; 456 } 457 #endif 458 459 } while (1); 460 } 461 462 static int __init armada_370_xp_mpic_of_init(struct device_node *node, 463 struct device_node *parent) 464 { 465 struct resource main_int_res, per_cpu_int_res; 466 int parent_irq; 467 u32 control; 468 469 BUG_ON(of_address_to_resource(node, 0, &main_int_res)); 470 BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); 471 472 BUG_ON(!request_mem_region(main_int_res.start, 473 resource_size(&main_int_res), 474 node->full_name)); 475 BUG_ON(!request_mem_region(per_cpu_int_res.start, 476 resource_size(&per_cpu_int_res), 477 node->full_name)); 478 479 main_int_base = ioremap(main_int_res.start, 480 resource_size(&main_int_res)); 481 BUG_ON(!main_int_base); 482 483 per_cpu_int_base = ioremap(per_cpu_int_res.start, 484 resource_size(&per_cpu_int_res)); 485 BUG_ON(!per_cpu_int_base); 486 487 control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); 488 489 armada_370_xp_mpic_domain = 490 irq_domain_add_linear(node, (control >> 2) & 0x3ff, 491 &armada_370_xp_mpic_irq_ops, NULL); 492 493 BUG_ON(!armada_370_xp_mpic_domain); 494 495 #ifdef CONFIG_SMP 496 armada_xp_mpic_smp_cpu_init(); 497 498 /* 499 * Set the default affinity from all CPUs to the boot cpu. 500 * This is required since the MPIC doesn't limit several CPUs 501 * from acknowledging the same interrupt. 502 */ 503 cpumask_clear(irq_default_affinity); 504 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 505 506 #endif 507 508 armada_370_xp_msi_init(node, main_int_res.start); 509 510 parent_irq = irq_of_parse_and_map(node, 0); 511 if (parent_irq <= 0) { 512 irq_set_default_host(armada_370_xp_mpic_domain); 513 set_handle_irq(armada_370_xp_handle_irq); 514 } else { 515 irq_set_chained_handler(parent_irq, 516 armada_370_xp_mpic_handle_cascade_irq); 517 } 518 519 return 0; 520 } 521 522 IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init); 523