1 /* 2 * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. 3 * Author: Marc Zyngier <marc.zyngier@arm.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #include <linux/bitmap.h> 19 #include <linux/cpu.h> 20 #include <linux/delay.h> 21 #include <linux/interrupt.h> 22 #include <linux/log2.h> 23 #include <linux/mm.h> 24 #include <linux/msi.h> 25 #include <linux/of.h> 26 #include <linux/of_address.h> 27 #include <linux/of_irq.h> 28 #include <linux/of_pci.h> 29 #include <linux/of_platform.h> 30 #include <linux/percpu.h> 31 #include <linux/slab.h> 32 33 #include <linux/irqchip.h> 34 #include <linux/irqchip/arm-gic-v3.h> 35 36 #include <asm/cacheflush.h> 37 #include <asm/cputype.h> 38 #include <asm/exception.h> 39 40 #include "irq-gic-common.h" 41 42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) 43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) 44 45 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) 46 47 /* 48 * Collection structure - just an ID, and a redistributor address to 49 * ping. We use one per CPU as a bag of interrupts assigned to this 50 * CPU. 51 */ 52 struct its_collection { 53 u64 target_address; 54 u16 col_id; 55 }; 56 57 /* 58 * The ITS structure - contains most of the infrastructure, with the 59 * top-level MSI domain, the command queue, the collections, and the 60 * list of devices writing to it. 61 */ 62 struct its_node { 63 raw_spinlock_t lock; 64 struct list_head entry; 65 void __iomem *base; 66 unsigned long phys_base; 67 struct its_cmd_block *cmd_base; 68 struct its_cmd_block *cmd_write; 69 struct { 70 void *base; 71 u32 order; 72 } tables[GITS_BASER_NR_REGS]; 73 struct its_collection *collections; 74 struct list_head its_device_list; 75 u64 flags; 76 u32 ite_size; 77 }; 78 79 #define ITS_ITT_ALIGN SZ_256 80 81 /* Convert page order to size in bytes */ 82 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o)) 83 84 struct event_lpi_map { 85 unsigned long *lpi_map; 86 u16 *col_map; 87 irq_hw_number_t lpi_base; 88 int nr_lpis; 89 }; 90 91 /* 92 * The ITS view of a device - belongs to an ITS, a collection, owns an 93 * interrupt translation table, and a list of interrupts. 94 */ 95 struct its_device { 96 struct list_head entry; 97 struct its_node *its; 98 struct event_lpi_map event_map; 99 void *itt; 100 u32 nr_ites; 101 u32 device_id; 102 }; 103 104 static LIST_HEAD(its_nodes); 105 static DEFINE_SPINLOCK(its_lock); 106 static struct rdists *gic_rdists; 107 108 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) 109 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 110 111 static struct its_collection *dev_event_to_col(struct its_device *its_dev, 112 u32 event) 113 { 114 struct its_node *its = its_dev->its; 115 116 return its->collections + its_dev->event_map.col_map[event]; 117 } 118 119 /* 120 * ITS command descriptors - parameters to be encoded in a command 121 * block. 122 */ 123 struct its_cmd_desc { 124 union { 125 struct { 126 struct its_device *dev; 127 u32 event_id; 128 } its_inv_cmd; 129 130 struct { 131 struct its_device *dev; 132 u32 event_id; 133 } its_int_cmd; 134 135 struct { 136 struct its_device *dev; 137 int valid; 138 } its_mapd_cmd; 139 140 struct { 141 struct its_collection *col; 142 int valid; 143 } its_mapc_cmd; 144 145 struct { 146 struct its_device *dev; 147 u32 phys_id; 148 u32 event_id; 149 } its_mapvi_cmd; 150 151 struct { 152 struct its_device *dev; 153 struct its_collection *col; 154 u32 event_id; 155 } its_movi_cmd; 156 157 struct { 158 struct its_device *dev; 159 u32 event_id; 160 } its_discard_cmd; 161 162 struct { 163 struct its_collection *col; 164 } its_invall_cmd; 165 }; 166 }; 167 168 /* 169 * The ITS command block, which is what the ITS actually parses. 170 */ 171 struct its_cmd_block { 172 u64 raw_cmd[4]; 173 }; 174 175 #define ITS_CMD_QUEUE_SZ SZ_64K 176 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) 177 178 typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, 179 struct its_cmd_desc *); 180 181 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 182 { 183 cmd->raw_cmd[0] &= ~0xffUL; 184 cmd->raw_cmd[0] |= cmd_nr; 185 } 186 187 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 188 { 189 cmd->raw_cmd[0] &= BIT_ULL(32) - 1; 190 cmd->raw_cmd[0] |= ((u64)devid) << 32; 191 } 192 193 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 194 { 195 cmd->raw_cmd[1] &= ~0xffffffffUL; 196 cmd->raw_cmd[1] |= id; 197 } 198 199 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 200 { 201 cmd->raw_cmd[1] &= 0xffffffffUL; 202 cmd->raw_cmd[1] |= ((u64)phys_id) << 32; 203 } 204 205 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 206 { 207 cmd->raw_cmd[1] &= ~0x1fUL; 208 cmd->raw_cmd[1] |= size & 0x1f; 209 } 210 211 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 212 { 213 cmd->raw_cmd[2] &= ~0xffffffffffffUL; 214 cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL; 215 } 216 217 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 218 { 219 cmd->raw_cmd[2] &= ~(1UL << 63); 220 cmd->raw_cmd[2] |= ((u64)!!valid) << 63; 221 } 222 223 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 224 { 225 cmd->raw_cmd[2] &= ~(0xffffffffUL << 16); 226 cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16)); 227 } 228 229 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 230 { 231 cmd->raw_cmd[2] &= ~0xffffUL; 232 cmd->raw_cmd[2] |= col; 233 } 234 235 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 236 { 237 /* Let's fixup BE commands */ 238 cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); 239 cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); 240 cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); 241 cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); 242 } 243 244 static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, 245 struct its_cmd_desc *desc) 246 { 247 unsigned long itt_addr; 248 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); 249 250 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); 251 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); 252 253 its_encode_cmd(cmd, GITS_CMD_MAPD); 254 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); 255 its_encode_size(cmd, size - 1); 256 its_encode_itt(cmd, itt_addr); 257 its_encode_valid(cmd, desc->its_mapd_cmd.valid); 258 259 its_fixup_cmd(cmd); 260 261 return NULL; 262 } 263 264 static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, 265 struct its_cmd_desc *desc) 266 { 267 its_encode_cmd(cmd, GITS_CMD_MAPC); 268 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 269 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); 270 its_encode_valid(cmd, desc->its_mapc_cmd.valid); 271 272 its_fixup_cmd(cmd); 273 274 return desc->its_mapc_cmd.col; 275 } 276 277 static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd, 278 struct its_cmd_desc *desc) 279 { 280 struct its_collection *col; 281 282 col = dev_event_to_col(desc->its_mapvi_cmd.dev, 283 desc->its_mapvi_cmd.event_id); 284 285 its_encode_cmd(cmd, GITS_CMD_MAPVI); 286 its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id); 287 its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id); 288 its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id); 289 its_encode_collection(cmd, col->col_id); 290 291 its_fixup_cmd(cmd); 292 293 return col; 294 } 295 296 static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, 297 struct its_cmd_desc *desc) 298 { 299 struct its_collection *col; 300 301 col = dev_event_to_col(desc->its_movi_cmd.dev, 302 desc->its_movi_cmd.event_id); 303 304 its_encode_cmd(cmd, GITS_CMD_MOVI); 305 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); 306 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); 307 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); 308 309 its_fixup_cmd(cmd); 310 311 return col; 312 } 313 314 static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, 315 struct its_cmd_desc *desc) 316 { 317 struct its_collection *col; 318 319 col = dev_event_to_col(desc->its_discard_cmd.dev, 320 desc->its_discard_cmd.event_id); 321 322 its_encode_cmd(cmd, GITS_CMD_DISCARD); 323 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); 324 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); 325 326 its_fixup_cmd(cmd); 327 328 return col; 329 } 330 331 static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, 332 struct its_cmd_desc *desc) 333 { 334 struct its_collection *col; 335 336 col = dev_event_to_col(desc->its_inv_cmd.dev, 337 desc->its_inv_cmd.event_id); 338 339 its_encode_cmd(cmd, GITS_CMD_INV); 340 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 341 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 342 343 its_fixup_cmd(cmd); 344 345 return col; 346 } 347 348 static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, 349 struct its_cmd_desc *desc) 350 { 351 its_encode_cmd(cmd, GITS_CMD_INVALL); 352 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 353 354 its_fixup_cmd(cmd); 355 356 return NULL; 357 } 358 359 static u64 its_cmd_ptr_to_offset(struct its_node *its, 360 struct its_cmd_block *ptr) 361 { 362 return (ptr - its->cmd_base) * sizeof(*ptr); 363 } 364 365 static int its_queue_full(struct its_node *its) 366 { 367 int widx; 368 int ridx; 369 370 widx = its->cmd_write - its->cmd_base; 371 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); 372 373 /* This is incredibly unlikely to happen, unless the ITS locks up. */ 374 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) 375 return 1; 376 377 return 0; 378 } 379 380 static struct its_cmd_block *its_allocate_entry(struct its_node *its) 381 { 382 struct its_cmd_block *cmd; 383 u32 count = 1000000; /* 1s! */ 384 385 while (its_queue_full(its)) { 386 count--; 387 if (!count) { 388 pr_err_ratelimited("ITS queue not draining\n"); 389 return NULL; 390 } 391 cpu_relax(); 392 udelay(1); 393 } 394 395 cmd = its->cmd_write++; 396 397 /* Handle queue wrapping */ 398 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) 399 its->cmd_write = its->cmd_base; 400 401 return cmd; 402 } 403 404 static struct its_cmd_block *its_post_commands(struct its_node *its) 405 { 406 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); 407 408 writel_relaxed(wr, its->base + GITS_CWRITER); 409 410 return its->cmd_write; 411 } 412 413 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) 414 { 415 /* 416 * Make sure the commands written to memory are observable by 417 * the ITS. 418 */ 419 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) 420 __flush_dcache_area(cmd, sizeof(*cmd)); 421 else 422 dsb(ishst); 423 } 424 425 static void its_wait_for_range_completion(struct its_node *its, 426 struct its_cmd_block *from, 427 struct its_cmd_block *to) 428 { 429 u64 rd_idx, from_idx, to_idx; 430 u32 count = 1000000; /* 1s! */ 431 432 from_idx = its_cmd_ptr_to_offset(its, from); 433 to_idx = its_cmd_ptr_to_offset(its, to); 434 435 while (1) { 436 rd_idx = readl_relaxed(its->base + GITS_CREADR); 437 if (rd_idx >= to_idx || rd_idx < from_idx) 438 break; 439 440 count--; 441 if (!count) { 442 pr_err_ratelimited("ITS queue timeout\n"); 443 return; 444 } 445 cpu_relax(); 446 udelay(1); 447 } 448 } 449 450 static void its_send_single_command(struct its_node *its, 451 its_cmd_builder_t builder, 452 struct its_cmd_desc *desc) 453 { 454 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; 455 struct its_collection *sync_col; 456 unsigned long flags; 457 458 raw_spin_lock_irqsave(&its->lock, flags); 459 460 cmd = its_allocate_entry(its); 461 if (!cmd) { /* We're soooooo screewed... */ 462 pr_err_ratelimited("ITS can't allocate, dropping command\n"); 463 raw_spin_unlock_irqrestore(&its->lock, flags); 464 return; 465 } 466 sync_col = builder(cmd, desc); 467 its_flush_cmd(its, cmd); 468 469 if (sync_col) { 470 sync_cmd = its_allocate_entry(its); 471 if (!sync_cmd) { 472 pr_err_ratelimited("ITS can't SYNC, skipping\n"); 473 goto post; 474 } 475 its_encode_cmd(sync_cmd, GITS_CMD_SYNC); 476 its_encode_target(sync_cmd, sync_col->target_address); 477 its_fixup_cmd(sync_cmd); 478 its_flush_cmd(its, sync_cmd); 479 } 480 481 post: 482 next_cmd = its_post_commands(its); 483 raw_spin_unlock_irqrestore(&its->lock, flags); 484 485 its_wait_for_range_completion(its, cmd, next_cmd); 486 } 487 488 static void its_send_inv(struct its_device *dev, u32 event_id) 489 { 490 struct its_cmd_desc desc; 491 492 desc.its_inv_cmd.dev = dev; 493 desc.its_inv_cmd.event_id = event_id; 494 495 its_send_single_command(dev->its, its_build_inv_cmd, &desc); 496 } 497 498 static void its_send_mapd(struct its_device *dev, int valid) 499 { 500 struct its_cmd_desc desc; 501 502 desc.its_mapd_cmd.dev = dev; 503 desc.its_mapd_cmd.valid = !!valid; 504 505 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); 506 } 507 508 static void its_send_mapc(struct its_node *its, struct its_collection *col, 509 int valid) 510 { 511 struct its_cmd_desc desc; 512 513 desc.its_mapc_cmd.col = col; 514 desc.its_mapc_cmd.valid = !!valid; 515 516 its_send_single_command(its, its_build_mapc_cmd, &desc); 517 } 518 519 static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id) 520 { 521 struct its_cmd_desc desc; 522 523 desc.its_mapvi_cmd.dev = dev; 524 desc.its_mapvi_cmd.phys_id = irq_id; 525 desc.its_mapvi_cmd.event_id = id; 526 527 its_send_single_command(dev->its, its_build_mapvi_cmd, &desc); 528 } 529 530 static void its_send_movi(struct its_device *dev, 531 struct its_collection *col, u32 id) 532 { 533 struct its_cmd_desc desc; 534 535 desc.its_movi_cmd.dev = dev; 536 desc.its_movi_cmd.col = col; 537 desc.its_movi_cmd.event_id = id; 538 539 its_send_single_command(dev->its, its_build_movi_cmd, &desc); 540 } 541 542 static void its_send_discard(struct its_device *dev, u32 id) 543 { 544 struct its_cmd_desc desc; 545 546 desc.its_discard_cmd.dev = dev; 547 desc.its_discard_cmd.event_id = id; 548 549 its_send_single_command(dev->its, its_build_discard_cmd, &desc); 550 } 551 552 static void its_send_invall(struct its_node *its, struct its_collection *col) 553 { 554 struct its_cmd_desc desc; 555 556 desc.its_invall_cmd.col = col; 557 558 its_send_single_command(its, its_build_invall_cmd, &desc); 559 } 560 561 /* 562 * irqchip functions - assumes MSI, mostly. 563 */ 564 565 static inline u32 its_get_event_id(struct irq_data *d) 566 { 567 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 568 return d->hwirq - its_dev->event_map.lpi_base; 569 } 570 571 static void lpi_set_config(struct irq_data *d, bool enable) 572 { 573 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 574 irq_hw_number_t hwirq = d->hwirq; 575 u32 id = its_get_event_id(d); 576 u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192; 577 578 if (enable) 579 *cfg |= LPI_PROP_ENABLED; 580 else 581 *cfg &= ~LPI_PROP_ENABLED; 582 583 /* 584 * Make the above write visible to the redistributors. 585 * And yes, we're flushing exactly: One. Single. Byte. 586 * Humpf... 587 */ 588 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) 589 __flush_dcache_area(cfg, sizeof(*cfg)); 590 else 591 dsb(ishst); 592 its_send_inv(its_dev, id); 593 } 594 595 static void its_mask_irq(struct irq_data *d) 596 { 597 lpi_set_config(d, false); 598 } 599 600 static void its_unmask_irq(struct irq_data *d) 601 { 602 lpi_set_config(d, true); 603 } 604 605 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 606 bool force) 607 { 608 unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask); 609 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 610 struct its_collection *target_col; 611 u32 id = its_get_event_id(d); 612 613 if (cpu >= nr_cpu_ids) 614 return -EINVAL; 615 616 target_col = &its_dev->its->collections[cpu]; 617 its_send_movi(its_dev, target_col, id); 618 its_dev->event_map.col_map[id] = cpu; 619 620 return IRQ_SET_MASK_OK_DONE; 621 } 622 623 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) 624 { 625 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 626 struct its_node *its; 627 u64 addr; 628 629 its = its_dev->its; 630 addr = its->phys_base + GITS_TRANSLATER; 631 632 msg->address_lo = addr & ((1UL << 32) - 1); 633 msg->address_hi = addr >> 32; 634 msg->data = its_get_event_id(d); 635 } 636 637 static struct irq_chip its_irq_chip = { 638 .name = "ITS", 639 .irq_mask = its_mask_irq, 640 .irq_unmask = its_unmask_irq, 641 .irq_eoi = irq_chip_eoi_parent, 642 .irq_set_affinity = its_set_affinity, 643 .irq_compose_msi_msg = its_irq_compose_msi_msg, 644 }; 645 646 /* 647 * How we allocate LPIs: 648 * 649 * The GIC has id_bits bits for interrupt identifiers. From there, we 650 * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as 651 * we allocate LPIs by chunks of 32, we can shift the whole thing by 5 652 * bits to the right. 653 * 654 * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations. 655 */ 656 #define IRQS_PER_CHUNK_SHIFT 5 657 #define IRQS_PER_CHUNK (1 << IRQS_PER_CHUNK_SHIFT) 658 659 static unsigned long *lpi_bitmap; 660 static u32 lpi_chunks; 661 static DEFINE_SPINLOCK(lpi_lock); 662 663 static int its_lpi_to_chunk(int lpi) 664 { 665 return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT; 666 } 667 668 static int its_chunk_to_lpi(int chunk) 669 { 670 return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192; 671 } 672 673 static int __init its_lpi_init(u32 id_bits) 674 { 675 lpi_chunks = its_lpi_to_chunk(1UL << id_bits); 676 677 lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long), 678 GFP_KERNEL); 679 if (!lpi_bitmap) { 680 lpi_chunks = 0; 681 return -ENOMEM; 682 } 683 684 pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks); 685 return 0; 686 } 687 688 static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids) 689 { 690 unsigned long *bitmap = NULL; 691 int chunk_id; 692 int nr_chunks; 693 int i; 694 695 nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK); 696 697 spin_lock(&lpi_lock); 698 699 do { 700 chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks, 701 0, nr_chunks, 0); 702 if (chunk_id < lpi_chunks) 703 break; 704 705 nr_chunks--; 706 } while (nr_chunks > 0); 707 708 if (!nr_chunks) 709 goto out; 710 711 bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long), 712 GFP_ATOMIC); 713 if (!bitmap) 714 goto out; 715 716 for (i = 0; i < nr_chunks; i++) 717 set_bit(chunk_id + i, lpi_bitmap); 718 719 *base = its_chunk_to_lpi(chunk_id); 720 *nr_ids = nr_chunks * IRQS_PER_CHUNK; 721 722 out: 723 spin_unlock(&lpi_lock); 724 725 if (!bitmap) 726 *base = *nr_ids = 0; 727 728 return bitmap; 729 } 730 731 static void its_lpi_free(struct event_lpi_map *map) 732 { 733 int base = map->lpi_base; 734 int nr_ids = map->nr_lpis; 735 int lpi; 736 737 spin_lock(&lpi_lock); 738 739 for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) { 740 int chunk = its_lpi_to_chunk(lpi); 741 BUG_ON(chunk > lpi_chunks); 742 if (test_bit(chunk, lpi_bitmap)) { 743 clear_bit(chunk, lpi_bitmap); 744 } else { 745 pr_err("Bad LPI chunk %d\n", chunk); 746 } 747 } 748 749 spin_unlock(&lpi_lock); 750 751 kfree(map->lpi_map); 752 kfree(map->col_map); 753 } 754 755 /* 756 * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to 757 * deal with (one configuration byte per interrupt). PENDBASE has to 758 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). 759 */ 760 #define LPI_PROPBASE_SZ SZ_64K 761 #define LPI_PENDBASE_SZ (LPI_PROPBASE_SZ / 8 + SZ_1K) 762 763 /* 764 * This is how many bits of ID we need, including the useless ones. 765 */ 766 #define LPI_NRBITS ilog2(LPI_PROPBASE_SZ + SZ_8K) 767 768 #define LPI_PROP_DEFAULT_PRIO 0xa0 769 770 static int __init its_alloc_lpi_tables(void) 771 { 772 phys_addr_t paddr; 773 774 gic_rdists->prop_page = alloc_pages(GFP_NOWAIT, 775 get_order(LPI_PROPBASE_SZ)); 776 if (!gic_rdists->prop_page) { 777 pr_err("Failed to allocate PROPBASE\n"); 778 return -ENOMEM; 779 } 780 781 paddr = page_to_phys(gic_rdists->prop_page); 782 pr_info("GIC: using LPI property table @%pa\n", &paddr); 783 784 /* Priority 0xa0, Group-1, disabled */ 785 memset(page_address(gic_rdists->prop_page), 786 LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, 787 LPI_PROPBASE_SZ); 788 789 /* Make sure the GIC will observe the written configuration */ 790 __flush_dcache_area(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ); 791 792 return 0; 793 } 794 795 static const char *its_base_type_string[] = { 796 [GITS_BASER_TYPE_DEVICE] = "Devices", 797 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs", 798 [GITS_BASER_TYPE_CPU] = "Physical CPUs", 799 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections", 800 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)", 801 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)", 802 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", 803 }; 804 805 static void its_free_tables(struct its_node *its) 806 { 807 int i; 808 809 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 810 if (its->tables[i].base) { 811 free_pages((unsigned long)its->tables[i].base, 812 its->tables[i].order); 813 its->tables[i].base = NULL; 814 } 815 } 816 } 817 818 static int its_alloc_tables(const char *node_name, struct its_node *its) 819 { 820 int err; 821 int i; 822 int psz = SZ_64K; 823 u64 shr = GITS_BASER_InnerShareable; 824 u64 cache; 825 u64 typer; 826 u32 ids; 827 828 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) { 829 /* 830 * erratum 22375: only alloc 8MB table size 831 * erratum 24313: ignore memory access type 832 */ 833 cache = 0; 834 ids = 0x14; /* 20 bits, 8MB */ 835 } else { 836 cache = GITS_BASER_WaWb; 837 typer = readq_relaxed(its->base + GITS_TYPER); 838 ids = GITS_TYPER_DEVBITS(typer); 839 } 840 841 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 842 u64 val = readq_relaxed(its->base + GITS_BASER + i * 8); 843 u64 type = GITS_BASER_TYPE(val); 844 u64 entry_size = GITS_BASER_ENTRY_SIZE(val); 845 int order = get_order(psz); 846 int alloc_pages; 847 u64 tmp; 848 void *base; 849 850 if (type == GITS_BASER_TYPE_NONE) 851 continue; 852 853 /* 854 * Allocate as many entries as required to fit the 855 * range of device IDs that the ITS can grok... The ID 856 * space being incredibly sparse, this results in a 857 * massive waste of memory. 858 * 859 * For other tables, only allocate a single page. 860 */ 861 if (type == GITS_BASER_TYPE_DEVICE) { 862 /* 863 * 'order' was initialized earlier to the default page 864 * granule of the the ITS. We can't have an allocation 865 * smaller than that. If the requested allocation 866 * is smaller, round up to the default page granule. 867 */ 868 order = max(get_order((1UL << ids) * entry_size), 869 order); 870 if (order >= MAX_ORDER) { 871 order = MAX_ORDER - 1; 872 pr_warn("%s: Device Table too large, reduce its page order to %u\n", 873 node_name, order); 874 } 875 } 876 877 retry_alloc_baser: 878 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); 879 if (alloc_pages > GITS_BASER_PAGES_MAX) { 880 alloc_pages = GITS_BASER_PAGES_MAX; 881 order = get_order(GITS_BASER_PAGES_MAX * psz); 882 pr_warn("%s: Device Table too large, reduce its page order to %u (%u pages)\n", 883 node_name, order, alloc_pages); 884 } 885 886 base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 887 if (!base) { 888 err = -ENOMEM; 889 goto out_free; 890 } 891 892 its->tables[i].base = base; 893 its->tables[i].order = order; 894 895 retry_baser: 896 val = (virt_to_phys(base) | 897 (type << GITS_BASER_TYPE_SHIFT) | 898 ((entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | 899 cache | 900 shr | 901 GITS_BASER_VALID); 902 903 switch (psz) { 904 case SZ_4K: 905 val |= GITS_BASER_PAGE_SIZE_4K; 906 break; 907 case SZ_16K: 908 val |= GITS_BASER_PAGE_SIZE_16K; 909 break; 910 case SZ_64K: 911 val |= GITS_BASER_PAGE_SIZE_64K; 912 break; 913 } 914 915 val |= alloc_pages - 1; 916 917 writeq_relaxed(val, its->base + GITS_BASER + i * 8); 918 tmp = readq_relaxed(its->base + GITS_BASER + i * 8); 919 920 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { 921 /* 922 * Shareability didn't stick. Just use 923 * whatever the read reported, which is likely 924 * to be the only thing this redistributor 925 * supports. If that's zero, make it 926 * non-cacheable as well. 927 */ 928 shr = tmp & GITS_BASER_SHAREABILITY_MASK; 929 if (!shr) { 930 cache = GITS_BASER_nC; 931 __flush_dcache_area(base, PAGE_ORDER_TO_SIZE(order)); 932 } 933 goto retry_baser; 934 } 935 936 if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { 937 /* 938 * Page size didn't stick. Let's try a smaller 939 * size and retry. If we reach 4K, then 940 * something is horribly wrong... 941 */ 942 free_pages((unsigned long)base, order); 943 its->tables[i].base = NULL; 944 945 switch (psz) { 946 case SZ_16K: 947 psz = SZ_4K; 948 goto retry_alloc_baser; 949 case SZ_64K: 950 psz = SZ_16K; 951 goto retry_alloc_baser; 952 } 953 } 954 955 if (val != tmp) { 956 pr_err("ITS: %s: GITS_BASER%d doesn't stick: %lx %lx\n", 957 node_name, i, 958 (unsigned long) val, (unsigned long) tmp); 959 err = -ENXIO; 960 goto out_free; 961 } 962 963 pr_info("ITS: allocated %d %s @%lx (psz %dK, shr %d)\n", 964 (int)(PAGE_ORDER_TO_SIZE(order) / entry_size), 965 its_base_type_string[type], 966 (unsigned long)virt_to_phys(base), 967 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); 968 } 969 970 return 0; 971 972 out_free: 973 its_free_tables(its); 974 975 return err; 976 } 977 978 static int its_alloc_collections(struct its_node *its) 979 { 980 its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections), 981 GFP_KERNEL); 982 if (!its->collections) 983 return -ENOMEM; 984 985 return 0; 986 } 987 988 static void its_cpu_init_lpis(void) 989 { 990 void __iomem *rbase = gic_data_rdist_rd_base(); 991 struct page *pend_page; 992 u64 val, tmp; 993 994 /* If we didn't allocate the pending table yet, do it now */ 995 pend_page = gic_data_rdist()->pend_page; 996 if (!pend_page) { 997 phys_addr_t paddr; 998 /* 999 * The pending pages have to be at least 64kB aligned, 1000 * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below. 1001 */ 1002 pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO, 1003 get_order(max(LPI_PENDBASE_SZ, SZ_64K))); 1004 if (!pend_page) { 1005 pr_err("Failed to allocate PENDBASE for CPU%d\n", 1006 smp_processor_id()); 1007 return; 1008 } 1009 1010 /* Make sure the GIC will observe the zero-ed page */ 1011 __flush_dcache_area(page_address(pend_page), LPI_PENDBASE_SZ); 1012 1013 paddr = page_to_phys(pend_page); 1014 pr_info("CPU%d: using LPI pending table @%pa\n", 1015 smp_processor_id(), &paddr); 1016 gic_data_rdist()->pend_page = pend_page; 1017 } 1018 1019 /* Disable LPIs */ 1020 val = readl_relaxed(rbase + GICR_CTLR); 1021 val &= ~GICR_CTLR_ENABLE_LPIS; 1022 writel_relaxed(val, rbase + GICR_CTLR); 1023 1024 /* 1025 * Make sure any change to the table is observable by the GIC. 1026 */ 1027 dsb(sy); 1028 1029 /* set PROPBASE */ 1030 val = (page_to_phys(gic_rdists->prop_page) | 1031 GICR_PROPBASER_InnerShareable | 1032 GICR_PROPBASER_WaWb | 1033 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); 1034 1035 writeq_relaxed(val, rbase + GICR_PROPBASER); 1036 tmp = readq_relaxed(rbase + GICR_PROPBASER); 1037 1038 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { 1039 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) { 1040 /* 1041 * The HW reports non-shareable, we must 1042 * remove the cacheability attributes as 1043 * well. 1044 */ 1045 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK | 1046 GICR_PROPBASER_CACHEABILITY_MASK); 1047 val |= GICR_PROPBASER_nC; 1048 writeq_relaxed(val, rbase + GICR_PROPBASER); 1049 } 1050 pr_info_once("GIC: using cache flushing for LPI property table\n"); 1051 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; 1052 } 1053 1054 /* set PENDBASE */ 1055 val = (page_to_phys(pend_page) | 1056 GICR_PENDBASER_InnerShareable | 1057 GICR_PENDBASER_WaWb); 1058 1059 writeq_relaxed(val, rbase + GICR_PENDBASER); 1060 tmp = readq_relaxed(rbase + GICR_PENDBASER); 1061 1062 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { 1063 /* 1064 * The HW reports non-shareable, we must remove the 1065 * cacheability attributes as well. 1066 */ 1067 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK | 1068 GICR_PENDBASER_CACHEABILITY_MASK); 1069 val |= GICR_PENDBASER_nC; 1070 writeq_relaxed(val, rbase + GICR_PENDBASER); 1071 } 1072 1073 /* Enable LPIs */ 1074 val = readl_relaxed(rbase + GICR_CTLR); 1075 val |= GICR_CTLR_ENABLE_LPIS; 1076 writel_relaxed(val, rbase + GICR_CTLR); 1077 1078 /* Make sure the GIC has seen the above */ 1079 dsb(sy); 1080 } 1081 1082 static void its_cpu_init_collection(void) 1083 { 1084 struct its_node *its; 1085 int cpu; 1086 1087 spin_lock(&its_lock); 1088 cpu = smp_processor_id(); 1089 1090 list_for_each_entry(its, &its_nodes, entry) { 1091 u64 target; 1092 1093 /* 1094 * We now have to bind each collection to its target 1095 * redistributor. 1096 */ 1097 if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) { 1098 /* 1099 * This ITS wants the physical address of the 1100 * redistributor. 1101 */ 1102 target = gic_data_rdist()->phys_base; 1103 } else { 1104 /* 1105 * This ITS wants a linear CPU number. 1106 */ 1107 target = readq_relaxed(gic_data_rdist_rd_base() + GICR_TYPER); 1108 target = GICR_TYPER_CPU_NUMBER(target) << 16; 1109 } 1110 1111 /* Perform collection mapping */ 1112 its->collections[cpu].target_address = target; 1113 its->collections[cpu].col_id = cpu; 1114 1115 its_send_mapc(its, &its->collections[cpu], 1); 1116 its_send_invall(its, &its->collections[cpu]); 1117 } 1118 1119 spin_unlock(&its_lock); 1120 } 1121 1122 static struct its_device *its_find_device(struct its_node *its, u32 dev_id) 1123 { 1124 struct its_device *its_dev = NULL, *tmp; 1125 unsigned long flags; 1126 1127 raw_spin_lock_irqsave(&its->lock, flags); 1128 1129 list_for_each_entry(tmp, &its->its_device_list, entry) { 1130 if (tmp->device_id == dev_id) { 1131 its_dev = tmp; 1132 break; 1133 } 1134 } 1135 1136 raw_spin_unlock_irqrestore(&its->lock, flags); 1137 1138 return its_dev; 1139 } 1140 1141 static struct its_device *its_create_device(struct its_node *its, u32 dev_id, 1142 int nvecs) 1143 { 1144 struct its_device *dev; 1145 unsigned long *lpi_map; 1146 unsigned long flags; 1147 u16 *col_map = NULL; 1148 void *itt; 1149 int lpi_base; 1150 int nr_lpis; 1151 int nr_ites; 1152 int sz; 1153 1154 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1155 /* 1156 * At least one bit of EventID is being used, hence a minimum 1157 * of two entries. No, the architecture doesn't let you 1158 * express an ITT with a single entry. 1159 */ 1160 nr_ites = max(2UL, roundup_pow_of_two(nvecs)); 1161 sz = nr_ites * its->ite_size; 1162 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; 1163 itt = kzalloc(sz, GFP_KERNEL); 1164 lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis); 1165 if (lpi_map) 1166 col_map = kzalloc(sizeof(*col_map) * nr_lpis, GFP_KERNEL); 1167 1168 if (!dev || !itt || !lpi_map || !col_map) { 1169 kfree(dev); 1170 kfree(itt); 1171 kfree(lpi_map); 1172 kfree(col_map); 1173 return NULL; 1174 } 1175 1176 __flush_dcache_area(itt, sz); 1177 1178 dev->its = its; 1179 dev->itt = itt; 1180 dev->nr_ites = nr_ites; 1181 dev->event_map.lpi_map = lpi_map; 1182 dev->event_map.col_map = col_map; 1183 dev->event_map.lpi_base = lpi_base; 1184 dev->event_map.nr_lpis = nr_lpis; 1185 dev->device_id = dev_id; 1186 INIT_LIST_HEAD(&dev->entry); 1187 1188 raw_spin_lock_irqsave(&its->lock, flags); 1189 list_add(&dev->entry, &its->its_device_list); 1190 raw_spin_unlock_irqrestore(&its->lock, flags); 1191 1192 /* Map device to its ITT */ 1193 its_send_mapd(dev, 1); 1194 1195 return dev; 1196 } 1197 1198 static void its_free_device(struct its_device *its_dev) 1199 { 1200 unsigned long flags; 1201 1202 raw_spin_lock_irqsave(&its_dev->its->lock, flags); 1203 list_del(&its_dev->entry); 1204 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); 1205 kfree(its_dev->itt); 1206 kfree(its_dev); 1207 } 1208 1209 static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq) 1210 { 1211 int idx; 1212 1213 idx = find_first_zero_bit(dev->event_map.lpi_map, 1214 dev->event_map.nr_lpis); 1215 if (idx == dev->event_map.nr_lpis) 1216 return -ENOSPC; 1217 1218 *hwirq = dev->event_map.lpi_base + idx; 1219 set_bit(idx, dev->event_map.lpi_map); 1220 1221 return 0; 1222 } 1223 1224 static int its_msi_prepare(struct irq_domain *domain, struct device *dev, 1225 int nvec, msi_alloc_info_t *info) 1226 { 1227 struct its_node *its; 1228 struct its_device *its_dev; 1229 struct msi_domain_info *msi_info; 1230 u32 dev_id; 1231 1232 /* 1233 * We ignore "dev" entierely, and rely on the dev_id that has 1234 * been passed via the scratchpad. This limits this domain's 1235 * usefulness to upper layers that definitely know that they 1236 * are built on top of the ITS. 1237 */ 1238 dev_id = info->scratchpad[0].ul; 1239 1240 msi_info = msi_get_domain_info(domain); 1241 its = msi_info->data; 1242 1243 its_dev = its_find_device(its, dev_id); 1244 if (its_dev) { 1245 /* 1246 * We already have seen this ID, probably through 1247 * another alias (PCI bridge of some sort). No need to 1248 * create the device. 1249 */ 1250 pr_debug("Reusing ITT for devID %x\n", dev_id); 1251 goto out; 1252 } 1253 1254 its_dev = its_create_device(its, dev_id, nvec); 1255 if (!its_dev) 1256 return -ENOMEM; 1257 1258 pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec)); 1259 out: 1260 info->scratchpad[0].ptr = its_dev; 1261 return 0; 1262 } 1263 1264 static struct msi_domain_ops its_msi_domain_ops = { 1265 .msi_prepare = its_msi_prepare, 1266 }; 1267 1268 static int its_irq_gic_domain_alloc(struct irq_domain *domain, 1269 unsigned int virq, 1270 irq_hw_number_t hwirq) 1271 { 1272 struct irq_fwspec fwspec; 1273 1274 if (irq_domain_get_of_node(domain->parent)) { 1275 fwspec.fwnode = domain->parent->fwnode; 1276 fwspec.param_count = 3; 1277 fwspec.param[0] = GIC_IRQ_TYPE_LPI; 1278 fwspec.param[1] = hwirq; 1279 fwspec.param[2] = IRQ_TYPE_EDGE_RISING; 1280 } else { 1281 return -EINVAL; 1282 } 1283 1284 return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); 1285 } 1286 1287 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1288 unsigned int nr_irqs, void *args) 1289 { 1290 msi_alloc_info_t *info = args; 1291 struct its_device *its_dev = info->scratchpad[0].ptr; 1292 irq_hw_number_t hwirq; 1293 int err; 1294 int i; 1295 1296 for (i = 0; i < nr_irqs; i++) { 1297 err = its_alloc_device_irq(its_dev, &hwirq); 1298 if (err) 1299 return err; 1300 1301 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq); 1302 if (err) 1303 return err; 1304 1305 irq_domain_set_hwirq_and_chip(domain, virq + i, 1306 hwirq, &its_irq_chip, its_dev); 1307 pr_debug("ID:%d pID:%d vID:%d\n", 1308 (int)(hwirq - its_dev->event_map.lpi_base), 1309 (int) hwirq, virq + i); 1310 } 1311 1312 return 0; 1313 } 1314 1315 static void its_irq_domain_activate(struct irq_domain *domain, 1316 struct irq_data *d) 1317 { 1318 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1319 u32 event = its_get_event_id(d); 1320 1321 /* Bind the LPI to the first possible CPU */ 1322 its_dev->event_map.col_map[event] = cpumask_first(cpu_online_mask); 1323 1324 /* Map the GIC IRQ and event to the device */ 1325 its_send_mapvi(its_dev, d->hwirq, event); 1326 } 1327 1328 static void its_irq_domain_deactivate(struct irq_domain *domain, 1329 struct irq_data *d) 1330 { 1331 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1332 u32 event = its_get_event_id(d); 1333 1334 /* Stop the delivery of interrupts */ 1335 its_send_discard(its_dev, event); 1336 } 1337 1338 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1339 unsigned int nr_irqs) 1340 { 1341 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 1342 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1343 int i; 1344 1345 for (i = 0; i < nr_irqs; i++) { 1346 struct irq_data *data = irq_domain_get_irq_data(domain, 1347 virq + i); 1348 u32 event = its_get_event_id(data); 1349 1350 /* Mark interrupt index as unused */ 1351 clear_bit(event, its_dev->event_map.lpi_map); 1352 1353 /* Nuke the entry in the domain */ 1354 irq_domain_reset_irq_data(data); 1355 } 1356 1357 /* If all interrupts have been freed, start mopping the floor */ 1358 if (bitmap_empty(its_dev->event_map.lpi_map, 1359 its_dev->event_map.nr_lpis)) { 1360 its_lpi_free(&its_dev->event_map); 1361 1362 /* Unmap device/itt */ 1363 its_send_mapd(its_dev, 0); 1364 its_free_device(its_dev); 1365 } 1366 1367 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 1368 } 1369 1370 static const struct irq_domain_ops its_domain_ops = { 1371 .alloc = its_irq_domain_alloc, 1372 .free = its_irq_domain_free, 1373 .activate = its_irq_domain_activate, 1374 .deactivate = its_irq_domain_deactivate, 1375 }; 1376 1377 static int its_force_quiescent(void __iomem *base) 1378 { 1379 u32 count = 1000000; /* 1s */ 1380 u32 val; 1381 1382 val = readl_relaxed(base + GITS_CTLR); 1383 if (val & GITS_CTLR_QUIESCENT) 1384 return 0; 1385 1386 /* Disable the generation of all interrupts to this ITS */ 1387 val &= ~GITS_CTLR_ENABLE; 1388 writel_relaxed(val, base + GITS_CTLR); 1389 1390 /* Poll GITS_CTLR and wait until ITS becomes quiescent */ 1391 while (1) { 1392 val = readl_relaxed(base + GITS_CTLR); 1393 if (val & GITS_CTLR_QUIESCENT) 1394 return 0; 1395 1396 count--; 1397 if (!count) 1398 return -EBUSY; 1399 1400 cpu_relax(); 1401 udelay(1); 1402 } 1403 } 1404 1405 static void __maybe_unused its_enable_quirk_cavium_22375(void *data) 1406 { 1407 struct its_node *its = data; 1408 1409 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; 1410 } 1411 1412 static const struct gic_quirk its_quirks[] = { 1413 #ifdef CONFIG_CAVIUM_ERRATUM_22375 1414 { 1415 .desc = "ITS: Cavium errata 22375, 24313", 1416 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 1417 .mask = 0xffff0fff, 1418 .init = its_enable_quirk_cavium_22375, 1419 }, 1420 #endif 1421 { 1422 } 1423 }; 1424 1425 static void its_enable_quirks(struct its_node *its) 1426 { 1427 u32 iidr = readl_relaxed(its->base + GITS_IIDR); 1428 1429 gic_enable_quirks(iidr, its_quirks, its); 1430 } 1431 1432 static int __init its_probe(struct device_node *node, 1433 struct irq_domain *parent) 1434 { 1435 struct resource res; 1436 struct its_node *its; 1437 void __iomem *its_base; 1438 struct irq_domain *inner_domain; 1439 u32 val; 1440 u64 baser, tmp; 1441 int err; 1442 1443 err = of_address_to_resource(node, 0, &res); 1444 if (err) { 1445 pr_warn("%s: no regs?\n", node->full_name); 1446 return -ENXIO; 1447 } 1448 1449 its_base = ioremap(res.start, resource_size(&res)); 1450 if (!its_base) { 1451 pr_warn("%s: unable to map registers\n", node->full_name); 1452 return -ENOMEM; 1453 } 1454 1455 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK; 1456 if (val != 0x30 && val != 0x40) { 1457 pr_warn("%s: no ITS detected, giving up\n", node->full_name); 1458 err = -ENODEV; 1459 goto out_unmap; 1460 } 1461 1462 err = its_force_quiescent(its_base); 1463 if (err) { 1464 pr_warn("%s: failed to quiesce, giving up\n", 1465 node->full_name); 1466 goto out_unmap; 1467 } 1468 1469 pr_info("ITS: %s\n", node->full_name); 1470 1471 its = kzalloc(sizeof(*its), GFP_KERNEL); 1472 if (!its) { 1473 err = -ENOMEM; 1474 goto out_unmap; 1475 } 1476 1477 raw_spin_lock_init(&its->lock); 1478 INIT_LIST_HEAD(&its->entry); 1479 INIT_LIST_HEAD(&its->its_device_list); 1480 its->base = its_base; 1481 its->phys_base = res.start; 1482 its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1; 1483 1484 its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL); 1485 if (!its->cmd_base) { 1486 err = -ENOMEM; 1487 goto out_free_its; 1488 } 1489 its->cmd_write = its->cmd_base; 1490 1491 its_enable_quirks(its); 1492 1493 err = its_alloc_tables(node->full_name, its); 1494 if (err) 1495 goto out_free_cmd; 1496 1497 err = its_alloc_collections(its); 1498 if (err) 1499 goto out_free_tables; 1500 1501 baser = (virt_to_phys(its->cmd_base) | 1502 GITS_CBASER_WaWb | 1503 GITS_CBASER_InnerShareable | 1504 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | 1505 GITS_CBASER_VALID); 1506 1507 writeq_relaxed(baser, its->base + GITS_CBASER); 1508 tmp = readq_relaxed(its->base + GITS_CBASER); 1509 1510 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) { 1511 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) { 1512 /* 1513 * The HW reports non-shareable, we must 1514 * remove the cacheability attributes as 1515 * well. 1516 */ 1517 baser &= ~(GITS_CBASER_SHAREABILITY_MASK | 1518 GITS_CBASER_CACHEABILITY_MASK); 1519 baser |= GITS_CBASER_nC; 1520 writeq_relaxed(baser, its->base + GITS_CBASER); 1521 } 1522 pr_info("ITS: using cache flushing for cmd queue\n"); 1523 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; 1524 } 1525 1526 writeq_relaxed(0, its->base + GITS_CWRITER); 1527 writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR); 1528 1529 if (of_property_read_bool(node, "msi-controller")) { 1530 struct msi_domain_info *info; 1531 1532 info = kzalloc(sizeof(*info), GFP_KERNEL); 1533 if (!info) { 1534 err = -ENOMEM; 1535 goto out_free_tables; 1536 } 1537 1538 inner_domain = irq_domain_add_tree(node, &its_domain_ops, its); 1539 if (!inner_domain) { 1540 err = -ENOMEM; 1541 kfree(info); 1542 goto out_free_tables; 1543 } 1544 1545 inner_domain->parent = parent; 1546 inner_domain->bus_token = DOMAIN_BUS_NEXUS; 1547 info->ops = &its_msi_domain_ops; 1548 info->data = its; 1549 inner_domain->host_data = info; 1550 } 1551 1552 spin_lock(&its_lock); 1553 list_add(&its->entry, &its_nodes); 1554 spin_unlock(&its_lock); 1555 1556 return 0; 1557 1558 out_free_tables: 1559 its_free_tables(its); 1560 out_free_cmd: 1561 kfree(its->cmd_base); 1562 out_free_its: 1563 kfree(its); 1564 out_unmap: 1565 iounmap(its_base); 1566 pr_err("ITS: failed probing %s (%d)\n", node->full_name, err); 1567 return err; 1568 } 1569 1570 static bool gic_rdists_supports_plpis(void) 1571 { 1572 return !!(readl_relaxed(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS); 1573 } 1574 1575 int its_cpu_init(void) 1576 { 1577 if (!list_empty(&its_nodes)) { 1578 if (!gic_rdists_supports_plpis()) { 1579 pr_info("CPU%d: LPIs not supported\n", smp_processor_id()); 1580 return -ENXIO; 1581 } 1582 its_cpu_init_lpis(); 1583 its_cpu_init_collection(); 1584 } 1585 1586 return 0; 1587 } 1588 1589 static struct of_device_id its_device_id[] = { 1590 { .compatible = "arm,gic-v3-its", }, 1591 {}, 1592 }; 1593 1594 int __init its_init(struct device_node *node, struct rdists *rdists, 1595 struct irq_domain *parent_domain) 1596 { 1597 struct device_node *np; 1598 1599 for (np = of_find_matching_node(node, its_device_id); np; 1600 np = of_find_matching_node(np, its_device_id)) { 1601 its_probe(np, parent_domain); 1602 } 1603 1604 if (list_empty(&its_nodes)) { 1605 pr_warn("ITS: No ITS available, not enabling LPIs\n"); 1606 return -ENXIO; 1607 } 1608 1609 gic_rdists = rdists; 1610 its_alloc_lpi_tables(); 1611 its_lpi_init(rdists->id_bits); 1612 1613 return 0; 1614 } 1615