1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/acpi.h> 8 #include <linux/acpi_iort.h> 9 #include <linux/bitfield.h> 10 #include <linux/bitmap.h> 11 #include <linux/cpu.h> 12 #include <linux/crash_dump.h> 13 #include <linux/delay.h> 14 #include <linux/dma-iommu.h> 15 #include <linux/efi.h> 16 #include <linux/interrupt.h> 17 #include <linux/iopoll.h> 18 #include <linux/irqdomain.h> 19 #include <linux/list.h> 20 #include <linux/log2.h> 21 #include <linux/memblock.h> 22 #include <linux/mm.h> 23 #include <linux/msi.h> 24 #include <linux/of.h> 25 #include <linux/of_address.h> 26 #include <linux/of_irq.h> 27 #include <linux/of_pci.h> 28 #include <linux/of_platform.h> 29 #include <linux/percpu.h> 30 #include <linux/slab.h> 31 #include <linux/syscore_ops.h> 32 33 #include <linux/irqchip.h> 34 #include <linux/irqchip/arm-gic-v3.h> 35 #include <linux/irqchip/arm-gic-v4.h> 36 37 #include <asm/cputype.h> 38 #include <asm/exception.h> 39 40 #include "irq-gic-common.h" 41 42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) 43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) 44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2) 45 46 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) 47 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) 48 49 static u32 lpi_id_bits; 50 51 /* 52 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to 53 * deal with (one configuration byte per interrupt). PENDBASE has to 54 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). 55 */ 56 #define LPI_NRBITS lpi_id_bits 57 #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K) 58 #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K) 59 60 #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI 61 62 /* 63 * Collection structure - just an ID, and a redistributor address to 64 * ping. We use one per CPU as a bag of interrupts assigned to this 65 * CPU. 66 */ 67 struct its_collection { 68 u64 target_address; 69 u16 col_id; 70 }; 71 72 /* 73 * The ITS_BASER structure - contains memory information, cached 74 * value of BASER register configuration and ITS page size. 75 */ 76 struct its_baser { 77 void *base; 78 u64 val; 79 u32 order; 80 u32 psz; 81 }; 82 83 struct its_device; 84 85 /* 86 * The ITS structure - contains most of the infrastructure, with the 87 * top-level MSI domain, the command queue, the collections, and the 88 * list of devices writing to it. 89 * 90 * dev_alloc_lock has to be taken for device allocations, while the 91 * spinlock must be taken to parse data structures such as the device 92 * list. 93 */ 94 struct its_node { 95 raw_spinlock_t lock; 96 struct mutex dev_alloc_lock; 97 struct list_head entry; 98 void __iomem *base; 99 void __iomem *sgir_base; 100 phys_addr_t phys_base; 101 struct its_cmd_block *cmd_base; 102 struct its_cmd_block *cmd_write; 103 struct its_baser tables[GITS_BASER_NR_REGS]; 104 struct its_collection *collections; 105 struct fwnode_handle *fwnode_handle; 106 u64 (*get_msi_base)(struct its_device *its_dev); 107 u64 typer; 108 u64 cbaser_save; 109 u32 ctlr_save; 110 u32 mpidr; 111 struct list_head its_device_list; 112 u64 flags; 113 unsigned long list_nr; 114 int numa_node; 115 unsigned int msi_domain_flags; 116 u32 pre_its_base; /* for Socionext Synquacer */ 117 int vlpi_redist_offset; 118 }; 119 120 #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) 121 #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) 122 #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) 123 124 #define ITS_ITT_ALIGN SZ_256 125 126 /* The maximum number of VPEID bits supported by VLPI commands */ 127 #define ITS_MAX_VPEID_BITS \ 128 ({ \ 129 int nvpeid = 16; \ 130 if (gic_rdists->has_rvpeid && \ 131 gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \ 132 nvpeid = 1 + (gic_rdists->gicd_typer2 & \ 133 GICD_TYPER2_VID); \ 134 \ 135 nvpeid; \ 136 }) 137 #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS)) 138 139 /* Convert page order to size in bytes */ 140 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o)) 141 142 struct event_lpi_map { 143 unsigned long *lpi_map; 144 u16 *col_map; 145 irq_hw_number_t lpi_base; 146 int nr_lpis; 147 raw_spinlock_t vlpi_lock; 148 struct its_vm *vm; 149 struct its_vlpi_map *vlpi_maps; 150 int nr_vlpis; 151 }; 152 153 /* 154 * The ITS view of a device - belongs to an ITS, owns an interrupt 155 * translation table, and a list of interrupts. If it some of its 156 * LPIs are injected into a guest (GICv4), the event_map.vm field 157 * indicates which one. 158 */ 159 struct its_device { 160 struct list_head entry; 161 struct its_node *its; 162 struct event_lpi_map event_map; 163 void *itt; 164 u32 nr_ites; 165 u32 device_id; 166 bool shared; 167 }; 168 169 static struct { 170 raw_spinlock_t lock; 171 struct its_device *dev; 172 struct its_vpe **vpes; 173 int next_victim; 174 } vpe_proxy; 175 176 struct cpu_lpi_count { 177 atomic_t managed; 178 atomic_t unmanaged; 179 }; 180 181 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count); 182 183 static LIST_HEAD(its_nodes); 184 static DEFINE_RAW_SPINLOCK(its_lock); 185 static struct rdists *gic_rdists; 186 static struct irq_domain *its_parent; 187 188 static unsigned long its_list_map; 189 static u16 vmovp_seq_num; 190 static DEFINE_RAW_SPINLOCK(vmovp_lock); 191 192 static DEFINE_IDA(its_vpeid_ida); 193 194 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) 195 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu)) 196 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 197 #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) 198 199 /* 200 * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we 201 * always have vSGIs mapped. 202 */ 203 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its) 204 { 205 return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]); 206 } 207 208 static u16 get_its_list(struct its_vm *vm) 209 { 210 struct its_node *its; 211 unsigned long its_list = 0; 212 213 list_for_each_entry(its, &its_nodes, entry) { 214 if (!is_v4(its)) 215 continue; 216 217 if (require_its_list_vmovp(vm, its)) 218 __set_bit(its->list_nr, &its_list); 219 } 220 221 return (u16)its_list; 222 } 223 224 static inline u32 its_get_event_id(struct irq_data *d) 225 { 226 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 227 return d->hwirq - its_dev->event_map.lpi_base; 228 } 229 230 static struct its_collection *dev_event_to_col(struct its_device *its_dev, 231 u32 event) 232 { 233 struct its_node *its = its_dev->its; 234 235 return its->collections + its_dev->event_map.col_map[event]; 236 } 237 238 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev, 239 u32 event) 240 { 241 if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis)) 242 return NULL; 243 244 return &its_dev->event_map.vlpi_maps[event]; 245 } 246 247 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) 248 { 249 if (irqd_is_forwarded_to_vcpu(d)) { 250 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 251 u32 event = its_get_event_id(d); 252 253 return dev_event_to_vlpi_map(its_dev, event); 254 } 255 256 return NULL; 257 } 258 259 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags) 260 { 261 raw_spin_lock_irqsave(&vpe->vpe_lock, *flags); 262 return vpe->col_idx; 263 } 264 265 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags) 266 { 267 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); 268 } 269 270 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags) 271 { 272 struct its_vlpi_map *map = get_vlpi_map(d); 273 int cpu; 274 275 if (map) { 276 cpu = vpe_to_cpuid_lock(map->vpe, flags); 277 } else { 278 /* Physical LPIs are already locked via the irq_desc lock */ 279 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 280 cpu = its_dev->event_map.col_map[its_get_event_id(d)]; 281 /* Keep GCC quiet... */ 282 *flags = 0; 283 } 284 285 return cpu; 286 } 287 288 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags) 289 { 290 struct its_vlpi_map *map = get_vlpi_map(d); 291 292 if (map) 293 vpe_to_cpuid_unlock(map->vpe, flags); 294 } 295 296 static struct its_collection *valid_col(struct its_collection *col) 297 { 298 if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0))) 299 return NULL; 300 301 return col; 302 } 303 304 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe) 305 { 306 if (valid_col(its->collections + vpe->col_idx)) 307 return vpe; 308 309 return NULL; 310 } 311 312 /* 313 * ITS command descriptors - parameters to be encoded in a command 314 * block. 315 */ 316 struct its_cmd_desc { 317 union { 318 struct { 319 struct its_device *dev; 320 u32 event_id; 321 } its_inv_cmd; 322 323 struct { 324 struct its_device *dev; 325 u32 event_id; 326 } its_clear_cmd; 327 328 struct { 329 struct its_device *dev; 330 u32 event_id; 331 } its_int_cmd; 332 333 struct { 334 struct its_device *dev; 335 int valid; 336 } its_mapd_cmd; 337 338 struct { 339 struct its_collection *col; 340 int valid; 341 } its_mapc_cmd; 342 343 struct { 344 struct its_device *dev; 345 u32 phys_id; 346 u32 event_id; 347 } its_mapti_cmd; 348 349 struct { 350 struct its_device *dev; 351 struct its_collection *col; 352 u32 event_id; 353 } its_movi_cmd; 354 355 struct { 356 struct its_device *dev; 357 u32 event_id; 358 } its_discard_cmd; 359 360 struct { 361 struct its_collection *col; 362 } its_invall_cmd; 363 364 struct { 365 struct its_vpe *vpe; 366 } its_vinvall_cmd; 367 368 struct { 369 struct its_vpe *vpe; 370 struct its_collection *col; 371 bool valid; 372 } its_vmapp_cmd; 373 374 struct { 375 struct its_vpe *vpe; 376 struct its_device *dev; 377 u32 virt_id; 378 u32 event_id; 379 bool db_enabled; 380 } its_vmapti_cmd; 381 382 struct { 383 struct its_vpe *vpe; 384 struct its_device *dev; 385 u32 event_id; 386 bool db_enabled; 387 } its_vmovi_cmd; 388 389 struct { 390 struct its_vpe *vpe; 391 struct its_collection *col; 392 u16 seq_num; 393 u16 its_list; 394 } its_vmovp_cmd; 395 396 struct { 397 struct its_vpe *vpe; 398 } its_invdb_cmd; 399 400 struct { 401 struct its_vpe *vpe; 402 u8 sgi; 403 u8 priority; 404 bool enable; 405 bool group; 406 bool clear; 407 } its_vsgi_cmd; 408 }; 409 }; 410 411 /* 412 * The ITS command block, which is what the ITS actually parses. 413 */ 414 struct its_cmd_block { 415 union { 416 u64 raw_cmd[4]; 417 __le64 raw_cmd_le[4]; 418 }; 419 }; 420 421 #define ITS_CMD_QUEUE_SZ SZ_64K 422 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) 423 424 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *, 425 struct its_cmd_block *, 426 struct its_cmd_desc *); 427 428 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *, 429 struct its_cmd_block *, 430 struct its_cmd_desc *); 431 432 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 433 { 434 u64 mask = GENMASK_ULL(h, l); 435 *raw_cmd &= ~mask; 436 *raw_cmd |= (val << l) & mask; 437 } 438 439 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 440 { 441 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); 442 } 443 444 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 445 { 446 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); 447 } 448 449 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 450 { 451 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); 452 } 453 454 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 455 { 456 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); 457 } 458 459 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 460 { 461 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); 462 } 463 464 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 465 { 466 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); 467 } 468 469 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 470 { 471 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); 472 } 473 474 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 475 { 476 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); 477 } 478 479 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 480 { 481 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); 482 } 483 484 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid) 485 { 486 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32); 487 } 488 489 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id) 490 { 491 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0); 492 } 493 494 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id) 495 { 496 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32); 497 } 498 499 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid) 500 { 501 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0); 502 } 503 504 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num) 505 { 506 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32); 507 } 508 509 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list) 510 { 511 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0); 512 } 513 514 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa) 515 { 516 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16); 517 } 518 519 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size) 520 { 521 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); 522 } 523 524 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa) 525 { 526 its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16); 527 } 528 529 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc) 530 { 531 its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8); 532 } 533 534 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz) 535 { 536 its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9); 537 } 538 539 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd, 540 u32 vpe_db_lpi) 541 { 542 its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0); 543 } 544 545 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd, 546 u32 vpe_db_lpi) 547 { 548 its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0); 549 } 550 551 static void its_encode_db(struct its_cmd_block *cmd, bool db) 552 { 553 its_mask_encode(&cmd->raw_cmd[2], db, 63, 63); 554 } 555 556 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi) 557 { 558 its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32); 559 } 560 561 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio) 562 { 563 its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20); 564 } 565 566 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp) 567 { 568 its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10); 569 } 570 571 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr) 572 { 573 its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9); 574 } 575 576 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en) 577 { 578 its_mask_encode(&cmd->raw_cmd[0], en, 8, 8); 579 } 580 581 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 582 { 583 /* Let's fixup BE commands */ 584 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]); 585 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]); 586 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]); 587 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]); 588 } 589 590 static struct its_collection *its_build_mapd_cmd(struct its_node *its, 591 struct its_cmd_block *cmd, 592 struct its_cmd_desc *desc) 593 { 594 unsigned long itt_addr; 595 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); 596 597 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); 598 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); 599 600 its_encode_cmd(cmd, GITS_CMD_MAPD); 601 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); 602 its_encode_size(cmd, size - 1); 603 its_encode_itt(cmd, itt_addr); 604 its_encode_valid(cmd, desc->its_mapd_cmd.valid); 605 606 its_fixup_cmd(cmd); 607 608 return NULL; 609 } 610 611 static struct its_collection *its_build_mapc_cmd(struct its_node *its, 612 struct its_cmd_block *cmd, 613 struct its_cmd_desc *desc) 614 { 615 its_encode_cmd(cmd, GITS_CMD_MAPC); 616 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 617 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); 618 its_encode_valid(cmd, desc->its_mapc_cmd.valid); 619 620 its_fixup_cmd(cmd); 621 622 return desc->its_mapc_cmd.col; 623 } 624 625 static struct its_collection *its_build_mapti_cmd(struct its_node *its, 626 struct its_cmd_block *cmd, 627 struct its_cmd_desc *desc) 628 { 629 struct its_collection *col; 630 631 col = dev_event_to_col(desc->its_mapti_cmd.dev, 632 desc->its_mapti_cmd.event_id); 633 634 its_encode_cmd(cmd, GITS_CMD_MAPTI); 635 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id); 636 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id); 637 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id); 638 its_encode_collection(cmd, col->col_id); 639 640 its_fixup_cmd(cmd); 641 642 return valid_col(col); 643 } 644 645 static struct its_collection *its_build_movi_cmd(struct its_node *its, 646 struct its_cmd_block *cmd, 647 struct its_cmd_desc *desc) 648 { 649 struct its_collection *col; 650 651 col = dev_event_to_col(desc->its_movi_cmd.dev, 652 desc->its_movi_cmd.event_id); 653 654 its_encode_cmd(cmd, GITS_CMD_MOVI); 655 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); 656 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); 657 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); 658 659 its_fixup_cmd(cmd); 660 661 return valid_col(col); 662 } 663 664 static struct its_collection *its_build_discard_cmd(struct its_node *its, 665 struct its_cmd_block *cmd, 666 struct its_cmd_desc *desc) 667 { 668 struct its_collection *col; 669 670 col = dev_event_to_col(desc->its_discard_cmd.dev, 671 desc->its_discard_cmd.event_id); 672 673 its_encode_cmd(cmd, GITS_CMD_DISCARD); 674 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); 675 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); 676 677 its_fixup_cmd(cmd); 678 679 return valid_col(col); 680 } 681 682 static struct its_collection *its_build_inv_cmd(struct its_node *its, 683 struct its_cmd_block *cmd, 684 struct its_cmd_desc *desc) 685 { 686 struct its_collection *col; 687 688 col = dev_event_to_col(desc->its_inv_cmd.dev, 689 desc->its_inv_cmd.event_id); 690 691 its_encode_cmd(cmd, GITS_CMD_INV); 692 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 693 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 694 695 its_fixup_cmd(cmd); 696 697 return valid_col(col); 698 } 699 700 static struct its_collection *its_build_int_cmd(struct its_node *its, 701 struct its_cmd_block *cmd, 702 struct its_cmd_desc *desc) 703 { 704 struct its_collection *col; 705 706 col = dev_event_to_col(desc->its_int_cmd.dev, 707 desc->its_int_cmd.event_id); 708 709 its_encode_cmd(cmd, GITS_CMD_INT); 710 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); 711 its_encode_event_id(cmd, desc->its_int_cmd.event_id); 712 713 its_fixup_cmd(cmd); 714 715 return valid_col(col); 716 } 717 718 static struct its_collection *its_build_clear_cmd(struct its_node *its, 719 struct its_cmd_block *cmd, 720 struct its_cmd_desc *desc) 721 { 722 struct its_collection *col; 723 724 col = dev_event_to_col(desc->its_clear_cmd.dev, 725 desc->its_clear_cmd.event_id); 726 727 its_encode_cmd(cmd, GITS_CMD_CLEAR); 728 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); 729 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); 730 731 its_fixup_cmd(cmd); 732 733 return valid_col(col); 734 } 735 736 static struct its_collection *its_build_invall_cmd(struct its_node *its, 737 struct its_cmd_block *cmd, 738 struct its_cmd_desc *desc) 739 { 740 its_encode_cmd(cmd, GITS_CMD_INVALL); 741 its_encode_collection(cmd, desc->its_invall_cmd.col->col_id); 742 743 its_fixup_cmd(cmd); 744 745 return NULL; 746 } 747 748 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its, 749 struct its_cmd_block *cmd, 750 struct its_cmd_desc *desc) 751 { 752 its_encode_cmd(cmd, GITS_CMD_VINVALL); 753 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id); 754 755 its_fixup_cmd(cmd); 756 757 return valid_vpe(its, desc->its_vinvall_cmd.vpe); 758 } 759 760 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, 761 struct its_cmd_block *cmd, 762 struct its_cmd_desc *desc) 763 { 764 unsigned long vpt_addr, vconf_addr; 765 u64 target; 766 bool alloc; 767 768 its_encode_cmd(cmd, GITS_CMD_VMAPP); 769 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); 770 its_encode_valid(cmd, desc->its_vmapp_cmd.valid); 771 772 if (!desc->its_vmapp_cmd.valid) { 773 if (is_v4_1(its)) { 774 alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); 775 its_encode_alloc(cmd, alloc); 776 } 777 778 goto out; 779 } 780 781 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); 782 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; 783 784 its_encode_target(cmd, target); 785 its_encode_vpt_addr(cmd, vpt_addr); 786 its_encode_vpt_size(cmd, LPI_NRBITS - 1); 787 788 if (!is_v4_1(its)) 789 goto out; 790 791 vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page)); 792 793 alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); 794 795 its_encode_alloc(cmd, alloc); 796 797 /* We can only signal PTZ when alloc==1. Why do we have two bits? */ 798 its_encode_ptz(cmd, alloc); 799 its_encode_vconf_addr(cmd, vconf_addr); 800 its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi); 801 802 out: 803 its_fixup_cmd(cmd); 804 805 return valid_vpe(its, desc->its_vmapp_cmd.vpe); 806 } 807 808 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, 809 struct its_cmd_block *cmd, 810 struct its_cmd_desc *desc) 811 { 812 u32 db; 813 814 if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled) 815 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; 816 else 817 db = 1023; 818 819 its_encode_cmd(cmd, GITS_CMD_VMAPTI); 820 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id); 821 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id); 822 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id); 823 its_encode_db_phys_id(cmd, db); 824 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id); 825 826 its_fixup_cmd(cmd); 827 828 return valid_vpe(its, desc->its_vmapti_cmd.vpe); 829 } 830 831 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, 832 struct its_cmd_block *cmd, 833 struct its_cmd_desc *desc) 834 { 835 u32 db; 836 837 if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled) 838 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; 839 else 840 db = 1023; 841 842 its_encode_cmd(cmd, GITS_CMD_VMOVI); 843 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id); 844 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id); 845 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id); 846 its_encode_db_phys_id(cmd, db); 847 its_encode_db_valid(cmd, true); 848 849 its_fixup_cmd(cmd); 850 851 return valid_vpe(its, desc->its_vmovi_cmd.vpe); 852 } 853 854 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, 855 struct its_cmd_block *cmd, 856 struct its_cmd_desc *desc) 857 { 858 u64 target; 859 860 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; 861 its_encode_cmd(cmd, GITS_CMD_VMOVP); 862 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); 863 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); 864 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); 865 its_encode_target(cmd, target); 866 867 if (is_v4_1(its)) { 868 its_encode_db(cmd, true); 869 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi); 870 } 871 872 its_fixup_cmd(cmd); 873 874 return valid_vpe(its, desc->its_vmovp_cmd.vpe); 875 } 876 877 static struct its_vpe *its_build_vinv_cmd(struct its_node *its, 878 struct its_cmd_block *cmd, 879 struct its_cmd_desc *desc) 880 { 881 struct its_vlpi_map *map; 882 883 map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev, 884 desc->its_inv_cmd.event_id); 885 886 its_encode_cmd(cmd, GITS_CMD_INV); 887 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 888 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 889 890 its_fixup_cmd(cmd); 891 892 return valid_vpe(its, map->vpe); 893 } 894 895 static struct its_vpe *its_build_vint_cmd(struct its_node *its, 896 struct its_cmd_block *cmd, 897 struct its_cmd_desc *desc) 898 { 899 struct its_vlpi_map *map; 900 901 map = dev_event_to_vlpi_map(desc->its_int_cmd.dev, 902 desc->its_int_cmd.event_id); 903 904 its_encode_cmd(cmd, GITS_CMD_INT); 905 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); 906 its_encode_event_id(cmd, desc->its_int_cmd.event_id); 907 908 its_fixup_cmd(cmd); 909 910 return valid_vpe(its, map->vpe); 911 } 912 913 static struct its_vpe *its_build_vclear_cmd(struct its_node *its, 914 struct its_cmd_block *cmd, 915 struct its_cmd_desc *desc) 916 { 917 struct its_vlpi_map *map; 918 919 map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev, 920 desc->its_clear_cmd.event_id); 921 922 its_encode_cmd(cmd, GITS_CMD_CLEAR); 923 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); 924 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); 925 926 its_fixup_cmd(cmd); 927 928 return valid_vpe(its, map->vpe); 929 } 930 931 static struct its_vpe *its_build_invdb_cmd(struct its_node *its, 932 struct its_cmd_block *cmd, 933 struct its_cmd_desc *desc) 934 { 935 if (WARN_ON(!is_v4_1(its))) 936 return NULL; 937 938 its_encode_cmd(cmd, GITS_CMD_INVDB); 939 its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id); 940 941 its_fixup_cmd(cmd); 942 943 return valid_vpe(its, desc->its_invdb_cmd.vpe); 944 } 945 946 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its, 947 struct its_cmd_block *cmd, 948 struct its_cmd_desc *desc) 949 { 950 if (WARN_ON(!is_v4_1(its))) 951 return NULL; 952 953 its_encode_cmd(cmd, GITS_CMD_VSGI); 954 its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id); 955 its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi); 956 its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority); 957 its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group); 958 its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear); 959 its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable); 960 961 its_fixup_cmd(cmd); 962 963 return valid_vpe(its, desc->its_vsgi_cmd.vpe); 964 } 965 966 static u64 its_cmd_ptr_to_offset(struct its_node *its, 967 struct its_cmd_block *ptr) 968 { 969 return (ptr - its->cmd_base) * sizeof(*ptr); 970 } 971 972 static int its_queue_full(struct its_node *its) 973 { 974 int widx; 975 int ridx; 976 977 widx = its->cmd_write - its->cmd_base; 978 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); 979 980 /* This is incredibly unlikely to happen, unless the ITS locks up. */ 981 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) 982 return 1; 983 984 return 0; 985 } 986 987 static struct its_cmd_block *its_allocate_entry(struct its_node *its) 988 { 989 struct its_cmd_block *cmd; 990 u32 count = 1000000; /* 1s! */ 991 992 while (its_queue_full(its)) { 993 count--; 994 if (!count) { 995 pr_err_ratelimited("ITS queue not draining\n"); 996 return NULL; 997 } 998 cpu_relax(); 999 udelay(1); 1000 } 1001 1002 cmd = its->cmd_write++; 1003 1004 /* Handle queue wrapping */ 1005 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) 1006 its->cmd_write = its->cmd_base; 1007 1008 /* Clear command */ 1009 cmd->raw_cmd[0] = 0; 1010 cmd->raw_cmd[1] = 0; 1011 cmd->raw_cmd[2] = 0; 1012 cmd->raw_cmd[3] = 0; 1013 1014 return cmd; 1015 } 1016 1017 static struct its_cmd_block *its_post_commands(struct its_node *its) 1018 { 1019 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); 1020 1021 writel_relaxed(wr, its->base + GITS_CWRITER); 1022 1023 return its->cmd_write; 1024 } 1025 1026 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) 1027 { 1028 /* 1029 * Make sure the commands written to memory are observable by 1030 * the ITS. 1031 */ 1032 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) 1033 gic_flush_dcache_to_poc(cmd, sizeof(*cmd)); 1034 else 1035 dsb(ishst); 1036 } 1037 1038 static int its_wait_for_range_completion(struct its_node *its, 1039 u64 prev_idx, 1040 struct its_cmd_block *to) 1041 { 1042 u64 rd_idx, to_idx, linear_idx; 1043 u32 count = 1000000; /* 1s! */ 1044 1045 /* Linearize to_idx if the command set has wrapped around */ 1046 to_idx = its_cmd_ptr_to_offset(its, to); 1047 if (to_idx < prev_idx) 1048 to_idx += ITS_CMD_QUEUE_SZ; 1049 1050 linear_idx = prev_idx; 1051 1052 while (1) { 1053 s64 delta; 1054 1055 rd_idx = readl_relaxed(its->base + GITS_CREADR); 1056 1057 /* 1058 * Compute the read pointer progress, taking the 1059 * potential wrap-around into account. 1060 */ 1061 delta = rd_idx - prev_idx; 1062 if (rd_idx < prev_idx) 1063 delta += ITS_CMD_QUEUE_SZ; 1064 1065 linear_idx += delta; 1066 if (linear_idx >= to_idx) 1067 break; 1068 1069 count--; 1070 if (!count) { 1071 pr_err_ratelimited("ITS queue timeout (%llu %llu)\n", 1072 to_idx, linear_idx); 1073 return -1; 1074 } 1075 prev_idx = rd_idx; 1076 cpu_relax(); 1077 udelay(1); 1078 } 1079 1080 return 0; 1081 } 1082 1083 /* Warning, macro hell follows */ 1084 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \ 1085 void name(struct its_node *its, \ 1086 buildtype builder, \ 1087 struct its_cmd_desc *desc) \ 1088 { \ 1089 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \ 1090 synctype *sync_obj; \ 1091 unsigned long flags; \ 1092 u64 rd_idx; \ 1093 \ 1094 raw_spin_lock_irqsave(&its->lock, flags); \ 1095 \ 1096 cmd = its_allocate_entry(its); \ 1097 if (!cmd) { /* We're soooooo screewed... */ \ 1098 raw_spin_unlock_irqrestore(&its->lock, flags); \ 1099 return; \ 1100 } \ 1101 sync_obj = builder(its, cmd, desc); \ 1102 its_flush_cmd(its, cmd); \ 1103 \ 1104 if (sync_obj) { \ 1105 sync_cmd = its_allocate_entry(its); \ 1106 if (!sync_cmd) \ 1107 goto post; \ 1108 \ 1109 buildfn(its, sync_cmd, sync_obj); \ 1110 its_flush_cmd(its, sync_cmd); \ 1111 } \ 1112 \ 1113 post: \ 1114 rd_idx = readl_relaxed(its->base + GITS_CREADR); \ 1115 next_cmd = its_post_commands(its); \ 1116 raw_spin_unlock_irqrestore(&its->lock, flags); \ 1117 \ 1118 if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \ 1119 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \ 1120 } 1121 1122 static void its_build_sync_cmd(struct its_node *its, 1123 struct its_cmd_block *sync_cmd, 1124 struct its_collection *sync_col) 1125 { 1126 its_encode_cmd(sync_cmd, GITS_CMD_SYNC); 1127 its_encode_target(sync_cmd, sync_col->target_address); 1128 1129 its_fixup_cmd(sync_cmd); 1130 } 1131 1132 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t, 1133 struct its_collection, its_build_sync_cmd) 1134 1135 static void its_build_vsync_cmd(struct its_node *its, 1136 struct its_cmd_block *sync_cmd, 1137 struct its_vpe *sync_vpe) 1138 { 1139 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC); 1140 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id); 1141 1142 its_fixup_cmd(sync_cmd); 1143 } 1144 1145 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t, 1146 struct its_vpe, its_build_vsync_cmd) 1147 1148 static void its_send_int(struct its_device *dev, u32 event_id) 1149 { 1150 struct its_cmd_desc desc; 1151 1152 desc.its_int_cmd.dev = dev; 1153 desc.its_int_cmd.event_id = event_id; 1154 1155 its_send_single_command(dev->its, its_build_int_cmd, &desc); 1156 } 1157 1158 static void its_send_clear(struct its_device *dev, u32 event_id) 1159 { 1160 struct its_cmd_desc desc; 1161 1162 desc.its_clear_cmd.dev = dev; 1163 desc.its_clear_cmd.event_id = event_id; 1164 1165 its_send_single_command(dev->its, its_build_clear_cmd, &desc); 1166 } 1167 1168 static void its_send_inv(struct its_device *dev, u32 event_id) 1169 { 1170 struct its_cmd_desc desc; 1171 1172 desc.its_inv_cmd.dev = dev; 1173 desc.its_inv_cmd.event_id = event_id; 1174 1175 its_send_single_command(dev->its, its_build_inv_cmd, &desc); 1176 } 1177 1178 static void its_send_mapd(struct its_device *dev, int valid) 1179 { 1180 struct its_cmd_desc desc; 1181 1182 desc.its_mapd_cmd.dev = dev; 1183 desc.its_mapd_cmd.valid = !!valid; 1184 1185 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); 1186 } 1187 1188 static void its_send_mapc(struct its_node *its, struct its_collection *col, 1189 int valid) 1190 { 1191 struct its_cmd_desc desc; 1192 1193 desc.its_mapc_cmd.col = col; 1194 desc.its_mapc_cmd.valid = !!valid; 1195 1196 its_send_single_command(its, its_build_mapc_cmd, &desc); 1197 } 1198 1199 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id) 1200 { 1201 struct its_cmd_desc desc; 1202 1203 desc.its_mapti_cmd.dev = dev; 1204 desc.its_mapti_cmd.phys_id = irq_id; 1205 desc.its_mapti_cmd.event_id = id; 1206 1207 its_send_single_command(dev->its, its_build_mapti_cmd, &desc); 1208 } 1209 1210 static void its_send_movi(struct its_device *dev, 1211 struct its_collection *col, u32 id) 1212 { 1213 struct its_cmd_desc desc; 1214 1215 desc.its_movi_cmd.dev = dev; 1216 desc.its_movi_cmd.col = col; 1217 desc.its_movi_cmd.event_id = id; 1218 1219 its_send_single_command(dev->its, its_build_movi_cmd, &desc); 1220 } 1221 1222 static void its_send_discard(struct its_device *dev, u32 id) 1223 { 1224 struct its_cmd_desc desc; 1225 1226 desc.its_discard_cmd.dev = dev; 1227 desc.its_discard_cmd.event_id = id; 1228 1229 its_send_single_command(dev->its, its_build_discard_cmd, &desc); 1230 } 1231 1232 static void its_send_invall(struct its_node *its, struct its_collection *col) 1233 { 1234 struct its_cmd_desc desc; 1235 1236 desc.its_invall_cmd.col = col; 1237 1238 its_send_single_command(its, its_build_invall_cmd, &desc); 1239 } 1240 1241 static void its_send_vmapti(struct its_device *dev, u32 id) 1242 { 1243 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id); 1244 struct its_cmd_desc desc; 1245 1246 desc.its_vmapti_cmd.vpe = map->vpe; 1247 desc.its_vmapti_cmd.dev = dev; 1248 desc.its_vmapti_cmd.virt_id = map->vintid; 1249 desc.its_vmapti_cmd.event_id = id; 1250 desc.its_vmapti_cmd.db_enabled = map->db_enabled; 1251 1252 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); 1253 } 1254 1255 static void its_send_vmovi(struct its_device *dev, u32 id) 1256 { 1257 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id); 1258 struct its_cmd_desc desc; 1259 1260 desc.its_vmovi_cmd.vpe = map->vpe; 1261 desc.its_vmovi_cmd.dev = dev; 1262 desc.its_vmovi_cmd.event_id = id; 1263 desc.its_vmovi_cmd.db_enabled = map->db_enabled; 1264 1265 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); 1266 } 1267 1268 static void its_send_vmapp(struct its_node *its, 1269 struct its_vpe *vpe, bool valid) 1270 { 1271 struct its_cmd_desc desc; 1272 1273 desc.its_vmapp_cmd.vpe = vpe; 1274 desc.its_vmapp_cmd.valid = valid; 1275 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; 1276 1277 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); 1278 } 1279 1280 static void its_send_vmovp(struct its_vpe *vpe) 1281 { 1282 struct its_cmd_desc desc = {}; 1283 struct its_node *its; 1284 unsigned long flags; 1285 int col_id = vpe->col_idx; 1286 1287 desc.its_vmovp_cmd.vpe = vpe; 1288 1289 if (!its_list_map) { 1290 its = list_first_entry(&its_nodes, struct its_node, entry); 1291 desc.its_vmovp_cmd.col = &its->collections[col_id]; 1292 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 1293 return; 1294 } 1295 1296 /* 1297 * Yet another marvel of the architecture. If using the 1298 * its_list "feature", we need to make sure that all ITSs 1299 * receive all VMOVP commands in the same order. The only way 1300 * to guarantee this is to make vmovp a serialization point. 1301 * 1302 * Wall <-- Head. 1303 */ 1304 raw_spin_lock_irqsave(&vmovp_lock, flags); 1305 1306 desc.its_vmovp_cmd.seq_num = vmovp_seq_num++; 1307 desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm); 1308 1309 /* Emit VMOVPs */ 1310 list_for_each_entry(its, &its_nodes, entry) { 1311 if (!is_v4(its)) 1312 continue; 1313 1314 if (!require_its_list_vmovp(vpe->its_vm, its)) 1315 continue; 1316 1317 desc.its_vmovp_cmd.col = &its->collections[col_id]; 1318 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 1319 } 1320 1321 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1322 } 1323 1324 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) 1325 { 1326 struct its_cmd_desc desc; 1327 1328 desc.its_vinvall_cmd.vpe = vpe; 1329 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); 1330 } 1331 1332 static void its_send_vinv(struct its_device *dev, u32 event_id) 1333 { 1334 struct its_cmd_desc desc; 1335 1336 /* 1337 * There is no real VINV command. This is just a normal INV, 1338 * with a VSYNC instead of a SYNC. 1339 */ 1340 desc.its_inv_cmd.dev = dev; 1341 desc.its_inv_cmd.event_id = event_id; 1342 1343 its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc); 1344 } 1345 1346 static void its_send_vint(struct its_device *dev, u32 event_id) 1347 { 1348 struct its_cmd_desc desc; 1349 1350 /* 1351 * There is no real VINT command. This is just a normal INT, 1352 * with a VSYNC instead of a SYNC. 1353 */ 1354 desc.its_int_cmd.dev = dev; 1355 desc.its_int_cmd.event_id = event_id; 1356 1357 its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc); 1358 } 1359 1360 static void its_send_vclear(struct its_device *dev, u32 event_id) 1361 { 1362 struct its_cmd_desc desc; 1363 1364 /* 1365 * There is no real VCLEAR command. This is just a normal CLEAR, 1366 * with a VSYNC instead of a SYNC. 1367 */ 1368 desc.its_clear_cmd.dev = dev; 1369 desc.its_clear_cmd.event_id = event_id; 1370 1371 its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); 1372 } 1373 1374 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) 1375 { 1376 struct its_cmd_desc desc; 1377 1378 desc.its_invdb_cmd.vpe = vpe; 1379 its_send_single_vcommand(its, its_build_invdb_cmd, &desc); 1380 } 1381 1382 /* 1383 * irqchip functions - assumes MSI, mostly. 1384 */ 1385 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) 1386 { 1387 struct its_vlpi_map *map = get_vlpi_map(d); 1388 irq_hw_number_t hwirq; 1389 void *va; 1390 u8 *cfg; 1391 1392 if (map) { 1393 va = page_address(map->vm->vprop_page); 1394 hwirq = map->vintid; 1395 1396 /* Remember the updated property */ 1397 map->properties &= ~clr; 1398 map->properties |= set | LPI_PROP_GROUP1; 1399 } else { 1400 va = gic_rdists->prop_table_va; 1401 hwirq = d->hwirq; 1402 } 1403 1404 cfg = va + hwirq - 8192; 1405 *cfg &= ~clr; 1406 *cfg |= set | LPI_PROP_GROUP1; 1407 1408 /* 1409 * Make the above write visible to the redistributors. 1410 * And yes, we're flushing exactly: One. Single. Byte. 1411 * Humpf... 1412 */ 1413 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) 1414 gic_flush_dcache_to_poc(cfg, sizeof(*cfg)); 1415 else 1416 dsb(ishst); 1417 } 1418 1419 static void wait_for_syncr(void __iomem *rdbase) 1420 { 1421 while (readl_relaxed(rdbase + GICR_SYNCR) & 1) 1422 cpu_relax(); 1423 } 1424 1425 static void direct_lpi_inv(struct irq_data *d) 1426 { 1427 struct its_vlpi_map *map = get_vlpi_map(d); 1428 void __iomem *rdbase; 1429 unsigned long flags; 1430 u64 val; 1431 int cpu; 1432 1433 if (map) { 1434 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1435 1436 WARN_ON(!is_v4_1(its_dev->its)); 1437 1438 val = GICR_INVLPIR_V; 1439 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id); 1440 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid); 1441 } else { 1442 val = d->hwirq; 1443 } 1444 1445 /* Target the redistributor this LPI is currently routed to */ 1446 cpu = irq_to_cpuid_lock(d, &flags); 1447 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); 1448 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; 1449 gic_write_lpir(val, rdbase + GICR_INVLPIR); 1450 1451 wait_for_syncr(rdbase); 1452 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); 1453 irq_to_cpuid_unlock(d, flags); 1454 } 1455 1456 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) 1457 { 1458 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1459 1460 lpi_write_config(d, clr, set); 1461 if (gic_rdists->has_direct_lpi && 1462 (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d))) 1463 direct_lpi_inv(d); 1464 else if (!irqd_is_forwarded_to_vcpu(d)) 1465 its_send_inv(its_dev, its_get_event_id(d)); 1466 else 1467 its_send_vinv(its_dev, its_get_event_id(d)); 1468 } 1469 1470 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) 1471 { 1472 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1473 u32 event = its_get_event_id(d); 1474 struct its_vlpi_map *map; 1475 1476 /* 1477 * GICv4.1 does away with the per-LPI nonsense, nothing to do 1478 * here. 1479 */ 1480 if (is_v4_1(its_dev->its)) 1481 return; 1482 1483 map = dev_event_to_vlpi_map(its_dev, event); 1484 1485 if (map->db_enabled == enable) 1486 return; 1487 1488 map->db_enabled = enable; 1489 1490 /* 1491 * More fun with the architecture: 1492 * 1493 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI 1494 * value or to 1023, depending on the enable bit. But that 1495 * would be issueing a mapping for an /existing/ DevID+EventID 1496 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI 1497 * to the /same/ vPE, using this opportunity to adjust the 1498 * doorbell. Mouahahahaha. We loves it, Precious. 1499 */ 1500 its_send_vmovi(its_dev, event); 1501 } 1502 1503 static void its_mask_irq(struct irq_data *d) 1504 { 1505 if (irqd_is_forwarded_to_vcpu(d)) 1506 its_vlpi_set_doorbell(d, false); 1507 1508 lpi_update_config(d, LPI_PROP_ENABLED, 0); 1509 } 1510 1511 static void its_unmask_irq(struct irq_data *d) 1512 { 1513 if (irqd_is_forwarded_to_vcpu(d)) 1514 its_vlpi_set_doorbell(d, true); 1515 1516 lpi_update_config(d, 0, LPI_PROP_ENABLED); 1517 } 1518 1519 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu) 1520 { 1521 if (irqd_affinity_is_managed(d)) 1522 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); 1523 1524 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); 1525 } 1526 1527 static void its_inc_lpi_count(struct irq_data *d, int cpu) 1528 { 1529 if (irqd_affinity_is_managed(d)) 1530 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); 1531 else 1532 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); 1533 } 1534 1535 static void its_dec_lpi_count(struct irq_data *d, int cpu) 1536 { 1537 if (irqd_affinity_is_managed(d)) 1538 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); 1539 else 1540 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); 1541 } 1542 1543 static unsigned int cpumask_pick_least_loaded(struct irq_data *d, 1544 const struct cpumask *cpu_mask) 1545 { 1546 unsigned int cpu = nr_cpu_ids, tmp; 1547 int count = S32_MAX; 1548 1549 for_each_cpu(tmp, cpu_mask) { 1550 int this_count = its_read_lpi_count(d, tmp); 1551 if (this_count < count) { 1552 cpu = tmp; 1553 count = this_count; 1554 } 1555 } 1556 1557 return cpu; 1558 } 1559 1560 /* 1561 * As suggested by Thomas Gleixner in: 1562 * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de 1563 */ 1564 static int its_select_cpu(struct irq_data *d, 1565 const struct cpumask *aff_mask) 1566 { 1567 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1568 cpumask_var_t tmpmask; 1569 int cpu, node; 1570 1571 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC)) 1572 return -ENOMEM; 1573 1574 node = its_dev->its->numa_node; 1575 1576 if (!irqd_affinity_is_managed(d)) { 1577 /* First try the NUMA node */ 1578 if (node != NUMA_NO_NODE) { 1579 /* 1580 * Try the intersection of the affinity mask and the 1581 * node mask (and the online mask, just to be safe). 1582 */ 1583 cpumask_and(tmpmask, cpumask_of_node(node), aff_mask); 1584 cpumask_and(tmpmask, tmpmask, cpu_online_mask); 1585 1586 /* 1587 * Ideally, we would check if the mask is empty, and 1588 * try again on the full node here. 1589 * 1590 * But it turns out that the way ACPI describes the 1591 * affinity for ITSs only deals about memory, and 1592 * not target CPUs, so it cannot describe a single 1593 * ITS placed next to two NUMA nodes. 1594 * 1595 * Instead, just fallback on the online mask. This 1596 * diverges from Thomas' suggestion above. 1597 */ 1598 cpu = cpumask_pick_least_loaded(d, tmpmask); 1599 if (cpu < nr_cpu_ids) 1600 goto out; 1601 1602 /* If we can't cross sockets, give up */ 1603 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)) 1604 goto out; 1605 1606 /* If the above failed, expand the search */ 1607 } 1608 1609 /* Try the intersection of the affinity and online masks */ 1610 cpumask_and(tmpmask, aff_mask, cpu_online_mask); 1611 1612 /* If that doesn't fly, the online mask is the last resort */ 1613 if (cpumask_empty(tmpmask)) 1614 cpumask_copy(tmpmask, cpu_online_mask); 1615 1616 cpu = cpumask_pick_least_loaded(d, tmpmask); 1617 } else { 1618 cpumask_and(tmpmask, irq_data_get_affinity_mask(d), cpu_online_mask); 1619 1620 /* If we cannot cross sockets, limit the search to that node */ 1621 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) && 1622 node != NUMA_NO_NODE) 1623 cpumask_and(tmpmask, tmpmask, cpumask_of_node(node)); 1624 1625 cpu = cpumask_pick_least_loaded(d, tmpmask); 1626 } 1627 out: 1628 free_cpumask_var(tmpmask); 1629 1630 pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu); 1631 return cpu; 1632 } 1633 1634 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1635 bool force) 1636 { 1637 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1638 struct its_collection *target_col; 1639 u32 id = its_get_event_id(d); 1640 int cpu, prev_cpu; 1641 1642 /* A forwarded interrupt should use irq_set_vcpu_affinity */ 1643 if (irqd_is_forwarded_to_vcpu(d)) 1644 return -EINVAL; 1645 1646 prev_cpu = its_dev->event_map.col_map[id]; 1647 its_dec_lpi_count(d, prev_cpu); 1648 1649 if (!force) 1650 cpu = its_select_cpu(d, mask_val); 1651 else 1652 cpu = cpumask_pick_least_loaded(d, mask_val); 1653 1654 if (cpu < 0 || cpu >= nr_cpu_ids) 1655 goto err; 1656 1657 /* don't set the affinity when the target cpu is same as current one */ 1658 if (cpu != prev_cpu) { 1659 target_col = &its_dev->its->collections[cpu]; 1660 its_send_movi(its_dev, target_col, id); 1661 its_dev->event_map.col_map[id] = cpu; 1662 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1663 } 1664 1665 its_inc_lpi_count(d, cpu); 1666 1667 return IRQ_SET_MASK_OK_DONE; 1668 1669 err: 1670 its_inc_lpi_count(d, prev_cpu); 1671 return -EINVAL; 1672 } 1673 1674 static u64 its_irq_get_msi_base(struct its_device *its_dev) 1675 { 1676 struct its_node *its = its_dev->its; 1677 1678 return its->phys_base + GITS_TRANSLATER; 1679 } 1680 1681 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) 1682 { 1683 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1684 struct its_node *its; 1685 u64 addr; 1686 1687 its = its_dev->its; 1688 addr = its->get_msi_base(its_dev); 1689 1690 msg->address_lo = lower_32_bits(addr); 1691 msg->address_hi = upper_32_bits(addr); 1692 msg->data = its_get_event_id(d); 1693 1694 iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg); 1695 } 1696 1697 static int its_irq_set_irqchip_state(struct irq_data *d, 1698 enum irqchip_irq_state which, 1699 bool state) 1700 { 1701 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1702 u32 event = its_get_event_id(d); 1703 1704 if (which != IRQCHIP_STATE_PENDING) 1705 return -EINVAL; 1706 1707 if (irqd_is_forwarded_to_vcpu(d)) { 1708 if (state) 1709 its_send_vint(its_dev, event); 1710 else 1711 its_send_vclear(its_dev, event); 1712 } else { 1713 if (state) 1714 its_send_int(its_dev, event); 1715 else 1716 its_send_clear(its_dev, event); 1717 } 1718 1719 return 0; 1720 } 1721 1722 static int its_irq_retrigger(struct irq_data *d) 1723 { 1724 return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true); 1725 } 1726 1727 /* 1728 * Two favourable cases: 1729 * 1730 * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times 1731 * for vSGI delivery 1732 * 1733 * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough 1734 * and we're better off mapping all VPEs always 1735 * 1736 * If neither (a) nor (b) is true, then we map vPEs on demand. 1737 * 1738 */ 1739 static bool gic_requires_eager_mapping(void) 1740 { 1741 if (!its_list_map || gic_rdists->has_rvpeid) 1742 return true; 1743 1744 return false; 1745 } 1746 1747 static void its_map_vm(struct its_node *its, struct its_vm *vm) 1748 { 1749 unsigned long flags; 1750 1751 if (gic_requires_eager_mapping()) 1752 return; 1753 1754 raw_spin_lock_irqsave(&vmovp_lock, flags); 1755 1756 /* 1757 * If the VM wasn't mapped yet, iterate over the vpes and get 1758 * them mapped now. 1759 */ 1760 vm->vlpi_count[its->list_nr]++; 1761 1762 if (vm->vlpi_count[its->list_nr] == 1) { 1763 int i; 1764 1765 for (i = 0; i < vm->nr_vpes; i++) { 1766 struct its_vpe *vpe = vm->vpes[i]; 1767 struct irq_data *d = irq_get_irq_data(vpe->irq); 1768 1769 /* Map the VPE to the first possible CPU */ 1770 vpe->col_idx = cpumask_first(cpu_online_mask); 1771 its_send_vmapp(its, vpe, true); 1772 its_send_vinvall(its, vpe); 1773 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 1774 } 1775 } 1776 1777 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1778 } 1779 1780 static void its_unmap_vm(struct its_node *its, struct its_vm *vm) 1781 { 1782 unsigned long flags; 1783 1784 /* Not using the ITS list? Everything is always mapped. */ 1785 if (gic_requires_eager_mapping()) 1786 return; 1787 1788 raw_spin_lock_irqsave(&vmovp_lock, flags); 1789 1790 if (!--vm->vlpi_count[its->list_nr]) { 1791 int i; 1792 1793 for (i = 0; i < vm->nr_vpes; i++) 1794 its_send_vmapp(its, vm->vpes[i], false); 1795 } 1796 1797 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1798 } 1799 1800 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) 1801 { 1802 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1803 u32 event = its_get_event_id(d); 1804 int ret = 0; 1805 1806 if (!info->map) 1807 return -EINVAL; 1808 1809 raw_spin_lock(&its_dev->event_map.vlpi_lock); 1810 1811 if (!its_dev->event_map.vm) { 1812 struct its_vlpi_map *maps; 1813 1814 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps), 1815 GFP_ATOMIC); 1816 if (!maps) { 1817 ret = -ENOMEM; 1818 goto out; 1819 } 1820 1821 its_dev->event_map.vm = info->map->vm; 1822 its_dev->event_map.vlpi_maps = maps; 1823 } else if (its_dev->event_map.vm != info->map->vm) { 1824 ret = -EINVAL; 1825 goto out; 1826 } 1827 1828 /* Get our private copy of the mapping information */ 1829 its_dev->event_map.vlpi_maps[event] = *info->map; 1830 1831 if (irqd_is_forwarded_to_vcpu(d)) { 1832 /* Already mapped, move it around */ 1833 its_send_vmovi(its_dev, event); 1834 } else { 1835 /* Ensure all the VPEs are mapped on this ITS */ 1836 its_map_vm(its_dev->its, info->map->vm); 1837 1838 /* 1839 * Flag the interrupt as forwarded so that we can 1840 * start poking the virtual property table. 1841 */ 1842 irqd_set_forwarded_to_vcpu(d); 1843 1844 /* Write out the property to the prop table */ 1845 lpi_write_config(d, 0xff, info->map->properties); 1846 1847 /* Drop the physical mapping */ 1848 its_send_discard(its_dev, event); 1849 1850 /* and install the virtual one */ 1851 its_send_vmapti(its_dev, event); 1852 1853 /* Increment the number of VLPIs */ 1854 its_dev->event_map.nr_vlpis++; 1855 } 1856 1857 out: 1858 raw_spin_unlock(&its_dev->event_map.vlpi_lock); 1859 return ret; 1860 } 1861 1862 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) 1863 { 1864 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1865 struct its_vlpi_map *map; 1866 int ret = 0; 1867 1868 raw_spin_lock(&its_dev->event_map.vlpi_lock); 1869 1870 map = get_vlpi_map(d); 1871 1872 if (!its_dev->event_map.vm || !map) { 1873 ret = -EINVAL; 1874 goto out; 1875 } 1876 1877 /* Copy our mapping information to the incoming request */ 1878 *info->map = *map; 1879 1880 out: 1881 raw_spin_unlock(&its_dev->event_map.vlpi_lock); 1882 return ret; 1883 } 1884 1885 static int its_vlpi_unmap(struct irq_data *d) 1886 { 1887 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1888 u32 event = its_get_event_id(d); 1889 int ret = 0; 1890 1891 raw_spin_lock(&its_dev->event_map.vlpi_lock); 1892 1893 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) { 1894 ret = -EINVAL; 1895 goto out; 1896 } 1897 1898 /* Drop the virtual mapping */ 1899 its_send_discard(its_dev, event); 1900 1901 /* and restore the physical one */ 1902 irqd_clr_forwarded_to_vcpu(d); 1903 its_send_mapti(its_dev, d->hwirq, event); 1904 lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO | 1905 LPI_PROP_ENABLED | 1906 LPI_PROP_GROUP1)); 1907 1908 /* Potentially unmap the VM from this ITS */ 1909 its_unmap_vm(its_dev->its, its_dev->event_map.vm); 1910 1911 /* 1912 * Drop the refcount and make the device available again if 1913 * this was the last VLPI. 1914 */ 1915 if (!--its_dev->event_map.nr_vlpis) { 1916 its_dev->event_map.vm = NULL; 1917 kfree(its_dev->event_map.vlpi_maps); 1918 } 1919 1920 out: 1921 raw_spin_unlock(&its_dev->event_map.vlpi_lock); 1922 return ret; 1923 } 1924 1925 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info) 1926 { 1927 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1928 1929 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) 1930 return -EINVAL; 1931 1932 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI) 1933 lpi_update_config(d, 0xff, info->config); 1934 else 1935 lpi_write_config(d, 0xff, info->config); 1936 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED)); 1937 1938 return 0; 1939 } 1940 1941 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 1942 { 1943 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1944 struct its_cmd_info *info = vcpu_info; 1945 1946 /* Need a v4 ITS */ 1947 if (!is_v4(its_dev->its)) 1948 return -EINVAL; 1949 1950 /* Unmap request? */ 1951 if (!info) 1952 return its_vlpi_unmap(d); 1953 1954 switch (info->cmd_type) { 1955 case MAP_VLPI: 1956 return its_vlpi_map(d, info); 1957 1958 case GET_VLPI: 1959 return its_vlpi_get(d, info); 1960 1961 case PROP_UPDATE_VLPI: 1962 case PROP_UPDATE_AND_INV_VLPI: 1963 return its_vlpi_prop_update(d, info); 1964 1965 default: 1966 return -EINVAL; 1967 } 1968 } 1969 1970 static struct irq_chip its_irq_chip = { 1971 .name = "ITS", 1972 .irq_mask = its_mask_irq, 1973 .irq_unmask = its_unmask_irq, 1974 .irq_eoi = irq_chip_eoi_parent, 1975 .irq_set_affinity = its_set_affinity, 1976 .irq_compose_msi_msg = its_irq_compose_msi_msg, 1977 .irq_set_irqchip_state = its_irq_set_irqchip_state, 1978 .irq_retrigger = its_irq_retrigger, 1979 .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity, 1980 }; 1981 1982 1983 /* 1984 * How we allocate LPIs: 1985 * 1986 * lpi_range_list contains ranges of LPIs that are to available to 1987 * allocate from. To allocate LPIs, just pick the first range that 1988 * fits the required allocation, and reduce it by the required 1989 * amount. Once empty, remove the range from the list. 1990 * 1991 * To free a range of LPIs, add a free range to the list, sort it and 1992 * merge the result if the new range happens to be adjacent to an 1993 * already free block. 1994 * 1995 * The consequence of the above is that allocation is cost is low, but 1996 * freeing is expensive. We assumes that freeing rarely occurs. 1997 */ 1998 #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */ 1999 2000 static DEFINE_MUTEX(lpi_range_lock); 2001 static LIST_HEAD(lpi_range_list); 2002 2003 struct lpi_range { 2004 struct list_head entry; 2005 u32 base_id; 2006 u32 span; 2007 }; 2008 2009 static struct lpi_range *mk_lpi_range(u32 base, u32 span) 2010 { 2011 struct lpi_range *range; 2012 2013 range = kmalloc(sizeof(*range), GFP_KERNEL); 2014 if (range) { 2015 range->base_id = base; 2016 range->span = span; 2017 } 2018 2019 return range; 2020 } 2021 2022 static int alloc_lpi_range(u32 nr_lpis, u32 *base) 2023 { 2024 struct lpi_range *range, *tmp; 2025 int err = -ENOSPC; 2026 2027 mutex_lock(&lpi_range_lock); 2028 2029 list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) { 2030 if (range->span >= nr_lpis) { 2031 *base = range->base_id; 2032 range->base_id += nr_lpis; 2033 range->span -= nr_lpis; 2034 2035 if (range->span == 0) { 2036 list_del(&range->entry); 2037 kfree(range); 2038 } 2039 2040 err = 0; 2041 break; 2042 } 2043 } 2044 2045 mutex_unlock(&lpi_range_lock); 2046 2047 pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis); 2048 return err; 2049 } 2050 2051 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b) 2052 { 2053 if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list) 2054 return; 2055 if (a->base_id + a->span != b->base_id) 2056 return; 2057 b->base_id = a->base_id; 2058 b->span += a->span; 2059 list_del(&a->entry); 2060 kfree(a); 2061 } 2062 2063 static int free_lpi_range(u32 base, u32 nr_lpis) 2064 { 2065 struct lpi_range *new, *old; 2066 2067 new = mk_lpi_range(base, nr_lpis); 2068 if (!new) 2069 return -ENOMEM; 2070 2071 mutex_lock(&lpi_range_lock); 2072 2073 list_for_each_entry_reverse(old, &lpi_range_list, entry) { 2074 if (old->base_id < base) 2075 break; 2076 } 2077 /* 2078 * old is the last element with ->base_id smaller than base, 2079 * so new goes right after it. If there are no elements with 2080 * ->base_id smaller than base, &old->entry ends up pointing 2081 * at the head of the list, and inserting new it the start of 2082 * the list is the right thing to do in that case as well. 2083 */ 2084 list_add(&new->entry, &old->entry); 2085 /* 2086 * Now check if we can merge with the preceding and/or 2087 * following ranges. 2088 */ 2089 merge_lpi_ranges(old, new); 2090 merge_lpi_ranges(new, list_next_entry(new, entry)); 2091 2092 mutex_unlock(&lpi_range_lock); 2093 return 0; 2094 } 2095 2096 static int __init its_lpi_init(u32 id_bits) 2097 { 2098 u32 lpis = (1UL << id_bits) - 8192; 2099 u32 numlpis; 2100 int err; 2101 2102 numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer); 2103 2104 if (numlpis > 2 && !WARN_ON(numlpis > lpis)) { 2105 lpis = numlpis; 2106 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n", 2107 lpis); 2108 } 2109 2110 /* 2111 * Initializing the allocator is just the same as freeing the 2112 * full range of LPIs. 2113 */ 2114 err = free_lpi_range(8192, lpis); 2115 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis); 2116 return err; 2117 } 2118 2119 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids) 2120 { 2121 unsigned long *bitmap = NULL; 2122 int err = 0; 2123 2124 do { 2125 err = alloc_lpi_range(nr_irqs, base); 2126 if (!err) 2127 break; 2128 2129 nr_irqs /= 2; 2130 } while (nr_irqs > 0); 2131 2132 if (!nr_irqs) 2133 err = -ENOSPC; 2134 2135 if (err) 2136 goto out; 2137 2138 bitmap = kcalloc(BITS_TO_LONGS(nr_irqs), sizeof (long), GFP_ATOMIC); 2139 if (!bitmap) 2140 goto out; 2141 2142 *nr_ids = nr_irqs; 2143 2144 out: 2145 if (!bitmap) 2146 *base = *nr_ids = 0; 2147 2148 return bitmap; 2149 } 2150 2151 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids) 2152 { 2153 WARN_ON(free_lpi_range(base, nr_ids)); 2154 kfree(bitmap); 2155 } 2156 2157 static void gic_reset_prop_table(void *va) 2158 { 2159 /* Priority 0xa0, Group-1, disabled */ 2160 memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ); 2161 2162 /* Make sure the GIC will observe the written configuration */ 2163 gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ); 2164 } 2165 2166 static struct page *its_allocate_prop_table(gfp_t gfp_flags) 2167 { 2168 struct page *prop_page; 2169 2170 prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); 2171 if (!prop_page) 2172 return NULL; 2173 2174 gic_reset_prop_table(page_address(prop_page)); 2175 2176 return prop_page; 2177 } 2178 2179 static void its_free_prop_table(struct page *prop_page) 2180 { 2181 free_pages((unsigned long)page_address(prop_page), 2182 get_order(LPI_PROPBASE_SZ)); 2183 } 2184 2185 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size) 2186 { 2187 phys_addr_t start, end, addr_end; 2188 u64 i; 2189 2190 /* 2191 * We don't bother checking for a kdump kernel as by 2192 * construction, the LPI tables are out of this kernel's 2193 * memory map. 2194 */ 2195 if (is_kdump_kernel()) 2196 return true; 2197 2198 addr_end = addr + size - 1; 2199 2200 for_each_reserved_mem_range(i, &start, &end) { 2201 if (addr >= start && addr_end <= end) 2202 return true; 2203 } 2204 2205 /* Not found, not a good sign... */ 2206 pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n", 2207 &addr, &addr_end); 2208 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 2209 return false; 2210 } 2211 2212 static int gic_reserve_range(phys_addr_t addr, unsigned long size) 2213 { 2214 if (efi_enabled(EFI_CONFIG_TABLES)) 2215 return efi_mem_reserve_persistent(addr, size); 2216 2217 return 0; 2218 } 2219 2220 static int __init its_setup_lpi_prop_table(void) 2221 { 2222 if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) { 2223 u64 val; 2224 2225 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER); 2226 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1; 2227 2228 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12); 2229 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa, 2230 LPI_PROPBASE_SZ, 2231 MEMREMAP_WB); 2232 gic_reset_prop_table(gic_rdists->prop_table_va); 2233 } else { 2234 struct page *page; 2235 2236 lpi_id_bits = min_t(u32, 2237 GICD_TYPER_ID_BITS(gic_rdists->gicd_typer), 2238 ITS_MAX_LPI_NRBITS); 2239 page = its_allocate_prop_table(GFP_NOWAIT); 2240 if (!page) { 2241 pr_err("Failed to allocate PROPBASE\n"); 2242 return -ENOMEM; 2243 } 2244 2245 gic_rdists->prop_table_pa = page_to_phys(page); 2246 gic_rdists->prop_table_va = page_address(page); 2247 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa, 2248 LPI_PROPBASE_SZ)); 2249 } 2250 2251 pr_info("GICv3: using LPI property table @%pa\n", 2252 &gic_rdists->prop_table_pa); 2253 2254 return its_lpi_init(lpi_id_bits); 2255 } 2256 2257 static const char *its_base_type_string[] = { 2258 [GITS_BASER_TYPE_DEVICE] = "Devices", 2259 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs", 2260 [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)", 2261 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections", 2262 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)", 2263 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)", 2264 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", 2265 }; 2266 2267 static u64 its_read_baser(struct its_node *its, struct its_baser *baser) 2268 { 2269 u32 idx = baser - its->tables; 2270 2271 return gits_read_baser(its->base + GITS_BASER + (idx << 3)); 2272 } 2273 2274 static void its_write_baser(struct its_node *its, struct its_baser *baser, 2275 u64 val) 2276 { 2277 u32 idx = baser - its->tables; 2278 2279 gits_write_baser(val, its->base + GITS_BASER + (idx << 3)); 2280 baser->val = its_read_baser(its, baser); 2281 } 2282 2283 static int its_setup_baser(struct its_node *its, struct its_baser *baser, 2284 u64 cache, u64 shr, u32 order, bool indirect) 2285 { 2286 u64 val = its_read_baser(its, baser); 2287 u64 esz = GITS_BASER_ENTRY_SIZE(val); 2288 u64 type = GITS_BASER_TYPE(val); 2289 u64 baser_phys, tmp; 2290 u32 alloc_pages, psz; 2291 struct page *page; 2292 void *base; 2293 2294 psz = baser->psz; 2295 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); 2296 if (alloc_pages > GITS_BASER_PAGES_MAX) { 2297 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", 2298 &its->phys_base, its_base_type_string[type], 2299 alloc_pages, GITS_BASER_PAGES_MAX); 2300 alloc_pages = GITS_BASER_PAGES_MAX; 2301 order = get_order(GITS_BASER_PAGES_MAX * psz); 2302 } 2303 2304 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); 2305 if (!page) 2306 return -ENOMEM; 2307 2308 base = (void *)page_address(page); 2309 baser_phys = virt_to_phys(base); 2310 2311 /* Check if the physical address of the memory is above 48bits */ 2312 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) { 2313 2314 /* 52bit PA is supported only when PageSize=64K */ 2315 if (psz != SZ_64K) { 2316 pr_err("ITS: no 52bit PA support when psz=%d\n", psz); 2317 free_pages((unsigned long)base, order); 2318 return -ENXIO; 2319 } 2320 2321 /* Convert 52bit PA to 48bit field */ 2322 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys); 2323 } 2324 2325 retry_baser: 2326 val = (baser_phys | 2327 (type << GITS_BASER_TYPE_SHIFT) | 2328 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | 2329 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) | 2330 cache | 2331 shr | 2332 GITS_BASER_VALID); 2333 2334 val |= indirect ? GITS_BASER_INDIRECT : 0x0; 2335 2336 switch (psz) { 2337 case SZ_4K: 2338 val |= GITS_BASER_PAGE_SIZE_4K; 2339 break; 2340 case SZ_16K: 2341 val |= GITS_BASER_PAGE_SIZE_16K; 2342 break; 2343 case SZ_64K: 2344 val |= GITS_BASER_PAGE_SIZE_64K; 2345 break; 2346 } 2347 2348 its_write_baser(its, baser, val); 2349 tmp = baser->val; 2350 2351 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { 2352 /* 2353 * Shareability didn't stick. Just use 2354 * whatever the read reported, which is likely 2355 * to be the only thing this redistributor 2356 * supports. If that's zero, make it 2357 * non-cacheable as well. 2358 */ 2359 shr = tmp & GITS_BASER_SHAREABILITY_MASK; 2360 if (!shr) { 2361 cache = GITS_BASER_nC; 2362 gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order)); 2363 } 2364 goto retry_baser; 2365 } 2366 2367 if (val != tmp) { 2368 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", 2369 &its->phys_base, its_base_type_string[type], 2370 val, tmp); 2371 free_pages((unsigned long)base, order); 2372 return -ENXIO; 2373 } 2374 2375 baser->order = order; 2376 baser->base = base; 2377 baser->psz = psz; 2378 tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz; 2379 2380 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n", 2381 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), 2382 its_base_type_string[type], 2383 (unsigned long)virt_to_phys(base), 2384 indirect ? "indirect" : "flat", (int)esz, 2385 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); 2386 2387 return 0; 2388 } 2389 2390 static bool its_parse_indirect_baser(struct its_node *its, 2391 struct its_baser *baser, 2392 u32 *order, u32 ids) 2393 { 2394 u64 tmp = its_read_baser(its, baser); 2395 u64 type = GITS_BASER_TYPE(tmp); 2396 u64 esz = GITS_BASER_ENTRY_SIZE(tmp); 2397 u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb; 2398 u32 new_order = *order; 2399 u32 psz = baser->psz; 2400 bool indirect = false; 2401 2402 /* No need to enable Indirection if memory requirement < (psz*2)bytes */ 2403 if ((esz << ids) > (psz * 2)) { 2404 /* 2405 * Find out whether hw supports a single or two-level table by 2406 * table by reading bit at offset '62' after writing '1' to it. 2407 */ 2408 its_write_baser(its, baser, val | GITS_BASER_INDIRECT); 2409 indirect = !!(baser->val & GITS_BASER_INDIRECT); 2410 2411 if (indirect) { 2412 /* 2413 * The size of the lvl2 table is equal to ITS page size 2414 * which is 'psz'. For computing lvl1 table size, 2415 * subtract ID bits that sparse lvl2 table from 'ids' 2416 * which is reported by ITS hardware times lvl1 table 2417 * entry size. 2418 */ 2419 ids -= ilog2(psz / (int)esz); 2420 esz = GITS_LVL1_ENTRY_SIZE; 2421 } 2422 } 2423 2424 /* 2425 * Allocate as many entries as required to fit the 2426 * range of device IDs that the ITS can grok... The ID 2427 * space being incredibly sparse, this results in a 2428 * massive waste of memory if two-level device table 2429 * feature is not supported by hardware. 2430 */ 2431 new_order = max_t(u32, get_order(esz << ids), new_order); 2432 if (new_order >= MAX_ORDER) { 2433 new_order = MAX_ORDER - 1; 2434 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); 2435 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n", 2436 &its->phys_base, its_base_type_string[type], 2437 device_ids(its), ids); 2438 } 2439 2440 *order = new_order; 2441 2442 return indirect; 2443 } 2444 2445 static u32 compute_common_aff(u64 val) 2446 { 2447 u32 aff, clpiaff; 2448 2449 aff = FIELD_GET(GICR_TYPER_AFFINITY, val); 2450 clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val); 2451 2452 return aff & ~(GENMASK(31, 0) >> (clpiaff * 8)); 2453 } 2454 2455 static u32 compute_its_aff(struct its_node *its) 2456 { 2457 u64 val; 2458 u32 svpet; 2459 2460 /* 2461 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute 2462 * the resulting affinity. We then use that to see if this match 2463 * our own affinity. 2464 */ 2465 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); 2466 val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet); 2467 val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr); 2468 return compute_common_aff(val); 2469 } 2470 2471 static struct its_node *find_sibling_its(struct its_node *cur_its) 2472 { 2473 struct its_node *its; 2474 u32 aff; 2475 2476 if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer)) 2477 return NULL; 2478 2479 aff = compute_its_aff(cur_its); 2480 2481 list_for_each_entry(its, &its_nodes, entry) { 2482 u64 baser; 2483 2484 if (!is_v4_1(its) || its == cur_its) 2485 continue; 2486 2487 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) 2488 continue; 2489 2490 if (aff != compute_its_aff(its)) 2491 continue; 2492 2493 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ 2494 baser = its->tables[2].val; 2495 if (!(baser & GITS_BASER_VALID)) 2496 continue; 2497 2498 return its; 2499 } 2500 2501 return NULL; 2502 } 2503 2504 static void its_free_tables(struct its_node *its) 2505 { 2506 int i; 2507 2508 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 2509 if (its->tables[i].base) { 2510 free_pages((unsigned long)its->tables[i].base, 2511 its->tables[i].order); 2512 its->tables[i].base = NULL; 2513 } 2514 } 2515 } 2516 2517 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser) 2518 { 2519 u64 psz = SZ_64K; 2520 2521 while (psz) { 2522 u64 val, gpsz; 2523 2524 val = its_read_baser(its, baser); 2525 val &= ~GITS_BASER_PAGE_SIZE_MASK; 2526 2527 switch (psz) { 2528 case SZ_64K: 2529 gpsz = GITS_BASER_PAGE_SIZE_64K; 2530 break; 2531 case SZ_16K: 2532 gpsz = GITS_BASER_PAGE_SIZE_16K; 2533 break; 2534 case SZ_4K: 2535 default: 2536 gpsz = GITS_BASER_PAGE_SIZE_4K; 2537 break; 2538 } 2539 2540 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT; 2541 2542 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz); 2543 its_write_baser(its, baser, val); 2544 2545 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz) 2546 break; 2547 2548 switch (psz) { 2549 case SZ_64K: 2550 psz = SZ_16K; 2551 break; 2552 case SZ_16K: 2553 psz = SZ_4K; 2554 break; 2555 case SZ_4K: 2556 default: 2557 return -1; 2558 } 2559 } 2560 2561 baser->psz = psz; 2562 return 0; 2563 } 2564 2565 static int its_alloc_tables(struct its_node *its) 2566 { 2567 u64 shr = GITS_BASER_InnerShareable; 2568 u64 cache = GITS_BASER_RaWaWb; 2569 int err, i; 2570 2571 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) 2572 /* erratum 24313: ignore memory access type */ 2573 cache = GITS_BASER_nCnB; 2574 2575 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 2576 struct its_baser *baser = its->tables + i; 2577 u64 val = its_read_baser(its, baser); 2578 u64 type = GITS_BASER_TYPE(val); 2579 bool indirect = false; 2580 u32 order; 2581 2582 if (type == GITS_BASER_TYPE_NONE) 2583 continue; 2584 2585 if (its_probe_baser_psz(its, baser)) { 2586 its_free_tables(its); 2587 return -ENXIO; 2588 } 2589 2590 order = get_order(baser->psz); 2591 2592 switch (type) { 2593 case GITS_BASER_TYPE_DEVICE: 2594 indirect = its_parse_indirect_baser(its, baser, &order, 2595 device_ids(its)); 2596 break; 2597 2598 case GITS_BASER_TYPE_VCPU: 2599 if (is_v4_1(its)) { 2600 struct its_node *sibling; 2601 2602 WARN_ON(i != 2); 2603 if ((sibling = find_sibling_its(its))) { 2604 *baser = sibling->tables[2]; 2605 its_write_baser(its, baser, baser->val); 2606 continue; 2607 } 2608 } 2609 2610 indirect = its_parse_indirect_baser(its, baser, &order, 2611 ITS_MAX_VPEID_BITS); 2612 break; 2613 } 2614 2615 err = its_setup_baser(its, baser, cache, shr, order, indirect); 2616 if (err < 0) { 2617 its_free_tables(its); 2618 return err; 2619 } 2620 2621 /* Update settings which will be used for next BASERn */ 2622 cache = baser->val & GITS_BASER_CACHEABILITY_MASK; 2623 shr = baser->val & GITS_BASER_SHAREABILITY_MASK; 2624 } 2625 2626 return 0; 2627 } 2628 2629 static u64 inherit_vpe_l1_table_from_its(void) 2630 { 2631 struct its_node *its; 2632 u64 val; 2633 u32 aff; 2634 2635 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 2636 aff = compute_common_aff(val); 2637 2638 list_for_each_entry(its, &its_nodes, entry) { 2639 u64 baser, addr; 2640 2641 if (!is_v4_1(its)) 2642 continue; 2643 2644 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) 2645 continue; 2646 2647 if (aff != compute_its_aff(its)) 2648 continue; 2649 2650 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ 2651 baser = its->tables[2].val; 2652 if (!(baser & GITS_BASER_VALID)) 2653 continue; 2654 2655 /* We have a winner! */ 2656 gic_data_rdist()->vpe_l1_base = its->tables[2].base; 2657 2658 val = GICR_VPROPBASER_4_1_VALID; 2659 if (baser & GITS_BASER_INDIRECT) 2660 val |= GICR_VPROPBASER_4_1_INDIRECT; 2661 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, 2662 FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)); 2663 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) { 2664 case GIC_PAGE_SIZE_64K: 2665 addr = GITS_BASER_ADDR_48_to_52(baser); 2666 break; 2667 default: 2668 addr = baser & GENMASK_ULL(47, 12); 2669 break; 2670 } 2671 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12); 2672 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK, 2673 FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser)); 2674 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK, 2675 FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser)); 2676 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1); 2677 2678 return val; 2679 } 2680 2681 return 0; 2682 } 2683 2684 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) 2685 { 2686 u32 aff; 2687 u64 val; 2688 int cpu; 2689 2690 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 2691 aff = compute_common_aff(val); 2692 2693 for_each_possible_cpu(cpu) { 2694 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; 2695 2696 if (!base || cpu == smp_processor_id()) 2697 continue; 2698 2699 val = gic_read_typer(base + GICR_TYPER); 2700 if (aff != compute_common_aff(val)) 2701 continue; 2702 2703 /* 2704 * At this point, we have a victim. This particular CPU 2705 * has already booted, and has an affinity that matches 2706 * ours wrt CommonLPIAff. Let's use its own VPROPBASER. 2707 * Make sure we don't write the Z bit in that case. 2708 */ 2709 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); 2710 val &= ~GICR_VPROPBASER_4_1_Z; 2711 2712 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; 2713 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; 2714 2715 return val; 2716 } 2717 2718 return 0; 2719 } 2720 2721 static bool allocate_vpe_l2_table(int cpu, u32 id) 2722 { 2723 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; 2724 unsigned int psz, esz, idx, npg, gpsz; 2725 u64 val; 2726 struct page *page; 2727 __le64 *table; 2728 2729 if (!gic_rdists->has_rvpeid) 2730 return true; 2731 2732 /* Skip non-present CPUs */ 2733 if (!base) 2734 return true; 2735 2736 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); 2737 2738 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1; 2739 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); 2740 npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1; 2741 2742 switch (gpsz) { 2743 default: 2744 WARN_ON(1); 2745 fallthrough; 2746 case GIC_PAGE_SIZE_4K: 2747 psz = SZ_4K; 2748 break; 2749 case GIC_PAGE_SIZE_16K: 2750 psz = SZ_16K; 2751 break; 2752 case GIC_PAGE_SIZE_64K: 2753 psz = SZ_64K; 2754 break; 2755 } 2756 2757 /* Don't allow vpe_id that exceeds single, flat table limit */ 2758 if (!(val & GICR_VPROPBASER_4_1_INDIRECT)) 2759 return (id < (npg * psz / (esz * SZ_8))); 2760 2761 /* Compute 1st level table index & check if that exceeds table limit */ 2762 idx = id >> ilog2(psz / (esz * SZ_8)); 2763 if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE)) 2764 return false; 2765 2766 table = gic_data_rdist_cpu(cpu)->vpe_l1_base; 2767 2768 /* Allocate memory for 2nd level table */ 2769 if (!table[idx]) { 2770 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); 2771 if (!page) 2772 return false; 2773 2774 /* Flush Lvl2 table to PoC if hw doesn't support coherency */ 2775 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) 2776 gic_flush_dcache_to_poc(page_address(page), psz); 2777 2778 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); 2779 2780 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ 2781 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) 2782 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); 2783 2784 /* Ensure updated table contents are visible to RD hardware */ 2785 dsb(sy); 2786 } 2787 2788 return true; 2789 } 2790 2791 static int allocate_vpe_l1_table(void) 2792 { 2793 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 2794 u64 val, gpsz, npg, pa; 2795 unsigned int psz = SZ_64K; 2796 unsigned int np, epp, esz; 2797 struct page *page; 2798 2799 if (!gic_rdists->has_rvpeid) 2800 return 0; 2801 2802 /* 2803 * if VPENDBASER.Valid is set, disable any previously programmed 2804 * VPE by setting PendingLast while clearing Valid. This has the 2805 * effect of making sure no doorbell will be generated and we can 2806 * then safely clear VPROPBASER.Valid. 2807 */ 2808 if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) 2809 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 2810 vlpi_base + GICR_VPENDBASER); 2811 2812 /* 2813 * If we can inherit the configuration from another RD, let's do 2814 * so. Otherwise, we have to go through the allocation process. We 2815 * assume that all RDs have the exact same requirements, as 2816 * nothing will work otherwise. 2817 */ 2818 val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask); 2819 if (val & GICR_VPROPBASER_4_1_VALID) 2820 goto out; 2821 2822 gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC); 2823 if (!gic_data_rdist()->vpe_table_mask) 2824 return -ENOMEM; 2825 2826 val = inherit_vpe_l1_table_from_its(); 2827 if (val & GICR_VPROPBASER_4_1_VALID) 2828 goto out; 2829 2830 /* First probe the page size */ 2831 val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K); 2832 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 2833 val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER); 2834 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); 2835 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val); 2836 2837 switch (gpsz) { 2838 default: 2839 gpsz = GIC_PAGE_SIZE_4K; 2840 fallthrough; 2841 case GIC_PAGE_SIZE_4K: 2842 psz = SZ_4K; 2843 break; 2844 case GIC_PAGE_SIZE_16K: 2845 psz = SZ_16K; 2846 break; 2847 case GIC_PAGE_SIZE_64K: 2848 psz = SZ_64K; 2849 break; 2850 } 2851 2852 /* 2853 * Start populating the register from scratch, including RO fields 2854 * (which we want to print in debug cases...) 2855 */ 2856 val = 0; 2857 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz); 2858 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz); 2859 2860 /* How many entries per GIC page? */ 2861 esz++; 2862 epp = psz / (esz * SZ_8); 2863 2864 /* 2865 * If we need more than just a single L1 page, flag the table 2866 * as indirect and compute the number of required L1 pages. 2867 */ 2868 if (epp < ITS_MAX_VPEID) { 2869 int nl2; 2870 2871 val |= GICR_VPROPBASER_4_1_INDIRECT; 2872 2873 /* Number of L2 pages required to cover the VPEID space */ 2874 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp); 2875 2876 /* Number of L1 pages to point to the L2 pages */ 2877 npg = DIV_ROUND_UP(nl2 * SZ_8, psz); 2878 } else { 2879 npg = 1; 2880 } 2881 2882 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1); 2883 2884 /* Right, that's the number of CPU pages we need for L1 */ 2885 np = DIV_ROUND_UP(npg * psz, PAGE_SIZE); 2886 2887 pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", 2888 np, npg, psz, epp, esz); 2889 page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); 2890 if (!page) 2891 return -ENOMEM; 2892 2893 gic_data_rdist()->vpe_l1_base = page_address(page); 2894 pa = virt_to_phys(page_address(page)); 2895 WARN_ON(!IS_ALIGNED(pa, psz)); 2896 2897 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12); 2898 val |= GICR_VPROPBASER_RaWb; 2899 val |= GICR_VPROPBASER_InnerShareable; 2900 val |= GICR_VPROPBASER_4_1_Z; 2901 val |= GICR_VPROPBASER_4_1_VALID; 2902 2903 out: 2904 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 2905 cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); 2906 2907 pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n", 2908 smp_processor_id(), val, 2909 cpumask_pr_args(gic_data_rdist()->vpe_table_mask)); 2910 2911 return 0; 2912 } 2913 2914 static int its_alloc_collections(struct its_node *its) 2915 { 2916 int i; 2917 2918 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections), 2919 GFP_KERNEL); 2920 if (!its->collections) 2921 return -ENOMEM; 2922 2923 for (i = 0; i < nr_cpu_ids; i++) 2924 its->collections[i].target_address = ~0ULL; 2925 2926 return 0; 2927 } 2928 2929 static struct page *its_allocate_pending_table(gfp_t gfp_flags) 2930 { 2931 struct page *pend_page; 2932 2933 pend_page = alloc_pages(gfp_flags | __GFP_ZERO, 2934 get_order(LPI_PENDBASE_SZ)); 2935 if (!pend_page) 2936 return NULL; 2937 2938 /* Make sure the GIC will observe the zero-ed page */ 2939 gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ); 2940 2941 return pend_page; 2942 } 2943 2944 static void its_free_pending_table(struct page *pt) 2945 { 2946 free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); 2947 } 2948 2949 /* 2950 * Booting with kdump and LPIs enabled is generally fine. Any other 2951 * case is wrong in the absence of firmware/EFI support. 2952 */ 2953 static bool enabled_lpis_allowed(void) 2954 { 2955 phys_addr_t addr; 2956 u64 val; 2957 2958 /* Check whether the property table is in a reserved region */ 2959 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER); 2960 addr = val & GENMASK_ULL(51, 12); 2961 2962 return gic_check_reserved_range(addr, LPI_PROPBASE_SZ); 2963 } 2964 2965 static int __init allocate_lpi_tables(void) 2966 { 2967 u64 val; 2968 int err, cpu; 2969 2970 /* 2971 * If LPIs are enabled while we run this from the boot CPU, 2972 * flag the RD tables as pre-allocated if the stars do align. 2973 */ 2974 val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR); 2975 if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) { 2976 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED | 2977 RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING); 2978 pr_info("GICv3: Using preallocated redistributor tables\n"); 2979 } 2980 2981 err = its_setup_lpi_prop_table(); 2982 if (err) 2983 return err; 2984 2985 /* 2986 * We allocate all the pending tables anyway, as we may have a 2987 * mix of RDs that have had LPIs enabled, and some that 2988 * don't. We'll free the unused ones as each CPU comes online. 2989 */ 2990 for_each_possible_cpu(cpu) { 2991 struct page *pend_page; 2992 2993 pend_page = its_allocate_pending_table(GFP_NOWAIT); 2994 if (!pend_page) { 2995 pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu); 2996 return -ENOMEM; 2997 } 2998 2999 gic_data_rdist_cpu(cpu)->pend_page = pend_page; 3000 } 3001 3002 return 0; 3003 } 3004 3005 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) 3006 { 3007 u32 count = 1000000; /* 1s! */ 3008 bool clean; 3009 u64 val; 3010 3011 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); 3012 val &= ~GICR_VPENDBASER_Valid; 3013 val &= ~clr; 3014 val |= set; 3015 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 3016 3017 do { 3018 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); 3019 clean = !(val & GICR_VPENDBASER_Dirty); 3020 if (!clean) { 3021 count--; 3022 cpu_relax(); 3023 udelay(1); 3024 } 3025 } while (!clean && count); 3026 3027 if (unlikely(val & GICR_VPENDBASER_Dirty)) { 3028 pr_err_ratelimited("ITS virtual pending table not cleaning\n"); 3029 val |= GICR_VPENDBASER_PendingLast; 3030 } 3031 3032 return val; 3033 } 3034 3035 static void its_cpu_init_lpis(void) 3036 { 3037 void __iomem *rbase = gic_data_rdist_rd_base(); 3038 struct page *pend_page; 3039 phys_addr_t paddr; 3040 u64 val, tmp; 3041 3042 if (gic_data_rdist()->lpi_enabled) 3043 return; 3044 3045 val = readl_relaxed(rbase + GICR_CTLR); 3046 if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) && 3047 (val & GICR_CTLR_ENABLE_LPIS)) { 3048 /* 3049 * Check that we get the same property table on all 3050 * RDs. If we don't, this is hopeless. 3051 */ 3052 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER); 3053 paddr &= GENMASK_ULL(51, 12); 3054 if (WARN_ON(gic_rdists->prop_table_pa != paddr)) 3055 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 3056 3057 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER); 3058 paddr &= GENMASK_ULL(51, 16); 3059 3060 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ)); 3061 its_free_pending_table(gic_data_rdist()->pend_page); 3062 gic_data_rdist()->pend_page = NULL; 3063 3064 goto out; 3065 } 3066 3067 pend_page = gic_data_rdist()->pend_page; 3068 paddr = page_to_phys(pend_page); 3069 WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ)); 3070 3071 /* set PROPBASE */ 3072 val = (gic_rdists->prop_table_pa | 3073 GICR_PROPBASER_InnerShareable | 3074 GICR_PROPBASER_RaWaWb | 3075 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); 3076 3077 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 3078 tmp = gicr_read_propbaser(rbase + GICR_PROPBASER); 3079 3080 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { 3081 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) { 3082 /* 3083 * The HW reports non-shareable, we must 3084 * remove the cacheability attributes as 3085 * well. 3086 */ 3087 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK | 3088 GICR_PROPBASER_CACHEABILITY_MASK); 3089 val |= GICR_PROPBASER_nC; 3090 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 3091 } 3092 pr_info_once("GIC: using cache flushing for LPI property table\n"); 3093 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; 3094 } 3095 3096 /* set PENDBASE */ 3097 val = (page_to_phys(pend_page) | 3098 GICR_PENDBASER_InnerShareable | 3099 GICR_PENDBASER_RaWaWb); 3100 3101 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 3102 tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER); 3103 3104 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { 3105 /* 3106 * The HW reports non-shareable, we must remove the 3107 * cacheability attributes as well. 3108 */ 3109 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK | 3110 GICR_PENDBASER_CACHEABILITY_MASK); 3111 val |= GICR_PENDBASER_nC; 3112 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 3113 } 3114 3115 /* Enable LPIs */ 3116 val = readl_relaxed(rbase + GICR_CTLR); 3117 val |= GICR_CTLR_ENABLE_LPIS; 3118 writel_relaxed(val, rbase + GICR_CTLR); 3119 3120 if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) { 3121 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3122 3123 /* 3124 * It's possible for CPU to receive VLPIs before it is 3125 * sheduled as a vPE, especially for the first CPU, and the 3126 * VLPI with INTID larger than 2^(IDbits+1) will be considered 3127 * as out of range and dropped by GIC. 3128 * So we initialize IDbits to known value to avoid VLPI drop. 3129 */ 3130 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; 3131 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n", 3132 smp_processor_id(), val); 3133 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 3134 3135 /* 3136 * Also clear Valid bit of GICR_VPENDBASER, in case some 3137 * ancient programming gets left in and has possibility of 3138 * corrupting memory. 3139 */ 3140 val = its_clear_vpend_valid(vlpi_base, 0, 0); 3141 } 3142 3143 if (allocate_vpe_l1_table()) { 3144 /* 3145 * If the allocation has failed, we're in massive trouble. 3146 * Disable direct injection, and pray that no VM was 3147 * already running... 3148 */ 3149 gic_rdists->has_rvpeid = false; 3150 gic_rdists->has_vlpis = false; 3151 } 3152 3153 /* Make sure the GIC has seen the above */ 3154 dsb(sy); 3155 out: 3156 gic_data_rdist()->lpi_enabled = true; 3157 pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n", 3158 smp_processor_id(), 3159 gic_data_rdist()->pend_page ? "allocated" : "reserved", 3160 &paddr); 3161 } 3162 3163 static void its_cpu_init_collection(struct its_node *its) 3164 { 3165 int cpu = smp_processor_id(); 3166 u64 target; 3167 3168 /* avoid cross node collections and its mapping */ 3169 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { 3170 struct device_node *cpu_node; 3171 3172 cpu_node = of_get_cpu_node(cpu, NULL); 3173 if (its->numa_node != NUMA_NO_NODE && 3174 its->numa_node != of_node_to_nid(cpu_node)) 3175 return; 3176 } 3177 3178 /* 3179 * We now have to bind each collection to its target 3180 * redistributor. 3181 */ 3182 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) { 3183 /* 3184 * This ITS wants the physical address of the 3185 * redistributor. 3186 */ 3187 target = gic_data_rdist()->phys_base; 3188 } else { 3189 /* This ITS wants a linear CPU number. */ 3190 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 3191 target = GICR_TYPER_CPU_NUMBER(target) << 16; 3192 } 3193 3194 /* Perform collection mapping */ 3195 its->collections[cpu].target_address = target; 3196 its->collections[cpu].col_id = cpu; 3197 3198 its_send_mapc(its, &its->collections[cpu], 1); 3199 its_send_invall(its, &its->collections[cpu]); 3200 } 3201 3202 static void its_cpu_init_collections(void) 3203 { 3204 struct its_node *its; 3205 3206 raw_spin_lock(&its_lock); 3207 3208 list_for_each_entry(its, &its_nodes, entry) 3209 its_cpu_init_collection(its); 3210 3211 raw_spin_unlock(&its_lock); 3212 } 3213 3214 static struct its_device *its_find_device(struct its_node *its, u32 dev_id) 3215 { 3216 struct its_device *its_dev = NULL, *tmp; 3217 unsigned long flags; 3218 3219 raw_spin_lock_irqsave(&its->lock, flags); 3220 3221 list_for_each_entry(tmp, &its->its_device_list, entry) { 3222 if (tmp->device_id == dev_id) { 3223 its_dev = tmp; 3224 break; 3225 } 3226 } 3227 3228 raw_spin_unlock_irqrestore(&its->lock, flags); 3229 3230 return its_dev; 3231 } 3232 3233 static struct its_baser *its_get_baser(struct its_node *its, u32 type) 3234 { 3235 int i; 3236 3237 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 3238 if (GITS_BASER_TYPE(its->tables[i].val) == type) 3239 return &its->tables[i]; 3240 } 3241 3242 return NULL; 3243 } 3244 3245 static bool its_alloc_table_entry(struct its_node *its, 3246 struct its_baser *baser, u32 id) 3247 { 3248 struct page *page; 3249 u32 esz, idx; 3250 __le64 *table; 3251 3252 /* Don't allow device id that exceeds single, flat table limit */ 3253 esz = GITS_BASER_ENTRY_SIZE(baser->val); 3254 if (!(baser->val & GITS_BASER_INDIRECT)) 3255 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); 3256 3257 /* Compute 1st level table index & check if that exceeds table limit */ 3258 idx = id >> ilog2(baser->psz / esz); 3259 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) 3260 return false; 3261 3262 table = baser->base; 3263 3264 /* Allocate memory for 2nd level table */ 3265 if (!table[idx]) { 3266 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, 3267 get_order(baser->psz)); 3268 if (!page) 3269 return false; 3270 3271 /* Flush Lvl2 table to PoC if hw doesn't support coherency */ 3272 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 3273 gic_flush_dcache_to_poc(page_address(page), baser->psz); 3274 3275 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); 3276 3277 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ 3278 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 3279 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); 3280 3281 /* Ensure updated table contents are visible to ITS hardware */ 3282 dsb(sy); 3283 } 3284 3285 return true; 3286 } 3287 3288 static bool its_alloc_device_table(struct its_node *its, u32 dev_id) 3289 { 3290 struct its_baser *baser; 3291 3292 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); 3293 3294 /* Don't allow device id that exceeds ITS hardware limit */ 3295 if (!baser) 3296 return (ilog2(dev_id) < device_ids(its)); 3297 3298 return its_alloc_table_entry(its, baser, dev_id); 3299 } 3300 3301 static bool its_alloc_vpe_table(u32 vpe_id) 3302 { 3303 struct its_node *its; 3304 int cpu; 3305 3306 /* 3307 * Make sure the L2 tables are allocated on *all* v4 ITSs. We 3308 * could try and only do it on ITSs corresponding to devices 3309 * that have interrupts targeted at this VPE, but the 3310 * complexity becomes crazy (and you have tons of memory 3311 * anyway, right?). 3312 */ 3313 list_for_each_entry(its, &its_nodes, entry) { 3314 struct its_baser *baser; 3315 3316 if (!is_v4(its)) 3317 continue; 3318 3319 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU); 3320 if (!baser) 3321 return false; 3322 3323 if (!its_alloc_table_entry(its, baser, vpe_id)) 3324 return false; 3325 } 3326 3327 /* Non v4.1? No need to iterate RDs and go back early. */ 3328 if (!gic_rdists->has_rvpeid) 3329 return true; 3330 3331 /* 3332 * Make sure the L2 tables are allocated for all copies of 3333 * the L1 table on *all* v4.1 RDs. 3334 */ 3335 for_each_possible_cpu(cpu) { 3336 if (!allocate_vpe_l2_table(cpu, vpe_id)) 3337 return false; 3338 } 3339 3340 return true; 3341 } 3342 3343 static struct its_device *its_create_device(struct its_node *its, u32 dev_id, 3344 int nvecs, bool alloc_lpis) 3345 { 3346 struct its_device *dev; 3347 unsigned long *lpi_map = NULL; 3348 unsigned long flags; 3349 u16 *col_map = NULL; 3350 void *itt; 3351 int lpi_base; 3352 int nr_lpis; 3353 int nr_ites; 3354 int sz; 3355 3356 if (!its_alloc_device_table(its, dev_id)) 3357 return NULL; 3358 3359 if (WARN_ON(!is_power_of_2(nvecs))) 3360 nvecs = roundup_pow_of_two(nvecs); 3361 3362 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 3363 /* 3364 * Even if the device wants a single LPI, the ITT must be 3365 * sized as a power of two (and you need at least one bit...). 3366 */ 3367 nr_ites = max(2, nvecs); 3368 sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); 3369 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; 3370 itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); 3371 if (alloc_lpis) { 3372 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis); 3373 if (lpi_map) 3374 col_map = kcalloc(nr_lpis, sizeof(*col_map), 3375 GFP_KERNEL); 3376 } else { 3377 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL); 3378 nr_lpis = 0; 3379 lpi_base = 0; 3380 } 3381 3382 if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { 3383 kfree(dev); 3384 kfree(itt); 3385 kfree(lpi_map); 3386 kfree(col_map); 3387 return NULL; 3388 } 3389 3390 gic_flush_dcache_to_poc(itt, sz); 3391 3392 dev->its = its; 3393 dev->itt = itt; 3394 dev->nr_ites = nr_ites; 3395 dev->event_map.lpi_map = lpi_map; 3396 dev->event_map.col_map = col_map; 3397 dev->event_map.lpi_base = lpi_base; 3398 dev->event_map.nr_lpis = nr_lpis; 3399 raw_spin_lock_init(&dev->event_map.vlpi_lock); 3400 dev->device_id = dev_id; 3401 INIT_LIST_HEAD(&dev->entry); 3402 3403 raw_spin_lock_irqsave(&its->lock, flags); 3404 list_add(&dev->entry, &its->its_device_list); 3405 raw_spin_unlock_irqrestore(&its->lock, flags); 3406 3407 /* Map device to its ITT */ 3408 its_send_mapd(dev, 1); 3409 3410 return dev; 3411 } 3412 3413 static void its_free_device(struct its_device *its_dev) 3414 { 3415 unsigned long flags; 3416 3417 raw_spin_lock_irqsave(&its_dev->its->lock, flags); 3418 list_del(&its_dev->entry); 3419 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); 3420 kfree(its_dev->event_map.col_map); 3421 kfree(its_dev->itt); 3422 kfree(its_dev); 3423 } 3424 3425 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq) 3426 { 3427 int idx; 3428 3429 /* Find a free LPI region in lpi_map and allocate them. */ 3430 idx = bitmap_find_free_region(dev->event_map.lpi_map, 3431 dev->event_map.nr_lpis, 3432 get_count_order(nvecs)); 3433 if (idx < 0) 3434 return -ENOSPC; 3435 3436 *hwirq = dev->event_map.lpi_base + idx; 3437 3438 return 0; 3439 } 3440 3441 static int its_msi_prepare(struct irq_domain *domain, struct device *dev, 3442 int nvec, msi_alloc_info_t *info) 3443 { 3444 struct its_node *its; 3445 struct its_device *its_dev; 3446 struct msi_domain_info *msi_info; 3447 u32 dev_id; 3448 int err = 0; 3449 3450 /* 3451 * We ignore "dev" entirely, and rely on the dev_id that has 3452 * been passed via the scratchpad. This limits this domain's 3453 * usefulness to upper layers that definitely know that they 3454 * are built on top of the ITS. 3455 */ 3456 dev_id = info->scratchpad[0].ul; 3457 3458 msi_info = msi_get_domain_info(domain); 3459 its = msi_info->data; 3460 3461 if (!gic_rdists->has_direct_lpi && 3462 vpe_proxy.dev && 3463 vpe_proxy.dev->its == its && 3464 dev_id == vpe_proxy.dev->device_id) { 3465 /* Bad luck. Get yourself a better implementation */ 3466 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n", 3467 dev_id); 3468 return -EINVAL; 3469 } 3470 3471 mutex_lock(&its->dev_alloc_lock); 3472 its_dev = its_find_device(its, dev_id); 3473 if (its_dev) { 3474 /* 3475 * We already have seen this ID, probably through 3476 * another alias (PCI bridge of some sort). No need to 3477 * create the device. 3478 */ 3479 its_dev->shared = true; 3480 pr_debug("Reusing ITT for devID %x\n", dev_id); 3481 goto out; 3482 } 3483 3484 its_dev = its_create_device(its, dev_id, nvec, true); 3485 if (!its_dev) { 3486 err = -ENOMEM; 3487 goto out; 3488 } 3489 3490 if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE) 3491 its_dev->shared = true; 3492 3493 pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec)); 3494 out: 3495 mutex_unlock(&its->dev_alloc_lock); 3496 info->scratchpad[0].ptr = its_dev; 3497 return err; 3498 } 3499 3500 static struct msi_domain_ops its_msi_domain_ops = { 3501 .msi_prepare = its_msi_prepare, 3502 }; 3503 3504 static int its_irq_gic_domain_alloc(struct irq_domain *domain, 3505 unsigned int virq, 3506 irq_hw_number_t hwirq) 3507 { 3508 struct irq_fwspec fwspec; 3509 3510 if (irq_domain_get_of_node(domain->parent)) { 3511 fwspec.fwnode = domain->parent->fwnode; 3512 fwspec.param_count = 3; 3513 fwspec.param[0] = GIC_IRQ_TYPE_LPI; 3514 fwspec.param[1] = hwirq; 3515 fwspec.param[2] = IRQ_TYPE_EDGE_RISING; 3516 } else if (is_fwnode_irqchip(domain->parent->fwnode)) { 3517 fwspec.fwnode = domain->parent->fwnode; 3518 fwspec.param_count = 2; 3519 fwspec.param[0] = hwirq; 3520 fwspec.param[1] = IRQ_TYPE_EDGE_RISING; 3521 } else { 3522 return -EINVAL; 3523 } 3524 3525 return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); 3526 } 3527 3528 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 3529 unsigned int nr_irqs, void *args) 3530 { 3531 msi_alloc_info_t *info = args; 3532 struct its_device *its_dev = info->scratchpad[0].ptr; 3533 struct its_node *its = its_dev->its; 3534 struct irq_data *irqd; 3535 irq_hw_number_t hwirq; 3536 int err; 3537 int i; 3538 3539 err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq); 3540 if (err) 3541 return err; 3542 3543 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev)); 3544 if (err) 3545 return err; 3546 3547 for (i = 0; i < nr_irqs; i++) { 3548 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i); 3549 if (err) 3550 return err; 3551 3552 irq_domain_set_hwirq_and_chip(domain, virq + i, 3553 hwirq + i, &its_irq_chip, its_dev); 3554 irqd = irq_get_irq_data(virq + i); 3555 irqd_set_single_target(irqd); 3556 irqd_set_affinity_on_activate(irqd); 3557 pr_debug("ID:%d pID:%d vID:%d\n", 3558 (int)(hwirq + i - its_dev->event_map.lpi_base), 3559 (int)(hwirq + i), virq + i); 3560 } 3561 3562 return 0; 3563 } 3564 3565 static int its_irq_domain_activate(struct irq_domain *domain, 3566 struct irq_data *d, bool reserve) 3567 { 3568 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 3569 u32 event = its_get_event_id(d); 3570 int cpu; 3571 3572 cpu = its_select_cpu(d, cpu_online_mask); 3573 if (cpu < 0 || cpu >= nr_cpu_ids) 3574 return -EINVAL; 3575 3576 its_inc_lpi_count(d, cpu); 3577 its_dev->event_map.col_map[event] = cpu; 3578 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 3579 3580 /* Map the GIC IRQ and event to the device */ 3581 its_send_mapti(its_dev, d->hwirq, event); 3582 return 0; 3583 } 3584 3585 static void its_irq_domain_deactivate(struct irq_domain *domain, 3586 struct irq_data *d) 3587 { 3588 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 3589 u32 event = its_get_event_id(d); 3590 3591 its_dec_lpi_count(d, its_dev->event_map.col_map[event]); 3592 /* Stop the delivery of interrupts */ 3593 its_send_discard(its_dev, event); 3594 } 3595 3596 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, 3597 unsigned int nr_irqs) 3598 { 3599 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 3600 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 3601 struct its_node *its = its_dev->its; 3602 int i; 3603 3604 bitmap_release_region(its_dev->event_map.lpi_map, 3605 its_get_event_id(irq_domain_get_irq_data(domain, virq)), 3606 get_count_order(nr_irqs)); 3607 3608 for (i = 0; i < nr_irqs; i++) { 3609 struct irq_data *data = irq_domain_get_irq_data(domain, 3610 virq + i); 3611 /* Nuke the entry in the domain */ 3612 irq_domain_reset_irq_data(data); 3613 } 3614 3615 mutex_lock(&its->dev_alloc_lock); 3616 3617 /* 3618 * If all interrupts have been freed, start mopping the 3619 * floor. This is conditionned on the device not being shared. 3620 */ 3621 if (!its_dev->shared && 3622 bitmap_empty(its_dev->event_map.lpi_map, 3623 its_dev->event_map.nr_lpis)) { 3624 its_lpi_free(its_dev->event_map.lpi_map, 3625 its_dev->event_map.lpi_base, 3626 its_dev->event_map.nr_lpis); 3627 3628 /* Unmap device/itt */ 3629 its_send_mapd(its_dev, 0); 3630 its_free_device(its_dev); 3631 } 3632 3633 mutex_unlock(&its->dev_alloc_lock); 3634 3635 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 3636 } 3637 3638 static const struct irq_domain_ops its_domain_ops = { 3639 .alloc = its_irq_domain_alloc, 3640 .free = its_irq_domain_free, 3641 .activate = its_irq_domain_activate, 3642 .deactivate = its_irq_domain_deactivate, 3643 }; 3644 3645 /* 3646 * This is insane. 3647 * 3648 * If a GICv4.0 doesn't implement Direct LPIs (which is extremely 3649 * likely), the only way to perform an invalidate is to use a fake 3650 * device to issue an INV command, implying that the LPI has first 3651 * been mapped to some event on that device. Since this is not exactly 3652 * cheap, we try to keep that mapping around as long as possible, and 3653 * only issue an UNMAP if we're short on available slots. 3654 * 3655 * Broken by design(tm). 3656 * 3657 * GICv4.1, on the other hand, mandates that we're able to invalidate 3658 * by writing to a MMIO register. It doesn't implement the whole of 3659 * DirectLPI, but that's good enough. And most of the time, we don't 3660 * even have to invalidate anything, as the redistributor can be told 3661 * whether to generate a doorbell or not (we thus leave it enabled, 3662 * always). 3663 */ 3664 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe) 3665 { 3666 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3667 if (gic_rdists->has_rvpeid) 3668 return; 3669 3670 /* Already unmapped? */ 3671 if (vpe->vpe_proxy_event == -1) 3672 return; 3673 3674 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event); 3675 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL; 3676 3677 /* 3678 * We don't track empty slots at all, so let's move the 3679 * next_victim pointer if we can quickly reuse that slot 3680 * instead of nuking an existing entry. Not clear that this is 3681 * always a win though, and this might just generate a ripple 3682 * effect... Let's just hope VPEs don't migrate too often. 3683 */ 3684 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 3685 vpe_proxy.next_victim = vpe->vpe_proxy_event; 3686 3687 vpe->vpe_proxy_event = -1; 3688 } 3689 3690 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe) 3691 { 3692 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3693 if (gic_rdists->has_rvpeid) 3694 return; 3695 3696 if (!gic_rdists->has_direct_lpi) { 3697 unsigned long flags; 3698 3699 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 3700 its_vpe_db_proxy_unmap_locked(vpe); 3701 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 3702 } 3703 } 3704 3705 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe) 3706 { 3707 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3708 if (gic_rdists->has_rvpeid) 3709 return; 3710 3711 /* Already mapped? */ 3712 if (vpe->vpe_proxy_event != -1) 3713 return; 3714 3715 /* This slot was already allocated. Kick the other VPE out. */ 3716 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 3717 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]); 3718 3719 /* Map the new VPE instead */ 3720 vpe_proxy.vpes[vpe_proxy.next_victim] = vpe; 3721 vpe->vpe_proxy_event = vpe_proxy.next_victim; 3722 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites; 3723 3724 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx; 3725 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event); 3726 } 3727 3728 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to) 3729 { 3730 unsigned long flags; 3731 struct its_collection *target_col; 3732 3733 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3734 if (gic_rdists->has_rvpeid) 3735 return; 3736 3737 if (gic_rdists->has_direct_lpi) { 3738 void __iomem *rdbase; 3739 3740 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base; 3741 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 3742 wait_for_syncr(rdbase); 3743 3744 return; 3745 } 3746 3747 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 3748 3749 its_vpe_db_proxy_map_locked(vpe); 3750 3751 target_col = &vpe_proxy.dev->its->collections[to]; 3752 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event); 3753 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to; 3754 3755 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 3756 } 3757 3758 static int its_vpe_set_affinity(struct irq_data *d, 3759 const struct cpumask *mask_val, 3760 bool force) 3761 { 3762 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3763 int from, cpu = cpumask_first(mask_val); 3764 unsigned long flags; 3765 3766 /* 3767 * Changing affinity is mega expensive, so let's be as lazy as 3768 * we can and only do it if we really have to. Also, if mapped 3769 * into the proxy device, we need to move the doorbell 3770 * interrupt to its new location. 3771 * 3772 * Another thing is that changing the affinity of a vPE affects 3773 * *other interrupts* such as all the vLPIs that are routed to 3774 * this vPE. This means that the irq_desc lock is not enough to 3775 * protect us, and that we must ensure nobody samples vpe->col_idx 3776 * during the update, hence the lock below which must also be 3777 * taken on any vLPI handling path that evaluates vpe->col_idx. 3778 */ 3779 from = vpe_to_cpuid_lock(vpe, &flags); 3780 if (from == cpu) 3781 goto out; 3782 3783 vpe->col_idx = cpu; 3784 3785 /* 3786 * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD 3787 * is sharing its VPE table with the current one. 3788 */ 3789 if (gic_data_rdist_cpu(cpu)->vpe_table_mask && 3790 cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask)) 3791 goto out; 3792 3793 its_send_vmovp(vpe); 3794 its_vpe_db_proxy_move(vpe, from, cpu); 3795 3796 out: 3797 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 3798 vpe_to_cpuid_unlock(vpe, flags); 3799 3800 return IRQ_SET_MASK_OK_DONE; 3801 } 3802 3803 static void its_wait_vpt_parse_complete(void) 3804 { 3805 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3806 u64 val; 3807 3808 if (!gic_rdists->has_vpend_valid_dirty) 3809 return; 3810 3811 WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER, 3812 val, 3813 !(val & GICR_VPENDBASER_Dirty), 3814 1, 500)); 3815 } 3816 3817 static void its_vpe_schedule(struct its_vpe *vpe) 3818 { 3819 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3820 u64 val; 3821 3822 /* Schedule the VPE */ 3823 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) & 3824 GENMASK_ULL(51, 12); 3825 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; 3826 val |= GICR_VPROPBASER_RaWb; 3827 val |= GICR_VPROPBASER_InnerShareable; 3828 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 3829 3830 val = virt_to_phys(page_address(vpe->vpt_page)) & 3831 GENMASK_ULL(51, 16); 3832 val |= GICR_VPENDBASER_RaWaWb; 3833 val |= GICR_VPENDBASER_InnerShareable; 3834 /* 3835 * There is no good way of finding out if the pending table is 3836 * empty as we can race against the doorbell interrupt very 3837 * easily. So in the end, vpe->pending_last is only an 3838 * indication that the vcpu has something pending, not one 3839 * that the pending table is empty. A good implementation 3840 * would be able to read its coarse map pretty quickly anyway, 3841 * making this a tolerable issue. 3842 */ 3843 val |= GICR_VPENDBASER_PendingLast; 3844 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; 3845 val |= GICR_VPENDBASER_Valid; 3846 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 3847 } 3848 3849 static void its_vpe_deschedule(struct its_vpe *vpe) 3850 { 3851 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3852 u64 val; 3853 3854 val = its_clear_vpend_valid(vlpi_base, 0, 0); 3855 3856 vpe->idai = !!(val & GICR_VPENDBASER_IDAI); 3857 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); 3858 } 3859 3860 static void its_vpe_invall(struct its_vpe *vpe) 3861 { 3862 struct its_node *its; 3863 3864 list_for_each_entry(its, &its_nodes, entry) { 3865 if (!is_v4(its)) 3866 continue; 3867 3868 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) 3869 continue; 3870 3871 /* 3872 * Sending a VINVALL to a single ITS is enough, as all 3873 * we need is to reach the redistributors. 3874 */ 3875 its_send_vinvall(its, vpe); 3876 return; 3877 } 3878 } 3879 3880 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 3881 { 3882 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3883 struct its_cmd_info *info = vcpu_info; 3884 3885 switch (info->cmd_type) { 3886 case SCHEDULE_VPE: 3887 its_vpe_schedule(vpe); 3888 return 0; 3889 3890 case DESCHEDULE_VPE: 3891 its_vpe_deschedule(vpe); 3892 return 0; 3893 3894 case COMMIT_VPE: 3895 its_wait_vpt_parse_complete(); 3896 return 0; 3897 3898 case INVALL_VPE: 3899 its_vpe_invall(vpe); 3900 return 0; 3901 3902 default: 3903 return -EINVAL; 3904 } 3905 } 3906 3907 static void its_vpe_send_cmd(struct its_vpe *vpe, 3908 void (*cmd)(struct its_device *, u32)) 3909 { 3910 unsigned long flags; 3911 3912 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 3913 3914 its_vpe_db_proxy_map_locked(vpe); 3915 cmd(vpe_proxy.dev, vpe->vpe_proxy_event); 3916 3917 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 3918 } 3919 3920 static void its_vpe_send_inv(struct irq_data *d) 3921 { 3922 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3923 3924 if (gic_rdists->has_direct_lpi) { 3925 void __iomem *rdbase; 3926 3927 /* Target the redistributor this VPE is currently known on */ 3928 raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); 3929 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 3930 gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR); 3931 wait_for_syncr(rdbase); 3932 raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); 3933 } else { 3934 its_vpe_send_cmd(vpe, its_send_inv); 3935 } 3936 } 3937 3938 static void its_vpe_mask_irq(struct irq_data *d) 3939 { 3940 /* 3941 * We need to unmask the LPI, which is described by the parent 3942 * irq_data. Instead of calling into the parent (which won't 3943 * exactly do the right thing, let's simply use the 3944 * parent_data pointer. Yes, I'm naughty. 3945 */ 3946 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); 3947 its_vpe_send_inv(d); 3948 } 3949 3950 static void its_vpe_unmask_irq(struct irq_data *d) 3951 { 3952 /* Same hack as above... */ 3953 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); 3954 its_vpe_send_inv(d); 3955 } 3956 3957 static int its_vpe_set_irqchip_state(struct irq_data *d, 3958 enum irqchip_irq_state which, 3959 bool state) 3960 { 3961 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3962 3963 if (which != IRQCHIP_STATE_PENDING) 3964 return -EINVAL; 3965 3966 if (gic_rdists->has_direct_lpi) { 3967 void __iomem *rdbase; 3968 3969 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 3970 if (state) { 3971 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR); 3972 } else { 3973 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 3974 wait_for_syncr(rdbase); 3975 } 3976 } else { 3977 if (state) 3978 its_vpe_send_cmd(vpe, its_send_int); 3979 else 3980 its_vpe_send_cmd(vpe, its_send_clear); 3981 } 3982 3983 return 0; 3984 } 3985 3986 static int its_vpe_retrigger(struct irq_data *d) 3987 { 3988 return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true); 3989 } 3990 3991 static struct irq_chip its_vpe_irq_chip = { 3992 .name = "GICv4-vpe", 3993 .irq_mask = its_vpe_mask_irq, 3994 .irq_unmask = its_vpe_unmask_irq, 3995 .irq_eoi = irq_chip_eoi_parent, 3996 .irq_set_affinity = its_vpe_set_affinity, 3997 .irq_retrigger = its_vpe_retrigger, 3998 .irq_set_irqchip_state = its_vpe_set_irqchip_state, 3999 .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity, 4000 }; 4001 4002 static struct its_node *find_4_1_its(void) 4003 { 4004 static struct its_node *its = NULL; 4005 4006 if (!its) { 4007 list_for_each_entry(its, &its_nodes, entry) { 4008 if (is_v4_1(its)) 4009 return its; 4010 } 4011 4012 /* Oops? */ 4013 its = NULL; 4014 } 4015 4016 return its; 4017 } 4018 4019 static void its_vpe_4_1_send_inv(struct irq_data *d) 4020 { 4021 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4022 struct its_node *its; 4023 4024 /* 4025 * GICv4.1 wants doorbells to be invalidated using the 4026 * INVDB command in order to be broadcast to all RDs. Send 4027 * it to the first valid ITS, and let the HW do its magic. 4028 */ 4029 its = find_4_1_its(); 4030 if (its) 4031 its_send_invdb(its, vpe); 4032 } 4033 4034 static void its_vpe_4_1_mask_irq(struct irq_data *d) 4035 { 4036 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); 4037 its_vpe_4_1_send_inv(d); 4038 } 4039 4040 static void its_vpe_4_1_unmask_irq(struct irq_data *d) 4041 { 4042 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); 4043 its_vpe_4_1_send_inv(d); 4044 } 4045 4046 static void its_vpe_4_1_schedule(struct its_vpe *vpe, 4047 struct its_cmd_info *info) 4048 { 4049 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 4050 u64 val = 0; 4051 4052 /* Schedule the VPE */ 4053 val |= GICR_VPENDBASER_Valid; 4054 val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0; 4055 val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; 4056 val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); 4057 4058 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 4059 } 4060 4061 static void its_vpe_4_1_deschedule(struct its_vpe *vpe, 4062 struct its_cmd_info *info) 4063 { 4064 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 4065 u64 val; 4066 4067 if (info->req_db) { 4068 unsigned long flags; 4069 4070 /* 4071 * vPE is going to block: make the vPE non-resident with 4072 * PendingLast clear and DB set. The GIC guarantees that if 4073 * we read-back PendingLast clear, then a doorbell will be 4074 * delivered when an interrupt comes. 4075 * 4076 * Note the locking to deal with the concurrent update of 4077 * pending_last from the doorbell interrupt handler that can 4078 * run concurrently. 4079 */ 4080 raw_spin_lock_irqsave(&vpe->vpe_lock, flags); 4081 val = its_clear_vpend_valid(vlpi_base, 4082 GICR_VPENDBASER_PendingLast, 4083 GICR_VPENDBASER_4_1_DB); 4084 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); 4085 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); 4086 } else { 4087 /* 4088 * We're not blocking, so just make the vPE non-resident 4089 * with PendingLast set, indicating that we'll be back. 4090 */ 4091 val = its_clear_vpend_valid(vlpi_base, 4092 0, 4093 GICR_VPENDBASER_PendingLast); 4094 vpe->pending_last = true; 4095 } 4096 } 4097 4098 static void its_vpe_4_1_invall(struct its_vpe *vpe) 4099 { 4100 void __iomem *rdbase; 4101 unsigned long flags; 4102 u64 val; 4103 int cpu; 4104 4105 val = GICR_INVALLR_V; 4106 val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); 4107 4108 /* Target the redistributor this vPE is currently known on */ 4109 cpu = vpe_to_cpuid_lock(vpe, &flags); 4110 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); 4111 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; 4112 gic_write_lpir(val, rdbase + GICR_INVALLR); 4113 4114 wait_for_syncr(rdbase); 4115 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); 4116 vpe_to_cpuid_unlock(vpe, flags); 4117 } 4118 4119 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 4120 { 4121 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4122 struct its_cmd_info *info = vcpu_info; 4123 4124 switch (info->cmd_type) { 4125 case SCHEDULE_VPE: 4126 its_vpe_4_1_schedule(vpe, info); 4127 return 0; 4128 4129 case DESCHEDULE_VPE: 4130 its_vpe_4_1_deschedule(vpe, info); 4131 return 0; 4132 4133 case COMMIT_VPE: 4134 its_wait_vpt_parse_complete(); 4135 return 0; 4136 4137 case INVALL_VPE: 4138 its_vpe_4_1_invall(vpe); 4139 return 0; 4140 4141 default: 4142 return -EINVAL; 4143 } 4144 } 4145 4146 static struct irq_chip its_vpe_4_1_irq_chip = { 4147 .name = "GICv4.1-vpe", 4148 .irq_mask = its_vpe_4_1_mask_irq, 4149 .irq_unmask = its_vpe_4_1_unmask_irq, 4150 .irq_eoi = irq_chip_eoi_parent, 4151 .irq_set_affinity = its_vpe_set_affinity, 4152 .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity, 4153 }; 4154 4155 static void its_configure_sgi(struct irq_data *d, bool clear) 4156 { 4157 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4158 struct its_cmd_desc desc; 4159 4160 desc.its_vsgi_cmd.vpe = vpe; 4161 desc.its_vsgi_cmd.sgi = d->hwirq; 4162 desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority; 4163 desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled; 4164 desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group; 4165 desc.its_vsgi_cmd.clear = clear; 4166 4167 /* 4168 * GICv4.1 allows us to send VSGI commands to any ITS as long as the 4169 * destination VPE is mapped there. Since we map them eagerly at 4170 * activation time, we're pretty sure the first GICv4.1 ITS will do. 4171 */ 4172 its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc); 4173 } 4174 4175 static void its_sgi_mask_irq(struct irq_data *d) 4176 { 4177 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4178 4179 vpe->sgi_config[d->hwirq].enabled = false; 4180 its_configure_sgi(d, false); 4181 } 4182 4183 static void its_sgi_unmask_irq(struct irq_data *d) 4184 { 4185 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4186 4187 vpe->sgi_config[d->hwirq].enabled = true; 4188 its_configure_sgi(d, false); 4189 } 4190 4191 static int its_sgi_set_affinity(struct irq_data *d, 4192 const struct cpumask *mask_val, 4193 bool force) 4194 { 4195 /* 4196 * There is no notion of affinity for virtual SGIs, at least 4197 * not on the host (since they can only be targetting a vPE). 4198 * Tell the kernel we've done whatever it asked for. 4199 */ 4200 irq_data_update_effective_affinity(d, mask_val); 4201 return IRQ_SET_MASK_OK; 4202 } 4203 4204 static int its_sgi_set_irqchip_state(struct irq_data *d, 4205 enum irqchip_irq_state which, 4206 bool state) 4207 { 4208 if (which != IRQCHIP_STATE_PENDING) 4209 return -EINVAL; 4210 4211 if (state) { 4212 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4213 struct its_node *its = find_4_1_its(); 4214 u64 val; 4215 4216 val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id); 4217 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq); 4218 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K); 4219 } else { 4220 its_configure_sgi(d, true); 4221 } 4222 4223 return 0; 4224 } 4225 4226 static int its_sgi_get_irqchip_state(struct irq_data *d, 4227 enum irqchip_irq_state which, bool *val) 4228 { 4229 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4230 void __iomem *base; 4231 unsigned long flags; 4232 u32 count = 1000000; /* 1s! */ 4233 u32 status; 4234 int cpu; 4235 4236 if (which != IRQCHIP_STATE_PENDING) 4237 return -EINVAL; 4238 4239 /* 4240 * Locking galore! We can race against two different events: 4241 * 4242 * - Concurent vPE affinity change: we must make sure it cannot 4243 * happen, or we'll talk to the wrong redistributor. This is 4244 * identical to what happens with vLPIs. 4245 * 4246 * - Concurrent VSGIPENDR access: As it involves accessing two 4247 * MMIO registers, this must be made atomic one way or another. 4248 */ 4249 cpu = vpe_to_cpuid_lock(vpe, &flags); 4250 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); 4251 base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K; 4252 writel_relaxed(vpe->vpe_id, base + GICR_VSGIR); 4253 do { 4254 status = readl_relaxed(base + GICR_VSGIPENDR); 4255 if (!(status & GICR_VSGIPENDR_BUSY)) 4256 goto out; 4257 4258 count--; 4259 if (!count) { 4260 pr_err_ratelimited("Unable to get SGI status\n"); 4261 goto out; 4262 } 4263 cpu_relax(); 4264 udelay(1); 4265 } while (count); 4266 4267 out: 4268 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); 4269 vpe_to_cpuid_unlock(vpe, flags); 4270 4271 if (!count) 4272 return -ENXIO; 4273 4274 *val = !!(status & (1 << d->hwirq)); 4275 4276 return 0; 4277 } 4278 4279 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 4280 { 4281 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4282 struct its_cmd_info *info = vcpu_info; 4283 4284 switch (info->cmd_type) { 4285 case PROP_UPDATE_VSGI: 4286 vpe->sgi_config[d->hwirq].priority = info->priority; 4287 vpe->sgi_config[d->hwirq].group = info->group; 4288 its_configure_sgi(d, false); 4289 return 0; 4290 4291 default: 4292 return -EINVAL; 4293 } 4294 } 4295 4296 static struct irq_chip its_sgi_irq_chip = { 4297 .name = "GICv4.1-sgi", 4298 .irq_mask = its_sgi_mask_irq, 4299 .irq_unmask = its_sgi_unmask_irq, 4300 .irq_set_affinity = its_sgi_set_affinity, 4301 .irq_set_irqchip_state = its_sgi_set_irqchip_state, 4302 .irq_get_irqchip_state = its_sgi_get_irqchip_state, 4303 .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity, 4304 }; 4305 4306 static int its_sgi_irq_domain_alloc(struct irq_domain *domain, 4307 unsigned int virq, unsigned int nr_irqs, 4308 void *args) 4309 { 4310 struct its_vpe *vpe = args; 4311 int i; 4312 4313 /* Yes, we do want 16 SGIs */ 4314 WARN_ON(nr_irqs != 16); 4315 4316 for (i = 0; i < 16; i++) { 4317 vpe->sgi_config[i].priority = 0; 4318 vpe->sgi_config[i].enabled = false; 4319 vpe->sgi_config[i].group = false; 4320 4321 irq_domain_set_hwirq_and_chip(domain, virq + i, i, 4322 &its_sgi_irq_chip, vpe); 4323 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY); 4324 } 4325 4326 return 0; 4327 } 4328 4329 static void its_sgi_irq_domain_free(struct irq_domain *domain, 4330 unsigned int virq, 4331 unsigned int nr_irqs) 4332 { 4333 /* Nothing to do */ 4334 } 4335 4336 static int its_sgi_irq_domain_activate(struct irq_domain *domain, 4337 struct irq_data *d, bool reserve) 4338 { 4339 /* Write out the initial SGI configuration */ 4340 its_configure_sgi(d, false); 4341 return 0; 4342 } 4343 4344 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain, 4345 struct irq_data *d) 4346 { 4347 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4348 4349 /* 4350 * The VSGI command is awkward: 4351 * 4352 * - To change the configuration, CLEAR must be set to false, 4353 * leaving the pending bit unchanged. 4354 * - To clear the pending bit, CLEAR must be set to true, leaving 4355 * the configuration unchanged. 4356 * 4357 * You just can't do both at once, hence the two commands below. 4358 */ 4359 vpe->sgi_config[d->hwirq].enabled = false; 4360 its_configure_sgi(d, false); 4361 its_configure_sgi(d, true); 4362 } 4363 4364 static const struct irq_domain_ops its_sgi_domain_ops = { 4365 .alloc = its_sgi_irq_domain_alloc, 4366 .free = its_sgi_irq_domain_free, 4367 .activate = its_sgi_irq_domain_activate, 4368 .deactivate = its_sgi_irq_domain_deactivate, 4369 }; 4370 4371 static int its_vpe_id_alloc(void) 4372 { 4373 return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL); 4374 } 4375 4376 static void its_vpe_id_free(u16 id) 4377 { 4378 ida_simple_remove(&its_vpeid_ida, id); 4379 } 4380 4381 static int its_vpe_init(struct its_vpe *vpe) 4382 { 4383 struct page *vpt_page; 4384 int vpe_id; 4385 4386 /* Allocate vpe_id */ 4387 vpe_id = its_vpe_id_alloc(); 4388 if (vpe_id < 0) 4389 return vpe_id; 4390 4391 /* Allocate VPT */ 4392 vpt_page = its_allocate_pending_table(GFP_KERNEL); 4393 if (!vpt_page) { 4394 its_vpe_id_free(vpe_id); 4395 return -ENOMEM; 4396 } 4397 4398 if (!its_alloc_vpe_table(vpe_id)) { 4399 its_vpe_id_free(vpe_id); 4400 its_free_pending_table(vpt_page); 4401 return -ENOMEM; 4402 } 4403 4404 raw_spin_lock_init(&vpe->vpe_lock); 4405 vpe->vpe_id = vpe_id; 4406 vpe->vpt_page = vpt_page; 4407 if (gic_rdists->has_rvpeid) 4408 atomic_set(&vpe->vmapp_count, 0); 4409 else 4410 vpe->vpe_proxy_event = -1; 4411 4412 return 0; 4413 } 4414 4415 static void its_vpe_teardown(struct its_vpe *vpe) 4416 { 4417 its_vpe_db_proxy_unmap(vpe); 4418 its_vpe_id_free(vpe->vpe_id); 4419 its_free_pending_table(vpe->vpt_page); 4420 } 4421 4422 static void its_vpe_irq_domain_free(struct irq_domain *domain, 4423 unsigned int virq, 4424 unsigned int nr_irqs) 4425 { 4426 struct its_vm *vm = domain->host_data; 4427 int i; 4428 4429 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 4430 4431 for (i = 0; i < nr_irqs; i++) { 4432 struct irq_data *data = irq_domain_get_irq_data(domain, 4433 virq + i); 4434 struct its_vpe *vpe = irq_data_get_irq_chip_data(data); 4435 4436 BUG_ON(vm != vpe->its_vm); 4437 4438 clear_bit(data->hwirq, vm->db_bitmap); 4439 its_vpe_teardown(vpe); 4440 irq_domain_reset_irq_data(data); 4441 } 4442 4443 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) { 4444 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); 4445 its_free_prop_table(vm->vprop_page); 4446 } 4447 } 4448 4449 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 4450 unsigned int nr_irqs, void *args) 4451 { 4452 struct irq_chip *irqchip = &its_vpe_irq_chip; 4453 struct its_vm *vm = args; 4454 unsigned long *bitmap; 4455 struct page *vprop_page; 4456 int base, nr_ids, i, err = 0; 4457 4458 BUG_ON(!vm); 4459 4460 bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids); 4461 if (!bitmap) 4462 return -ENOMEM; 4463 4464 if (nr_ids < nr_irqs) { 4465 its_lpi_free(bitmap, base, nr_ids); 4466 return -ENOMEM; 4467 } 4468 4469 vprop_page = its_allocate_prop_table(GFP_KERNEL); 4470 if (!vprop_page) { 4471 its_lpi_free(bitmap, base, nr_ids); 4472 return -ENOMEM; 4473 } 4474 4475 vm->db_bitmap = bitmap; 4476 vm->db_lpi_base = base; 4477 vm->nr_db_lpis = nr_ids; 4478 vm->vprop_page = vprop_page; 4479 4480 if (gic_rdists->has_rvpeid) 4481 irqchip = &its_vpe_4_1_irq_chip; 4482 4483 for (i = 0; i < nr_irqs; i++) { 4484 vm->vpes[i]->vpe_db_lpi = base + i; 4485 err = its_vpe_init(vm->vpes[i]); 4486 if (err) 4487 break; 4488 err = its_irq_gic_domain_alloc(domain, virq + i, 4489 vm->vpes[i]->vpe_db_lpi); 4490 if (err) 4491 break; 4492 irq_domain_set_hwirq_and_chip(domain, virq + i, i, 4493 irqchip, vm->vpes[i]); 4494 set_bit(i, bitmap); 4495 } 4496 4497 if (err) { 4498 if (i > 0) 4499 its_vpe_irq_domain_free(domain, virq, i - 1); 4500 4501 its_lpi_free(bitmap, base, nr_ids); 4502 its_free_prop_table(vprop_page); 4503 } 4504 4505 return err; 4506 } 4507 4508 static int its_vpe_irq_domain_activate(struct irq_domain *domain, 4509 struct irq_data *d, bool reserve) 4510 { 4511 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4512 struct its_node *its; 4513 4514 /* 4515 * If we use the list map, we issue VMAPP on demand... Unless 4516 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs 4517 * so that VSGIs can work. 4518 */ 4519 if (!gic_requires_eager_mapping()) 4520 return 0; 4521 4522 /* Map the VPE to the first possible CPU */ 4523 vpe->col_idx = cpumask_first(cpu_online_mask); 4524 4525 list_for_each_entry(its, &its_nodes, entry) { 4526 if (!is_v4(its)) 4527 continue; 4528 4529 its_send_vmapp(its, vpe, true); 4530 its_send_vinvall(its, vpe); 4531 } 4532 4533 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 4534 4535 return 0; 4536 } 4537 4538 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain, 4539 struct irq_data *d) 4540 { 4541 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4542 struct its_node *its; 4543 4544 /* 4545 * If we use the list map on GICv4.0, we unmap the VPE once no 4546 * VLPIs are associated with the VM. 4547 */ 4548 if (!gic_requires_eager_mapping()) 4549 return; 4550 4551 list_for_each_entry(its, &its_nodes, entry) { 4552 if (!is_v4(its)) 4553 continue; 4554 4555 its_send_vmapp(its, vpe, false); 4556 } 4557 } 4558 4559 static const struct irq_domain_ops its_vpe_domain_ops = { 4560 .alloc = its_vpe_irq_domain_alloc, 4561 .free = its_vpe_irq_domain_free, 4562 .activate = its_vpe_irq_domain_activate, 4563 .deactivate = its_vpe_irq_domain_deactivate, 4564 }; 4565 4566 static int its_force_quiescent(void __iomem *base) 4567 { 4568 u32 count = 1000000; /* 1s */ 4569 u32 val; 4570 4571 val = readl_relaxed(base + GITS_CTLR); 4572 /* 4573 * GIC architecture specification requires the ITS to be both 4574 * disabled and quiescent for writes to GITS_BASER<n> or 4575 * GITS_CBASER to not have UNPREDICTABLE results. 4576 */ 4577 if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE)) 4578 return 0; 4579 4580 /* Disable the generation of all interrupts to this ITS */ 4581 val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe); 4582 writel_relaxed(val, base + GITS_CTLR); 4583 4584 /* Poll GITS_CTLR and wait until ITS becomes quiescent */ 4585 while (1) { 4586 val = readl_relaxed(base + GITS_CTLR); 4587 if (val & GITS_CTLR_QUIESCENT) 4588 return 0; 4589 4590 count--; 4591 if (!count) 4592 return -EBUSY; 4593 4594 cpu_relax(); 4595 udelay(1); 4596 } 4597 } 4598 4599 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data) 4600 { 4601 struct its_node *its = data; 4602 4603 /* erratum 22375: only alloc 8MB table size (20 bits) */ 4604 its->typer &= ~GITS_TYPER_DEVBITS; 4605 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1); 4606 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; 4607 4608 return true; 4609 } 4610 4611 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data) 4612 { 4613 struct its_node *its = data; 4614 4615 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; 4616 4617 return true; 4618 } 4619 4620 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data) 4621 { 4622 struct its_node *its = data; 4623 4624 /* On QDF2400, the size of the ITE is 16Bytes */ 4625 its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE; 4626 its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1); 4627 4628 return true; 4629 } 4630 4631 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev) 4632 { 4633 struct its_node *its = its_dev->its; 4634 4635 /* 4636 * The Socionext Synquacer SoC has a so-called 'pre-ITS', 4637 * which maps 32-bit writes targeted at a separate window of 4638 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER 4639 * with device ID taken from bits [device_id_bits + 1:2] of 4640 * the window offset. 4641 */ 4642 return its->pre_its_base + (its_dev->device_id << 2); 4643 } 4644 4645 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data) 4646 { 4647 struct its_node *its = data; 4648 u32 pre_its_window[2]; 4649 u32 ids; 4650 4651 if (!fwnode_property_read_u32_array(its->fwnode_handle, 4652 "socionext,synquacer-pre-its", 4653 pre_its_window, 4654 ARRAY_SIZE(pre_its_window))) { 4655 4656 its->pre_its_base = pre_its_window[0]; 4657 its->get_msi_base = its_irq_get_msi_base_pre_its; 4658 4659 ids = ilog2(pre_its_window[1]) - 2; 4660 if (device_ids(its) > ids) { 4661 its->typer &= ~GITS_TYPER_DEVBITS; 4662 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1); 4663 } 4664 4665 /* the pre-ITS breaks isolation, so disable MSI remapping */ 4666 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP; 4667 return true; 4668 } 4669 return false; 4670 } 4671 4672 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data) 4673 { 4674 struct its_node *its = data; 4675 4676 /* 4677 * Hip07 insists on using the wrong address for the VLPI 4678 * page. Trick it into doing the right thing... 4679 */ 4680 its->vlpi_redist_offset = SZ_128K; 4681 return true; 4682 } 4683 4684 static const struct gic_quirk its_quirks[] = { 4685 #ifdef CONFIG_CAVIUM_ERRATUM_22375 4686 { 4687 .desc = "ITS: Cavium errata 22375, 24313", 4688 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 4689 .mask = 0xffff0fff, 4690 .init = its_enable_quirk_cavium_22375, 4691 }, 4692 #endif 4693 #ifdef CONFIG_CAVIUM_ERRATUM_23144 4694 { 4695 .desc = "ITS: Cavium erratum 23144", 4696 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 4697 .mask = 0xffff0fff, 4698 .init = its_enable_quirk_cavium_23144, 4699 }, 4700 #endif 4701 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065 4702 { 4703 .desc = "ITS: QDF2400 erratum 0065", 4704 .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */ 4705 .mask = 0xffffffff, 4706 .init = its_enable_quirk_qdf2400_e0065, 4707 }, 4708 #endif 4709 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS 4710 { 4711 /* 4712 * The Socionext Synquacer SoC incorporates ARM's own GIC-500 4713 * implementation, but with a 'pre-ITS' added that requires 4714 * special handling in software. 4715 */ 4716 .desc = "ITS: Socionext Synquacer pre-ITS", 4717 .iidr = 0x0001143b, 4718 .mask = 0xffffffff, 4719 .init = its_enable_quirk_socionext_synquacer, 4720 }, 4721 #endif 4722 #ifdef CONFIG_HISILICON_ERRATUM_161600802 4723 { 4724 .desc = "ITS: Hip07 erratum 161600802", 4725 .iidr = 0x00000004, 4726 .mask = 0xffffffff, 4727 .init = its_enable_quirk_hip07_161600802, 4728 }, 4729 #endif 4730 { 4731 } 4732 }; 4733 4734 static void its_enable_quirks(struct its_node *its) 4735 { 4736 u32 iidr = readl_relaxed(its->base + GITS_IIDR); 4737 4738 gic_enable_quirks(iidr, its_quirks, its); 4739 } 4740 4741 static int its_save_disable(void) 4742 { 4743 struct its_node *its; 4744 int err = 0; 4745 4746 raw_spin_lock(&its_lock); 4747 list_for_each_entry(its, &its_nodes, entry) { 4748 void __iomem *base; 4749 4750 base = its->base; 4751 its->ctlr_save = readl_relaxed(base + GITS_CTLR); 4752 err = its_force_quiescent(base); 4753 if (err) { 4754 pr_err("ITS@%pa: failed to quiesce: %d\n", 4755 &its->phys_base, err); 4756 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 4757 goto err; 4758 } 4759 4760 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER); 4761 } 4762 4763 err: 4764 if (err) { 4765 list_for_each_entry_continue_reverse(its, &its_nodes, entry) { 4766 void __iomem *base; 4767 4768 base = its->base; 4769 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 4770 } 4771 } 4772 raw_spin_unlock(&its_lock); 4773 4774 return err; 4775 } 4776 4777 static void its_restore_enable(void) 4778 { 4779 struct its_node *its; 4780 int ret; 4781 4782 raw_spin_lock(&its_lock); 4783 list_for_each_entry(its, &its_nodes, entry) { 4784 void __iomem *base; 4785 int i; 4786 4787 base = its->base; 4788 4789 /* 4790 * Make sure that the ITS is disabled. If it fails to quiesce, 4791 * don't restore it since writing to CBASER or BASER<n> 4792 * registers is undefined according to the GIC v3 ITS 4793 * Specification. 4794 * 4795 * Firmware resuming with the ITS enabled is terminally broken. 4796 */ 4797 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE); 4798 ret = its_force_quiescent(base); 4799 if (ret) { 4800 pr_err("ITS@%pa: failed to quiesce on resume: %d\n", 4801 &its->phys_base, ret); 4802 continue; 4803 } 4804 4805 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER); 4806 4807 /* 4808 * Writing CBASER resets CREADR to 0, so make CWRITER and 4809 * cmd_write line up with it. 4810 */ 4811 its->cmd_write = its->cmd_base; 4812 gits_write_cwriter(0, base + GITS_CWRITER); 4813 4814 /* Restore GITS_BASER from the value cache. */ 4815 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 4816 struct its_baser *baser = &its->tables[i]; 4817 4818 if (!(baser->val & GITS_BASER_VALID)) 4819 continue; 4820 4821 its_write_baser(its, baser, baser->val); 4822 } 4823 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 4824 4825 /* 4826 * Reinit the collection if it's stored in the ITS. This is 4827 * indicated by the col_id being less than the HCC field. 4828 * CID < HCC as specified in the GIC v3 Documentation. 4829 */ 4830 if (its->collections[smp_processor_id()].col_id < 4831 GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER))) 4832 its_cpu_init_collection(its); 4833 } 4834 raw_spin_unlock(&its_lock); 4835 } 4836 4837 static struct syscore_ops its_syscore_ops = { 4838 .suspend = its_save_disable, 4839 .resume = its_restore_enable, 4840 }; 4841 4842 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) 4843 { 4844 struct irq_domain *inner_domain; 4845 struct msi_domain_info *info; 4846 4847 info = kzalloc(sizeof(*info), GFP_KERNEL); 4848 if (!info) 4849 return -ENOMEM; 4850 4851 inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its); 4852 if (!inner_domain) { 4853 kfree(info); 4854 return -ENOMEM; 4855 } 4856 4857 inner_domain->parent = its_parent; 4858 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); 4859 inner_domain->flags |= its->msi_domain_flags; 4860 info->ops = &its_msi_domain_ops; 4861 info->data = its; 4862 inner_domain->host_data = info; 4863 4864 return 0; 4865 } 4866 4867 static int its_init_vpe_domain(void) 4868 { 4869 struct its_node *its; 4870 u32 devid; 4871 int entries; 4872 4873 if (gic_rdists->has_direct_lpi) { 4874 pr_info("ITS: Using DirectLPI for VPE invalidation\n"); 4875 return 0; 4876 } 4877 4878 /* Any ITS will do, even if not v4 */ 4879 its = list_first_entry(&its_nodes, struct its_node, entry); 4880 4881 entries = roundup_pow_of_two(nr_cpu_ids); 4882 vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes), 4883 GFP_KERNEL); 4884 if (!vpe_proxy.vpes) { 4885 pr_err("ITS: Can't allocate GICv4 proxy device array\n"); 4886 return -ENOMEM; 4887 } 4888 4889 /* Use the last possible DevID */ 4890 devid = GENMASK(device_ids(its) - 1, 0); 4891 vpe_proxy.dev = its_create_device(its, devid, entries, false); 4892 if (!vpe_proxy.dev) { 4893 kfree(vpe_proxy.vpes); 4894 pr_err("ITS: Can't allocate GICv4 proxy device\n"); 4895 return -ENOMEM; 4896 } 4897 4898 BUG_ON(entries > vpe_proxy.dev->nr_ites); 4899 4900 raw_spin_lock_init(&vpe_proxy.lock); 4901 vpe_proxy.next_victim = 0; 4902 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n", 4903 devid, vpe_proxy.dev->nr_ites); 4904 4905 return 0; 4906 } 4907 4908 static int __init its_compute_its_list_map(struct resource *res, 4909 void __iomem *its_base) 4910 { 4911 int its_number; 4912 u32 ctlr; 4913 4914 /* 4915 * This is assumed to be done early enough that we're 4916 * guaranteed to be single-threaded, hence no 4917 * locking. Should this change, we should address 4918 * this. 4919 */ 4920 its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX); 4921 if (its_number >= GICv4_ITS_LIST_MAX) { 4922 pr_err("ITS@%pa: No ITSList entry available!\n", 4923 &res->start); 4924 return -EINVAL; 4925 } 4926 4927 ctlr = readl_relaxed(its_base + GITS_CTLR); 4928 ctlr &= ~GITS_CTLR_ITS_NUMBER; 4929 ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT; 4930 writel_relaxed(ctlr, its_base + GITS_CTLR); 4931 ctlr = readl_relaxed(its_base + GITS_CTLR); 4932 if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) { 4933 its_number = ctlr & GITS_CTLR_ITS_NUMBER; 4934 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT; 4935 } 4936 4937 if (test_and_set_bit(its_number, &its_list_map)) { 4938 pr_err("ITS@%pa: Duplicate ITSList entry %d\n", 4939 &res->start, its_number); 4940 return -EINVAL; 4941 } 4942 4943 return its_number; 4944 } 4945 4946 static int __init its_probe_one(struct resource *res, 4947 struct fwnode_handle *handle, int numa_node) 4948 { 4949 struct its_node *its; 4950 void __iomem *its_base; 4951 u32 val, ctlr; 4952 u64 baser, tmp, typer; 4953 struct page *page; 4954 int err; 4955 4956 its_base = ioremap(res->start, SZ_64K); 4957 if (!its_base) { 4958 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start); 4959 return -ENOMEM; 4960 } 4961 4962 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK; 4963 if (val != 0x30 && val != 0x40) { 4964 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start); 4965 err = -ENODEV; 4966 goto out_unmap; 4967 } 4968 4969 err = its_force_quiescent(its_base); 4970 if (err) { 4971 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start); 4972 goto out_unmap; 4973 } 4974 4975 pr_info("ITS %pR\n", res); 4976 4977 its = kzalloc(sizeof(*its), GFP_KERNEL); 4978 if (!its) { 4979 err = -ENOMEM; 4980 goto out_unmap; 4981 } 4982 4983 raw_spin_lock_init(&its->lock); 4984 mutex_init(&its->dev_alloc_lock); 4985 INIT_LIST_HEAD(&its->entry); 4986 INIT_LIST_HEAD(&its->its_device_list); 4987 typer = gic_read_typer(its_base + GITS_TYPER); 4988 its->typer = typer; 4989 its->base = its_base; 4990 its->phys_base = res->start; 4991 if (is_v4(its)) { 4992 if (!(typer & GITS_TYPER_VMOVP)) { 4993 err = its_compute_its_list_map(res, its_base); 4994 if (err < 0) 4995 goto out_free_its; 4996 4997 its->list_nr = err; 4998 4999 pr_info("ITS@%pa: Using ITS number %d\n", 5000 &res->start, err); 5001 } else { 5002 pr_info("ITS@%pa: Single VMOVP capable\n", &res->start); 5003 } 5004 5005 if (is_v4_1(its)) { 5006 u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer); 5007 5008 its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K); 5009 if (!its->sgir_base) { 5010 err = -ENOMEM; 5011 goto out_free_its; 5012 } 5013 5014 its->mpidr = readl_relaxed(its_base + GITS_MPIDR); 5015 5016 pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n", 5017 &res->start, its->mpidr, svpet); 5018 } 5019 } 5020 5021 its->numa_node = numa_node; 5022 5023 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, 5024 get_order(ITS_CMD_QUEUE_SZ)); 5025 if (!page) { 5026 err = -ENOMEM; 5027 goto out_unmap_sgir; 5028 } 5029 its->cmd_base = (void *)page_address(page); 5030 its->cmd_write = its->cmd_base; 5031 its->fwnode_handle = handle; 5032 its->get_msi_base = its_irq_get_msi_base; 5033 its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP; 5034 5035 its_enable_quirks(its); 5036 5037 err = its_alloc_tables(its); 5038 if (err) 5039 goto out_free_cmd; 5040 5041 err = its_alloc_collections(its); 5042 if (err) 5043 goto out_free_tables; 5044 5045 baser = (virt_to_phys(its->cmd_base) | 5046 GITS_CBASER_RaWaWb | 5047 GITS_CBASER_InnerShareable | 5048 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | 5049 GITS_CBASER_VALID); 5050 5051 gits_write_cbaser(baser, its->base + GITS_CBASER); 5052 tmp = gits_read_cbaser(its->base + GITS_CBASER); 5053 5054 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) { 5055 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) { 5056 /* 5057 * The HW reports non-shareable, we must 5058 * remove the cacheability attributes as 5059 * well. 5060 */ 5061 baser &= ~(GITS_CBASER_SHAREABILITY_MASK | 5062 GITS_CBASER_CACHEABILITY_MASK); 5063 baser |= GITS_CBASER_nC; 5064 gits_write_cbaser(baser, its->base + GITS_CBASER); 5065 } 5066 pr_info("ITS: using cache flushing for cmd queue\n"); 5067 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; 5068 } 5069 5070 gits_write_cwriter(0, its->base + GITS_CWRITER); 5071 ctlr = readl_relaxed(its->base + GITS_CTLR); 5072 ctlr |= GITS_CTLR_ENABLE; 5073 if (is_v4(its)) 5074 ctlr |= GITS_CTLR_ImDe; 5075 writel_relaxed(ctlr, its->base + GITS_CTLR); 5076 5077 err = its_init_domain(handle, its); 5078 if (err) 5079 goto out_free_tables; 5080 5081 raw_spin_lock(&its_lock); 5082 list_add(&its->entry, &its_nodes); 5083 raw_spin_unlock(&its_lock); 5084 5085 return 0; 5086 5087 out_free_tables: 5088 its_free_tables(its); 5089 out_free_cmd: 5090 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); 5091 out_unmap_sgir: 5092 if (its->sgir_base) 5093 iounmap(its->sgir_base); 5094 out_free_its: 5095 kfree(its); 5096 out_unmap: 5097 iounmap(its_base); 5098 pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err); 5099 return err; 5100 } 5101 5102 static bool gic_rdists_supports_plpis(void) 5103 { 5104 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS); 5105 } 5106 5107 static int redist_disable_lpis(void) 5108 { 5109 void __iomem *rbase = gic_data_rdist_rd_base(); 5110 u64 timeout = USEC_PER_SEC; 5111 u64 val; 5112 5113 if (!gic_rdists_supports_plpis()) { 5114 pr_info("CPU%d: LPIs not supported\n", smp_processor_id()); 5115 return -ENXIO; 5116 } 5117 5118 val = readl_relaxed(rbase + GICR_CTLR); 5119 if (!(val & GICR_CTLR_ENABLE_LPIS)) 5120 return 0; 5121 5122 /* 5123 * If coming via a CPU hotplug event, we don't need to disable 5124 * LPIs before trying to re-enable them. They are already 5125 * configured and all is well in the world. 5126 * 5127 * If running with preallocated tables, there is nothing to do. 5128 */ 5129 if (gic_data_rdist()->lpi_enabled || 5130 (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) 5131 return 0; 5132 5133 /* 5134 * From that point on, we only try to do some damage control. 5135 */ 5136 pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n", 5137 smp_processor_id()); 5138 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 5139 5140 /* Disable LPIs */ 5141 val &= ~GICR_CTLR_ENABLE_LPIS; 5142 writel_relaxed(val, rbase + GICR_CTLR); 5143 5144 /* Make sure any change to GICR_CTLR is observable by the GIC */ 5145 dsb(sy); 5146 5147 /* 5148 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs 5149 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers. 5150 * Error out if we time out waiting for RWP to clear. 5151 */ 5152 while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) { 5153 if (!timeout) { 5154 pr_err("CPU%d: Timeout while disabling LPIs\n", 5155 smp_processor_id()); 5156 return -ETIMEDOUT; 5157 } 5158 udelay(1); 5159 timeout--; 5160 } 5161 5162 /* 5163 * After it has been written to 1, it is IMPLEMENTATION 5164 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be 5165 * cleared to 0. Error out if clearing the bit failed. 5166 */ 5167 if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) { 5168 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id()); 5169 return -EBUSY; 5170 } 5171 5172 return 0; 5173 } 5174 5175 int its_cpu_init(void) 5176 { 5177 if (!list_empty(&its_nodes)) { 5178 int ret; 5179 5180 ret = redist_disable_lpis(); 5181 if (ret) 5182 return ret; 5183 5184 its_cpu_init_lpis(); 5185 its_cpu_init_collections(); 5186 } 5187 5188 return 0; 5189 } 5190 5191 static const struct of_device_id its_device_id[] = { 5192 { .compatible = "arm,gic-v3-its", }, 5193 {}, 5194 }; 5195 5196 static int __init its_of_probe(struct device_node *node) 5197 { 5198 struct device_node *np; 5199 struct resource res; 5200 5201 for (np = of_find_matching_node(node, its_device_id); np; 5202 np = of_find_matching_node(np, its_device_id)) { 5203 if (!of_device_is_available(np)) 5204 continue; 5205 if (!of_property_read_bool(np, "msi-controller")) { 5206 pr_warn("%pOF: no msi-controller property, ITS ignored\n", 5207 np); 5208 continue; 5209 } 5210 5211 if (of_address_to_resource(np, 0, &res)) { 5212 pr_warn("%pOF: no regs?\n", np); 5213 continue; 5214 } 5215 5216 its_probe_one(&res, &np->fwnode, of_node_to_nid(np)); 5217 } 5218 return 0; 5219 } 5220 5221 #ifdef CONFIG_ACPI 5222 5223 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K) 5224 5225 #ifdef CONFIG_ACPI_NUMA 5226 struct its_srat_map { 5227 /* numa node id */ 5228 u32 numa_node; 5229 /* GIC ITS ID */ 5230 u32 its_id; 5231 }; 5232 5233 static struct its_srat_map *its_srat_maps __initdata; 5234 static int its_in_srat __initdata; 5235 5236 static int __init acpi_get_its_numa_node(u32 its_id) 5237 { 5238 int i; 5239 5240 for (i = 0; i < its_in_srat; i++) { 5241 if (its_id == its_srat_maps[i].its_id) 5242 return its_srat_maps[i].numa_node; 5243 } 5244 return NUMA_NO_NODE; 5245 } 5246 5247 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header, 5248 const unsigned long end) 5249 { 5250 return 0; 5251 } 5252 5253 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header, 5254 const unsigned long end) 5255 { 5256 int node; 5257 struct acpi_srat_gic_its_affinity *its_affinity; 5258 5259 its_affinity = (struct acpi_srat_gic_its_affinity *)header; 5260 if (!its_affinity) 5261 return -EINVAL; 5262 5263 if (its_affinity->header.length < sizeof(*its_affinity)) { 5264 pr_err("SRAT: Invalid header length %d in ITS affinity\n", 5265 its_affinity->header.length); 5266 return -EINVAL; 5267 } 5268 5269 /* 5270 * Note that in theory a new proximity node could be created by this 5271 * entry as it is an SRAT resource allocation structure. 5272 * We do not currently support doing so. 5273 */ 5274 node = pxm_to_node(its_affinity->proximity_domain); 5275 5276 if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { 5277 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node); 5278 return 0; 5279 } 5280 5281 its_srat_maps[its_in_srat].numa_node = node; 5282 its_srat_maps[its_in_srat].its_id = its_affinity->its_id; 5283 its_in_srat++; 5284 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n", 5285 its_affinity->proximity_domain, its_affinity->its_id, node); 5286 5287 return 0; 5288 } 5289 5290 static void __init acpi_table_parse_srat_its(void) 5291 { 5292 int count; 5293 5294 count = acpi_table_parse_entries(ACPI_SIG_SRAT, 5295 sizeof(struct acpi_table_srat), 5296 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 5297 gic_acpi_match_srat_its, 0); 5298 if (count <= 0) 5299 return; 5300 5301 its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map), 5302 GFP_KERNEL); 5303 if (!its_srat_maps) { 5304 pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n"); 5305 return; 5306 } 5307 5308 acpi_table_parse_entries(ACPI_SIG_SRAT, 5309 sizeof(struct acpi_table_srat), 5310 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 5311 gic_acpi_parse_srat_its, 0); 5312 } 5313 5314 /* free the its_srat_maps after ITS probing */ 5315 static void __init acpi_its_srat_maps_free(void) 5316 { 5317 kfree(its_srat_maps); 5318 } 5319 #else 5320 static void __init acpi_table_parse_srat_its(void) { } 5321 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; } 5322 static void __init acpi_its_srat_maps_free(void) { } 5323 #endif 5324 5325 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, 5326 const unsigned long end) 5327 { 5328 struct acpi_madt_generic_translator *its_entry; 5329 struct fwnode_handle *dom_handle; 5330 struct resource res; 5331 int err; 5332 5333 its_entry = (struct acpi_madt_generic_translator *)header; 5334 memset(&res, 0, sizeof(res)); 5335 res.start = its_entry->base_address; 5336 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1; 5337 res.flags = IORESOURCE_MEM; 5338 5339 dom_handle = irq_domain_alloc_fwnode(&res.start); 5340 if (!dom_handle) { 5341 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n", 5342 &res.start); 5343 return -ENOMEM; 5344 } 5345 5346 err = iort_register_domain_token(its_entry->translation_id, res.start, 5347 dom_handle); 5348 if (err) { 5349 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n", 5350 &res.start, its_entry->translation_id); 5351 goto dom_err; 5352 } 5353 5354 err = its_probe_one(&res, dom_handle, 5355 acpi_get_its_numa_node(its_entry->translation_id)); 5356 if (!err) 5357 return 0; 5358 5359 iort_deregister_domain_token(its_entry->translation_id); 5360 dom_err: 5361 irq_domain_free_fwnode(dom_handle); 5362 return err; 5363 } 5364 5365 static void __init its_acpi_probe(void) 5366 { 5367 acpi_table_parse_srat_its(); 5368 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, 5369 gic_acpi_parse_madt_its, 0); 5370 acpi_its_srat_maps_free(); 5371 } 5372 #else 5373 static void __init its_acpi_probe(void) { } 5374 #endif 5375 5376 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, 5377 struct irq_domain *parent_domain) 5378 { 5379 struct device_node *of_node; 5380 struct its_node *its; 5381 bool has_v4 = false; 5382 bool has_v4_1 = false; 5383 int err; 5384 5385 gic_rdists = rdists; 5386 5387 its_parent = parent_domain; 5388 of_node = to_of_node(handle); 5389 if (of_node) 5390 its_of_probe(of_node); 5391 else 5392 its_acpi_probe(); 5393 5394 if (list_empty(&its_nodes)) { 5395 pr_warn("ITS: No ITS available, not enabling LPIs\n"); 5396 return -ENXIO; 5397 } 5398 5399 err = allocate_lpi_tables(); 5400 if (err) 5401 return err; 5402 5403 list_for_each_entry(its, &its_nodes, entry) { 5404 has_v4 |= is_v4(its); 5405 has_v4_1 |= is_v4_1(its); 5406 } 5407 5408 /* Don't bother with inconsistent systems */ 5409 if (WARN_ON(!has_v4_1 && rdists->has_rvpeid)) 5410 rdists->has_rvpeid = false; 5411 5412 if (has_v4 & rdists->has_vlpis) { 5413 const struct irq_domain_ops *sgi_ops; 5414 5415 if (has_v4_1) 5416 sgi_ops = &its_sgi_domain_ops; 5417 else 5418 sgi_ops = NULL; 5419 5420 if (its_init_vpe_domain() || 5421 its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) { 5422 rdists->has_vlpis = false; 5423 pr_err("ITS: Disabling GICv4 support\n"); 5424 } 5425 } 5426 5427 register_syscore_ops(&its_syscore_ops); 5428 5429 return 0; 5430 } 5431