1 /* 2 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 3 * Author: Marc Zyngier <marc.zyngier@arm.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #include <linux/acpi.h> 19 #include <linux/acpi_iort.h> 20 #include <linux/bitmap.h> 21 #include <linux/cpu.h> 22 #include <linux/crash_dump.h> 23 #include <linux/delay.h> 24 #include <linux/dma-iommu.h> 25 #include <linux/efi.h> 26 #include <linux/interrupt.h> 27 #include <linux/irqdomain.h> 28 #include <linux/list.h> 29 #include <linux/log2.h> 30 #include <linux/memblock.h> 31 #include <linux/mm.h> 32 #include <linux/msi.h> 33 #include <linux/of.h> 34 #include <linux/of_address.h> 35 #include <linux/of_irq.h> 36 #include <linux/of_pci.h> 37 #include <linux/of_platform.h> 38 #include <linux/percpu.h> 39 #include <linux/slab.h> 40 #include <linux/syscore_ops.h> 41 42 #include <linux/irqchip.h> 43 #include <linux/irqchip/arm-gic-v3.h> 44 #include <linux/irqchip/arm-gic-v4.h> 45 46 #include <asm/cputype.h> 47 #include <asm/exception.h> 48 49 #include "irq-gic-common.h" 50 51 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) 52 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) 53 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2) 54 #define ITS_FLAGS_SAVE_SUSPEND_STATE (1ULL << 3) 55 56 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) 57 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) 58 59 static u32 lpi_id_bits; 60 61 /* 62 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to 63 * deal with (one configuration byte per interrupt). PENDBASE has to 64 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). 65 */ 66 #define LPI_NRBITS lpi_id_bits 67 #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K) 68 #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K) 69 70 #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI 71 72 /* 73 * Collection structure - just an ID, and a redistributor address to 74 * ping. We use one per CPU as a bag of interrupts assigned to this 75 * CPU. 76 */ 77 struct its_collection { 78 u64 target_address; 79 u16 col_id; 80 }; 81 82 /* 83 * The ITS_BASER structure - contains memory information, cached 84 * value of BASER register configuration and ITS page size. 85 */ 86 struct its_baser { 87 void *base; 88 u64 val; 89 u32 order; 90 u32 psz; 91 }; 92 93 struct its_device; 94 95 /* 96 * The ITS structure - contains most of the infrastructure, with the 97 * top-level MSI domain, the command queue, the collections, and the 98 * list of devices writing to it. 99 * 100 * dev_alloc_lock has to be taken for device allocations, while the 101 * spinlock must be taken to parse data structures such as the device 102 * list. 103 */ 104 struct its_node { 105 raw_spinlock_t lock; 106 struct mutex dev_alloc_lock; 107 struct list_head entry; 108 void __iomem *base; 109 phys_addr_t phys_base; 110 struct its_cmd_block *cmd_base; 111 struct its_cmd_block *cmd_write; 112 struct its_baser tables[GITS_BASER_NR_REGS]; 113 struct its_collection *collections; 114 struct fwnode_handle *fwnode_handle; 115 u64 (*get_msi_base)(struct its_device *its_dev); 116 u64 cbaser_save; 117 u32 ctlr_save; 118 struct list_head its_device_list; 119 u64 flags; 120 unsigned long list_nr; 121 u32 ite_size; 122 u32 device_ids; 123 int numa_node; 124 unsigned int msi_domain_flags; 125 u32 pre_its_base; /* for Socionext Synquacer */ 126 bool is_v4; 127 int vlpi_redist_offset; 128 }; 129 130 #define ITS_ITT_ALIGN SZ_256 131 132 /* The maximum number of VPEID bits supported by VLPI commands */ 133 #define ITS_MAX_VPEID_BITS (16) 134 #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS)) 135 136 /* Convert page order to size in bytes */ 137 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o)) 138 139 struct event_lpi_map { 140 unsigned long *lpi_map; 141 u16 *col_map; 142 irq_hw_number_t lpi_base; 143 int nr_lpis; 144 struct mutex vlpi_lock; 145 struct its_vm *vm; 146 struct its_vlpi_map *vlpi_maps; 147 int nr_vlpis; 148 }; 149 150 /* 151 * The ITS view of a device - belongs to an ITS, owns an interrupt 152 * translation table, and a list of interrupts. If it some of its 153 * LPIs are injected into a guest (GICv4), the event_map.vm field 154 * indicates which one. 155 */ 156 struct its_device { 157 struct list_head entry; 158 struct its_node *its; 159 struct event_lpi_map event_map; 160 void *itt; 161 u32 nr_ites; 162 u32 device_id; 163 bool shared; 164 }; 165 166 static struct { 167 raw_spinlock_t lock; 168 struct its_device *dev; 169 struct its_vpe **vpes; 170 int next_victim; 171 } vpe_proxy; 172 173 static LIST_HEAD(its_nodes); 174 static DEFINE_RAW_SPINLOCK(its_lock); 175 static struct rdists *gic_rdists; 176 static struct irq_domain *its_parent; 177 178 static unsigned long its_list_map; 179 static u16 vmovp_seq_num; 180 static DEFINE_RAW_SPINLOCK(vmovp_lock); 181 182 static DEFINE_IDA(its_vpeid_ida); 183 184 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) 185 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu)) 186 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 187 #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) 188 189 static struct its_collection *dev_event_to_col(struct its_device *its_dev, 190 u32 event) 191 { 192 struct its_node *its = its_dev->its; 193 194 return its->collections + its_dev->event_map.col_map[event]; 195 } 196 197 static struct its_collection *valid_col(struct its_collection *col) 198 { 199 if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(0, 15))) 200 return NULL; 201 202 return col; 203 } 204 205 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe) 206 { 207 if (valid_col(its->collections + vpe->col_idx)) 208 return vpe; 209 210 return NULL; 211 } 212 213 /* 214 * ITS command descriptors - parameters to be encoded in a command 215 * block. 216 */ 217 struct its_cmd_desc { 218 union { 219 struct { 220 struct its_device *dev; 221 u32 event_id; 222 } its_inv_cmd; 223 224 struct { 225 struct its_device *dev; 226 u32 event_id; 227 } its_clear_cmd; 228 229 struct { 230 struct its_device *dev; 231 u32 event_id; 232 } its_int_cmd; 233 234 struct { 235 struct its_device *dev; 236 int valid; 237 } its_mapd_cmd; 238 239 struct { 240 struct its_collection *col; 241 int valid; 242 } its_mapc_cmd; 243 244 struct { 245 struct its_device *dev; 246 u32 phys_id; 247 u32 event_id; 248 } its_mapti_cmd; 249 250 struct { 251 struct its_device *dev; 252 struct its_collection *col; 253 u32 event_id; 254 } its_movi_cmd; 255 256 struct { 257 struct its_device *dev; 258 u32 event_id; 259 } its_discard_cmd; 260 261 struct { 262 struct its_collection *col; 263 } its_invall_cmd; 264 265 struct { 266 struct its_vpe *vpe; 267 } its_vinvall_cmd; 268 269 struct { 270 struct its_vpe *vpe; 271 struct its_collection *col; 272 bool valid; 273 } its_vmapp_cmd; 274 275 struct { 276 struct its_vpe *vpe; 277 struct its_device *dev; 278 u32 virt_id; 279 u32 event_id; 280 bool db_enabled; 281 } its_vmapti_cmd; 282 283 struct { 284 struct its_vpe *vpe; 285 struct its_device *dev; 286 u32 event_id; 287 bool db_enabled; 288 } its_vmovi_cmd; 289 290 struct { 291 struct its_vpe *vpe; 292 struct its_collection *col; 293 u16 seq_num; 294 u16 its_list; 295 } its_vmovp_cmd; 296 }; 297 }; 298 299 /* 300 * The ITS command block, which is what the ITS actually parses. 301 */ 302 struct its_cmd_block { 303 u64 raw_cmd[4]; 304 }; 305 306 #define ITS_CMD_QUEUE_SZ SZ_64K 307 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) 308 309 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *, 310 struct its_cmd_block *, 311 struct its_cmd_desc *); 312 313 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *, 314 struct its_cmd_block *, 315 struct its_cmd_desc *); 316 317 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 318 { 319 u64 mask = GENMASK_ULL(h, l); 320 *raw_cmd &= ~mask; 321 *raw_cmd |= (val << l) & mask; 322 } 323 324 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 325 { 326 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); 327 } 328 329 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 330 { 331 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); 332 } 333 334 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 335 { 336 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); 337 } 338 339 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 340 { 341 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); 342 } 343 344 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 345 { 346 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); 347 } 348 349 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 350 { 351 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); 352 } 353 354 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 355 { 356 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); 357 } 358 359 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 360 { 361 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); 362 } 363 364 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 365 { 366 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); 367 } 368 369 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid) 370 { 371 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32); 372 } 373 374 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id) 375 { 376 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0); 377 } 378 379 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id) 380 { 381 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32); 382 } 383 384 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid) 385 { 386 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0); 387 } 388 389 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num) 390 { 391 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32); 392 } 393 394 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list) 395 { 396 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0); 397 } 398 399 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa) 400 { 401 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16); 402 } 403 404 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size) 405 { 406 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); 407 } 408 409 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 410 { 411 /* Let's fixup BE commands */ 412 cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); 413 cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); 414 cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); 415 cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); 416 } 417 418 static struct its_collection *its_build_mapd_cmd(struct its_node *its, 419 struct its_cmd_block *cmd, 420 struct its_cmd_desc *desc) 421 { 422 unsigned long itt_addr; 423 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); 424 425 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); 426 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); 427 428 its_encode_cmd(cmd, GITS_CMD_MAPD); 429 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); 430 its_encode_size(cmd, size - 1); 431 its_encode_itt(cmd, itt_addr); 432 its_encode_valid(cmd, desc->its_mapd_cmd.valid); 433 434 its_fixup_cmd(cmd); 435 436 return NULL; 437 } 438 439 static struct its_collection *its_build_mapc_cmd(struct its_node *its, 440 struct its_cmd_block *cmd, 441 struct its_cmd_desc *desc) 442 { 443 its_encode_cmd(cmd, GITS_CMD_MAPC); 444 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 445 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); 446 its_encode_valid(cmd, desc->its_mapc_cmd.valid); 447 448 its_fixup_cmd(cmd); 449 450 return desc->its_mapc_cmd.col; 451 } 452 453 static struct its_collection *its_build_mapti_cmd(struct its_node *its, 454 struct its_cmd_block *cmd, 455 struct its_cmd_desc *desc) 456 { 457 struct its_collection *col; 458 459 col = dev_event_to_col(desc->its_mapti_cmd.dev, 460 desc->its_mapti_cmd.event_id); 461 462 its_encode_cmd(cmd, GITS_CMD_MAPTI); 463 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id); 464 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id); 465 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id); 466 its_encode_collection(cmd, col->col_id); 467 468 its_fixup_cmd(cmd); 469 470 return valid_col(col); 471 } 472 473 static struct its_collection *its_build_movi_cmd(struct its_node *its, 474 struct its_cmd_block *cmd, 475 struct its_cmd_desc *desc) 476 { 477 struct its_collection *col; 478 479 col = dev_event_to_col(desc->its_movi_cmd.dev, 480 desc->its_movi_cmd.event_id); 481 482 its_encode_cmd(cmd, GITS_CMD_MOVI); 483 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); 484 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); 485 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); 486 487 its_fixup_cmd(cmd); 488 489 return valid_col(col); 490 } 491 492 static struct its_collection *its_build_discard_cmd(struct its_node *its, 493 struct its_cmd_block *cmd, 494 struct its_cmd_desc *desc) 495 { 496 struct its_collection *col; 497 498 col = dev_event_to_col(desc->its_discard_cmd.dev, 499 desc->its_discard_cmd.event_id); 500 501 its_encode_cmd(cmd, GITS_CMD_DISCARD); 502 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); 503 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); 504 505 its_fixup_cmd(cmd); 506 507 return valid_col(col); 508 } 509 510 static struct its_collection *its_build_inv_cmd(struct its_node *its, 511 struct its_cmd_block *cmd, 512 struct its_cmd_desc *desc) 513 { 514 struct its_collection *col; 515 516 col = dev_event_to_col(desc->its_inv_cmd.dev, 517 desc->its_inv_cmd.event_id); 518 519 its_encode_cmd(cmd, GITS_CMD_INV); 520 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 521 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 522 523 its_fixup_cmd(cmd); 524 525 return valid_col(col); 526 } 527 528 static struct its_collection *its_build_int_cmd(struct its_node *its, 529 struct its_cmd_block *cmd, 530 struct its_cmd_desc *desc) 531 { 532 struct its_collection *col; 533 534 col = dev_event_to_col(desc->its_int_cmd.dev, 535 desc->its_int_cmd.event_id); 536 537 its_encode_cmd(cmd, GITS_CMD_INT); 538 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); 539 its_encode_event_id(cmd, desc->its_int_cmd.event_id); 540 541 its_fixup_cmd(cmd); 542 543 return valid_col(col); 544 } 545 546 static struct its_collection *its_build_clear_cmd(struct its_node *its, 547 struct its_cmd_block *cmd, 548 struct its_cmd_desc *desc) 549 { 550 struct its_collection *col; 551 552 col = dev_event_to_col(desc->its_clear_cmd.dev, 553 desc->its_clear_cmd.event_id); 554 555 its_encode_cmd(cmd, GITS_CMD_CLEAR); 556 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); 557 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); 558 559 its_fixup_cmd(cmd); 560 561 return valid_col(col); 562 } 563 564 static struct its_collection *its_build_invall_cmd(struct its_node *its, 565 struct its_cmd_block *cmd, 566 struct its_cmd_desc *desc) 567 { 568 its_encode_cmd(cmd, GITS_CMD_INVALL); 569 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 570 571 its_fixup_cmd(cmd); 572 573 return NULL; 574 } 575 576 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its, 577 struct its_cmd_block *cmd, 578 struct its_cmd_desc *desc) 579 { 580 its_encode_cmd(cmd, GITS_CMD_VINVALL); 581 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id); 582 583 its_fixup_cmd(cmd); 584 585 return valid_vpe(its, desc->its_vinvall_cmd.vpe); 586 } 587 588 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, 589 struct its_cmd_block *cmd, 590 struct its_cmd_desc *desc) 591 { 592 unsigned long vpt_addr; 593 u64 target; 594 595 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); 596 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; 597 598 its_encode_cmd(cmd, GITS_CMD_VMAPP); 599 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); 600 its_encode_valid(cmd, desc->its_vmapp_cmd.valid); 601 its_encode_target(cmd, target); 602 its_encode_vpt_addr(cmd, vpt_addr); 603 its_encode_vpt_size(cmd, LPI_NRBITS - 1); 604 605 its_fixup_cmd(cmd); 606 607 return valid_vpe(its, desc->its_vmapp_cmd.vpe); 608 } 609 610 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, 611 struct its_cmd_block *cmd, 612 struct its_cmd_desc *desc) 613 { 614 u32 db; 615 616 if (desc->its_vmapti_cmd.db_enabled) 617 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; 618 else 619 db = 1023; 620 621 its_encode_cmd(cmd, GITS_CMD_VMAPTI); 622 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id); 623 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id); 624 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id); 625 its_encode_db_phys_id(cmd, db); 626 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id); 627 628 its_fixup_cmd(cmd); 629 630 return valid_vpe(its, desc->its_vmapti_cmd.vpe); 631 } 632 633 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, 634 struct its_cmd_block *cmd, 635 struct its_cmd_desc *desc) 636 { 637 u32 db; 638 639 if (desc->its_vmovi_cmd.db_enabled) 640 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; 641 else 642 db = 1023; 643 644 its_encode_cmd(cmd, GITS_CMD_VMOVI); 645 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id); 646 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id); 647 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id); 648 its_encode_db_phys_id(cmd, db); 649 its_encode_db_valid(cmd, true); 650 651 its_fixup_cmd(cmd); 652 653 return valid_vpe(its, desc->its_vmovi_cmd.vpe); 654 } 655 656 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, 657 struct its_cmd_block *cmd, 658 struct its_cmd_desc *desc) 659 { 660 u64 target; 661 662 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; 663 its_encode_cmd(cmd, GITS_CMD_VMOVP); 664 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); 665 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); 666 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); 667 its_encode_target(cmd, target); 668 669 its_fixup_cmd(cmd); 670 671 return valid_vpe(its, desc->its_vmovp_cmd.vpe); 672 } 673 674 static u64 its_cmd_ptr_to_offset(struct its_node *its, 675 struct its_cmd_block *ptr) 676 { 677 return (ptr - its->cmd_base) * sizeof(*ptr); 678 } 679 680 static int its_queue_full(struct its_node *its) 681 { 682 int widx; 683 int ridx; 684 685 widx = its->cmd_write - its->cmd_base; 686 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); 687 688 /* This is incredibly unlikely to happen, unless the ITS locks up. */ 689 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) 690 return 1; 691 692 return 0; 693 } 694 695 static struct its_cmd_block *its_allocate_entry(struct its_node *its) 696 { 697 struct its_cmd_block *cmd; 698 u32 count = 1000000; /* 1s! */ 699 700 while (its_queue_full(its)) { 701 count--; 702 if (!count) { 703 pr_err_ratelimited("ITS queue not draining\n"); 704 return NULL; 705 } 706 cpu_relax(); 707 udelay(1); 708 } 709 710 cmd = its->cmd_write++; 711 712 /* Handle queue wrapping */ 713 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) 714 its->cmd_write = its->cmd_base; 715 716 /* Clear command */ 717 cmd->raw_cmd[0] = 0; 718 cmd->raw_cmd[1] = 0; 719 cmd->raw_cmd[2] = 0; 720 cmd->raw_cmd[3] = 0; 721 722 return cmd; 723 } 724 725 static struct its_cmd_block *its_post_commands(struct its_node *its) 726 { 727 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); 728 729 writel_relaxed(wr, its->base + GITS_CWRITER); 730 731 return its->cmd_write; 732 } 733 734 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) 735 { 736 /* 737 * Make sure the commands written to memory are observable by 738 * the ITS. 739 */ 740 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) 741 gic_flush_dcache_to_poc(cmd, sizeof(*cmd)); 742 else 743 dsb(ishst); 744 } 745 746 static int its_wait_for_range_completion(struct its_node *its, 747 struct its_cmd_block *from, 748 struct its_cmd_block *to) 749 { 750 u64 rd_idx, from_idx, to_idx; 751 u32 count = 1000000; /* 1s! */ 752 753 from_idx = its_cmd_ptr_to_offset(its, from); 754 to_idx = its_cmd_ptr_to_offset(its, to); 755 756 while (1) { 757 rd_idx = readl_relaxed(its->base + GITS_CREADR); 758 759 /* Direct case */ 760 if (from_idx < to_idx && rd_idx >= to_idx) 761 break; 762 763 /* Wrapped case */ 764 if (from_idx >= to_idx && rd_idx >= to_idx && rd_idx < from_idx) 765 break; 766 767 count--; 768 if (!count) { 769 pr_err_ratelimited("ITS queue timeout (%llu %llu %llu)\n", 770 from_idx, to_idx, rd_idx); 771 return -1; 772 } 773 cpu_relax(); 774 udelay(1); 775 } 776 777 return 0; 778 } 779 780 /* Warning, macro hell follows */ 781 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \ 782 void name(struct its_node *its, \ 783 buildtype builder, \ 784 struct its_cmd_desc *desc) \ 785 { \ 786 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \ 787 synctype *sync_obj; \ 788 unsigned long flags; \ 789 \ 790 raw_spin_lock_irqsave(&its->lock, flags); \ 791 \ 792 cmd = its_allocate_entry(its); \ 793 if (!cmd) { /* We're soooooo screewed... */ \ 794 raw_spin_unlock_irqrestore(&its->lock, flags); \ 795 return; \ 796 } \ 797 sync_obj = builder(its, cmd, desc); \ 798 its_flush_cmd(its, cmd); \ 799 \ 800 if (sync_obj) { \ 801 sync_cmd = its_allocate_entry(its); \ 802 if (!sync_cmd) \ 803 goto post; \ 804 \ 805 buildfn(its, sync_cmd, sync_obj); \ 806 its_flush_cmd(its, sync_cmd); \ 807 } \ 808 \ 809 post: \ 810 next_cmd = its_post_commands(its); \ 811 raw_spin_unlock_irqrestore(&its->lock, flags); \ 812 \ 813 if (its_wait_for_range_completion(its, cmd, next_cmd)) \ 814 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \ 815 } 816 817 static void its_build_sync_cmd(struct its_node *its, 818 struct its_cmd_block *sync_cmd, 819 struct its_collection *sync_col) 820 { 821 its_encode_cmd(sync_cmd, GITS_CMD_SYNC); 822 its_encode_target(sync_cmd, sync_col->target_address); 823 824 its_fixup_cmd(sync_cmd); 825 } 826 827 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t, 828 struct its_collection, its_build_sync_cmd) 829 830 static void its_build_vsync_cmd(struct its_node *its, 831 struct its_cmd_block *sync_cmd, 832 struct its_vpe *sync_vpe) 833 { 834 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC); 835 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id); 836 837 its_fixup_cmd(sync_cmd); 838 } 839 840 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t, 841 struct its_vpe, its_build_vsync_cmd) 842 843 static void its_send_int(struct its_device *dev, u32 event_id) 844 { 845 struct its_cmd_desc desc; 846 847 desc.its_int_cmd.dev = dev; 848 desc.its_int_cmd.event_id = event_id; 849 850 its_send_single_command(dev->its, its_build_int_cmd, &desc); 851 } 852 853 static void its_send_clear(struct its_device *dev, u32 event_id) 854 { 855 struct its_cmd_desc desc; 856 857 desc.its_clear_cmd.dev = dev; 858 desc.its_clear_cmd.event_id = event_id; 859 860 its_send_single_command(dev->its, its_build_clear_cmd, &desc); 861 } 862 863 static void its_send_inv(struct its_device *dev, u32 event_id) 864 { 865 struct its_cmd_desc desc; 866 867 desc.its_inv_cmd.dev = dev; 868 desc.its_inv_cmd.event_id = event_id; 869 870 its_send_single_command(dev->its, its_build_inv_cmd, &desc); 871 } 872 873 static void its_send_mapd(struct its_device *dev, int valid) 874 { 875 struct its_cmd_desc desc; 876 877 desc.its_mapd_cmd.dev = dev; 878 desc.its_mapd_cmd.valid = !!valid; 879 880 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); 881 } 882 883 static void its_send_mapc(struct its_node *its, struct its_collection *col, 884 int valid) 885 { 886 struct its_cmd_desc desc; 887 888 desc.its_mapc_cmd.col = col; 889 desc.its_mapc_cmd.valid = !!valid; 890 891 its_send_single_command(its, its_build_mapc_cmd, &desc); 892 } 893 894 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id) 895 { 896 struct its_cmd_desc desc; 897 898 desc.its_mapti_cmd.dev = dev; 899 desc.its_mapti_cmd.phys_id = irq_id; 900 desc.its_mapti_cmd.event_id = id; 901 902 its_send_single_command(dev->its, its_build_mapti_cmd, &desc); 903 } 904 905 static void its_send_movi(struct its_device *dev, 906 struct its_collection *col, u32 id) 907 { 908 struct its_cmd_desc desc; 909 910 desc.its_movi_cmd.dev = dev; 911 desc.its_movi_cmd.col = col; 912 desc.its_movi_cmd.event_id = id; 913 914 its_send_single_command(dev->its, its_build_movi_cmd, &desc); 915 } 916 917 static void its_send_discard(struct its_device *dev, u32 id) 918 { 919 struct its_cmd_desc desc; 920 921 desc.its_discard_cmd.dev = dev; 922 desc.its_discard_cmd.event_id = id; 923 924 its_send_single_command(dev->its, its_build_discard_cmd, &desc); 925 } 926 927 static void its_send_invall(struct its_node *its, struct its_collection *col) 928 { 929 struct its_cmd_desc desc; 930 931 desc.its_invall_cmd.col = col; 932 933 its_send_single_command(its, its_build_invall_cmd, &desc); 934 } 935 936 static void its_send_vmapti(struct its_device *dev, u32 id) 937 { 938 struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; 939 struct its_cmd_desc desc; 940 941 desc.its_vmapti_cmd.vpe = map->vpe; 942 desc.its_vmapti_cmd.dev = dev; 943 desc.its_vmapti_cmd.virt_id = map->vintid; 944 desc.its_vmapti_cmd.event_id = id; 945 desc.its_vmapti_cmd.db_enabled = map->db_enabled; 946 947 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); 948 } 949 950 static void its_send_vmovi(struct its_device *dev, u32 id) 951 { 952 struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; 953 struct its_cmd_desc desc; 954 955 desc.its_vmovi_cmd.vpe = map->vpe; 956 desc.its_vmovi_cmd.dev = dev; 957 desc.its_vmovi_cmd.event_id = id; 958 desc.its_vmovi_cmd.db_enabled = map->db_enabled; 959 960 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); 961 } 962 963 static void its_send_vmapp(struct its_node *its, 964 struct its_vpe *vpe, bool valid) 965 { 966 struct its_cmd_desc desc; 967 968 desc.its_vmapp_cmd.vpe = vpe; 969 desc.its_vmapp_cmd.valid = valid; 970 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; 971 972 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); 973 } 974 975 static void its_send_vmovp(struct its_vpe *vpe) 976 { 977 struct its_cmd_desc desc; 978 struct its_node *its; 979 unsigned long flags; 980 int col_id = vpe->col_idx; 981 982 desc.its_vmovp_cmd.vpe = vpe; 983 desc.its_vmovp_cmd.its_list = (u16)its_list_map; 984 985 if (!its_list_map) { 986 its = list_first_entry(&its_nodes, struct its_node, entry); 987 desc.its_vmovp_cmd.seq_num = 0; 988 desc.its_vmovp_cmd.col = &its->collections[col_id]; 989 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 990 return; 991 } 992 993 /* 994 * Yet another marvel of the architecture. If using the 995 * its_list "feature", we need to make sure that all ITSs 996 * receive all VMOVP commands in the same order. The only way 997 * to guarantee this is to make vmovp a serialization point. 998 * 999 * Wall <-- Head. 1000 */ 1001 raw_spin_lock_irqsave(&vmovp_lock, flags); 1002 1003 desc.its_vmovp_cmd.seq_num = vmovp_seq_num++; 1004 1005 /* Emit VMOVPs */ 1006 list_for_each_entry(its, &its_nodes, entry) { 1007 if (!its->is_v4) 1008 continue; 1009 1010 if (!vpe->its_vm->vlpi_count[its->list_nr]) 1011 continue; 1012 1013 desc.its_vmovp_cmd.col = &its->collections[col_id]; 1014 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 1015 } 1016 1017 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1018 } 1019 1020 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) 1021 { 1022 struct its_cmd_desc desc; 1023 1024 desc.its_vinvall_cmd.vpe = vpe; 1025 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); 1026 } 1027 1028 /* 1029 * irqchip functions - assumes MSI, mostly. 1030 */ 1031 1032 static inline u32 its_get_event_id(struct irq_data *d) 1033 { 1034 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1035 return d->hwirq - its_dev->event_map.lpi_base; 1036 } 1037 1038 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) 1039 { 1040 irq_hw_number_t hwirq; 1041 void *va; 1042 u8 *cfg; 1043 1044 if (irqd_is_forwarded_to_vcpu(d)) { 1045 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1046 u32 event = its_get_event_id(d); 1047 struct its_vlpi_map *map; 1048 1049 va = page_address(its_dev->event_map.vm->vprop_page); 1050 map = &its_dev->event_map.vlpi_maps[event]; 1051 hwirq = map->vintid; 1052 1053 /* Remember the updated property */ 1054 map->properties &= ~clr; 1055 map->properties |= set | LPI_PROP_GROUP1; 1056 } else { 1057 va = gic_rdists->prop_table_va; 1058 hwirq = d->hwirq; 1059 } 1060 1061 cfg = va + hwirq - 8192; 1062 *cfg &= ~clr; 1063 *cfg |= set | LPI_PROP_GROUP1; 1064 1065 /* 1066 * Make the above write visible to the redistributors. 1067 * And yes, we're flushing exactly: One. Single. Byte. 1068 * Humpf... 1069 */ 1070 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) 1071 gic_flush_dcache_to_poc(cfg, sizeof(*cfg)); 1072 else 1073 dsb(ishst); 1074 } 1075 1076 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) 1077 { 1078 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1079 1080 lpi_write_config(d, clr, set); 1081 its_send_inv(its_dev, its_get_event_id(d)); 1082 } 1083 1084 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) 1085 { 1086 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1087 u32 event = its_get_event_id(d); 1088 1089 if (its_dev->event_map.vlpi_maps[event].db_enabled == enable) 1090 return; 1091 1092 its_dev->event_map.vlpi_maps[event].db_enabled = enable; 1093 1094 /* 1095 * More fun with the architecture: 1096 * 1097 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI 1098 * value or to 1023, depending on the enable bit. But that 1099 * would be issueing a mapping for an /existing/ DevID+EventID 1100 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI 1101 * to the /same/ vPE, using this opportunity to adjust the 1102 * doorbell. Mouahahahaha. We loves it, Precious. 1103 */ 1104 its_send_vmovi(its_dev, event); 1105 } 1106 1107 static void its_mask_irq(struct irq_data *d) 1108 { 1109 if (irqd_is_forwarded_to_vcpu(d)) 1110 its_vlpi_set_doorbell(d, false); 1111 1112 lpi_update_config(d, LPI_PROP_ENABLED, 0); 1113 } 1114 1115 static void its_unmask_irq(struct irq_data *d) 1116 { 1117 if (irqd_is_forwarded_to_vcpu(d)) 1118 its_vlpi_set_doorbell(d, true); 1119 1120 lpi_update_config(d, 0, LPI_PROP_ENABLED); 1121 } 1122 1123 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1124 bool force) 1125 { 1126 unsigned int cpu; 1127 const struct cpumask *cpu_mask = cpu_online_mask; 1128 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1129 struct its_collection *target_col; 1130 u32 id = its_get_event_id(d); 1131 1132 /* A forwarded interrupt should use irq_set_vcpu_affinity */ 1133 if (irqd_is_forwarded_to_vcpu(d)) 1134 return -EINVAL; 1135 1136 /* lpi cannot be routed to a redistributor that is on a foreign node */ 1137 if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { 1138 if (its_dev->its->numa_node >= 0) { 1139 cpu_mask = cpumask_of_node(its_dev->its->numa_node); 1140 if (!cpumask_intersects(mask_val, cpu_mask)) 1141 return -EINVAL; 1142 } 1143 } 1144 1145 cpu = cpumask_any_and(mask_val, cpu_mask); 1146 1147 if (cpu >= nr_cpu_ids) 1148 return -EINVAL; 1149 1150 /* don't set the affinity when the target cpu is same as current one */ 1151 if (cpu != its_dev->event_map.col_map[id]) { 1152 target_col = &its_dev->its->collections[cpu]; 1153 its_send_movi(its_dev, target_col, id); 1154 its_dev->event_map.col_map[id] = cpu; 1155 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1156 } 1157 1158 return IRQ_SET_MASK_OK_DONE; 1159 } 1160 1161 static u64 its_irq_get_msi_base(struct its_device *its_dev) 1162 { 1163 struct its_node *its = its_dev->its; 1164 1165 return its->phys_base + GITS_TRANSLATER; 1166 } 1167 1168 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) 1169 { 1170 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1171 struct its_node *its; 1172 u64 addr; 1173 1174 its = its_dev->its; 1175 addr = its->get_msi_base(its_dev); 1176 1177 msg->address_lo = lower_32_bits(addr); 1178 msg->address_hi = upper_32_bits(addr); 1179 msg->data = its_get_event_id(d); 1180 1181 iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg); 1182 } 1183 1184 static int its_irq_set_irqchip_state(struct irq_data *d, 1185 enum irqchip_irq_state which, 1186 bool state) 1187 { 1188 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1189 u32 event = its_get_event_id(d); 1190 1191 if (which != IRQCHIP_STATE_PENDING) 1192 return -EINVAL; 1193 1194 if (state) 1195 its_send_int(its_dev, event); 1196 else 1197 its_send_clear(its_dev, event); 1198 1199 return 0; 1200 } 1201 1202 static void its_map_vm(struct its_node *its, struct its_vm *vm) 1203 { 1204 unsigned long flags; 1205 1206 /* Not using the ITS list? Everything is always mapped. */ 1207 if (!its_list_map) 1208 return; 1209 1210 raw_spin_lock_irqsave(&vmovp_lock, flags); 1211 1212 /* 1213 * If the VM wasn't mapped yet, iterate over the vpes and get 1214 * them mapped now. 1215 */ 1216 vm->vlpi_count[its->list_nr]++; 1217 1218 if (vm->vlpi_count[its->list_nr] == 1) { 1219 int i; 1220 1221 for (i = 0; i < vm->nr_vpes; i++) { 1222 struct its_vpe *vpe = vm->vpes[i]; 1223 struct irq_data *d = irq_get_irq_data(vpe->irq); 1224 1225 /* Map the VPE to the first possible CPU */ 1226 vpe->col_idx = cpumask_first(cpu_online_mask); 1227 its_send_vmapp(its, vpe, true); 1228 its_send_vinvall(its, vpe); 1229 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 1230 } 1231 } 1232 1233 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1234 } 1235 1236 static void its_unmap_vm(struct its_node *its, struct its_vm *vm) 1237 { 1238 unsigned long flags; 1239 1240 /* Not using the ITS list? Everything is always mapped. */ 1241 if (!its_list_map) 1242 return; 1243 1244 raw_spin_lock_irqsave(&vmovp_lock, flags); 1245 1246 if (!--vm->vlpi_count[its->list_nr]) { 1247 int i; 1248 1249 for (i = 0; i < vm->nr_vpes; i++) 1250 its_send_vmapp(its, vm->vpes[i], false); 1251 } 1252 1253 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1254 } 1255 1256 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) 1257 { 1258 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1259 u32 event = its_get_event_id(d); 1260 int ret = 0; 1261 1262 if (!info->map) 1263 return -EINVAL; 1264 1265 mutex_lock(&its_dev->event_map.vlpi_lock); 1266 1267 if (!its_dev->event_map.vm) { 1268 struct its_vlpi_map *maps; 1269 1270 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps), 1271 GFP_KERNEL); 1272 if (!maps) { 1273 ret = -ENOMEM; 1274 goto out; 1275 } 1276 1277 its_dev->event_map.vm = info->map->vm; 1278 its_dev->event_map.vlpi_maps = maps; 1279 } else if (its_dev->event_map.vm != info->map->vm) { 1280 ret = -EINVAL; 1281 goto out; 1282 } 1283 1284 /* Get our private copy of the mapping information */ 1285 its_dev->event_map.vlpi_maps[event] = *info->map; 1286 1287 if (irqd_is_forwarded_to_vcpu(d)) { 1288 /* Already mapped, move it around */ 1289 its_send_vmovi(its_dev, event); 1290 } else { 1291 /* Ensure all the VPEs are mapped on this ITS */ 1292 its_map_vm(its_dev->its, info->map->vm); 1293 1294 /* 1295 * Flag the interrupt as forwarded so that we can 1296 * start poking the virtual property table. 1297 */ 1298 irqd_set_forwarded_to_vcpu(d); 1299 1300 /* Write out the property to the prop table */ 1301 lpi_write_config(d, 0xff, info->map->properties); 1302 1303 /* Drop the physical mapping */ 1304 its_send_discard(its_dev, event); 1305 1306 /* and install the virtual one */ 1307 its_send_vmapti(its_dev, event); 1308 1309 /* Increment the number of VLPIs */ 1310 its_dev->event_map.nr_vlpis++; 1311 } 1312 1313 out: 1314 mutex_unlock(&its_dev->event_map.vlpi_lock); 1315 return ret; 1316 } 1317 1318 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) 1319 { 1320 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1321 u32 event = its_get_event_id(d); 1322 int ret = 0; 1323 1324 mutex_lock(&its_dev->event_map.vlpi_lock); 1325 1326 if (!its_dev->event_map.vm || 1327 !its_dev->event_map.vlpi_maps[event].vm) { 1328 ret = -EINVAL; 1329 goto out; 1330 } 1331 1332 /* Copy our mapping information to the incoming request */ 1333 *info->map = its_dev->event_map.vlpi_maps[event]; 1334 1335 out: 1336 mutex_unlock(&its_dev->event_map.vlpi_lock); 1337 return ret; 1338 } 1339 1340 static int its_vlpi_unmap(struct irq_data *d) 1341 { 1342 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1343 u32 event = its_get_event_id(d); 1344 int ret = 0; 1345 1346 mutex_lock(&its_dev->event_map.vlpi_lock); 1347 1348 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) { 1349 ret = -EINVAL; 1350 goto out; 1351 } 1352 1353 /* Drop the virtual mapping */ 1354 its_send_discard(its_dev, event); 1355 1356 /* and restore the physical one */ 1357 irqd_clr_forwarded_to_vcpu(d); 1358 its_send_mapti(its_dev, d->hwirq, event); 1359 lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO | 1360 LPI_PROP_ENABLED | 1361 LPI_PROP_GROUP1)); 1362 1363 /* Potentially unmap the VM from this ITS */ 1364 its_unmap_vm(its_dev->its, its_dev->event_map.vm); 1365 1366 /* 1367 * Drop the refcount and make the device available again if 1368 * this was the last VLPI. 1369 */ 1370 if (!--its_dev->event_map.nr_vlpis) { 1371 its_dev->event_map.vm = NULL; 1372 kfree(its_dev->event_map.vlpi_maps); 1373 } 1374 1375 out: 1376 mutex_unlock(&its_dev->event_map.vlpi_lock); 1377 return ret; 1378 } 1379 1380 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info) 1381 { 1382 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1383 1384 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) 1385 return -EINVAL; 1386 1387 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI) 1388 lpi_update_config(d, 0xff, info->config); 1389 else 1390 lpi_write_config(d, 0xff, info->config); 1391 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED)); 1392 1393 return 0; 1394 } 1395 1396 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 1397 { 1398 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1399 struct its_cmd_info *info = vcpu_info; 1400 1401 /* Need a v4 ITS */ 1402 if (!its_dev->its->is_v4) 1403 return -EINVAL; 1404 1405 /* Unmap request? */ 1406 if (!info) 1407 return its_vlpi_unmap(d); 1408 1409 switch (info->cmd_type) { 1410 case MAP_VLPI: 1411 return its_vlpi_map(d, info); 1412 1413 case GET_VLPI: 1414 return its_vlpi_get(d, info); 1415 1416 case PROP_UPDATE_VLPI: 1417 case PROP_UPDATE_AND_INV_VLPI: 1418 return its_vlpi_prop_update(d, info); 1419 1420 default: 1421 return -EINVAL; 1422 } 1423 } 1424 1425 static struct irq_chip its_irq_chip = { 1426 .name = "ITS", 1427 .irq_mask = its_mask_irq, 1428 .irq_unmask = its_unmask_irq, 1429 .irq_eoi = irq_chip_eoi_parent, 1430 .irq_set_affinity = its_set_affinity, 1431 .irq_compose_msi_msg = its_irq_compose_msi_msg, 1432 .irq_set_irqchip_state = its_irq_set_irqchip_state, 1433 .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity, 1434 }; 1435 1436 1437 /* 1438 * How we allocate LPIs: 1439 * 1440 * lpi_range_list contains ranges of LPIs that are to available to 1441 * allocate from. To allocate LPIs, just pick the first range that 1442 * fits the required allocation, and reduce it by the required 1443 * amount. Once empty, remove the range from the list. 1444 * 1445 * To free a range of LPIs, add a free range to the list, sort it and 1446 * merge the result if the new range happens to be adjacent to an 1447 * already free block. 1448 * 1449 * The consequence of the above is that allocation is cost is low, but 1450 * freeing is expensive. We assumes that freeing rarely occurs. 1451 */ 1452 #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */ 1453 1454 static DEFINE_MUTEX(lpi_range_lock); 1455 static LIST_HEAD(lpi_range_list); 1456 1457 struct lpi_range { 1458 struct list_head entry; 1459 u32 base_id; 1460 u32 span; 1461 }; 1462 1463 static struct lpi_range *mk_lpi_range(u32 base, u32 span) 1464 { 1465 struct lpi_range *range; 1466 1467 range = kmalloc(sizeof(*range), GFP_KERNEL); 1468 if (range) { 1469 range->base_id = base; 1470 range->span = span; 1471 } 1472 1473 return range; 1474 } 1475 1476 static int alloc_lpi_range(u32 nr_lpis, u32 *base) 1477 { 1478 struct lpi_range *range, *tmp; 1479 int err = -ENOSPC; 1480 1481 mutex_lock(&lpi_range_lock); 1482 1483 list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) { 1484 if (range->span >= nr_lpis) { 1485 *base = range->base_id; 1486 range->base_id += nr_lpis; 1487 range->span -= nr_lpis; 1488 1489 if (range->span == 0) { 1490 list_del(&range->entry); 1491 kfree(range); 1492 } 1493 1494 err = 0; 1495 break; 1496 } 1497 } 1498 1499 mutex_unlock(&lpi_range_lock); 1500 1501 pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis); 1502 return err; 1503 } 1504 1505 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b) 1506 { 1507 if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list) 1508 return; 1509 if (a->base_id + a->span != b->base_id) 1510 return; 1511 b->base_id = a->base_id; 1512 b->span += a->span; 1513 list_del(&a->entry); 1514 kfree(a); 1515 } 1516 1517 static int free_lpi_range(u32 base, u32 nr_lpis) 1518 { 1519 struct lpi_range *new, *old; 1520 1521 new = mk_lpi_range(base, nr_lpis); 1522 if (!new) 1523 return -ENOMEM; 1524 1525 mutex_lock(&lpi_range_lock); 1526 1527 list_for_each_entry_reverse(old, &lpi_range_list, entry) { 1528 if (old->base_id < base) 1529 break; 1530 } 1531 /* 1532 * old is the last element with ->base_id smaller than base, 1533 * so new goes right after it. If there are no elements with 1534 * ->base_id smaller than base, &old->entry ends up pointing 1535 * at the head of the list, and inserting new it the start of 1536 * the list is the right thing to do in that case as well. 1537 */ 1538 list_add(&new->entry, &old->entry); 1539 /* 1540 * Now check if we can merge with the preceding and/or 1541 * following ranges. 1542 */ 1543 merge_lpi_ranges(old, new); 1544 merge_lpi_ranges(new, list_next_entry(new, entry)); 1545 1546 mutex_unlock(&lpi_range_lock); 1547 return 0; 1548 } 1549 1550 static int __init its_lpi_init(u32 id_bits) 1551 { 1552 u32 lpis = (1UL << id_bits) - 8192; 1553 u32 numlpis; 1554 int err; 1555 1556 numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer); 1557 1558 if (numlpis > 2 && !WARN_ON(numlpis > lpis)) { 1559 lpis = numlpis; 1560 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n", 1561 lpis); 1562 } 1563 1564 /* 1565 * Initializing the allocator is just the same as freeing the 1566 * full range of LPIs. 1567 */ 1568 err = free_lpi_range(8192, lpis); 1569 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis); 1570 return err; 1571 } 1572 1573 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids) 1574 { 1575 unsigned long *bitmap = NULL; 1576 int err = 0; 1577 1578 do { 1579 err = alloc_lpi_range(nr_irqs, base); 1580 if (!err) 1581 break; 1582 1583 nr_irqs /= 2; 1584 } while (nr_irqs > 0); 1585 1586 if (!nr_irqs) 1587 err = -ENOSPC; 1588 1589 if (err) 1590 goto out; 1591 1592 bitmap = kcalloc(BITS_TO_LONGS(nr_irqs), sizeof (long), GFP_ATOMIC); 1593 if (!bitmap) 1594 goto out; 1595 1596 *nr_ids = nr_irqs; 1597 1598 out: 1599 if (!bitmap) 1600 *base = *nr_ids = 0; 1601 1602 return bitmap; 1603 } 1604 1605 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids) 1606 { 1607 WARN_ON(free_lpi_range(base, nr_ids)); 1608 kfree(bitmap); 1609 } 1610 1611 static void gic_reset_prop_table(void *va) 1612 { 1613 /* Priority 0xa0, Group-1, disabled */ 1614 memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ); 1615 1616 /* Make sure the GIC will observe the written configuration */ 1617 gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ); 1618 } 1619 1620 static struct page *its_allocate_prop_table(gfp_t gfp_flags) 1621 { 1622 struct page *prop_page; 1623 1624 prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); 1625 if (!prop_page) 1626 return NULL; 1627 1628 gic_reset_prop_table(page_address(prop_page)); 1629 1630 return prop_page; 1631 } 1632 1633 static void its_free_prop_table(struct page *prop_page) 1634 { 1635 free_pages((unsigned long)page_address(prop_page), 1636 get_order(LPI_PROPBASE_SZ)); 1637 } 1638 1639 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size) 1640 { 1641 phys_addr_t start, end, addr_end; 1642 u64 i; 1643 1644 /* 1645 * We don't bother checking for a kdump kernel as by 1646 * construction, the LPI tables are out of this kernel's 1647 * memory map. 1648 */ 1649 if (is_kdump_kernel()) 1650 return true; 1651 1652 addr_end = addr + size - 1; 1653 1654 for_each_reserved_mem_region(i, &start, &end) { 1655 if (addr >= start && addr_end <= end) 1656 return true; 1657 } 1658 1659 /* Not found, not a good sign... */ 1660 pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n", 1661 &addr, &addr_end); 1662 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 1663 return false; 1664 } 1665 1666 static int gic_reserve_range(phys_addr_t addr, unsigned long size) 1667 { 1668 if (efi_enabled(EFI_CONFIG_TABLES)) 1669 return efi_mem_reserve_persistent(addr, size); 1670 1671 return 0; 1672 } 1673 1674 static int __init its_setup_lpi_prop_table(void) 1675 { 1676 if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) { 1677 u64 val; 1678 1679 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER); 1680 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1; 1681 1682 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12); 1683 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa, 1684 LPI_PROPBASE_SZ, 1685 MEMREMAP_WB); 1686 gic_reset_prop_table(gic_rdists->prop_table_va); 1687 } else { 1688 struct page *page; 1689 1690 lpi_id_bits = min_t(u32, 1691 GICD_TYPER_ID_BITS(gic_rdists->gicd_typer), 1692 ITS_MAX_LPI_NRBITS); 1693 page = its_allocate_prop_table(GFP_NOWAIT); 1694 if (!page) { 1695 pr_err("Failed to allocate PROPBASE\n"); 1696 return -ENOMEM; 1697 } 1698 1699 gic_rdists->prop_table_pa = page_to_phys(page); 1700 gic_rdists->prop_table_va = page_address(page); 1701 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa, 1702 LPI_PROPBASE_SZ)); 1703 } 1704 1705 pr_info("GICv3: using LPI property table @%pa\n", 1706 &gic_rdists->prop_table_pa); 1707 1708 return its_lpi_init(lpi_id_bits); 1709 } 1710 1711 static const char *its_base_type_string[] = { 1712 [GITS_BASER_TYPE_DEVICE] = "Devices", 1713 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs", 1714 [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)", 1715 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections", 1716 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)", 1717 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)", 1718 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", 1719 }; 1720 1721 static u64 its_read_baser(struct its_node *its, struct its_baser *baser) 1722 { 1723 u32 idx = baser - its->tables; 1724 1725 return gits_read_baser(its->base + GITS_BASER + (idx << 3)); 1726 } 1727 1728 static void its_write_baser(struct its_node *its, struct its_baser *baser, 1729 u64 val) 1730 { 1731 u32 idx = baser - its->tables; 1732 1733 gits_write_baser(val, its->base + GITS_BASER + (idx << 3)); 1734 baser->val = its_read_baser(its, baser); 1735 } 1736 1737 static int its_setup_baser(struct its_node *its, struct its_baser *baser, 1738 u64 cache, u64 shr, u32 psz, u32 order, 1739 bool indirect) 1740 { 1741 u64 val = its_read_baser(its, baser); 1742 u64 esz = GITS_BASER_ENTRY_SIZE(val); 1743 u64 type = GITS_BASER_TYPE(val); 1744 u64 baser_phys, tmp; 1745 u32 alloc_pages; 1746 struct page *page; 1747 void *base; 1748 1749 retry_alloc_baser: 1750 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); 1751 if (alloc_pages > GITS_BASER_PAGES_MAX) { 1752 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", 1753 &its->phys_base, its_base_type_string[type], 1754 alloc_pages, GITS_BASER_PAGES_MAX); 1755 alloc_pages = GITS_BASER_PAGES_MAX; 1756 order = get_order(GITS_BASER_PAGES_MAX * psz); 1757 } 1758 1759 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); 1760 if (!page) 1761 return -ENOMEM; 1762 1763 base = (void *)page_address(page); 1764 baser_phys = virt_to_phys(base); 1765 1766 /* Check if the physical address of the memory is above 48bits */ 1767 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) { 1768 1769 /* 52bit PA is supported only when PageSize=64K */ 1770 if (psz != SZ_64K) { 1771 pr_err("ITS: no 52bit PA support when psz=%d\n", psz); 1772 free_pages((unsigned long)base, order); 1773 return -ENXIO; 1774 } 1775 1776 /* Convert 52bit PA to 48bit field */ 1777 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys); 1778 } 1779 1780 retry_baser: 1781 val = (baser_phys | 1782 (type << GITS_BASER_TYPE_SHIFT) | 1783 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | 1784 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) | 1785 cache | 1786 shr | 1787 GITS_BASER_VALID); 1788 1789 val |= indirect ? GITS_BASER_INDIRECT : 0x0; 1790 1791 switch (psz) { 1792 case SZ_4K: 1793 val |= GITS_BASER_PAGE_SIZE_4K; 1794 break; 1795 case SZ_16K: 1796 val |= GITS_BASER_PAGE_SIZE_16K; 1797 break; 1798 case SZ_64K: 1799 val |= GITS_BASER_PAGE_SIZE_64K; 1800 break; 1801 } 1802 1803 its_write_baser(its, baser, val); 1804 tmp = baser->val; 1805 1806 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { 1807 /* 1808 * Shareability didn't stick. Just use 1809 * whatever the read reported, which is likely 1810 * to be the only thing this redistributor 1811 * supports. If that's zero, make it 1812 * non-cacheable as well. 1813 */ 1814 shr = tmp & GITS_BASER_SHAREABILITY_MASK; 1815 if (!shr) { 1816 cache = GITS_BASER_nC; 1817 gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order)); 1818 } 1819 goto retry_baser; 1820 } 1821 1822 if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { 1823 /* 1824 * Page size didn't stick. Let's try a smaller 1825 * size and retry. If we reach 4K, then 1826 * something is horribly wrong... 1827 */ 1828 free_pages((unsigned long)base, order); 1829 baser->base = NULL; 1830 1831 switch (psz) { 1832 case SZ_16K: 1833 psz = SZ_4K; 1834 goto retry_alloc_baser; 1835 case SZ_64K: 1836 psz = SZ_16K; 1837 goto retry_alloc_baser; 1838 } 1839 } 1840 1841 if (val != tmp) { 1842 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", 1843 &its->phys_base, its_base_type_string[type], 1844 val, tmp); 1845 free_pages((unsigned long)base, order); 1846 return -ENXIO; 1847 } 1848 1849 baser->order = order; 1850 baser->base = base; 1851 baser->psz = psz; 1852 tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz; 1853 1854 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n", 1855 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), 1856 its_base_type_string[type], 1857 (unsigned long)virt_to_phys(base), 1858 indirect ? "indirect" : "flat", (int)esz, 1859 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); 1860 1861 return 0; 1862 } 1863 1864 static bool its_parse_indirect_baser(struct its_node *its, 1865 struct its_baser *baser, 1866 u32 psz, u32 *order, u32 ids) 1867 { 1868 u64 tmp = its_read_baser(its, baser); 1869 u64 type = GITS_BASER_TYPE(tmp); 1870 u64 esz = GITS_BASER_ENTRY_SIZE(tmp); 1871 u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb; 1872 u32 new_order = *order; 1873 bool indirect = false; 1874 1875 /* No need to enable Indirection if memory requirement < (psz*2)bytes */ 1876 if ((esz << ids) > (psz * 2)) { 1877 /* 1878 * Find out whether hw supports a single or two-level table by 1879 * table by reading bit at offset '62' after writing '1' to it. 1880 */ 1881 its_write_baser(its, baser, val | GITS_BASER_INDIRECT); 1882 indirect = !!(baser->val & GITS_BASER_INDIRECT); 1883 1884 if (indirect) { 1885 /* 1886 * The size of the lvl2 table is equal to ITS page size 1887 * which is 'psz'. For computing lvl1 table size, 1888 * subtract ID bits that sparse lvl2 table from 'ids' 1889 * which is reported by ITS hardware times lvl1 table 1890 * entry size. 1891 */ 1892 ids -= ilog2(psz / (int)esz); 1893 esz = GITS_LVL1_ENTRY_SIZE; 1894 } 1895 } 1896 1897 /* 1898 * Allocate as many entries as required to fit the 1899 * range of device IDs that the ITS can grok... The ID 1900 * space being incredibly sparse, this results in a 1901 * massive waste of memory if two-level device table 1902 * feature is not supported by hardware. 1903 */ 1904 new_order = max_t(u32, get_order(esz << ids), new_order); 1905 if (new_order >= MAX_ORDER) { 1906 new_order = MAX_ORDER - 1; 1907 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); 1908 pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n", 1909 &its->phys_base, its_base_type_string[type], 1910 its->device_ids, ids); 1911 } 1912 1913 *order = new_order; 1914 1915 return indirect; 1916 } 1917 1918 static void its_free_tables(struct its_node *its) 1919 { 1920 int i; 1921 1922 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 1923 if (its->tables[i].base) { 1924 free_pages((unsigned long)its->tables[i].base, 1925 its->tables[i].order); 1926 its->tables[i].base = NULL; 1927 } 1928 } 1929 } 1930 1931 static int its_alloc_tables(struct its_node *its) 1932 { 1933 u64 shr = GITS_BASER_InnerShareable; 1934 u64 cache = GITS_BASER_RaWaWb; 1935 u32 psz = SZ_64K; 1936 int err, i; 1937 1938 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) 1939 /* erratum 24313: ignore memory access type */ 1940 cache = GITS_BASER_nCnB; 1941 1942 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 1943 struct its_baser *baser = its->tables + i; 1944 u64 val = its_read_baser(its, baser); 1945 u64 type = GITS_BASER_TYPE(val); 1946 u32 order = get_order(psz); 1947 bool indirect = false; 1948 1949 switch (type) { 1950 case GITS_BASER_TYPE_NONE: 1951 continue; 1952 1953 case GITS_BASER_TYPE_DEVICE: 1954 indirect = its_parse_indirect_baser(its, baser, 1955 psz, &order, 1956 its->device_ids); 1957 break; 1958 1959 case GITS_BASER_TYPE_VCPU: 1960 indirect = its_parse_indirect_baser(its, baser, 1961 psz, &order, 1962 ITS_MAX_VPEID_BITS); 1963 break; 1964 } 1965 1966 err = its_setup_baser(its, baser, cache, shr, psz, order, indirect); 1967 if (err < 0) { 1968 its_free_tables(its); 1969 return err; 1970 } 1971 1972 /* Update settings which will be used for next BASERn */ 1973 psz = baser->psz; 1974 cache = baser->val & GITS_BASER_CACHEABILITY_MASK; 1975 shr = baser->val & GITS_BASER_SHAREABILITY_MASK; 1976 } 1977 1978 return 0; 1979 } 1980 1981 static int its_alloc_collections(struct its_node *its) 1982 { 1983 int i; 1984 1985 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections), 1986 GFP_KERNEL); 1987 if (!its->collections) 1988 return -ENOMEM; 1989 1990 for (i = 0; i < nr_cpu_ids; i++) 1991 its->collections[i].target_address = ~0ULL; 1992 1993 return 0; 1994 } 1995 1996 static struct page *its_allocate_pending_table(gfp_t gfp_flags) 1997 { 1998 struct page *pend_page; 1999 2000 pend_page = alloc_pages(gfp_flags | __GFP_ZERO, 2001 get_order(LPI_PENDBASE_SZ)); 2002 if (!pend_page) 2003 return NULL; 2004 2005 /* Make sure the GIC will observe the zero-ed page */ 2006 gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ); 2007 2008 return pend_page; 2009 } 2010 2011 static void its_free_pending_table(struct page *pt) 2012 { 2013 free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); 2014 } 2015 2016 /* 2017 * Booting with kdump and LPIs enabled is generally fine. Any other 2018 * case is wrong in the absence of firmware/EFI support. 2019 */ 2020 static bool enabled_lpis_allowed(void) 2021 { 2022 phys_addr_t addr; 2023 u64 val; 2024 2025 /* Check whether the property table is in a reserved region */ 2026 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER); 2027 addr = val & GENMASK_ULL(51, 12); 2028 2029 return gic_check_reserved_range(addr, LPI_PROPBASE_SZ); 2030 } 2031 2032 static int __init allocate_lpi_tables(void) 2033 { 2034 u64 val; 2035 int err, cpu; 2036 2037 /* 2038 * If LPIs are enabled while we run this from the boot CPU, 2039 * flag the RD tables as pre-allocated if the stars do align. 2040 */ 2041 val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR); 2042 if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) { 2043 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED | 2044 RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING); 2045 pr_info("GICv3: Using preallocated redistributor tables\n"); 2046 } 2047 2048 err = its_setup_lpi_prop_table(); 2049 if (err) 2050 return err; 2051 2052 /* 2053 * We allocate all the pending tables anyway, as we may have a 2054 * mix of RDs that have had LPIs enabled, and some that 2055 * don't. We'll free the unused ones as each CPU comes online. 2056 */ 2057 for_each_possible_cpu(cpu) { 2058 struct page *pend_page; 2059 2060 pend_page = its_allocate_pending_table(GFP_NOWAIT); 2061 if (!pend_page) { 2062 pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu); 2063 return -ENOMEM; 2064 } 2065 2066 gic_data_rdist_cpu(cpu)->pend_page = pend_page; 2067 } 2068 2069 return 0; 2070 } 2071 2072 static u64 its_clear_vpend_valid(void __iomem *vlpi_base) 2073 { 2074 u32 count = 1000000; /* 1s! */ 2075 bool clean; 2076 u64 val; 2077 2078 val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); 2079 val &= ~GICR_VPENDBASER_Valid; 2080 gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 2081 2082 do { 2083 val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); 2084 clean = !(val & GICR_VPENDBASER_Dirty); 2085 if (!clean) { 2086 count--; 2087 cpu_relax(); 2088 udelay(1); 2089 } 2090 } while (!clean && count); 2091 2092 return val; 2093 } 2094 2095 static void its_cpu_init_lpis(void) 2096 { 2097 void __iomem *rbase = gic_data_rdist_rd_base(); 2098 struct page *pend_page; 2099 phys_addr_t paddr; 2100 u64 val, tmp; 2101 2102 if (gic_data_rdist()->lpi_enabled) 2103 return; 2104 2105 val = readl_relaxed(rbase + GICR_CTLR); 2106 if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) && 2107 (val & GICR_CTLR_ENABLE_LPIS)) { 2108 /* 2109 * Check that we get the same property table on all 2110 * RDs. If we don't, this is hopeless. 2111 */ 2112 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER); 2113 paddr &= GENMASK_ULL(51, 12); 2114 if (WARN_ON(gic_rdists->prop_table_pa != paddr)) 2115 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 2116 2117 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER); 2118 paddr &= GENMASK_ULL(51, 16); 2119 2120 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ)); 2121 its_free_pending_table(gic_data_rdist()->pend_page); 2122 gic_data_rdist()->pend_page = NULL; 2123 2124 goto out; 2125 } 2126 2127 pend_page = gic_data_rdist()->pend_page; 2128 paddr = page_to_phys(pend_page); 2129 WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ)); 2130 2131 /* set PROPBASE */ 2132 val = (gic_rdists->prop_table_pa | 2133 GICR_PROPBASER_InnerShareable | 2134 GICR_PROPBASER_RaWaWb | 2135 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); 2136 2137 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 2138 tmp = gicr_read_propbaser(rbase + GICR_PROPBASER); 2139 2140 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { 2141 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) { 2142 /* 2143 * The HW reports non-shareable, we must 2144 * remove the cacheability attributes as 2145 * well. 2146 */ 2147 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK | 2148 GICR_PROPBASER_CACHEABILITY_MASK); 2149 val |= GICR_PROPBASER_nC; 2150 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 2151 } 2152 pr_info_once("GIC: using cache flushing for LPI property table\n"); 2153 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; 2154 } 2155 2156 /* set PENDBASE */ 2157 val = (page_to_phys(pend_page) | 2158 GICR_PENDBASER_InnerShareable | 2159 GICR_PENDBASER_RaWaWb); 2160 2161 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 2162 tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER); 2163 2164 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { 2165 /* 2166 * The HW reports non-shareable, we must remove the 2167 * cacheability attributes as well. 2168 */ 2169 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK | 2170 GICR_PENDBASER_CACHEABILITY_MASK); 2171 val |= GICR_PENDBASER_nC; 2172 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 2173 } 2174 2175 /* Enable LPIs */ 2176 val = readl_relaxed(rbase + GICR_CTLR); 2177 val |= GICR_CTLR_ENABLE_LPIS; 2178 writel_relaxed(val, rbase + GICR_CTLR); 2179 2180 if (gic_rdists->has_vlpis) { 2181 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 2182 2183 /* 2184 * It's possible for CPU to receive VLPIs before it is 2185 * sheduled as a vPE, especially for the first CPU, and the 2186 * VLPI with INTID larger than 2^(IDbits+1) will be considered 2187 * as out of range and dropped by GIC. 2188 * So we initialize IDbits to known value to avoid VLPI drop. 2189 */ 2190 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; 2191 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n", 2192 smp_processor_id(), val); 2193 gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 2194 2195 /* 2196 * Also clear Valid bit of GICR_VPENDBASER, in case some 2197 * ancient programming gets left in and has possibility of 2198 * corrupting memory. 2199 */ 2200 val = its_clear_vpend_valid(vlpi_base); 2201 WARN_ON(val & GICR_VPENDBASER_Dirty); 2202 } 2203 2204 /* Make sure the GIC has seen the above */ 2205 dsb(sy); 2206 out: 2207 gic_data_rdist()->lpi_enabled = true; 2208 pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n", 2209 smp_processor_id(), 2210 gic_data_rdist()->pend_page ? "allocated" : "reserved", 2211 &paddr); 2212 } 2213 2214 static void its_cpu_init_collection(struct its_node *its) 2215 { 2216 int cpu = smp_processor_id(); 2217 u64 target; 2218 2219 /* avoid cross node collections and its mapping */ 2220 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { 2221 struct device_node *cpu_node; 2222 2223 cpu_node = of_get_cpu_node(cpu, NULL); 2224 if (its->numa_node != NUMA_NO_NODE && 2225 its->numa_node != of_node_to_nid(cpu_node)) 2226 return; 2227 } 2228 2229 /* 2230 * We now have to bind each collection to its target 2231 * redistributor. 2232 */ 2233 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) { 2234 /* 2235 * This ITS wants the physical address of the 2236 * redistributor. 2237 */ 2238 target = gic_data_rdist()->phys_base; 2239 } else { 2240 /* This ITS wants a linear CPU number. */ 2241 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 2242 target = GICR_TYPER_CPU_NUMBER(target) << 16; 2243 } 2244 2245 /* Perform collection mapping */ 2246 its->collections[cpu].target_address = target; 2247 its->collections[cpu].col_id = cpu; 2248 2249 its_send_mapc(its, &its->collections[cpu], 1); 2250 its_send_invall(its, &its->collections[cpu]); 2251 } 2252 2253 static void its_cpu_init_collections(void) 2254 { 2255 struct its_node *its; 2256 2257 raw_spin_lock(&its_lock); 2258 2259 list_for_each_entry(its, &its_nodes, entry) 2260 its_cpu_init_collection(its); 2261 2262 raw_spin_unlock(&its_lock); 2263 } 2264 2265 static struct its_device *its_find_device(struct its_node *its, u32 dev_id) 2266 { 2267 struct its_device *its_dev = NULL, *tmp; 2268 unsigned long flags; 2269 2270 raw_spin_lock_irqsave(&its->lock, flags); 2271 2272 list_for_each_entry(tmp, &its->its_device_list, entry) { 2273 if (tmp->device_id == dev_id) { 2274 its_dev = tmp; 2275 break; 2276 } 2277 } 2278 2279 raw_spin_unlock_irqrestore(&its->lock, flags); 2280 2281 return its_dev; 2282 } 2283 2284 static struct its_baser *its_get_baser(struct its_node *its, u32 type) 2285 { 2286 int i; 2287 2288 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 2289 if (GITS_BASER_TYPE(its->tables[i].val) == type) 2290 return &its->tables[i]; 2291 } 2292 2293 return NULL; 2294 } 2295 2296 static bool its_alloc_table_entry(struct its_node *its, 2297 struct its_baser *baser, u32 id) 2298 { 2299 struct page *page; 2300 u32 esz, idx; 2301 __le64 *table; 2302 2303 /* Don't allow device id that exceeds single, flat table limit */ 2304 esz = GITS_BASER_ENTRY_SIZE(baser->val); 2305 if (!(baser->val & GITS_BASER_INDIRECT)) 2306 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); 2307 2308 /* Compute 1st level table index & check if that exceeds table limit */ 2309 idx = id >> ilog2(baser->psz / esz); 2310 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) 2311 return false; 2312 2313 table = baser->base; 2314 2315 /* Allocate memory for 2nd level table */ 2316 if (!table[idx]) { 2317 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, 2318 get_order(baser->psz)); 2319 if (!page) 2320 return false; 2321 2322 /* Flush Lvl2 table to PoC if hw doesn't support coherency */ 2323 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 2324 gic_flush_dcache_to_poc(page_address(page), baser->psz); 2325 2326 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); 2327 2328 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ 2329 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 2330 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); 2331 2332 /* Ensure updated table contents are visible to ITS hardware */ 2333 dsb(sy); 2334 } 2335 2336 return true; 2337 } 2338 2339 static bool its_alloc_device_table(struct its_node *its, u32 dev_id) 2340 { 2341 struct its_baser *baser; 2342 2343 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); 2344 2345 /* Don't allow device id that exceeds ITS hardware limit */ 2346 if (!baser) 2347 return (ilog2(dev_id) < its->device_ids); 2348 2349 return its_alloc_table_entry(its, baser, dev_id); 2350 } 2351 2352 static bool its_alloc_vpe_table(u32 vpe_id) 2353 { 2354 struct its_node *its; 2355 2356 /* 2357 * Make sure the L2 tables are allocated on *all* v4 ITSs. We 2358 * could try and only do it on ITSs corresponding to devices 2359 * that have interrupts targeted at this VPE, but the 2360 * complexity becomes crazy (and you have tons of memory 2361 * anyway, right?). 2362 */ 2363 list_for_each_entry(its, &its_nodes, entry) { 2364 struct its_baser *baser; 2365 2366 if (!its->is_v4) 2367 continue; 2368 2369 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU); 2370 if (!baser) 2371 return false; 2372 2373 if (!its_alloc_table_entry(its, baser, vpe_id)) 2374 return false; 2375 } 2376 2377 return true; 2378 } 2379 2380 static struct its_device *its_create_device(struct its_node *its, u32 dev_id, 2381 int nvecs, bool alloc_lpis) 2382 { 2383 struct its_device *dev; 2384 unsigned long *lpi_map = NULL; 2385 unsigned long flags; 2386 u16 *col_map = NULL; 2387 void *itt; 2388 int lpi_base; 2389 int nr_lpis; 2390 int nr_ites; 2391 int sz; 2392 2393 if (!its_alloc_device_table(its, dev_id)) 2394 return NULL; 2395 2396 if (WARN_ON(!is_power_of_2(nvecs))) 2397 nvecs = roundup_pow_of_two(nvecs); 2398 2399 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2400 /* 2401 * Even if the device wants a single LPI, the ITT must be 2402 * sized as a power of two (and you need at least one bit...). 2403 */ 2404 nr_ites = max(2, nvecs); 2405 sz = nr_ites * its->ite_size; 2406 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; 2407 itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); 2408 if (alloc_lpis) { 2409 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis); 2410 if (lpi_map) 2411 col_map = kcalloc(nr_lpis, sizeof(*col_map), 2412 GFP_KERNEL); 2413 } else { 2414 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL); 2415 nr_lpis = 0; 2416 lpi_base = 0; 2417 } 2418 2419 if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { 2420 kfree(dev); 2421 kfree(itt); 2422 kfree(lpi_map); 2423 kfree(col_map); 2424 return NULL; 2425 } 2426 2427 gic_flush_dcache_to_poc(itt, sz); 2428 2429 dev->its = its; 2430 dev->itt = itt; 2431 dev->nr_ites = nr_ites; 2432 dev->event_map.lpi_map = lpi_map; 2433 dev->event_map.col_map = col_map; 2434 dev->event_map.lpi_base = lpi_base; 2435 dev->event_map.nr_lpis = nr_lpis; 2436 mutex_init(&dev->event_map.vlpi_lock); 2437 dev->device_id = dev_id; 2438 INIT_LIST_HEAD(&dev->entry); 2439 2440 raw_spin_lock_irqsave(&its->lock, flags); 2441 list_add(&dev->entry, &its->its_device_list); 2442 raw_spin_unlock_irqrestore(&its->lock, flags); 2443 2444 /* Map device to its ITT */ 2445 its_send_mapd(dev, 1); 2446 2447 return dev; 2448 } 2449 2450 static void its_free_device(struct its_device *its_dev) 2451 { 2452 unsigned long flags; 2453 2454 raw_spin_lock_irqsave(&its_dev->its->lock, flags); 2455 list_del(&its_dev->entry); 2456 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); 2457 kfree(its_dev->itt); 2458 kfree(its_dev); 2459 } 2460 2461 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq) 2462 { 2463 int idx; 2464 2465 idx = bitmap_find_free_region(dev->event_map.lpi_map, 2466 dev->event_map.nr_lpis, 2467 get_count_order(nvecs)); 2468 if (idx < 0) 2469 return -ENOSPC; 2470 2471 *hwirq = dev->event_map.lpi_base + idx; 2472 set_bit(idx, dev->event_map.lpi_map); 2473 2474 return 0; 2475 } 2476 2477 static int its_msi_prepare(struct irq_domain *domain, struct device *dev, 2478 int nvec, msi_alloc_info_t *info) 2479 { 2480 struct its_node *its; 2481 struct its_device *its_dev; 2482 struct msi_domain_info *msi_info; 2483 u32 dev_id; 2484 int err = 0; 2485 2486 /* 2487 * We ignore "dev" entirely, and rely on the dev_id that has 2488 * been passed via the scratchpad. This limits this domain's 2489 * usefulness to upper layers that definitely know that they 2490 * are built on top of the ITS. 2491 */ 2492 dev_id = info->scratchpad[0].ul; 2493 2494 msi_info = msi_get_domain_info(domain); 2495 its = msi_info->data; 2496 2497 if (!gic_rdists->has_direct_lpi && 2498 vpe_proxy.dev && 2499 vpe_proxy.dev->its == its && 2500 dev_id == vpe_proxy.dev->device_id) { 2501 /* Bad luck. Get yourself a better implementation */ 2502 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n", 2503 dev_id); 2504 return -EINVAL; 2505 } 2506 2507 mutex_lock(&its->dev_alloc_lock); 2508 its_dev = its_find_device(its, dev_id); 2509 if (its_dev) { 2510 /* 2511 * We already have seen this ID, probably through 2512 * another alias (PCI bridge of some sort). No need to 2513 * create the device. 2514 */ 2515 its_dev->shared = true; 2516 pr_debug("Reusing ITT for devID %x\n", dev_id); 2517 goto out; 2518 } 2519 2520 its_dev = its_create_device(its, dev_id, nvec, true); 2521 if (!its_dev) { 2522 err = -ENOMEM; 2523 goto out; 2524 } 2525 2526 pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec)); 2527 out: 2528 mutex_unlock(&its->dev_alloc_lock); 2529 info->scratchpad[0].ptr = its_dev; 2530 return err; 2531 } 2532 2533 static struct msi_domain_ops its_msi_domain_ops = { 2534 .msi_prepare = its_msi_prepare, 2535 }; 2536 2537 static int its_irq_gic_domain_alloc(struct irq_domain *domain, 2538 unsigned int virq, 2539 irq_hw_number_t hwirq) 2540 { 2541 struct irq_fwspec fwspec; 2542 2543 if (irq_domain_get_of_node(domain->parent)) { 2544 fwspec.fwnode = domain->parent->fwnode; 2545 fwspec.param_count = 3; 2546 fwspec.param[0] = GIC_IRQ_TYPE_LPI; 2547 fwspec.param[1] = hwirq; 2548 fwspec.param[2] = IRQ_TYPE_EDGE_RISING; 2549 } else if (is_fwnode_irqchip(domain->parent->fwnode)) { 2550 fwspec.fwnode = domain->parent->fwnode; 2551 fwspec.param_count = 2; 2552 fwspec.param[0] = hwirq; 2553 fwspec.param[1] = IRQ_TYPE_EDGE_RISING; 2554 } else { 2555 return -EINVAL; 2556 } 2557 2558 return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); 2559 } 2560 2561 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 2562 unsigned int nr_irqs, void *args) 2563 { 2564 msi_alloc_info_t *info = args; 2565 struct its_device *its_dev = info->scratchpad[0].ptr; 2566 struct its_node *its = its_dev->its; 2567 irq_hw_number_t hwirq; 2568 int err; 2569 int i; 2570 2571 err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq); 2572 if (err) 2573 return err; 2574 2575 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev)); 2576 if (err) 2577 return err; 2578 2579 for (i = 0; i < nr_irqs; i++) { 2580 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i); 2581 if (err) 2582 return err; 2583 2584 irq_domain_set_hwirq_and_chip(domain, virq + i, 2585 hwirq + i, &its_irq_chip, its_dev); 2586 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq + i))); 2587 pr_debug("ID:%d pID:%d vID:%d\n", 2588 (int)(hwirq + i - its_dev->event_map.lpi_base), 2589 (int)(hwirq + i), virq + i); 2590 } 2591 2592 return 0; 2593 } 2594 2595 static int its_irq_domain_activate(struct irq_domain *domain, 2596 struct irq_data *d, bool reserve) 2597 { 2598 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 2599 u32 event = its_get_event_id(d); 2600 const struct cpumask *cpu_mask = cpu_online_mask; 2601 int cpu; 2602 2603 /* get the cpu_mask of local node */ 2604 if (its_dev->its->numa_node >= 0) 2605 cpu_mask = cpumask_of_node(its_dev->its->numa_node); 2606 2607 /* Bind the LPI to the first possible CPU */ 2608 cpu = cpumask_first_and(cpu_mask, cpu_online_mask); 2609 if (cpu >= nr_cpu_ids) { 2610 if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) 2611 return -EINVAL; 2612 2613 cpu = cpumask_first(cpu_online_mask); 2614 } 2615 2616 its_dev->event_map.col_map[event] = cpu; 2617 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 2618 2619 /* Map the GIC IRQ and event to the device */ 2620 its_send_mapti(its_dev, d->hwirq, event); 2621 return 0; 2622 } 2623 2624 static void its_irq_domain_deactivate(struct irq_domain *domain, 2625 struct irq_data *d) 2626 { 2627 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 2628 u32 event = its_get_event_id(d); 2629 2630 /* Stop the delivery of interrupts */ 2631 its_send_discard(its_dev, event); 2632 } 2633 2634 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, 2635 unsigned int nr_irqs) 2636 { 2637 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 2638 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 2639 struct its_node *its = its_dev->its; 2640 int i; 2641 2642 for (i = 0; i < nr_irqs; i++) { 2643 struct irq_data *data = irq_domain_get_irq_data(domain, 2644 virq + i); 2645 u32 event = its_get_event_id(data); 2646 2647 /* Mark interrupt index as unused */ 2648 clear_bit(event, its_dev->event_map.lpi_map); 2649 2650 /* Nuke the entry in the domain */ 2651 irq_domain_reset_irq_data(data); 2652 } 2653 2654 mutex_lock(&its->dev_alloc_lock); 2655 2656 /* 2657 * If all interrupts have been freed, start mopping the 2658 * floor. This is conditionned on the device not being shared. 2659 */ 2660 if (!its_dev->shared && 2661 bitmap_empty(its_dev->event_map.lpi_map, 2662 its_dev->event_map.nr_lpis)) { 2663 its_lpi_free(its_dev->event_map.lpi_map, 2664 its_dev->event_map.lpi_base, 2665 its_dev->event_map.nr_lpis); 2666 kfree(its_dev->event_map.col_map); 2667 2668 /* Unmap device/itt */ 2669 its_send_mapd(its_dev, 0); 2670 its_free_device(its_dev); 2671 } 2672 2673 mutex_unlock(&its->dev_alloc_lock); 2674 2675 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 2676 } 2677 2678 static const struct irq_domain_ops its_domain_ops = { 2679 .alloc = its_irq_domain_alloc, 2680 .free = its_irq_domain_free, 2681 .activate = its_irq_domain_activate, 2682 .deactivate = its_irq_domain_deactivate, 2683 }; 2684 2685 /* 2686 * This is insane. 2687 * 2688 * If a GICv4 doesn't implement Direct LPIs (which is extremely 2689 * likely), the only way to perform an invalidate is to use a fake 2690 * device to issue an INV command, implying that the LPI has first 2691 * been mapped to some event on that device. Since this is not exactly 2692 * cheap, we try to keep that mapping around as long as possible, and 2693 * only issue an UNMAP if we're short on available slots. 2694 * 2695 * Broken by design(tm). 2696 */ 2697 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe) 2698 { 2699 /* Already unmapped? */ 2700 if (vpe->vpe_proxy_event == -1) 2701 return; 2702 2703 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event); 2704 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL; 2705 2706 /* 2707 * We don't track empty slots at all, so let's move the 2708 * next_victim pointer if we can quickly reuse that slot 2709 * instead of nuking an existing entry. Not clear that this is 2710 * always a win though, and this might just generate a ripple 2711 * effect... Let's just hope VPEs don't migrate too often. 2712 */ 2713 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 2714 vpe_proxy.next_victim = vpe->vpe_proxy_event; 2715 2716 vpe->vpe_proxy_event = -1; 2717 } 2718 2719 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe) 2720 { 2721 if (!gic_rdists->has_direct_lpi) { 2722 unsigned long flags; 2723 2724 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 2725 its_vpe_db_proxy_unmap_locked(vpe); 2726 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 2727 } 2728 } 2729 2730 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe) 2731 { 2732 /* Already mapped? */ 2733 if (vpe->vpe_proxy_event != -1) 2734 return; 2735 2736 /* This slot was already allocated. Kick the other VPE out. */ 2737 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 2738 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]); 2739 2740 /* Map the new VPE instead */ 2741 vpe_proxy.vpes[vpe_proxy.next_victim] = vpe; 2742 vpe->vpe_proxy_event = vpe_proxy.next_victim; 2743 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites; 2744 2745 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx; 2746 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event); 2747 } 2748 2749 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to) 2750 { 2751 unsigned long flags; 2752 struct its_collection *target_col; 2753 2754 if (gic_rdists->has_direct_lpi) { 2755 void __iomem *rdbase; 2756 2757 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base; 2758 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 2759 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) 2760 cpu_relax(); 2761 2762 return; 2763 } 2764 2765 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 2766 2767 its_vpe_db_proxy_map_locked(vpe); 2768 2769 target_col = &vpe_proxy.dev->its->collections[to]; 2770 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event); 2771 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to; 2772 2773 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 2774 } 2775 2776 static int its_vpe_set_affinity(struct irq_data *d, 2777 const struct cpumask *mask_val, 2778 bool force) 2779 { 2780 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2781 int cpu = cpumask_first(mask_val); 2782 2783 /* 2784 * Changing affinity is mega expensive, so let's be as lazy as 2785 * we can and only do it if we really have to. Also, if mapped 2786 * into the proxy device, we need to move the doorbell 2787 * interrupt to its new location. 2788 */ 2789 if (vpe->col_idx != cpu) { 2790 int from = vpe->col_idx; 2791 2792 vpe->col_idx = cpu; 2793 its_send_vmovp(vpe); 2794 its_vpe_db_proxy_move(vpe, from, cpu); 2795 } 2796 2797 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 2798 2799 return IRQ_SET_MASK_OK_DONE; 2800 } 2801 2802 static void its_vpe_schedule(struct its_vpe *vpe) 2803 { 2804 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 2805 u64 val; 2806 2807 /* Schedule the VPE */ 2808 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) & 2809 GENMASK_ULL(51, 12); 2810 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; 2811 val |= GICR_VPROPBASER_RaWb; 2812 val |= GICR_VPROPBASER_InnerShareable; 2813 gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 2814 2815 val = virt_to_phys(page_address(vpe->vpt_page)) & 2816 GENMASK_ULL(51, 16); 2817 val |= GICR_VPENDBASER_RaWaWb; 2818 val |= GICR_VPENDBASER_NonShareable; 2819 /* 2820 * There is no good way of finding out if the pending table is 2821 * empty as we can race against the doorbell interrupt very 2822 * easily. So in the end, vpe->pending_last is only an 2823 * indication that the vcpu has something pending, not one 2824 * that the pending table is empty. A good implementation 2825 * would be able to read its coarse map pretty quickly anyway, 2826 * making this a tolerable issue. 2827 */ 2828 val |= GICR_VPENDBASER_PendingLast; 2829 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; 2830 val |= GICR_VPENDBASER_Valid; 2831 gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 2832 } 2833 2834 static void its_vpe_deschedule(struct its_vpe *vpe) 2835 { 2836 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 2837 u64 val; 2838 2839 val = its_clear_vpend_valid(vlpi_base); 2840 2841 if (unlikely(val & GICR_VPENDBASER_Dirty)) { 2842 pr_err_ratelimited("ITS virtual pending table not cleaning\n"); 2843 vpe->idai = false; 2844 vpe->pending_last = true; 2845 } else { 2846 vpe->idai = !!(val & GICR_VPENDBASER_IDAI); 2847 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); 2848 } 2849 } 2850 2851 static void its_vpe_invall(struct its_vpe *vpe) 2852 { 2853 struct its_node *its; 2854 2855 list_for_each_entry(its, &its_nodes, entry) { 2856 if (!its->is_v4) 2857 continue; 2858 2859 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) 2860 continue; 2861 2862 /* 2863 * Sending a VINVALL to a single ITS is enough, as all 2864 * we need is to reach the redistributors. 2865 */ 2866 its_send_vinvall(its, vpe); 2867 return; 2868 } 2869 } 2870 2871 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 2872 { 2873 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2874 struct its_cmd_info *info = vcpu_info; 2875 2876 switch (info->cmd_type) { 2877 case SCHEDULE_VPE: 2878 its_vpe_schedule(vpe); 2879 return 0; 2880 2881 case DESCHEDULE_VPE: 2882 its_vpe_deschedule(vpe); 2883 return 0; 2884 2885 case INVALL_VPE: 2886 its_vpe_invall(vpe); 2887 return 0; 2888 2889 default: 2890 return -EINVAL; 2891 } 2892 } 2893 2894 static void its_vpe_send_cmd(struct its_vpe *vpe, 2895 void (*cmd)(struct its_device *, u32)) 2896 { 2897 unsigned long flags; 2898 2899 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 2900 2901 its_vpe_db_proxy_map_locked(vpe); 2902 cmd(vpe_proxy.dev, vpe->vpe_proxy_event); 2903 2904 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 2905 } 2906 2907 static void its_vpe_send_inv(struct irq_data *d) 2908 { 2909 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2910 2911 if (gic_rdists->has_direct_lpi) { 2912 void __iomem *rdbase; 2913 2914 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 2915 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR); 2916 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) 2917 cpu_relax(); 2918 } else { 2919 its_vpe_send_cmd(vpe, its_send_inv); 2920 } 2921 } 2922 2923 static void its_vpe_mask_irq(struct irq_data *d) 2924 { 2925 /* 2926 * We need to unmask the LPI, which is described by the parent 2927 * irq_data. Instead of calling into the parent (which won't 2928 * exactly do the right thing, let's simply use the 2929 * parent_data pointer. Yes, I'm naughty. 2930 */ 2931 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); 2932 its_vpe_send_inv(d); 2933 } 2934 2935 static void its_vpe_unmask_irq(struct irq_data *d) 2936 { 2937 /* Same hack as above... */ 2938 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); 2939 its_vpe_send_inv(d); 2940 } 2941 2942 static int its_vpe_set_irqchip_state(struct irq_data *d, 2943 enum irqchip_irq_state which, 2944 bool state) 2945 { 2946 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2947 2948 if (which != IRQCHIP_STATE_PENDING) 2949 return -EINVAL; 2950 2951 if (gic_rdists->has_direct_lpi) { 2952 void __iomem *rdbase; 2953 2954 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 2955 if (state) { 2956 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR); 2957 } else { 2958 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 2959 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) 2960 cpu_relax(); 2961 } 2962 } else { 2963 if (state) 2964 its_vpe_send_cmd(vpe, its_send_int); 2965 else 2966 its_vpe_send_cmd(vpe, its_send_clear); 2967 } 2968 2969 return 0; 2970 } 2971 2972 static struct irq_chip its_vpe_irq_chip = { 2973 .name = "GICv4-vpe", 2974 .irq_mask = its_vpe_mask_irq, 2975 .irq_unmask = its_vpe_unmask_irq, 2976 .irq_eoi = irq_chip_eoi_parent, 2977 .irq_set_affinity = its_vpe_set_affinity, 2978 .irq_set_irqchip_state = its_vpe_set_irqchip_state, 2979 .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity, 2980 }; 2981 2982 static int its_vpe_id_alloc(void) 2983 { 2984 return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL); 2985 } 2986 2987 static void its_vpe_id_free(u16 id) 2988 { 2989 ida_simple_remove(&its_vpeid_ida, id); 2990 } 2991 2992 static int its_vpe_init(struct its_vpe *vpe) 2993 { 2994 struct page *vpt_page; 2995 int vpe_id; 2996 2997 /* Allocate vpe_id */ 2998 vpe_id = its_vpe_id_alloc(); 2999 if (vpe_id < 0) 3000 return vpe_id; 3001 3002 /* Allocate VPT */ 3003 vpt_page = its_allocate_pending_table(GFP_KERNEL); 3004 if (!vpt_page) { 3005 its_vpe_id_free(vpe_id); 3006 return -ENOMEM; 3007 } 3008 3009 if (!its_alloc_vpe_table(vpe_id)) { 3010 its_vpe_id_free(vpe_id); 3011 its_free_pending_table(vpe->vpt_page); 3012 return -ENOMEM; 3013 } 3014 3015 vpe->vpe_id = vpe_id; 3016 vpe->vpt_page = vpt_page; 3017 vpe->vpe_proxy_event = -1; 3018 3019 return 0; 3020 } 3021 3022 static void its_vpe_teardown(struct its_vpe *vpe) 3023 { 3024 its_vpe_db_proxy_unmap(vpe); 3025 its_vpe_id_free(vpe->vpe_id); 3026 its_free_pending_table(vpe->vpt_page); 3027 } 3028 3029 static void its_vpe_irq_domain_free(struct irq_domain *domain, 3030 unsigned int virq, 3031 unsigned int nr_irqs) 3032 { 3033 struct its_vm *vm = domain->host_data; 3034 int i; 3035 3036 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 3037 3038 for (i = 0; i < nr_irqs; i++) { 3039 struct irq_data *data = irq_domain_get_irq_data(domain, 3040 virq + i); 3041 struct its_vpe *vpe = irq_data_get_irq_chip_data(data); 3042 3043 BUG_ON(vm != vpe->its_vm); 3044 3045 clear_bit(data->hwirq, vm->db_bitmap); 3046 its_vpe_teardown(vpe); 3047 irq_domain_reset_irq_data(data); 3048 } 3049 3050 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) { 3051 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); 3052 its_free_prop_table(vm->vprop_page); 3053 } 3054 } 3055 3056 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 3057 unsigned int nr_irqs, void *args) 3058 { 3059 struct its_vm *vm = args; 3060 unsigned long *bitmap; 3061 struct page *vprop_page; 3062 int base, nr_ids, i, err = 0; 3063 3064 BUG_ON(!vm); 3065 3066 bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids); 3067 if (!bitmap) 3068 return -ENOMEM; 3069 3070 if (nr_ids < nr_irqs) { 3071 its_lpi_free(bitmap, base, nr_ids); 3072 return -ENOMEM; 3073 } 3074 3075 vprop_page = its_allocate_prop_table(GFP_KERNEL); 3076 if (!vprop_page) { 3077 its_lpi_free(bitmap, base, nr_ids); 3078 return -ENOMEM; 3079 } 3080 3081 vm->db_bitmap = bitmap; 3082 vm->db_lpi_base = base; 3083 vm->nr_db_lpis = nr_ids; 3084 vm->vprop_page = vprop_page; 3085 3086 for (i = 0; i < nr_irqs; i++) { 3087 vm->vpes[i]->vpe_db_lpi = base + i; 3088 err = its_vpe_init(vm->vpes[i]); 3089 if (err) 3090 break; 3091 err = its_irq_gic_domain_alloc(domain, virq + i, 3092 vm->vpes[i]->vpe_db_lpi); 3093 if (err) 3094 break; 3095 irq_domain_set_hwirq_and_chip(domain, virq + i, i, 3096 &its_vpe_irq_chip, vm->vpes[i]); 3097 set_bit(i, bitmap); 3098 } 3099 3100 if (err) { 3101 if (i > 0) 3102 its_vpe_irq_domain_free(domain, virq, i - 1); 3103 3104 its_lpi_free(bitmap, base, nr_ids); 3105 its_free_prop_table(vprop_page); 3106 } 3107 3108 return err; 3109 } 3110 3111 static int its_vpe_irq_domain_activate(struct irq_domain *domain, 3112 struct irq_data *d, bool reserve) 3113 { 3114 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3115 struct its_node *its; 3116 3117 /* If we use the list map, we issue VMAPP on demand... */ 3118 if (its_list_map) 3119 return 0; 3120 3121 /* Map the VPE to the first possible CPU */ 3122 vpe->col_idx = cpumask_first(cpu_online_mask); 3123 3124 list_for_each_entry(its, &its_nodes, entry) { 3125 if (!its->is_v4) 3126 continue; 3127 3128 its_send_vmapp(its, vpe, true); 3129 its_send_vinvall(its, vpe); 3130 } 3131 3132 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 3133 3134 return 0; 3135 } 3136 3137 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain, 3138 struct irq_data *d) 3139 { 3140 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3141 struct its_node *its; 3142 3143 /* 3144 * If we use the list map, we unmap the VPE once no VLPIs are 3145 * associated with the VM. 3146 */ 3147 if (its_list_map) 3148 return; 3149 3150 list_for_each_entry(its, &its_nodes, entry) { 3151 if (!its->is_v4) 3152 continue; 3153 3154 its_send_vmapp(its, vpe, false); 3155 } 3156 } 3157 3158 static const struct irq_domain_ops its_vpe_domain_ops = { 3159 .alloc = its_vpe_irq_domain_alloc, 3160 .free = its_vpe_irq_domain_free, 3161 .activate = its_vpe_irq_domain_activate, 3162 .deactivate = its_vpe_irq_domain_deactivate, 3163 }; 3164 3165 static int its_force_quiescent(void __iomem *base) 3166 { 3167 u32 count = 1000000; /* 1s */ 3168 u32 val; 3169 3170 val = readl_relaxed(base + GITS_CTLR); 3171 /* 3172 * GIC architecture specification requires the ITS to be both 3173 * disabled and quiescent for writes to GITS_BASER<n> or 3174 * GITS_CBASER to not have UNPREDICTABLE results. 3175 */ 3176 if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE)) 3177 return 0; 3178 3179 /* Disable the generation of all interrupts to this ITS */ 3180 val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe); 3181 writel_relaxed(val, base + GITS_CTLR); 3182 3183 /* Poll GITS_CTLR and wait until ITS becomes quiescent */ 3184 while (1) { 3185 val = readl_relaxed(base + GITS_CTLR); 3186 if (val & GITS_CTLR_QUIESCENT) 3187 return 0; 3188 3189 count--; 3190 if (!count) 3191 return -EBUSY; 3192 3193 cpu_relax(); 3194 udelay(1); 3195 } 3196 } 3197 3198 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data) 3199 { 3200 struct its_node *its = data; 3201 3202 /* erratum 22375: only alloc 8MB table size */ 3203 its->device_ids = 0x14; /* 20 bits, 8MB */ 3204 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; 3205 3206 return true; 3207 } 3208 3209 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data) 3210 { 3211 struct its_node *its = data; 3212 3213 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; 3214 3215 return true; 3216 } 3217 3218 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data) 3219 { 3220 struct its_node *its = data; 3221 3222 /* On QDF2400, the size of the ITE is 16Bytes */ 3223 its->ite_size = 16; 3224 3225 return true; 3226 } 3227 3228 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev) 3229 { 3230 struct its_node *its = its_dev->its; 3231 3232 /* 3233 * The Socionext Synquacer SoC has a so-called 'pre-ITS', 3234 * which maps 32-bit writes targeted at a separate window of 3235 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER 3236 * with device ID taken from bits [device_id_bits + 1:2] of 3237 * the window offset. 3238 */ 3239 return its->pre_its_base + (its_dev->device_id << 2); 3240 } 3241 3242 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data) 3243 { 3244 struct its_node *its = data; 3245 u32 pre_its_window[2]; 3246 u32 ids; 3247 3248 if (!fwnode_property_read_u32_array(its->fwnode_handle, 3249 "socionext,synquacer-pre-its", 3250 pre_its_window, 3251 ARRAY_SIZE(pre_its_window))) { 3252 3253 its->pre_its_base = pre_its_window[0]; 3254 its->get_msi_base = its_irq_get_msi_base_pre_its; 3255 3256 ids = ilog2(pre_its_window[1]) - 2; 3257 if (its->device_ids > ids) 3258 its->device_ids = ids; 3259 3260 /* the pre-ITS breaks isolation, so disable MSI remapping */ 3261 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP; 3262 return true; 3263 } 3264 return false; 3265 } 3266 3267 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data) 3268 { 3269 struct its_node *its = data; 3270 3271 /* 3272 * Hip07 insists on using the wrong address for the VLPI 3273 * page. Trick it into doing the right thing... 3274 */ 3275 its->vlpi_redist_offset = SZ_128K; 3276 return true; 3277 } 3278 3279 static const struct gic_quirk its_quirks[] = { 3280 #ifdef CONFIG_CAVIUM_ERRATUM_22375 3281 { 3282 .desc = "ITS: Cavium errata 22375, 24313", 3283 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 3284 .mask = 0xffff0fff, 3285 .init = its_enable_quirk_cavium_22375, 3286 }, 3287 #endif 3288 #ifdef CONFIG_CAVIUM_ERRATUM_23144 3289 { 3290 .desc = "ITS: Cavium erratum 23144", 3291 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 3292 .mask = 0xffff0fff, 3293 .init = its_enable_quirk_cavium_23144, 3294 }, 3295 #endif 3296 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065 3297 { 3298 .desc = "ITS: QDF2400 erratum 0065", 3299 .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */ 3300 .mask = 0xffffffff, 3301 .init = its_enable_quirk_qdf2400_e0065, 3302 }, 3303 #endif 3304 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS 3305 { 3306 /* 3307 * The Socionext Synquacer SoC incorporates ARM's own GIC-500 3308 * implementation, but with a 'pre-ITS' added that requires 3309 * special handling in software. 3310 */ 3311 .desc = "ITS: Socionext Synquacer pre-ITS", 3312 .iidr = 0x0001143b, 3313 .mask = 0xffffffff, 3314 .init = its_enable_quirk_socionext_synquacer, 3315 }, 3316 #endif 3317 #ifdef CONFIG_HISILICON_ERRATUM_161600802 3318 { 3319 .desc = "ITS: Hip07 erratum 161600802", 3320 .iidr = 0x00000004, 3321 .mask = 0xffffffff, 3322 .init = its_enable_quirk_hip07_161600802, 3323 }, 3324 #endif 3325 { 3326 } 3327 }; 3328 3329 static void its_enable_quirks(struct its_node *its) 3330 { 3331 u32 iidr = readl_relaxed(its->base + GITS_IIDR); 3332 3333 gic_enable_quirks(iidr, its_quirks, its); 3334 } 3335 3336 static int its_save_disable(void) 3337 { 3338 struct its_node *its; 3339 int err = 0; 3340 3341 raw_spin_lock(&its_lock); 3342 list_for_each_entry(its, &its_nodes, entry) { 3343 void __iomem *base; 3344 3345 if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE)) 3346 continue; 3347 3348 base = its->base; 3349 its->ctlr_save = readl_relaxed(base + GITS_CTLR); 3350 err = its_force_quiescent(base); 3351 if (err) { 3352 pr_err("ITS@%pa: failed to quiesce: %d\n", 3353 &its->phys_base, err); 3354 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 3355 goto err; 3356 } 3357 3358 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER); 3359 } 3360 3361 err: 3362 if (err) { 3363 list_for_each_entry_continue_reverse(its, &its_nodes, entry) { 3364 void __iomem *base; 3365 3366 if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE)) 3367 continue; 3368 3369 base = its->base; 3370 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 3371 } 3372 } 3373 raw_spin_unlock(&its_lock); 3374 3375 return err; 3376 } 3377 3378 static void its_restore_enable(void) 3379 { 3380 struct its_node *its; 3381 int ret; 3382 3383 raw_spin_lock(&its_lock); 3384 list_for_each_entry(its, &its_nodes, entry) { 3385 void __iomem *base; 3386 int i; 3387 3388 if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE)) 3389 continue; 3390 3391 base = its->base; 3392 3393 /* 3394 * Make sure that the ITS is disabled. If it fails to quiesce, 3395 * don't restore it since writing to CBASER or BASER<n> 3396 * registers is undefined according to the GIC v3 ITS 3397 * Specification. 3398 */ 3399 ret = its_force_quiescent(base); 3400 if (ret) { 3401 pr_err("ITS@%pa: failed to quiesce on resume: %d\n", 3402 &its->phys_base, ret); 3403 continue; 3404 } 3405 3406 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER); 3407 3408 /* 3409 * Writing CBASER resets CREADR to 0, so make CWRITER and 3410 * cmd_write line up with it. 3411 */ 3412 its->cmd_write = its->cmd_base; 3413 gits_write_cwriter(0, base + GITS_CWRITER); 3414 3415 /* Restore GITS_BASER from the value cache. */ 3416 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 3417 struct its_baser *baser = &its->tables[i]; 3418 3419 if (!(baser->val & GITS_BASER_VALID)) 3420 continue; 3421 3422 its_write_baser(its, baser, baser->val); 3423 } 3424 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 3425 3426 /* 3427 * Reinit the collection if it's stored in the ITS. This is 3428 * indicated by the col_id being less than the HCC field. 3429 * CID < HCC as specified in the GIC v3 Documentation. 3430 */ 3431 if (its->collections[smp_processor_id()].col_id < 3432 GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER))) 3433 its_cpu_init_collection(its); 3434 } 3435 raw_spin_unlock(&its_lock); 3436 } 3437 3438 static struct syscore_ops its_syscore_ops = { 3439 .suspend = its_save_disable, 3440 .resume = its_restore_enable, 3441 }; 3442 3443 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) 3444 { 3445 struct irq_domain *inner_domain; 3446 struct msi_domain_info *info; 3447 3448 info = kzalloc(sizeof(*info), GFP_KERNEL); 3449 if (!info) 3450 return -ENOMEM; 3451 3452 inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its); 3453 if (!inner_domain) { 3454 kfree(info); 3455 return -ENOMEM; 3456 } 3457 3458 inner_domain->parent = its_parent; 3459 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); 3460 inner_domain->flags |= its->msi_domain_flags; 3461 info->ops = &its_msi_domain_ops; 3462 info->data = its; 3463 inner_domain->host_data = info; 3464 3465 return 0; 3466 } 3467 3468 static int its_init_vpe_domain(void) 3469 { 3470 struct its_node *its; 3471 u32 devid; 3472 int entries; 3473 3474 if (gic_rdists->has_direct_lpi) { 3475 pr_info("ITS: Using DirectLPI for VPE invalidation\n"); 3476 return 0; 3477 } 3478 3479 /* Any ITS will do, even if not v4 */ 3480 its = list_first_entry(&its_nodes, struct its_node, entry); 3481 3482 entries = roundup_pow_of_two(nr_cpu_ids); 3483 vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes), 3484 GFP_KERNEL); 3485 if (!vpe_proxy.vpes) { 3486 pr_err("ITS: Can't allocate GICv4 proxy device array\n"); 3487 return -ENOMEM; 3488 } 3489 3490 /* Use the last possible DevID */ 3491 devid = GENMASK(its->device_ids - 1, 0); 3492 vpe_proxy.dev = its_create_device(its, devid, entries, false); 3493 if (!vpe_proxy.dev) { 3494 kfree(vpe_proxy.vpes); 3495 pr_err("ITS: Can't allocate GICv4 proxy device\n"); 3496 return -ENOMEM; 3497 } 3498 3499 BUG_ON(entries > vpe_proxy.dev->nr_ites); 3500 3501 raw_spin_lock_init(&vpe_proxy.lock); 3502 vpe_proxy.next_victim = 0; 3503 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n", 3504 devid, vpe_proxy.dev->nr_ites); 3505 3506 return 0; 3507 } 3508 3509 static int __init its_compute_its_list_map(struct resource *res, 3510 void __iomem *its_base) 3511 { 3512 int its_number; 3513 u32 ctlr; 3514 3515 /* 3516 * This is assumed to be done early enough that we're 3517 * guaranteed to be single-threaded, hence no 3518 * locking. Should this change, we should address 3519 * this. 3520 */ 3521 its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX); 3522 if (its_number >= GICv4_ITS_LIST_MAX) { 3523 pr_err("ITS@%pa: No ITSList entry available!\n", 3524 &res->start); 3525 return -EINVAL; 3526 } 3527 3528 ctlr = readl_relaxed(its_base + GITS_CTLR); 3529 ctlr &= ~GITS_CTLR_ITS_NUMBER; 3530 ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT; 3531 writel_relaxed(ctlr, its_base + GITS_CTLR); 3532 ctlr = readl_relaxed(its_base + GITS_CTLR); 3533 if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) { 3534 its_number = ctlr & GITS_CTLR_ITS_NUMBER; 3535 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT; 3536 } 3537 3538 if (test_and_set_bit(its_number, &its_list_map)) { 3539 pr_err("ITS@%pa: Duplicate ITSList entry %d\n", 3540 &res->start, its_number); 3541 return -EINVAL; 3542 } 3543 3544 return its_number; 3545 } 3546 3547 static int __init its_probe_one(struct resource *res, 3548 struct fwnode_handle *handle, int numa_node) 3549 { 3550 struct its_node *its; 3551 void __iomem *its_base; 3552 u32 val, ctlr; 3553 u64 baser, tmp, typer; 3554 struct page *page; 3555 int err; 3556 3557 its_base = ioremap(res->start, resource_size(res)); 3558 if (!its_base) { 3559 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start); 3560 return -ENOMEM; 3561 } 3562 3563 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK; 3564 if (val != 0x30 && val != 0x40) { 3565 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start); 3566 err = -ENODEV; 3567 goto out_unmap; 3568 } 3569 3570 err = its_force_quiescent(its_base); 3571 if (err) { 3572 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start); 3573 goto out_unmap; 3574 } 3575 3576 pr_info("ITS %pR\n", res); 3577 3578 its = kzalloc(sizeof(*its), GFP_KERNEL); 3579 if (!its) { 3580 err = -ENOMEM; 3581 goto out_unmap; 3582 } 3583 3584 raw_spin_lock_init(&its->lock); 3585 mutex_init(&its->dev_alloc_lock); 3586 INIT_LIST_HEAD(&its->entry); 3587 INIT_LIST_HEAD(&its->its_device_list); 3588 typer = gic_read_typer(its_base + GITS_TYPER); 3589 its->base = its_base; 3590 its->phys_base = res->start; 3591 its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer); 3592 its->device_ids = GITS_TYPER_DEVBITS(typer); 3593 its->is_v4 = !!(typer & GITS_TYPER_VLPIS); 3594 if (its->is_v4) { 3595 if (!(typer & GITS_TYPER_VMOVP)) { 3596 err = its_compute_its_list_map(res, its_base); 3597 if (err < 0) 3598 goto out_free_its; 3599 3600 its->list_nr = err; 3601 3602 pr_info("ITS@%pa: Using ITS number %d\n", 3603 &res->start, err); 3604 } else { 3605 pr_info("ITS@%pa: Single VMOVP capable\n", &res->start); 3606 } 3607 } 3608 3609 its->numa_node = numa_node; 3610 3611 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, 3612 get_order(ITS_CMD_QUEUE_SZ)); 3613 if (!page) { 3614 err = -ENOMEM; 3615 goto out_free_its; 3616 } 3617 its->cmd_base = (void *)page_address(page); 3618 its->cmd_write = its->cmd_base; 3619 its->fwnode_handle = handle; 3620 its->get_msi_base = its_irq_get_msi_base; 3621 its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP; 3622 3623 its_enable_quirks(its); 3624 3625 err = its_alloc_tables(its); 3626 if (err) 3627 goto out_free_cmd; 3628 3629 err = its_alloc_collections(its); 3630 if (err) 3631 goto out_free_tables; 3632 3633 baser = (virt_to_phys(its->cmd_base) | 3634 GITS_CBASER_RaWaWb | 3635 GITS_CBASER_InnerShareable | 3636 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | 3637 GITS_CBASER_VALID); 3638 3639 gits_write_cbaser(baser, its->base + GITS_CBASER); 3640 tmp = gits_read_cbaser(its->base + GITS_CBASER); 3641 3642 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) { 3643 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) { 3644 /* 3645 * The HW reports non-shareable, we must 3646 * remove the cacheability attributes as 3647 * well. 3648 */ 3649 baser &= ~(GITS_CBASER_SHAREABILITY_MASK | 3650 GITS_CBASER_CACHEABILITY_MASK); 3651 baser |= GITS_CBASER_nC; 3652 gits_write_cbaser(baser, its->base + GITS_CBASER); 3653 } 3654 pr_info("ITS: using cache flushing for cmd queue\n"); 3655 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; 3656 } 3657 3658 gits_write_cwriter(0, its->base + GITS_CWRITER); 3659 ctlr = readl_relaxed(its->base + GITS_CTLR); 3660 ctlr |= GITS_CTLR_ENABLE; 3661 if (its->is_v4) 3662 ctlr |= GITS_CTLR_ImDe; 3663 writel_relaxed(ctlr, its->base + GITS_CTLR); 3664 3665 if (GITS_TYPER_HCC(typer)) 3666 its->flags |= ITS_FLAGS_SAVE_SUSPEND_STATE; 3667 3668 err = its_init_domain(handle, its); 3669 if (err) 3670 goto out_free_tables; 3671 3672 raw_spin_lock(&its_lock); 3673 list_add(&its->entry, &its_nodes); 3674 raw_spin_unlock(&its_lock); 3675 3676 return 0; 3677 3678 out_free_tables: 3679 its_free_tables(its); 3680 out_free_cmd: 3681 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); 3682 out_free_its: 3683 kfree(its); 3684 out_unmap: 3685 iounmap(its_base); 3686 pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err); 3687 return err; 3688 } 3689 3690 static bool gic_rdists_supports_plpis(void) 3691 { 3692 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS); 3693 } 3694 3695 static int redist_disable_lpis(void) 3696 { 3697 void __iomem *rbase = gic_data_rdist_rd_base(); 3698 u64 timeout = USEC_PER_SEC; 3699 u64 val; 3700 3701 if (!gic_rdists_supports_plpis()) { 3702 pr_info("CPU%d: LPIs not supported\n", smp_processor_id()); 3703 return -ENXIO; 3704 } 3705 3706 val = readl_relaxed(rbase + GICR_CTLR); 3707 if (!(val & GICR_CTLR_ENABLE_LPIS)) 3708 return 0; 3709 3710 /* 3711 * If coming via a CPU hotplug event, we don't need to disable 3712 * LPIs before trying to re-enable them. They are already 3713 * configured and all is well in the world. 3714 * 3715 * If running with preallocated tables, there is nothing to do. 3716 */ 3717 if (gic_data_rdist()->lpi_enabled || 3718 (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) 3719 return 0; 3720 3721 /* 3722 * From that point on, we only try to do some damage control. 3723 */ 3724 pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n", 3725 smp_processor_id()); 3726 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 3727 3728 /* Disable LPIs */ 3729 val &= ~GICR_CTLR_ENABLE_LPIS; 3730 writel_relaxed(val, rbase + GICR_CTLR); 3731 3732 /* Make sure any change to GICR_CTLR is observable by the GIC */ 3733 dsb(sy); 3734 3735 /* 3736 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs 3737 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers. 3738 * Error out if we time out waiting for RWP to clear. 3739 */ 3740 while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) { 3741 if (!timeout) { 3742 pr_err("CPU%d: Timeout while disabling LPIs\n", 3743 smp_processor_id()); 3744 return -ETIMEDOUT; 3745 } 3746 udelay(1); 3747 timeout--; 3748 } 3749 3750 /* 3751 * After it has been written to 1, it is IMPLEMENTATION 3752 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be 3753 * cleared to 0. Error out if clearing the bit failed. 3754 */ 3755 if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) { 3756 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id()); 3757 return -EBUSY; 3758 } 3759 3760 return 0; 3761 } 3762 3763 int its_cpu_init(void) 3764 { 3765 if (!list_empty(&its_nodes)) { 3766 int ret; 3767 3768 ret = redist_disable_lpis(); 3769 if (ret) 3770 return ret; 3771 3772 its_cpu_init_lpis(); 3773 its_cpu_init_collections(); 3774 } 3775 3776 return 0; 3777 } 3778 3779 static const struct of_device_id its_device_id[] = { 3780 { .compatible = "arm,gic-v3-its", }, 3781 {}, 3782 }; 3783 3784 static int __init its_of_probe(struct device_node *node) 3785 { 3786 struct device_node *np; 3787 struct resource res; 3788 3789 for (np = of_find_matching_node(node, its_device_id); np; 3790 np = of_find_matching_node(np, its_device_id)) { 3791 if (!of_device_is_available(np)) 3792 continue; 3793 if (!of_property_read_bool(np, "msi-controller")) { 3794 pr_warn("%pOF: no msi-controller property, ITS ignored\n", 3795 np); 3796 continue; 3797 } 3798 3799 if (of_address_to_resource(np, 0, &res)) { 3800 pr_warn("%pOF: no regs?\n", np); 3801 continue; 3802 } 3803 3804 its_probe_one(&res, &np->fwnode, of_node_to_nid(np)); 3805 } 3806 return 0; 3807 } 3808 3809 #ifdef CONFIG_ACPI 3810 3811 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K) 3812 3813 #ifdef CONFIG_ACPI_NUMA 3814 struct its_srat_map { 3815 /* numa node id */ 3816 u32 numa_node; 3817 /* GIC ITS ID */ 3818 u32 its_id; 3819 }; 3820 3821 static struct its_srat_map *its_srat_maps __initdata; 3822 static int its_in_srat __initdata; 3823 3824 static int __init acpi_get_its_numa_node(u32 its_id) 3825 { 3826 int i; 3827 3828 for (i = 0; i < its_in_srat; i++) { 3829 if (its_id == its_srat_maps[i].its_id) 3830 return its_srat_maps[i].numa_node; 3831 } 3832 return NUMA_NO_NODE; 3833 } 3834 3835 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header, 3836 const unsigned long end) 3837 { 3838 return 0; 3839 } 3840 3841 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header, 3842 const unsigned long end) 3843 { 3844 int node; 3845 struct acpi_srat_gic_its_affinity *its_affinity; 3846 3847 its_affinity = (struct acpi_srat_gic_its_affinity *)header; 3848 if (!its_affinity) 3849 return -EINVAL; 3850 3851 if (its_affinity->header.length < sizeof(*its_affinity)) { 3852 pr_err("SRAT: Invalid header length %d in ITS affinity\n", 3853 its_affinity->header.length); 3854 return -EINVAL; 3855 } 3856 3857 node = acpi_map_pxm_to_node(its_affinity->proximity_domain); 3858 3859 if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { 3860 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node); 3861 return 0; 3862 } 3863 3864 its_srat_maps[its_in_srat].numa_node = node; 3865 its_srat_maps[its_in_srat].its_id = its_affinity->its_id; 3866 its_in_srat++; 3867 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n", 3868 its_affinity->proximity_domain, its_affinity->its_id, node); 3869 3870 return 0; 3871 } 3872 3873 static void __init acpi_table_parse_srat_its(void) 3874 { 3875 int count; 3876 3877 count = acpi_table_parse_entries(ACPI_SIG_SRAT, 3878 sizeof(struct acpi_table_srat), 3879 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 3880 gic_acpi_match_srat_its, 0); 3881 if (count <= 0) 3882 return; 3883 3884 its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map), 3885 GFP_KERNEL); 3886 if (!its_srat_maps) { 3887 pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n"); 3888 return; 3889 } 3890 3891 acpi_table_parse_entries(ACPI_SIG_SRAT, 3892 sizeof(struct acpi_table_srat), 3893 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 3894 gic_acpi_parse_srat_its, 0); 3895 } 3896 3897 /* free the its_srat_maps after ITS probing */ 3898 static void __init acpi_its_srat_maps_free(void) 3899 { 3900 kfree(its_srat_maps); 3901 } 3902 #else 3903 static void __init acpi_table_parse_srat_its(void) { } 3904 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; } 3905 static void __init acpi_its_srat_maps_free(void) { } 3906 #endif 3907 3908 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, 3909 const unsigned long end) 3910 { 3911 struct acpi_madt_generic_translator *its_entry; 3912 struct fwnode_handle *dom_handle; 3913 struct resource res; 3914 int err; 3915 3916 its_entry = (struct acpi_madt_generic_translator *)header; 3917 memset(&res, 0, sizeof(res)); 3918 res.start = its_entry->base_address; 3919 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1; 3920 res.flags = IORESOURCE_MEM; 3921 3922 dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address); 3923 if (!dom_handle) { 3924 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n", 3925 &res.start); 3926 return -ENOMEM; 3927 } 3928 3929 err = iort_register_domain_token(its_entry->translation_id, res.start, 3930 dom_handle); 3931 if (err) { 3932 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n", 3933 &res.start, its_entry->translation_id); 3934 goto dom_err; 3935 } 3936 3937 err = its_probe_one(&res, dom_handle, 3938 acpi_get_its_numa_node(its_entry->translation_id)); 3939 if (!err) 3940 return 0; 3941 3942 iort_deregister_domain_token(its_entry->translation_id); 3943 dom_err: 3944 irq_domain_free_fwnode(dom_handle); 3945 return err; 3946 } 3947 3948 static void __init its_acpi_probe(void) 3949 { 3950 acpi_table_parse_srat_its(); 3951 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, 3952 gic_acpi_parse_madt_its, 0); 3953 acpi_its_srat_maps_free(); 3954 } 3955 #else 3956 static void __init its_acpi_probe(void) { } 3957 #endif 3958 3959 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, 3960 struct irq_domain *parent_domain) 3961 { 3962 struct device_node *of_node; 3963 struct its_node *its; 3964 bool has_v4 = false; 3965 int err; 3966 3967 its_parent = parent_domain; 3968 of_node = to_of_node(handle); 3969 if (of_node) 3970 its_of_probe(of_node); 3971 else 3972 its_acpi_probe(); 3973 3974 if (list_empty(&its_nodes)) { 3975 pr_warn("ITS: No ITS available, not enabling LPIs\n"); 3976 return -ENXIO; 3977 } 3978 3979 gic_rdists = rdists; 3980 3981 err = allocate_lpi_tables(); 3982 if (err) 3983 return err; 3984 3985 list_for_each_entry(its, &its_nodes, entry) 3986 has_v4 |= its->is_v4; 3987 3988 if (has_v4 & rdists->has_vlpis) { 3989 if (its_init_vpe_domain() || 3990 its_init_v4(parent_domain, &its_vpe_domain_ops)) { 3991 rdists->has_vlpis = false; 3992 pr_err("ITS: Disabling GICv4 support\n"); 3993 } 3994 } 3995 3996 register_syscore_ops(&its_syscore_ops); 3997 3998 return 0; 3999 } 4000