1 /* 2 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 3 * Author: Marc Zyngier <marc.zyngier@arm.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18 #include <linux/acpi.h> 19 #include <linux/acpi_iort.h> 20 #include <linux/bitmap.h> 21 #include <linux/cpu.h> 22 #include <linux/delay.h> 23 #include <linux/dma-iommu.h> 24 #include <linux/interrupt.h> 25 #include <linux/irqdomain.h> 26 #include <linux/log2.h> 27 #include <linux/mm.h> 28 #include <linux/msi.h> 29 #include <linux/of.h> 30 #include <linux/of_address.h> 31 #include <linux/of_irq.h> 32 #include <linux/of_pci.h> 33 #include <linux/of_platform.h> 34 #include <linux/percpu.h> 35 #include <linux/slab.h> 36 37 #include <linux/irqchip.h> 38 #include <linux/irqchip/arm-gic-v3.h> 39 #include <linux/irqchip/arm-gic-v4.h> 40 41 #include <asm/cputype.h> 42 #include <asm/exception.h> 43 44 #include "irq-gic-common.h" 45 46 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) 47 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) 48 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2) 49 50 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) 51 52 static u32 lpi_id_bits; 53 54 /* 55 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to 56 * deal with (one configuration byte per interrupt). PENDBASE has to 57 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). 58 */ 59 #define LPI_NRBITS lpi_id_bits 60 #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K) 61 #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K) 62 63 #define LPI_PROP_DEFAULT_PRIO 0xa0 64 65 /* 66 * Collection structure - just an ID, and a redistributor address to 67 * ping. We use one per CPU as a bag of interrupts assigned to this 68 * CPU. 69 */ 70 struct its_collection { 71 u64 target_address; 72 u16 col_id; 73 }; 74 75 /* 76 * The ITS_BASER structure - contains memory information, cached 77 * value of BASER register configuration and ITS page size. 78 */ 79 struct its_baser { 80 void *base; 81 u64 val; 82 u32 order; 83 u32 psz; 84 }; 85 86 struct its_device; 87 88 /* 89 * The ITS structure - contains most of the infrastructure, with the 90 * top-level MSI domain, the command queue, the collections, and the 91 * list of devices writing to it. 92 */ 93 struct its_node { 94 raw_spinlock_t lock; 95 struct list_head entry; 96 void __iomem *base; 97 phys_addr_t phys_base; 98 struct its_cmd_block *cmd_base; 99 struct its_cmd_block *cmd_write; 100 struct its_baser tables[GITS_BASER_NR_REGS]; 101 struct its_collection *collections; 102 struct fwnode_handle *fwnode_handle; 103 u64 (*get_msi_base)(struct its_device *its_dev); 104 struct list_head its_device_list; 105 u64 flags; 106 unsigned long list_nr; 107 u32 ite_size; 108 u32 device_ids; 109 int numa_node; 110 unsigned int msi_domain_flags; 111 u32 pre_its_base; /* for Socionext Synquacer */ 112 bool is_v4; 113 int vlpi_redist_offset; 114 }; 115 116 #define ITS_ITT_ALIGN SZ_256 117 118 /* The maximum number of VPEID bits supported by VLPI commands */ 119 #define ITS_MAX_VPEID_BITS (16) 120 #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS)) 121 122 /* Convert page order to size in bytes */ 123 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o)) 124 125 struct event_lpi_map { 126 unsigned long *lpi_map; 127 u16 *col_map; 128 irq_hw_number_t lpi_base; 129 int nr_lpis; 130 struct mutex vlpi_lock; 131 struct its_vm *vm; 132 struct its_vlpi_map *vlpi_maps; 133 int nr_vlpis; 134 }; 135 136 /* 137 * The ITS view of a device - belongs to an ITS, owns an interrupt 138 * translation table, and a list of interrupts. If it some of its 139 * LPIs are injected into a guest (GICv4), the event_map.vm field 140 * indicates which one. 141 */ 142 struct its_device { 143 struct list_head entry; 144 struct its_node *its; 145 struct event_lpi_map event_map; 146 void *itt; 147 u32 nr_ites; 148 u32 device_id; 149 }; 150 151 static struct { 152 raw_spinlock_t lock; 153 struct its_device *dev; 154 struct its_vpe **vpes; 155 int next_victim; 156 } vpe_proxy; 157 158 static LIST_HEAD(its_nodes); 159 static DEFINE_SPINLOCK(its_lock); 160 static struct rdists *gic_rdists; 161 static struct irq_domain *its_parent; 162 163 static unsigned long its_list_map; 164 static u16 vmovp_seq_num; 165 static DEFINE_RAW_SPINLOCK(vmovp_lock); 166 167 static DEFINE_IDA(its_vpeid_ida); 168 169 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) 170 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 171 #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) 172 173 static struct its_collection *dev_event_to_col(struct its_device *its_dev, 174 u32 event) 175 { 176 struct its_node *its = its_dev->its; 177 178 return its->collections + its_dev->event_map.col_map[event]; 179 } 180 181 /* 182 * ITS command descriptors - parameters to be encoded in a command 183 * block. 184 */ 185 struct its_cmd_desc { 186 union { 187 struct { 188 struct its_device *dev; 189 u32 event_id; 190 } its_inv_cmd; 191 192 struct { 193 struct its_device *dev; 194 u32 event_id; 195 } its_clear_cmd; 196 197 struct { 198 struct its_device *dev; 199 u32 event_id; 200 } its_int_cmd; 201 202 struct { 203 struct its_device *dev; 204 int valid; 205 } its_mapd_cmd; 206 207 struct { 208 struct its_collection *col; 209 int valid; 210 } its_mapc_cmd; 211 212 struct { 213 struct its_device *dev; 214 u32 phys_id; 215 u32 event_id; 216 } its_mapti_cmd; 217 218 struct { 219 struct its_device *dev; 220 struct its_collection *col; 221 u32 event_id; 222 } its_movi_cmd; 223 224 struct { 225 struct its_device *dev; 226 u32 event_id; 227 } its_discard_cmd; 228 229 struct { 230 struct its_collection *col; 231 } its_invall_cmd; 232 233 struct { 234 struct its_vpe *vpe; 235 } its_vinvall_cmd; 236 237 struct { 238 struct its_vpe *vpe; 239 struct its_collection *col; 240 bool valid; 241 } its_vmapp_cmd; 242 243 struct { 244 struct its_vpe *vpe; 245 struct its_device *dev; 246 u32 virt_id; 247 u32 event_id; 248 bool db_enabled; 249 } its_vmapti_cmd; 250 251 struct { 252 struct its_vpe *vpe; 253 struct its_device *dev; 254 u32 event_id; 255 bool db_enabled; 256 } its_vmovi_cmd; 257 258 struct { 259 struct its_vpe *vpe; 260 struct its_collection *col; 261 u16 seq_num; 262 u16 its_list; 263 } its_vmovp_cmd; 264 }; 265 }; 266 267 /* 268 * The ITS command block, which is what the ITS actually parses. 269 */ 270 struct its_cmd_block { 271 u64 raw_cmd[4]; 272 }; 273 274 #define ITS_CMD_QUEUE_SZ SZ_64K 275 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) 276 277 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *, 278 struct its_cmd_block *, 279 struct its_cmd_desc *); 280 281 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *, 282 struct its_cmd_block *, 283 struct its_cmd_desc *); 284 285 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 286 { 287 u64 mask = GENMASK_ULL(h, l); 288 *raw_cmd &= ~mask; 289 *raw_cmd |= (val << l) & mask; 290 } 291 292 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 293 { 294 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); 295 } 296 297 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 298 { 299 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); 300 } 301 302 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 303 { 304 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); 305 } 306 307 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 308 { 309 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); 310 } 311 312 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 313 { 314 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); 315 } 316 317 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 318 { 319 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); 320 } 321 322 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 323 { 324 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); 325 } 326 327 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 328 { 329 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); 330 } 331 332 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 333 { 334 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); 335 } 336 337 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid) 338 { 339 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32); 340 } 341 342 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id) 343 { 344 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0); 345 } 346 347 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id) 348 { 349 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32); 350 } 351 352 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid) 353 { 354 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0); 355 } 356 357 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num) 358 { 359 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32); 360 } 361 362 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list) 363 { 364 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0); 365 } 366 367 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa) 368 { 369 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16); 370 } 371 372 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size) 373 { 374 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); 375 } 376 377 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 378 { 379 /* Let's fixup BE commands */ 380 cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); 381 cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); 382 cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); 383 cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); 384 } 385 386 static struct its_collection *its_build_mapd_cmd(struct its_node *its, 387 struct its_cmd_block *cmd, 388 struct its_cmd_desc *desc) 389 { 390 unsigned long itt_addr; 391 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); 392 393 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); 394 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); 395 396 its_encode_cmd(cmd, GITS_CMD_MAPD); 397 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); 398 its_encode_size(cmd, size - 1); 399 its_encode_itt(cmd, itt_addr); 400 its_encode_valid(cmd, desc->its_mapd_cmd.valid); 401 402 its_fixup_cmd(cmd); 403 404 return NULL; 405 } 406 407 static struct its_collection *its_build_mapc_cmd(struct its_node *its, 408 struct its_cmd_block *cmd, 409 struct its_cmd_desc *desc) 410 { 411 its_encode_cmd(cmd, GITS_CMD_MAPC); 412 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 413 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); 414 its_encode_valid(cmd, desc->its_mapc_cmd.valid); 415 416 its_fixup_cmd(cmd); 417 418 return desc->its_mapc_cmd.col; 419 } 420 421 static struct its_collection *its_build_mapti_cmd(struct its_node *its, 422 struct its_cmd_block *cmd, 423 struct its_cmd_desc *desc) 424 { 425 struct its_collection *col; 426 427 col = dev_event_to_col(desc->its_mapti_cmd.dev, 428 desc->its_mapti_cmd.event_id); 429 430 its_encode_cmd(cmd, GITS_CMD_MAPTI); 431 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id); 432 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id); 433 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id); 434 its_encode_collection(cmd, col->col_id); 435 436 its_fixup_cmd(cmd); 437 438 return col; 439 } 440 441 static struct its_collection *its_build_movi_cmd(struct its_node *its, 442 struct its_cmd_block *cmd, 443 struct its_cmd_desc *desc) 444 { 445 struct its_collection *col; 446 447 col = dev_event_to_col(desc->its_movi_cmd.dev, 448 desc->its_movi_cmd.event_id); 449 450 its_encode_cmd(cmd, GITS_CMD_MOVI); 451 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); 452 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); 453 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); 454 455 its_fixup_cmd(cmd); 456 457 return col; 458 } 459 460 static struct its_collection *its_build_discard_cmd(struct its_node *its, 461 struct its_cmd_block *cmd, 462 struct its_cmd_desc *desc) 463 { 464 struct its_collection *col; 465 466 col = dev_event_to_col(desc->its_discard_cmd.dev, 467 desc->its_discard_cmd.event_id); 468 469 its_encode_cmd(cmd, GITS_CMD_DISCARD); 470 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); 471 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); 472 473 its_fixup_cmd(cmd); 474 475 return col; 476 } 477 478 static struct its_collection *its_build_inv_cmd(struct its_node *its, 479 struct its_cmd_block *cmd, 480 struct its_cmd_desc *desc) 481 { 482 struct its_collection *col; 483 484 col = dev_event_to_col(desc->its_inv_cmd.dev, 485 desc->its_inv_cmd.event_id); 486 487 its_encode_cmd(cmd, GITS_CMD_INV); 488 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 489 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 490 491 its_fixup_cmd(cmd); 492 493 return col; 494 } 495 496 static struct its_collection *its_build_int_cmd(struct its_node *its, 497 struct its_cmd_block *cmd, 498 struct its_cmd_desc *desc) 499 { 500 struct its_collection *col; 501 502 col = dev_event_to_col(desc->its_int_cmd.dev, 503 desc->its_int_cmd.event_id); 504 505 its_encode_cmd(cmd, GITS_CMD_INT); 506 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); 507 its_encode_event_id(cmd, desc->its_int_cmd.event_id); 508 509 its_fixup_cmd(cmd); 510 511 return col; 512 } 513 514 static struct its_collection *its_build_clear_cmd(struct its_node *its, 515 struct its_cmd_block *cmd, 516 struct its_cmd_desc *desc) 517 { 518 struct its_collection *col; 519 520 col = dev_event_to_col(desc->its_clear_cmd.dev, 521 desc->its_clear_cmd.event_id); 522 523 its_encode_cmd(cmd, GITS_CMD_CLEAR); 524 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); 525 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); 526 527 its_fixup_cmd(cmd); 528 529 return col; 530 } 531 532 static struct its_collection *its_build_invall_cmd(struct its_node *its, 533 struct its_cmd_block *cmd, 534 struct its_cmd_desc *desc) 535 { 536 its_encode_cmd(cmd, GITS_CMD_INVALL); 537 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 538 539 its_fixup_cmd(cmd); 540 541 return NULL; 542 } 543 544 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its, 545 struct its_cmd_block *cmd, 546 struct its_cmd_desc *desc) 547 { 548 its_encode_cmd(cmd, GITS_CMD_VINVALL); 549 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id); 550 551 its_fixup_cmd(cmd); 552 553 return desc->its_vinvall_cmd.vpe; 554 } 555 556 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, 557 struct its_cmd_block *cmd, 558 struct its_cmd_desc *desc) 559 { 560 unsigned long vpt_addr; 561 u64 target; 562 563 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); 564 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; 565 566 its_encode_cmd(cmd, GITS_CMD_VMAPP); 567 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); 568 its_encode_valid(cmd, desc->its_vmapp_cmd.valid); 569 its_encode_target(cmd, target); 570 its_encode_vpt_addr(cmd, vpt_addr); 571 its_encode_vpt_size(cmd, LPI_NRBITS - 1); 572 573 its_fixup_cmd(cmd); 574 575 return desc->its_vmapp_cmd.vpe; 576 } 577 578 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, 579 struct its_cmd_block *cmd, 580 struct its_cmd_desc *desc) 581 { 582 u32 db; 583 584 if (desc->its_vmapti_cmd.db_enabled) 585 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; 586 else 587 db = 1023; 588 589 its_encode_cmd(cmd, GITS_CMD_VMAPTI); 590 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id); 591 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id); 592 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id); 593 its_encode_db_phys_id(cmd, db); 594 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id); 595 596 its_fixup_cmd(cmd); 597 598 return desc->its_vmapti_cmd.vpe; 599 } 600 601 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, 602 struct its_cmd_block *cmd, 603 struct its_cmd_desc *desc) 604 { 605 u32 db; 606 607 if (desc->its_vmovi_cmd.db_enabled) 608 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; 609 else 610 db = 1023; 611 612 its_encode_cmd(cmd, GITS_CMD_VMOVI); 613 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id); 614 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id); 615 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id); 616 its_encode_db_phys_id(cmd, db); 617 its_encode_db_valid(cmd, true); 618 619 its_fixup_cmd(cmd); 620 621 return desc->its_vmovi_cmd.vpe; 622 } 623 624 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, 625 struct its_cmd_block *cmd, 626 struct its_cmd_desc *desc) 627 { 628 u64 target; 629 630 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; 631 its_encode_cmd(cmd, GITS_CMD_VMOVP); 632 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); 633 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); 634 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); 635 its_encode_target(cmd, target); 636 637 its_fixup_cmd(cmd); 638 639 return desc->its_vmovp_cmd.vpe; 640 } 641 642 static u64 its_cmd_ptr_to_offset(struct its_node *its, 643 struct its_cmd_block *ptr) 644 { 645 return (ptr - its->cmd_base) * sizeof(*ptr); 646 } 647 648 static int its_queue_full(struct its_node *its) 649 { 650 int widx; 651 int ridx; 652 653 widx = its->cmd_write - its->cmd_base; 654 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); 655 656 /* This is incredibly unlikely to happen, unless the ITS locks up. */ 657 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) 658 return 1; 659 660 return 0; 661 } 662 663 static struct its_cmd_block *its_allocate_entry(struct its_node *its) 664 { 665 struct its_cmd_block *cmd; 666 u32 count = 1000000; /* 1s! */ 667 668 while (its_queue_full(its)) { 669 count--; 670 if (!count) { 671 pr_err_ratelimited("ITS queue not draining\n"); 672 return NULL; 673 } 674 cpu_relax(); 675 udelay(1); 676 } 677 678 cmd = its->cmd_write++; 679 680 /* Handle queue wrapping */ 681 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) 682 its->cmd_write = its->cmd_base; 683 684 /* Clear command */ 685 cmd->raw_cmd[0] = 0; 686 cmd->raw_cmd[1] = 0; 687 cmd->raw_cmd[2] = 0; 688 cmd->raw_cmd[3] = 0; 689 690 return cmd; 691 } 692 693 static struct its_cmd_block *its_post_commands(struct its_node *its) 694 { 695 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); 696 697 writel_relaxed(wr, its->base + GITS_CWRITER); 698 699 return its->cmd_write; 700 } 701 702 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) 703 { 704 /* 705 * Make sure the commands written to memory are observable by 706 * the ITS. 707 */ 708 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) 709 gic_flush_dcache_to_poc(cmd, sizeof(*cmd)); 710 else 711 dsb(ishst); 712 } 713 714 static int its_wait_for_range_completion(struct its_node *its, 715 struct its_cmd_block *from, 716 struct its_cmd_block *to) 717 { 718 u64 rd_idx, from_idx, to_idx; 719 u32 count = 1000000; /* 1s! */ 720 721 from_idx = its_cmd_ptr_to_offset(its, from); 722 to_idx = its_cmd_ptr_to_offset(its, to); 723 724 while (1) { 725 rd_idx = readl_relaxed(its->base + GITS_CREADR); 726 727 /* Direct case */ 728 if (from_idx < to_idx && rd_idx >= to_idx) 729 break; 730 731 /* Wrapped case */ 732 if (from_idx >= to_idx && rd_idx >= to_idx && rd_idx < from_idx) 733 break; 734 735 count--; 736 if (!count) { 737 pr_err_ratelimited("ITS queue timeout (%llu %llu %llu)\n", 738 from_idx, to_idx, rd_idx); 739 return -1; 740 } 741 cpu_relax(); 742 udelay(1); 743 } 744 745 return 0; 746 } 747 748 /* Warning, macro hell follows */ 749 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \ 750 void name(struct its_node *its, \ 751 buildtype builder, \ 752 struct its_cmd_desc *desc) \ 753 { \ 754 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \ 755 synctype *sync_obj; \ 756 unsigned long flags; \ 757 \ 758 raw_spin_lock_irqsave(&its->lock, flags); \ 759 \ 760 cmd = its_allocate_entry(its); \ 761 if (!cmd) { /* We're soooooo screewed... */ \ 762 raw_spin_unlock_irqrestore(&its->lock, flags); \ 763 return; \ 764 } \ 765 sync_obj = builder(its, cmd, desc); \ 766 its_flush_cmd(its, cmd); \ 767 \ 768 if (sync_obj) { \ 769 sync_cmd = its_allocate_entry(its); \ 770 if (!sync_cmd) \ 771 goto post; \ 772 \ 773 buildfn(its, sync_cmd, sync_obj); \ 774 its_flush_cmd(its, sync_cmd); \ 775 } \ 776 \ 777 post: \ 778 next_cmd = its_post_commands(its); \ 779 raw_spin_unlock_irqrestore(&its->lock, flags); \ 780 \ 781 if (its_wait_for_range_completion(its, cmd, next_cmd)) \ 782 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \ 783 } 784 785 static void its_build_sync_cmd(struct its_node *its, 786 struct its_cmd_block *sync_cmd, 787 struct its_collection *sync_col) 788 { 789 its_encode_cmd(sync_cmd, GITS_CMD_SYNC); 790 its_encode_target(sync_cmd, sync_col->target_address); 791 792 its_fixup_cmd(sync_cmd); 793 } 794 795 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t, 796 struct its_collection, its_build_sync_cmd) 797 798 static void its_build_vsync_cmd(struct its_node *its, 799 struct its_cmd_block *sync_cmd, 800 struct its_vpe *sync_vpe) 801 { 802 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC); 803 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id); 804 805 its_fixup_cmd(sync_cmd); 806 } 807 808 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t, 809 struct its_vpe, its_build_vsync_cmd) 810 811 static void its_send_int(struct its_device *dev, u32 event_id) 812 { 813 struct its_cmd_desc desc; 814 815 desc.its_int_cmd.dev = dev; 816 desc.its_int_cmd.event_id = event_id; 817 818 its_send_single_command(dev->its, its_build_int_cmd, &desc); 819 } 820 821 static void its_send_clear(struct its_device *dev, u32 event_id) 822 { 823 struct its_cmd_desc desc; 824 825 desc.its_clear_cmd.dev = dev; 826 desc.its_clear_cmd.event_id = event_id; 827 828 its_send_single_command(dev->its, its_build_clear_cmd, &desc); 829 } 830 831 static void its_send_inv(struct its_device *dev, u32 event_id) 832 { 833 struct its_cmd_desc desc; 834 835 desc.its_inv_cmd.dev = dev; 836 desc.its_inv_cmd.event_id = event_id; 837 838 its_send_single_command(dev->its, its_build_inv_cmd, &desc); 839 } 840 841 static void its_send_mapd(struct its_device *dev, int valid) 842 { 843 struct its_cmd_desc desc; 844 845 desc.its_mapd_cmd.dev = dev; 846 desc.its_mapd_cmd.valid = !!valid; 847 848 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); 849 } 850 851 static void its_send_mapc(struct its_node *its, struct its_collection *col, 852 int valid) 853 { 854 struct its_cmd_desc desc; 855 856 desc.its_mapc_cmd.col = col; 857 desc.its_mapc_cmd.valid = !!valid; 858 859 its_send_single_command(its, its_build_mapc_cmd, &desc); 860 } 861 862 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id) 863 { 864 struct its_cmd_desc desc; 865 866 desc.its_mapti_cmd.dev = dev; 867 desc.its_mapti_cmd.phys_id = irq_id; 868 desc.its_mapti_cmd.event_id = id; 869 870 its_send_single_command(dev->its, its_build_mapti_cmd, &desc); 871 } 872 873 static void its_send_movi(struct its_device *dev, 874 struct its_collection *col, u32 id) 875 { 876 struct its_cmd_desc desc; 877 878 desc.its_movi_cmd.dev = dev; 879 desc.its_movi_cmd.col = col; 880 desc.its_movi_cmd.event_id = id; 881 882 its_send_single_command(dev->its, its_build_movi_cmd, &desc); 883 } 884 885 static void its_send_discard(struct its_device *dev, u32 id) 886 { 887 struct its_cmd_desc desc; 888 889 desc.its_discard_cmd.dev = dev; 890 desc.its_discard_cmd.event_id = id; 891 892 its_send_single_command(dev->its, its_build_discard_cmd, &desc); 893 } 894 895 static void its_send_invall(struct its_node *its, struct its_collection *col) 896 { 897 struct its_cmd_desc desc; 898 899 desc.its_invall_cmd.col = col; 900 901 its_send_single_command(its, its_build_invall_cmd, &desc); 902 } 903 904 static void its_send_vmapti(struct its_device *dev, u32 id) 905 { 906 struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; 907 struct its_cmd_desc desc; 908 909 desc.its_vmapti_cmd.vpe = map->vpe; 910 desc.its_vmapti_cmd.dev = dev; 911 desc.its_vmapti_cmd.virt_id = map->vintid; 912 desc.its_vmapti_cmd.event_id = id; 913 desc.its_vmapti_cmd.db_enabled = map->db_enabled; 914 915 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); 916 } 917 918 static void its_send_vmovi(struct its_device *dev, u32 id) 919 { 920 struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; 921 struct its_cmd_desc desc; 922 923 desc.its_vmovi_cmd.vpe = map->vpe; 924 desc.its_vmovi_cmd.dev = dev; 925 desc.its_vmovi_cmd.event_id = id; 926 desc.its_vmovi_cmd.db_enabled = map->db_enabled; 927 928 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); 929 } 930 931 static void its_send_vmapp(struct its_node *its, 932 struct its_vpe *vpe, bool valid) 933 { 934 struct its_cmd_desc desc; 935 936 desc.its_vmapp_cmd.vpe = vpe; 937 desc.its_vmapp_cmd.valid = valid; 938 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; 939 940 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); 941 } 942 943 static void its_send_vmovp(struct its_vpe *vpe) 944 { 945 struct its_cmd_desc desc; 946 struct its_node *its; 947 unsigned long flags; 948 int col_id = vpe->col_idx; 949 950 desc.its_vmovp_cmd.vpe = vpe; 951 desc.its_vmovp_cmd.its_list = (u16)its_list_map; 952 953 if (!its_list_map) { 954 its = list_first_entry(&its_nodes, struct its_node, entry); 955 desc.its_vmovp_cmd.seq_num = 0; 956 desc.its_vmovp_cmd.col = &its->collections[col_id]; 957 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 958 return; 959 } 960 961 /* 962 * Yet another marvel of the architecture. If using the 963 * its_list "feature", we need to make sure that all ITSs 964 * receive all VMOVP commands in the same order. The only way 965 * to guarantee this is to make vmovp a serialization point. 966 * 967 * Wall <-- Head. 968 */ 969 raw_spin_lock_irqsave(&vmovp_lock, flags); 970 971 desc.its_vmovp_cmd.seq_num = vmovp_seq_num++; 972 973 /* Emit VMOVPs */ 974 list_for_each_entry(its, &its_nodes, entry) { 975 if (!its->is_v4) 976 continue; 977 978 if (!vpe->its_vm->vlpi_count[its->list_nr]) 979 continue; 980 981 desc.its_vmovp_cmd.col = &its->collections[col_id]; 982 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 983 } 984 985 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 986 } 987 988 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) 989 { 990 struct its_cmd_desc desc; 991 992 desc.its_vinvall_cmd.vpe = vpe; 993 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); 994 } 995 996 /* 997 * irqchip functions - assumes MSI, mostly. 998 */ 999 1000 static inline u32 its_get_event_id(struct irq_data *d) 1001 { 1002 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1003 return d->hwirq - its_dev->event_map.lpi_base; 1004 } 1005 1006 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) 1007 { 1008 irq_hw_number_t hwirq; 1009 struct page *prop_page; 1010 u8 *cfg; 1011 1012 if (irqd_is_forwarded_to_vcpu(d)) { 1013 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1014 u32 event = its_get_event_id(d); 1015 struct its_vlpi_map *map; 1016 1017 prop_page = its_dev->event_map.vm->vprop_page; 1018 map = &its_dev->event_map.vlpi_maps[event]; 1019 hwirq = map->vintid; 1020 1021 /* Remember the updated property */ 1022 map->properties &= ~clr; 1023 map->properties |= set | LPI_PROP_GROUP1; 1024 } else { 1025 prop_page = gic_rdists->prop_page; 1026 hwirq = d->hwirq; 1027 } 1028 1029 cfg = page_address(prop_page) + hwirq - 8192; 1030 *cfg &= ~clr; 1031 *cfg |= set | LPI_PROP_GROUP1; 1032 1033 /* 1034 * Make the above write visible to the redistributors. 1035 * And yes, we're flushing exactly: One. Single. Byte. 1036 * Humpf... 1037 */ 1038 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) 1039 gic_flush_dcache_to_poc(cfg, sizeof(*cfg)); 1040 else 1041 dsb(ishst); 1042 } 1043 1044 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) 1045 { 1046 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1047 1048 lpi_write_config(d, clr, set); 1049 its_send_inv(its_dev, its_get_event_id(d)); 1050 } 1051 1052 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) 1053 { 1054 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1055 u32 event = its_get_event_id(d); 1056 1057 if (its_dev->event_map.vlpi_maps[event].db_enabled == enable) 1058 return; 1059 1060 its_dev->event_map.vlpi_maps[event].db_enabled = enable; 1061 1062 /* 1063 * More fun with the architecture: 1064 * 1065 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI 1066 * value or to 1023, depending on the enable bit. But that 1067 * would be issueing a mapping for an /existing/ DevID+EventID 1068 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI 1069 * to the /same/ vPE, using this opportunity to adjust the 1070 * doorbell. Mouahahahaha. We loves it, Precious. 1071 */ 1072 its_send_vmovi(its_dev, event); 1073 } 1074 1075 static void its_mask_irq(struct irq_data *d) 1076 { 1077 if (irqd_is_forwarded_to_vcpu(d)) 1078 its_vlpi_set_doorbell(d, false); 1079 1080 lpi_update_config(d, LPI_PROP_ENABLED, 0); 1081 } 1082 1083 static void its_unmask_irq(struct irq_data *d) 1084 { 1085 if (irqd_is_forwarded_to_vcpu(d)) 1086 its_vlpi_set_doorbell(d, true); 1087 1088 lpi_update_config(d, 0, LPI_PROP_ENABLED); 1089 } 1090 1091 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1092 bool force) 1093 { 1094 unsigned int cpu; 1095 const struct cpumask *cpu_mask = cpu_online_mask; 1096 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1097 struct its_collection *target_col; 1098 u32 id = its_get_event_id(d); 1099 1100 /* A forwarded interrupt should use irq_set_vcpu_affinity */ 1101 if (irqd_is_forwarded_to_vcpu(d)) 1102 return -EINVAL; 1103 1104 /* lpi cannot be routed to a redistributor that is on a foreign node */ 1105 if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { 1106 if (its_dev->its->numa_node >= 0) { 1107 cpu_mask = cpumask_of_node(its_dev->its->numa_node); 1108 if (!cpumask_intersects(mask_val, cpu_mask)) 1109 return -EINVAL; 1110 } 1111 } 1112 1113 cpu = cpumask_any_and(mask_val, cpu_mask); 1114 1115 if (cpu >= nr_cpu_ids) 1116 return -EINVAL; 1117 1118 /* don't set the affinity when the target cpu is same as current one */ 1119 if (cpu != its_dev->event_map.col_map[id]) { 1120 target_col = &its_dev->its->collections[cpu]; 1121 its_send_movi(its_dev, target_col, id); 1122 its_dev->event_map.col_map[id] = cpu; 1123 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1124 } 1125 1126 return IRQ_SET_MASK_OK_DONE; 1127 } 1128 1129 static u64 its_irq_get_msi_base(struct its_device *its_dev) 1130 { 1131 struct its_node *its = its_dev->its; 1132 1133 return its->phys_base + GITS_TRANSLATER; 1134 } 1135 1136 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) 1137 { 1138 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1139 struct its_node *its; 1140 u64 addr; 1141 1142 its = its_dev->its; 1143 addr = its->get_msi_base(its_dev); 1144 1145 msg->address_lo = lower_32_bits(addr); 1146 msg->address_hi = upper_32_bits(addr); 1147 msg->data = its_get_event_id(d); 1148 1149 iommu_dma_map_msi_msg(d->irq, msg); 1150 } 1151 1152 static int its_irq_set_irqchip_state(struct irq_data *d, 1153 enum irqchip_irq_state which, 1154 bool state) 1155 { 1156 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1157 u32 event = its_get_event_id(d); 1158 1159 if (which != IRQCHIP_STATE_PENDING) 1160 return -EINVAL; 1161 1162 if (state) 1163 its_send_int(its_dev, event); 1164 else 1165 its_send_clear(its_dev, event); 1166 1167 return 0; 1168 } 1169 1170 static void its_map_vm(struct its_node *its, struct its_vm *vm) 1171 { 1172 unsigned long flags; 1173 1174 /* Not using the ITS list? Everything is always mapped. */ 1175 if (!its_list_map) 1176 return; 1177 1178 raw_spin_lock_irqsave(&vmovp_lock, flags); 1179 1180 /* 1181 * If the VM wasn't mapped yet, iterate over the vpes and get 1182 * them mapped now. 1183 */ 1184 vm->vlpi_count[its->list_nr]++; 1185 1186 if (vm->vlpi_count[its->list_nr] == 1) { 1187 int i; 1188 1189 for (i = 0; i < vm->nr_vpes; i++) { 1190 struct its_vpe *vpe = vm->vpes[i]; 1191 struct irq_data *d = irq_get_irq_data(vpe->irq); 1192 1193 /* Map the VPE to the first possible CPU */ 1194 vpe->col_idx = cpumask_first(cpu_online_mask); 1195 its_send_vmapp(its, vpe, true); 1196 its_send_vinvall(its, vpe); 1197 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 1198 } 1199 } 1200 1201 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1202 } 1203 1204 static void its_unmap_vm(struct its_node *its, struct its_vm *vm) 1205 { 1206 unsigned long flags; 1207 1208 /* Not using the ITS list? Everything is always mapped. */ 1209 if (!its_list_map) 1210 return; 1211 1212 raw_spin_lock_irqsave(&vmovp_lock, flags); 1213 1214 if (!--vm->vlpi_count[its->list_nr]) { 1215 int i; 1216 1217 for (i = 0; i < vm->nr_vpes; i++) 1218 its_send_vmapp(its, vm->vpes[i], false); 1219 } 1220 1221 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1222 } 1223 1224 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) 1225 { 1226 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1227 u32 event = its_get_event_id(d); 1228 int ret = 0; 1229 1230 if (!info->map) 1231 return -EINVAL; 1232 1233 mutex_lock(&its_dev->event_map.vlpi_lock); 1234 1235 if (!its_dev->event_map.vm) { 1236 struct its_vlpi_map *maps; 1237 1238 maps = kzalloc(sizeof(*maps) * its_dev->event_map.nr_lpis, 1239 GFP_KERNEL); 1240 if (!maps) { 1241 ret = -ENOMEM; 1242 goto out; 1243 } 1244 1245 its_dev->event_map.vm = info->map->vm; 1246 its_dev->event_map.vlpi_maps = maps; 1247 } else if (its_dev->event_map.vm != info->map->vm) { 1248 ret = -EINVAL; 1249 goto out; 1250 } 1251 1252 /* Get our private copy of the mapping information */ 1253 its_dev->event_map.vlpi_maps[event] = *info->map; 1254 1255 if (irqd_is_forwarded_to_vcpu(d)) { 1256 /* Already mapped, move it around */ 1257 its_send_vmovi(its_dev, event); 1258 } else { 1259 /* Ensure all the VPEs are mapped on this ITS */ 1260 its_map_vm(its_dev->its, info->map->vm); 1261 1262 /* 1263 * Flag the interrupt as forwarded so that we can 1264 * start poking the virtual property table. 1265 */ 1266 irqd_set_forwarded_to_vcpu(d); 1267 1268 /* Write out the property to the prop table */ 1269 lpi_write_config(d, 0xff, info->map->properties); 1270 1271 /* Drop the physical mapping */ 1272 its_send_discard(its_dev, event); 1273 1274 /* and install the virtual one */ 1275 its_send_vmapti(its_dev, event); 1276 1277 /* Increment the number of VLPIs */ 1278 its_dev->event_map.nr_vlpis++; 1279 } 1280 1281 out: 1282 mutex_unlock(&its_dev->event_map.vlpi_lock); 1283 return ret; 1284 } 1285 1286 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) 1287 { 1288 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1289 u32 event = its_get_event_id(d); 1290 int ret = 0; 1291 1292 mutex_lock(&its_dev->event_map.vlpi_lock); 1293 1294 if (!its_dev->event_map.vm || 1295 !its_dev->event_map.vlpi_maps[event].vm) { 1296 ret = -EINVAL; 1297 goto out; 1298 } 1299 1300 /* Copy our mapping information to the incoming request */ 1301 *info->map = its_dev->event_map.vlpi_maps[event]; 1302 1303 out: 1304 mutex_unlock(&its_dev->event_map.vlpi_lock); 1305 return ret; 1306 } 1307 1308 static int its_vlpi_unmap(struct irq_data *d) 1309 { 1310 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1311 u32 event = its_get_event_id(d); 1312 int ret = 0; 1313 1314 mutex_lock(&its_dev->event_map.vlpi_lock); 1315 1316 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) { 1317 ret = -EINVAL; 1318 goto out; 1319 } 1320 1321 /* Drop the virtual mapping */ 1322 its_send_discard(its_dev, event); 1323 1324 /* and restore the physical one */ 1325 irqd_clr_forwarded_to_vcpu(d); 1326 its_send_mapti(its_dev, d->hwirq, event); 1327 lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO | 1328 LPI_PROP_ENABLED | 1329 LPI_PROP_GROUP1)); 1330 1331 /* Potentially unmap the VM from this ITS */ 1332 its_unmap_vm(its_dev->its, its_dev->event_map.vm); 1333 1334 /* 1335 * Drop the refcount and make the device available again if 1336 * this was the last VLPI. 1337 */ 1338 if (!--its_dev->event_map.nr_vlpis) { 1339 its_dev->event_map.vm = NULL; 1340 kfree(its_dev->event_map.vlpi_maps); 1341 } 1342 1343 out: 1344 mutex_unlock(&its_dev->event_map.vlpi_lock); 1345 return ret; 1346 } 1347 1348 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info) 1349 { 1350 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1351 1352 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) 1353 return -EINVAL; 1354 1355 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI) 1356 lpi_update_config(d, 0xff, info->config); 1357 else 1358 lpi_write_config(d, 0xff, info->config); 1359 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED)); 1360 1361 return 0; 1362 } 1363 1364 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 1365 { 1366 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1367 struct its_cmd_info *info = vcpu_info; 1368 1369 /* Need a v4 ITS */ 1370 if (!its_dev->its->is_v4) 1371 return -EINVAL; 1372 1373 /* Unmap request? */ 1374 if (!info) 1375 return its_vlpi_unmap(d); 1376 1377 switch (info->cmd_type) { 1378 case MAP_VLPI: 1379 return its_vlpi_map(d, info); 1380 1381 case GET_VLPI: 1382 return its_vlpi_get(d, info); 1383 1384 case PROP_UPDATE_VLPI: 1385 case PROP_UPDATE_AND_INV_VLPI: 1386 return its_vlpi_prop_update(d, info); 1387 1388 default: 1389 return -EINVAL; 1390 } 1391 } 1392 1393 static struct irq_chip its_irq_chip = { 1394 .name = "ITS", 1395 .irq_mask = its_mask_irq, 1396 .irq_unmask = its_unmask_irq, 1397 .irq_eoi = irq_chip_eoi_parent, 1398 .irq_set_affinity = its_set_affinity, 1399 .irq_compose_msi_msg = its_irq_compose_msi_msg, 1400 .irq_set_irqchip_state = its_irq_set_irqchip_state, 1401 .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity, 1402 }; 1403 1404 /* 1405 * How we allocate LPIs: 1406 * 1407 * The GIC has id_bits bits for interrupt identifiers. From there, we 1408 * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as 1409 * we allocate LPIs by chunks of 32, we can shift the whole thing by 5 1410 * bits to the right. 1411 * 1412 * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations. 1413 */ 1414 #define IRQS_PER_CHUNK_SHIFT 5 1415 #define IRQS_PER_CHUNK (1 << IRQS_PER_CHUNK_SHIFT) 1416 #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */ 1417 1418 static unsigned long *lpi_bitmap; 1419 static u32 lpi_chunks; 1420 static DEFINE_SPINLOCK(lpi_lock); 1421 1422 static int its_lpi_to_chunk(int lpi) 1423 { 1424 return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT; 1425 } 1426 1427 static int its_chunk_to_lpi(int chunk) 1428 { 1429 return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192; 1430 } 1431 1432 static int __init its_lpi_init(u32 id_bits) 1433 { 1434 lpi_chunks = its_lpi_to_chunk(1UL << id_bits); 1435 1436 lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long), 1437 GFP_KERNEL); 1438 if (!lpi_bitmap) { 1439 lpi_chunks = 0; 1440 return -ENOMEM; 1441 } 1442 1443 pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks); 1444 return 0; 1445 } 1446 1447 static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids) 1448 { 1449 unsigned long *bitmap = NULL; 1450 int chunk_id; 1451 int nr_chunks; 1452 int i; 1453 1454 nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK); 1455 1456 spin_lock(&lpi_lock); 1457 1458 do { 1459 chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks, 1460 0, nr_chunks, 0); 1461 if (chunk_id < lpi_chunks) 1462 break; 1463 1464 nr_chunks--; 1465 } while (nr_chunks > 0); 1466 1467 if (!nr_chunks) 1468 goto out; 1469 1470 bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long), 1471 GFP_ATOMIC); 1472 if (!bitmap) 1473 goto out; 1474 1475 for (i = 0; i < nr_chunks; i++) 1476 set_bit(chunk_id + i, lpi_bitmap); 1477 1478 *base = its_chunk_to_lpi(chunk_id); 1479 *nr_ids = nr_chunks * IRQS_PER_CHUNK; 1480 1481 out: 1482 spin_unlock(&lpi_lock); 1483 1484 if (!bitmap) 1485 *base = *nr_ids = 0; 1486 1487 return bitmap; 1488 } 1489 1490 static void its_lpi_free_chunks(unsigned long *bitmap, int base, int nr_ids) 1491 { 1492 int lpi; 1493 1494 spin_lock(&lpi_lock); 1495 1496 for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) { 1497 int chunk = its_lpi_to_chunk(lpi); 1498 1499 BUG_ON(chunk > lpi_chunks); 1500 if (test_bit(chunk, lpi_bitmap)) { 1501 clear_bit(chunk, lpi_bitmap); 1502 } else { 1503 pr_err("Bad LPI chunk %d\n", chunk); 1504 } 1505 } 1506 1507 spin_unlock(&lpi_lock); 1508 1509 kfree(bitmap); 1510 } 1511 1512 static struct page *its_allocate_prop_table(gfp_t gfp_flags) 1513 { 1514 struct page *prop_page; 1515 1516 prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); 1517 if (!prop_page) 1518 return NULL; 1519 1520 /* Priority 0xa0, Group-1, disabled */ 1521 memset(page_address(prop_page), 1522 LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, 1523 LPI_PROPBASE_SZ); 1524 1525 /* Make sure the GIC will observe the written configuration */ 1526 gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ); 1527 1528 return prop_page; 1529 } 1530 1531 static void its_free_prop_table(struct page *prop_page) 1532 { 1533 free_pages((unsigned long)page_address(prop_page), 1534 get_order(LPI_PROPBASE_SZ)); 1535 } 1536 1537 static int __init its_alloc_lpi_tables(void) 1538 { 1539 phys_addr_t paddr; 1540 1541 lpi_id_bits = min_t(u32, gic_rdists->id_bits, ITS_MAX_LPI_NRBITS); 1542 gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT); 1543 if (!gic_rdists->prop_page) { 1544 pr_err("Failed to allocate PROPBASE\n"); 1545 return -ENOMEM; 1546 } 1547 1548 paddr = page_to_phys(gic_rdists->prop_page); 1549 pr_info("GIC: using LPI property table @%pa\n", &paddr); 1550 1551 return its_lpi_init(lpi_id_bits); 1552 } 1553 1554 static const char *its_base_type_string[] = { 1555 [GITS_BASER_TYPE_DEVICE] = "Devices", 1556 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs", 1557 [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)", 1558 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections", 1559 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)", 1560 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)", 1561 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", 1562 }; 1563 1564 static u64 its_read_baser(struct its_node *its, struct its_baser *baser) 1565 { 1566 u32 idx = baser - its->tables; 1567 1568 return gits_read_baser(its->base + GITS_BASER + (idx << 3)); 1569 } 1570 1571 static void its_write_baser(struct its_node *its, struct its_baser *baser, 1572 u64 val) 1573 { 1574 u32 idx = baser - its->tables; 1575 1576 gits_write_baser(val, its->base + GITS_BASER + (idx << 3)); 1577 baser->val = its_read_baser(its, baser); 1578 } 1579 1580 static int its_setup_baser(struct its_node *its, struct its_baser *baser, 1581 u64 cache, u64 shr, u32 psz, u32 order, 1582 bool indirect) 1583 { 1584 u64 val = its_read_baser(its, baser); 1585 u64 esz = GITS_BASER_ENTRY_SIZE(val); 1586 u64 type = GITS_BASER_TYPE(val); 1587 u64 baser_phys, tmp; 1588 u32 alloc_pages; 1589 void *base; 1590 1591 retry_alloc_baser: 1592 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); 1593 if (alloc_pages > GITS_BASER_PAGES_MAX) { 1594 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", 1595 &its->phys_base, its_base_type_string[type], 1596 alloc_pages, GITS_BASER_PAGES_MAX); 1597 alloc_pages = GITS_BASER_PAGES_MAX; 1598 order = get_order(GITS_BASER_PAGES_MAX * psz); 1599 } 1600 1601 base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 1602 if (!base) 1603 return -ENOMEM; 1604 1605 baser_phys = virt_to_phys(base); 1606 1607 /* Check if the physical address of the memory is above 48bits */ 1608 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) { 1609 1610 /* 52bit PA is supported only when PageSize=64K */ 1611 if (psz != SZ_64K) { 1612 pr_err("ITS: no 52bit PA support when psz=%d\n", psz); 1613 free_pages((unsigned long)base, order); 1614 return -ENXIO; 1615 } 1616 1617 /* Convert 52bit PA to 48bit field */ 1618 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys); 1619 } 1620 1621 retry_baser: 1622 val = (baser_phys | 1623 (type << GITS_BASER_TYPE_SHIFT) | 1624 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | 1625 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) | 1626 cache | 1627 shr | 1628 GITS_BASER_VALID); 1629 1630 val |= indirect ? GITS_BASER_INDIRECT : 0x0; 1631 1632 switch (psz) { 1633 case SZ_4K: 1634 val |= GITS_BASER_PAGE_SIZE_4K; 1635 break; 1636 case SZ_16K: 1637 val |= GITS_BASER_PAGE_SIZE_16K; 1638 break; 1639 case SZ_64K: 1640 val |= GITS_BASER_PAGE_SIZE_64K; 1641 break; 1642 } 1643 1644 its_write_baser(its, baser, val); 1645 tmp = baser->val; 1646 1647 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { 1648 /* 1649 * Shareability didn't stick. Just use 1650 * whatever the read reported, which is likely 1651 * to be the only thing this redistributor 1652 * supports. If that's zero, make it 1653 * non-cacheable as well. 1654 */ 1655 shr = tmp & GITS_BASER_SHAREABILITY_MASK; 1656 if (!shr) { 1657 cache = GITS_BASER_nC; 1658 gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order)); 1659 } 1660 goto retry_baser; 1661 } 1662 1663 if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { 1664 /* 1665 * Page size didn't stick. Let's try a smaller 1666 * size and retry. If we reach 4K, then 1667 * something is horribly wrong... 1668 */ 1669 free_pages((unsigned long)base, order); 1670 baser->base = NULL; 1671 1672 switch (psz) { 1673 case SZ_16K: 1674 psz = SZ_4K; 1675 goto retry_alloc_baser; 1676 case SZ_64K: 1677 psz = SZ_16K; 1678 goto retry_alloc_baser; 1679 } 1680 } 1681 1682 if (val != tmp) { 1683 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", 1684 &its->phys_base, its_base_type_string[type], 1685 val, tmp); 1686 free_pages((unsigned long)base, order); 1687 return -ENXIO; 1688 } 1689 1690 baser->order = order; 1691 baser->base = base; 1692 baser->psz = psz; 1693 tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz; 1694 1695 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n", 1696 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), 1697 its_base_type_string[type], 1698 (unsigned long)virt_to_phys(base), 1699 indirect ? "indirect" : "flat", (int)esz, 1700 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); 1701 1702 return 0; 1703 } 1704 1705 static bool its_parse_indirect_baser(struct its_node *its, 1706 struct its_baser *baser, 1707 u32 psz, u32 *order, u32 ids) 1708 { 1709 u64 tmp = its_read_baser(its, baser); 1710 u64 type = GITS_BASER_TYPE(tmp); 1711 u64 esz = GITS_BASER_ENTRY_SIZE(tmp); 1712 u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb; 1713 u32 new_order = *order; 1714 bool indirect = false; 1715 1716 /* No need to enable Indirection if memory requirement < (psz*2)bytes */ 1717 if ((esz << ids) > (psz * 2)) { 1718 /* 1719 * Find out whether hw supports a single or two-level table by 1720 * table by reading bit at offset '62' after writing '1' to it. 1721 */ 1722 its_write_baser(its, baser, val | GITS_BASER_INDIRECT); 1723 indirect = !!(baser->val & GITS_BASER_INDIRECT); 1724 1725 if (indirect) { 1726 /* 1727 * The size of the lvl2 table is equal to ITS page size 1728 * which is 'psz'. For computing lvl1 table size, 1729 * subtract ID bits that sparse lvl2 table from 'ids' 1730 * which is reported by ITS hardware times lvl1 table 1731 * entry size. 1732 */ 1733 ids -= ilog2(psz / (int)esz); 1734 esz = GITS_LVL1_ENTRY_SIZE; 1735 } 1736 } 1737 1738 /* 1739 * Allocate as many entries as required to fit the 1740 * range of device IDs that the ITS can grok... The ID 1741 * space being incredibly sparse, this results in a 1742 * massive waste of memory if two-level device table 1743 * feature is not supported by hardware. 1744 */ 1745 new_order = max_t(u32, get_order(esz << ids), new_order); 1746 if (new_order >= MAX_ORDER) { 1747 new_order = MAX_ORDER - 1; 1748 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); 1749 pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n", 1750 &its->phys_base, its_base_type_string[type], 1751 its->device_ids, ids); 1752 } 1753 1754 *order = new_order; 1755 1756 return indirect; 1757 } 1758 1759 static void its_free_tables(struct its_node *its) 1760 { 1761 int i; 1762 1763 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 1764 if (its->tables[i].base) { 1765 free_pages((unsigned long)its->tables[i].base, 1766 its->tables[i].order); 1767 its->tables[i].base = NULL; 1768 } 1769 } 1770 } 1771 1772 static int its_alloc_tables(struct its_node *its) 1773 { 1774 u64 shr = GITS_BASER_InnerShareable; 1775 u64 cache = GITS_BASER_RaWaWb; 1776 u32 psz = SZ_64K; 1777 int err, i; 1778 1779 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) 1780 /* erratum 24313: ignore memory access type */ 1781 cache = GITS_BASER_nCnB; 1782 1783 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 1784 struct its_baser *baser = its->tables + i; 1785 u64 val = its_read_baser(its, baser); 1786 u64 type = GITS_BASER_TYPE(val); 1787 u32 order = get_order(psz); 1788 bool indirect = false; 1789 1790 switch (type) { 1791 case GITS_BASER_TYPE_NONE: 1792 continue; 1793 1794 case GITS_BASER_TYPE_DEVICE: 1795 indirect = its_parse_indirect_baser(its, baser, 1796 psz, &order, 1797 its->device_ids); 1798 case GITS_BASER_TYPE_VCPU: 1799 indirect = its_parse_indirect_baser(its, baser, 1800 psz, &order, 1801 ITS_MAX_VPEID_BITS); 1802 break; 1803 } 1804 1805 err = its_setup_baser(its, baser, cache, shr, psz, order, indirect); 1806 if (err < 0) { 1807 its_free_tables(its); 1808 return err; 1809 } 1810 1811 /* Update settings which will be used for next BASERn */ 1812 psz = baser->psz; 1813 cache = baser->val & GITS_BASER_CACHEABILITY_MASK; 1814 shr = baser->val & GITS_BASER_SHAREABILITY_MASK; 1815 } 1816 1817 return 0; 1818 } 1819 1820 static int its_alloc_collections(struct its_node *its) 1821 { 1822 its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections), 1823 GFP_KERNEL); 1824 if (!its->collections) 1825 return -ENOMEM; 1826 1827 return 0; 1828 } 1829 1830 static struct page *its_allocate_pending_table(gfp_t gfp_flags) 1831 { 1832 struct page *pend_page; 1833 /* 1834 * The pending pages have to be at least 64kB aligned, 1835 * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below. 1836 */ 1837 pend_page = alloc_pages(gfp_flags | __GFP_ZERO, 1838 get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K))); 1839 if (!pend_page) 1840 return NULL; 1841 1842 /* Make sure the GIC will observe the zero-ed page */ 1843 gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ); 1844 1845 return pend_page; 1846 } 1847 1848 static void its_free_pending_table(struct page *pt) 1849 { 1850 free_pages((unsigned long)page_address(pt), 1851 get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K))); 1852 } 1853 1854 static void its_cpu_init_lpis(void) 1855 { 1856 void __iomem *rbase = gic_data_rdist_rd_base(); 1857 struct page *pend_page; 1858 u64 val, tmp; 1859 1860 /* If we didn't allocate the pending table yet, do it now */ 1861 pend_page = gic_data_rdist()->pend_page; 1862 if (!pend_page) { 1863 phys_addr_t paddr; 1864 1865 pend_page = its_allocate_pending_table(GFP_NOWAIT); 1866 if (!pend_page) { 1867 pr_err("Failed to allocate PENDBASE for CPU%d\n", 1868 smp_processor_id()); 1869 return; 1870 } 1871 1872 paddr = page_to_phys(pend_page); 1873 pr_info("CPU%d: using LPI pending table @%pa\n", 1874 smp_processor_id(), &paddr); 1875 gic_data_rdist()->pend_page = pend_page; 1876 } 1877 1878 /* Disable LPIs */ 1879 val = readl_relaxed(rbase + GICR_CTLR); 1880 val &= ~GICR_CTLR_ENABLE_LPIS; 1881 writel_relaxed(val, rbase + GICR_CTLR); 1882 1883 /* 1884 * Make sure any change to the table is observable by the GIC. 1885 */ 1886 dsb(sy); 1887 1888 /* set PROPBASE */ 1889 val = (page_to_phys(gic_rdists->prop_page) | 1890 GICR_PROPBASER_InnerShareable | 1891 GICR_PROPBASER_RaWaWb | 1892 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); 1893 1894 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 1895 tmp = gicr_read_propbaser(rbase + GICR_PROPBASER); 1896 1897 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { 1898 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) { 1899 /* 1900 * The HW reports non-shareable, we must 1901 * remove the cacheability attributes as 1902 * well. 1903 */ 1904 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK | 1905 GICR_PROPBASER_CACHEABILITY_MASK); 1906 val |= GICR_PROPBASER_nC; 1907 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 1908 } 1909 pr_info_once("GIC: using cache flushing for LPI property table\n"); 1910 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; 1911 } 1912 1913 /* set PENDBASE */ 1914 val = (page_to_phys(pend_page) | 1915 GICR_PENDBASER_InnerShareable | 1916 GICR_PENDBASER_RaWaWb); 1917 1918 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 1919 tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER); 1920 1921 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { 1922 /* 1923 * The HW reports non-shareable, we must remove the 1924 * cacheability attributes as well. 1925 */ 1926 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK | 1927 GICR_PENDBASER_CACHEABILITY_MASK); 1928 val |= GICR_PENDBASER_nC; 1929 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 1930 } 1931 1932 /* Enable LPIs */ 1933 val = readl_relaxed(rbase + GICR_CTLR); 1934 val |= GICR_CTLR_ENABLE_LPIS; 1935 writel_relaxed(val, rbase + GICR_CTLR); 1936 1937 /* Make sure the GIC has seen the above */ 1938 dsb(sy); 1939 } 1940 1941 static void its_cpu_init_collection(void) 1942 { 1943 struct its_node *its; 1944 int cpu; 1945 1946 spin_lock(&its_lock); 1947 cpu = smp_processor_id(); 1948 1949 list_for_each_entry(its, &its_nodes, entry) { 1950 u64 target; 1951 1952 /* avoid cross node collections and its mapping */ 1953 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { 1954 struct device_node *cpu_node; 1955 1956 cpu_node = of_get_cpu_node(cpu, NULL); 1957 if (its->numa_node != NUMA_NO_NODE && 1958 its->numa_node != of_node_to_nid(cpu_node)) 1959 continue; 1960 } 1961 1962 /* 1963 * We now have to bind each collection to its target 1964 * redistributor. 1965 */ 1966 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) { 1967 /* 1968 * This ITS wants the physical address of the 1969 * redistributor. 1970 */ 1971 target = gic_data_rdist()->phys_base; 1972 } else { 1973 /* 1974 * This ITS wants a linear CPU number. 1975 */ 1976 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 1977 target = GICR_TYPER_CPU_NUMBER(target) << 16; 1978 } 1979 1980 /* Perform collection mapping */ 1981 its->collections[cpu].target_address = target; 1982 its->collections[cpu].col_id = cpu; 1983 1984 its_send_mapc(its, &its->collections[cpu], 1); 1985 its_send_invall(its, &its->collections[cpu]); 1986 } 1987 1988 spin_unlock(&its_lock); 1989 } 1990 1991 static struct its_device *its_find_device(struct its_node *its, u32 dev_id) 1992 { 1993 struct its_device *its_dev = NULL, *tmp; 1994 unsigned long flags; 1995 1996 raw_spin_lock_irqsave(&its->lock, flags); 1997 1998 list_for_each_entry(tmp, &its->its_device_list, entry) { 1999 if (tmp->device_id == dev_id) { 2000 its_dev = tmp; 2001 break; 2002 } 2003 } 2004 2005 raw_spin_unlock_irqrestore(&its->lock, flags); 2006 2007 return its_dev; 2008 } 2009 2010 static struct its_baser *its_get_baser(struct its_node *its, u32 type) 2011 { 2012 int i; 2013 2014 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 2015 if (GITS_BASER_TYPE(its->tables[i].val) == type) 2016 return &its->tables[i]; 2017 } 2018 2019 return NULL; 2020 } 2021 2022 static bool its_alloc_table_entry(struct its_baser *baser, u32 id) 2023 { 2024 struct page *page; 2025 u32 esz, idx; 2026 __le64 *table; 2027 2028 /* Don't allow device id that exceeds single, flat table limit */ 2029 esz = GITS_BASER_ENTRY_SIZE(baser->val); 2030 if (!(baser->val & GITS_BASER_INDIRECT)) 2031 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); 2032 2033 /* Compute 1st level table index & check if that exceeds table limit */ 2034 idx = id >> ilog2(baser->psz / esz); 2035 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) 2036 return false; 2037 2038 table = baser->base; 2039 2040 /* Allocate memory for 2nd level table */ 2041 if (!table[idx]) { 2042 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(baser->psz)); 2043 if (!page) 2044 return false; 2045 2046 /* Flush Lvl2 table to PoC if hw doesn't support coherency */ 2047 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 2048 gic_flush_dcache_to_poc(page_address(page), baser->psz); 2049 2050 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); 2051 2052 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ 2053 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 2054 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); 2055 2056 /* Ensure updated table contents are visible to ITS hardware */ 2057 dsb(sy); 2058 } 2059 2060 return true; 2061 } 2062 2063 static bool its_alloc_device_table(struct its_node *its, u32 dev_id) 2064 { 2065 struct its_baser *baser; 2066 2067 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); 2068 2069 /* Don't allow device id that exceeds ITS hardware limit */ 2070 if (!baser) 2071 return (ilog2(dev_id) < its->device_ids); 2072 2073 return its_alloc_table_entry(baser, dev_id); 2074 } 2075 2076 static bool its_alloc_vpe_table(u32 vpe_id) 2077 { 2078 struct its_node *its; 2079 2080 /* 2081 * Make sure the L2 tables are allocated on *all* v4 ITSs. We 2082 * could try and only do it on ITSs corresponding to devices 2083 * that have interrupts targeted at this VPE, but the 2084 * complexity becomes crazy (and you have tons of memory 2085 * anyway, right?). 2086 */ 2087 list_for_each_entry(its, &its_nodes, entry) { 2088 struct its_baser *baser; 2089 2090 if (!its->is_v4) 2091 continue; 2092 2093 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU); 2094 if (!baser) 2095 return false; 2096 2097 if (!its_alloc_table_entry(baser, vpe_id)) 2098 return false; 2099 } 2100 2101 return true; 2102 } 2103 2104 static struct its_device *its_create_device(struct its_node *its, u32 dev_id, 2105 int nvecs, bool alloc_lpis) 2106 { 2107 struct its_device *dev; 2108 unsigned long *lpi_map = NULL; 2109 unsigned long flags; 2110 u16 *col_map = NULL; 2111 void *itt; 2112 int lpi_base; 2113 int nr_lpis; 2114 int nr_ites; 2115 int sz; 2116 2117 if (!its_alloc_device_table(its, dev_id)) 2118 return NULL; 2119 2120 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2121 /* 2122 * At least one bit of EventID is being used, hence a minimum 2123 * of two entries. No, the architecture doesn't let you 2124 * express an ITT with a single entry. 2125 */ 2126 nr_ites = max(2UL, roundup_pow_of_two(nvecs)); 2127 sz = nr_ites * its->ite_size; 2128 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; 2129 itt = kzalloc(sz, GFP_KERNEL); 2130 if (alloc_lpis) { 2131 lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis); 2132 if (lpi_map) 2133 col_map = kzalloc(sizeof(*col_map) * nr_lpis, 2134 GFP_KERNEL); 2135 } else { 2136 col_map = kzalloc(sizeof(*col_map) * nr_ites, GFP_KERNEL); 2137 nr_lpis = 0; 2138 lpi_base = 0; 2139 } 2140 2141 if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { 2142 kfree(dev); 2143 kfree(itt); 2144 kfree(lpi_map); 2145 kfree(col_map); 2146 return NULL; 2147 } 2148 2149 gic_flush_dcache_to_poc(itt, sz); 2150 2151 dev->its = its; 2152 dev->itt = itt; 2153 dev->nr_ites = nr_ites; 2154 dev->event_map.lpi_map = lpi_map; 2155 dev->event_map.col_map = col_map; 2156 dev->event_map.lpi_base = lpi_base; 2157 dev->event_map.nr_lpis = nr_lpis; 2158 mutex_init(&dev->event_map.vlpi_lock); 2159 dev->device_id = dev_id; 2160 INIT_LIST_HEAD(&dev->entry); 2161 2162 raw_spin_lock_irqsave(&its->lock, flags); 2163 list_add(&dev->entry, &its->its_device_list); 2164 raw_spin_unlock_irqrestore(&its->lock, flags); 2165 2166 /* Map device to its ITT */ 2167 its_send_mapd(dev, 1); 2168 2169 return dev; 2170 } 2171 2172 static void its_free_device(struct its_device *its_dev) 2173 { 2174 unsigned long flags; 2175 2176 raw_spin_lock_irqsave(&its_dev->its->lock, flags); 2177 list_del(&its_dev->entry); 2178 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); 2179 kfree(its_dev->itt); 2180 kfree(its_dev); 2181 } 2182 2183 static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq) 2184 { 2185 int idx; 2186 2187 idx = find_first_zero_bit(dev->event_map.lpi_map, 2188 dev->event_map.nr_lpis); 2189 if (idx == dev->event_map.nr_lpis) 2190 return -ENOSPC; 2191 2192 *hwirq = dev->event_map.lpi_base + idx; 2193 set_bit(idx, dev->event_map.lpi_map); 2194 2195 return 0; 2196 } 2197 2198 static int its_msi_prepare(struct irq_domain *domain, struct device *dev, 2199 int nvec, msi_alloc_info_t *info) 2200 { 2201 struct its_node *its; 2202 struct its_device *its_dev; 2203 struct msi_domain_info *msi_info; 2204 u32 dev_id; 2205 2206 /* 2207 * We ignore "dev" entierely, and rely on the dev_id that has 2208 * been passed via the scratchpad. This limits this domain's 2209 * usefulness to upper layers that definitely know that they 2210 * are built on top of the ITS. 2211 */ 2212 dev_id = info->scratchpad[0].ul; 2213 2214 msi_info = msi_get_domain_info(domain); 2215 its = msi_info->data; 2216 2217 if (!gic_rdists->has_direct_lpi && 2218 vpe_proxy.dev && 2219 vpe_proxy.dev->its == its && 2220 dev_id == vpe_proxy.dev->device_id) { 2221 /* Bad luck. Get yourself a better implementation */ 2222 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n", 2223 dev_id); 2224 return -EINVAL; 2225 } 2226 2227 its_dev = its_find_device(its, dev_id); 2228 if (its_dev) { 2229 /* 2230 * We already have seen this ID, probably through 2231 * another alias (PCI bridge of some sort). No need to 2232 * create the device. 2233 */ 2234 pr_debug("Reusing ITT for devID %x\n", dev_id); 2235 goto out; 2236 } 2237 2238 its_dev = its_create_device(its, dev_id, nvec, true); 2239 if (!its_dev) 2240 return -ENOMEM; 2241 2242 pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec)); 2243 out: 2244 info->scratchpad[0].ptr = its_dev; 2245 return 0; 2246 } 2247 2248 static struct msi_domain_ops its_msi_domain_ops = { 2249 .msi_prepare = its_msi_prepare, 2250 }; 2251 2252 static int its_irq_gic_domain_alloc(struct irq_domain *domain, 2253 unsigned int virq, 2254 irq_hw_number_t hwirq) 2255 { 2256 struct irq_fwspec fwspec; 2257 2258 if (irq_domain_get_of_node(domain->parent)) { 2259 fwspec.fwnode = domain->parent->fwnode; 2260 fwspec.param_count = 3; 2261 fwspec.param[0] = GIC_IRQ_TYPE_LPI; 2262 fwspec.param[1] = hwirq; 2263 fwspec.param[2] = IRQ_TYPE_EDGE_RISING; 2264 } else if (is_fwnode_irqchip(domain->parent->fwnode)) { 2265 fwspec.fwnode = domain->parent->fwnode; 2266 fwspec.param_count = 2; 2267 fwspec.param[0] = hwirq; 2268 fwspec.param[1] = IRQ_TYPE_EDGE_RISING; 2269 } else { 2270 return -EINVAL; 2271 } 2272 2273 return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); 2274 } 2275 2276 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 2277 unsigned int nr_irqs, void *args) 2278 { 2279 msi_alloc_info_t *info = args; 2280 struct its_device *its_dev = info->scratchpad[0].ptr; 2281 irq_hw_number_t hwirq; 2282 int err; 2283 int i; 2284 2285 for (i = 0; i < nr_irqs; i++) { 2286 err = its_alloc_device_irq(its_dev, &hwirq); 2287 if (err) 2288 return err; 2289 2290 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq); 2291 if (err) 2292 return err; 2293 2294 irq_domain_set_hwirq_and_chip(domain, virq + i, 2295 hwirq, &its_irq_chip, its_dev); 2296 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq + i))); 2297 pr_debug("ID:%d pID:%d vID:%d\n", 2298 (int)(hwirq - its_dev->event_map.lpi_base), 2299 (int) hwirq, virq + i); 2300 } 2301 2302 return 0; 2303 } 2304 2305 static int its_irq_domain_activate(struct irq_domain *domain, 2306 struct irq_data *d, bool early) 2307 { 2308 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 2309 u32 event = its_get_event_id(d); 2310 const struct cpumask *cpu_mask = cpu_online_mask; 2311 int cpu; 2312 2313 /* get the cpu_mask of local node */ 2314 if (its_dev->its->numa_node >= 0) 2315 cpu_mask = cpumask_of_node(its_dev->its->numa_node); 2316 2317 /* Bind the LPI to the first possible CPU */ 2318 cpu = cpumask_first(cpu_mask); 2319 its_dev->event_map.col_map[event] = cpu; 2320 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 2321 2322 /* Map the GIC IRQ and event to the device */ 2323 its_send_mapti(its_dev, d->hwirq, event); 2324 return 0; 2325 } 2326 2327 static void its_irq_domain_deactivate(struct irq_domain *domain, 2328 struct irq_data *d) 2329 { 2330 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 2331 u32 event = its_get_event_id(d); 2332 2333 /* Stop the delivery of interrupts */ 2334 its_send_discard(its_dev, event); 2335 } 2336 2337 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, 2338 unsigned int nr_irqs) 2339 { 2340 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 2341 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 2342 int i; 2343 2344 for (i = 0; i < nr_irqs; i++) { 2345 struct irq_data *data = irq_domain_get_irq_data(domain, 2346 virq + i); 2347 u32 event = its_get_event_id(data); 2348 2349 /* Mark interrupt index as unused */ 2350 clear_bit(event, its_dev->event_map.lpi_map); 2351 2352 /* Nuke the entry in the domain */ 2353 irq_domain_reset_irq_data(data); 2354 } 2355 2356 /* If all interrupts have been freed, start mopping the floor */ 2357 if (bitmap_empty(its_dev->event_map.lpi_map, 2358 its_dev->event_map.nr_lpis)) { 2359 its_lpi_free_chunks(its_dev->event_map.lpi_map, 2360 its_dev->event_map.lpi_base, 2361 its_dev->event_map.nr_lpis); 2362 kfree(its_dev->event_map.col_map); 2363 2364 /* Unmap device/itt */ 2365 its_send_mapd(its_dev, 0); 2366 its_free_device(its_dev); 2367 } 2368 2369 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 2370 } 2371 2372 static const struct irq_domain_ops its_domain_ops = { 2373 .alloc = its_irq_domain_alloc, 2374 .free = its_irq_domain_free, 2375 .activate = its_irq_domain_activate, 2376 .deactivate = its_irq_domain_deactivate, 2377 }; 2378 2379 /* 2380 * This is insane. 2381 * 2382 * If a GICv4 doesn't implement Direct LPIs (which is extremely 2383 * likely), the only way to perform an invalidate is to use a fake 2384 * device to issue an INV command, implying that the LPI has first 2385 * been mapped to some event on that device. Since this is not exactly 2386 * cheap, we try to keep that mapping around as long as possible, and 2387 * only issue an UNMAP if we're short on available slots. 2388 * 2389 * Broken by design(tm). 2390 */ 2391 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe) 2392 { 2393 /* Already unmapped? */ 2394 if (vpe->vpe_proxy_event == -1) 2395 return; 2396 2397 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event); 2398 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL; 2399 2400 /* 2401 * We don't track empty slots at all, so let's move the 2402 * next_victim pointer if we can quickly reuse that slot 2403 * instead of nuking an existing entry. Not clear that this is 2404 * always a win though, and this might just generate a ripple 2405 * effect... Let's just hope VPEs don't migrate too often. 2406 */ 2407 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 2408 vpe_proxy.next_victim = vpe->vpe_proxy_event; 2409 2410 vpe->vpe_proxy_event = -1; 2411 } 2412 2413 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe) 2414 { 2415 if (!gic_rdists->has_direct_lpi) { 2416 unsigned long flags; 2417 2418 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 2419 its_vpe_db_proxy_unmap_locked(vpe); 2420 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 2421 } 2422 } 2423 2424 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe) 2425 { 2426 /* Already mapped? */ 2427 if (vpe->vpe_proxy_event != -1) 2428 return; 2429 2430 /* This slot was already allocated. Kick the other VPE out. */ 2431 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 2432 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]); 2433 2434 /* Map the new VPE instead */ 2435 vpe_proxy.vpes[vpe_proxy.next_victim] = vpe; 2436 vpe->vpe_proxy_event = vpe_proxy.next_victim; 2437 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites; 2438 2439 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx; 2440 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event); 2441 } 2442 2443 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to) 2444 { 2445 unsigned long flags; 2446 struct its_collection *target_col; 2447 2448 if (gic_rdists->has_direct_lpi) { 2449 void __iomem *rdbase; 2450 2451 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base; 2452 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 2453 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) 2454 cpu_relax(); 2455 2456 return; 2457 } 2458 2459 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 2460 2461 its_vpe_db_proxy_map_locked(vpe); 2462 2463 target_col = &vpe_proxy.dev->its->collections[to]; 2464 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event); 2465 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to; 2466 2467 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 2468 } 2469 2470 static int its_vpe_set_affinity(struct irq_data *d, 2471 const struct cpumask *mask_val, 2472 bool force) 2473 { 2474 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2475 int cpu = cpumask_first(mask_val); 2476 2477 /* 2478 * Changing affinity is mega expensive, so let's be as lazy as 2479 * we can and only do it if we really have to. Also, if mapped 2480 * into the proxy device, we need to move the doorbell 2481 * interrupt to its new location. 2482 */ 2483 if (vpe->col_idx != cpu) { 2484 int from = vpe->col_idx; 2485 2486 vpe->col_idx = cpu; 2487 its_send_vmovp(vpe); 2488 its_vpe_db_proxy_move(vpe, from, cpu); 2489 } 2490 2491 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 2492 2493 return IRQ_SET_MASK_OK_DONE; 2494 } 2495 2496 static void its_vpe_schedule(struct its_vpe *vpe) 2497 { 2498 void * __iomem vlpi_base = gic_data_rdist_vlpi_base(); 2499 u64 val; 2500 2501 /* Schedule the VPE */ 2502 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) & 2503 GENMASK_ULL(51, 12); 2504 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; 2505 val |= GICR_VPROPBASER_RaWb; 2506 val |= GICR_VPROPBASER_InnerShareable; 2507 gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 2508 2509 val = virt_to_phys(page_address(vpe->vpt_page)) & 2510 GENMASK_ULL(51, 16); 2511 val |= GICR_VPENDBASER_RaWaWb; 2512 val |= GICR_VPENDBASER_NonShareable; 2513 /* 2514 * There is no good way of finding out if the pending table is 2515 * empty as we can race against the doorbell interrupt very 2516 * easily. So in the end, vpe->pending_last is only an 2517 * indication that the vcpu has something pending, not one 2518 * that the pending table is empty. A good implementation 2519 * would be able to read its coarse map pretty quickly anyway, 2520 * making this a tolerable issue. 2521 */ 2522 val |= GICR_VPENDBASER_PendingLast; 2523 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; 2524 val |= GICR_VPENDBASER_Valid; 2525 gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 2526 } 2527 2528 static void its_vpe_deschedule(struct its_vpe *vpe) 2529 { 2530 void * __iomem vlpi_base = gic_data_rdist_vlpi_base(); 2531 u32 count = 1000000; /* 1s! */ 2532 bool clean; 2533 u64 val; 2534 2535 /* We're being scheduled out */ 2536 val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); 2537 val &= ~GICR_VPENDBASER_Valid; 2538 gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 2539 2540 do { 2541 val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); 2542 clean = !(val & GICR_VPENDBASER_Dirty); 2543 if (!clean) { 2544 count--; 2545 cpu_relax(); 2546 udelay(1); 2547 } 2548 } while (!clean && count); 2549 2550 if (unlikely(!clean && !count)) { 2551 pr_err_ratelimited("ITS virtual pending table not cleaning\n"); 2552 vpe->idai = false; 2553 vpe->pending_last = true; 2554 } else { 2555 vpe->idai = !!(val & GICR_VPENDBASER_IDAI); 2556 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); 2557 } 2558 } 2559 2560 static void its_vpe_invall(struct its_vpe *vpe) 2561 { 2562 struct its_node *its; 2563 2564 list_for_each_entry(its, &its_nodes, entry) { 2565 if (!its->is_v4) 2566 continue; 2567 2568 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) 2569 continue; 2570 2571 /* 2572 * Sending a VINVALL to a single ITS is enough, as all 2573 * we need is to reach the redistributors. 2574 */ 2575 its_send_vinvall(its, vpe); 2576 return; 2577 } 2578 } 2579 2580 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 2581 { 2582 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2583 struct its_cmd_info *info = vcpu_info; 2584 2585 switch (info->cmd_type) { 2586 case SCHEDULE_VPE: 2587 its_vpe_schedule(vpe); 2588 return 0; 2589 2590 case DESCHEDULE_VPE: 2591 its_vpe_deschedule(vpe); 2592 return 0; 2593 2594 case INVALL_VPE: 2595 its_vpe_invall(vpe); 2596 return 0; 2597 2598 default: 2599 return -EINVAL; 2600 } 2601 } 2602 2603 static void its_vpe_send_cmd(struct its_vpe *vpe, 2604 void (*cmd)(struct its_device *, u32)) 2605 { 2606 unsigned long flags; 2607 2608 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 2609 2610 its_vpe_db_proxy_map_locked(vpe); 2611 cmd(vpe_proxy.dev, vpe->vpe_proxy_event); 2612 2613 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 2614 } 2615 2616 static void its_vpe_send_inv(struct irq_data *d) 2617 { 2618 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2619 2620 if (gic_rdists->has_direct_lpi) { 2621 void __iomem *rdbase; 2622 2623 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 2624 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR); 2625 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) 2626 cpu_relax(); 2627 } else { 2628 its_vpe_send_cmd(vpe, its_send_inv); 2629 } 2630 } 2631 2632 static void its_vpe_mask_irq(struct irq_data *d) 2633 { 2634 /* 2635 * We need to unmask the LPI, which is described by the parent 2636 * irq_data. Instead of calling into the parent (which won't 2637 * exactly do the right thing, let's simply use the 2638 * parent_data pointer. Yes, I'm naughty. 2639 */ 2640 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); 2641 its_vpe_send_inv(d); 2642 } 2643 2644 static void its_vpe_unmask_irq(struct irq_data *d) 2645 { 2646 /* Same hack as above... */ 2647 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); 2648 its_vpe_send_inv(d); 2649 } 2650 2651 static int its_vpe_set_irqchip_state(struct irq_data *d, 2652 enum irqchip_irq_state which, 2653 bool state) 2654 { 2655 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2656 2657 if (which != IRQCHIP_STATE_PENDING) 2658 return -EINVAL; 2659 2660 if (gic_rdists->has_direct_lpi) { 2661 void __iomem *rdbase; 2662 2663 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 2664 if (state) { 2665 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR); 2666 } else { 2667 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 2668 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) 2669 cpu_relax(); 2670 } 2671 } else { 2672 if (state) 2673 its_vpe_send_cmd(vpe, its_send_int); 2674 else 2675 its_vpe_send_cmd(vpe, its_send_clear); 2676 } 2677 2678 return 0; 2679 } 2680 2681 static struct irq_chip its_vpe_irq_chip = { 2682 .name = "GICv4-vpe", 2683 .irq_mask = its_vpe_mask_irq, 2684 .irq_unmask = its_vpe_unmask_irq, 2685 .irq_eoi = irq_chip_eoi_parent, 2686 .irq_set_affinity = its_vpe_set_affinity, 2687 .irq_set_irqchip_state = its_vpe_set_irqchip_state, 2688 .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity, 2689 }; 2690 2691 static int its_vpe_id_alloc(void) 2692 { 2693 return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL); 2694 } 2695 2696 static void its_vpe_id_free(u16 id) 2697 { 2698 ida_simple_remove(&its_vpeid_ida, id); 2699 } 2700 2701 static int its_vpe_init(struct its_vpe *vpe) 2702 { 2703 struct page *vpt_page; 2704 int vpe_id; 2705 2706 /* Allocate vpe_id */ 2707 vpe_id = its_vpe_id_alloc(); 2708 if (vpe_id < 0) 2709 return vpe_id; 2710 2711 /* Allocate VPT */ 2712 vpt_page = its_allocate_pending_table(GFP_KERNEL); 2713 if (!vpt_page) { 2714 its_vpe_id_free(vpe_id); 2715 return -ENOMEM; 2716 } 2717 2718 if (!its_alloc_vpe_table(vpe_id)) { 2719 its_vpe_id_free(vpe_id); 2720 its_free_pending_table(vpe->vpt_page); 2721 return -ENOMEM; 2722 } 2723 2724 vpe->vpe_id = vpe_id; 2725 vpe->vpt_page = vpt_page; 2726 vpe->vpe_proxy_event = -1; 2727 2728 return 0; 2729 } 2730 2731 static void its_vpe_teardown(struct its_vpe *vpe) 2732 { 2733 its_vpe_db_proxy_unmap(vpe); 2734 its_vpe_id_free(vpe->vpe_id); 2735 its_free_pending_table(vpe->vpt_page); 2736 } 2737 2738 static void its_vpe_irq_domain_free(struct irq_domain *domain, 2739 unsigned int virq, 2740 unsigned int nr_irqs) 2741 { 2742 struct its_vm *vm = domain->host_data; 2743 int i; 2744 2745 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 2746 2747 for (i = 0; i < nr_irqs; i++) { 2748 struct irq_data *data = irq_domain_get_irq_data(domain, 2749 virq + i); 2750 struct its_vpe *vpe = irq_data_get_irq_chip_data(data); 2751 2752 BUG_ON(vm != vpe->its_vm); 2753 2754 clear_bit(data->hwirq, vm->db_bitmap); 2755 its_vpe_teardown(vpe); 2756 irq_domain_reset_irq_data(data); 2757 } 2758 2759 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) { 2760 its_lpi_free_chunks(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); 2761 its_free_prop_table(vm->vprop_page); 2762 } 2763 } 2764 2765 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 2766 unsigned int nr_irqs, void *args) 2767 { 2768 struct its_vm *vm = args; 2769 unsigned long *bitmap; 2770 struct page *vprop_page; 2771 int base, nr_ids, i, err = 0; 2772 2773 BUG_ON(!vm); 2774 2775 bitmap = its_lpi_alloc_chunks(nr_irqs, &base, &nr_ids); 2776 if (!bitmap) 2777 return -ENOMEM; 2778 2779 if (nr_ids < nr_irqs) { 2780 its_lpi_free_chunks(bitmap, base, nr_ids); 2781 return -ENOMEM; 2782 } 2783 2784 vprop_page = its_allocate_prop_table(GFP_KERNEL); 2785 if (!vprop_page) { 2786 its_lpi_free_chunks(bitmap, base, nr_ids); 2787 return -ENOMEM; 2788 } 2789 2790 vm->db_bitmap = bitmap; 2791 vm->db_lpi_base = base; 2792 vm->nr_db_lpis = nr_ids; 2793 vm->vprop_page = vprop_page; 2794 2795 for (i = 0; i < nr_irqs; i++) { 2796 vm->vpes[i]->vpe_db_lpi = base + i; 2797 err = its_vpe_init(vm->vpes[i]); 2798 if (err) 2799 break; 2800 err = its_irq_gic_domain_alloc(domain, virq + i, 2801 vm->vpes[i]->vpe_db_lpi); 2802 if (err) 2803 break; 2804 irq_domain_set_hwirq_and_chip(domain, virq + i, i, 2805 &its_vpe_irq_chip, vm->vpes[i]); 2806 set_bit(i, bitmap); 2807 } 2808 2809 if (err) { 2810 if (i > 0) 2811 its_vpe_irq_domain_free(domain, virq, i - 1); 2812 2813 its_lpi_free_chunks(bitmap, base, nr_ids); 2814 its_free_prop_table(vprop_page); 2815 } 2816 2817 return err; 2818 } 2819 2820 static int its_vpe_irq_domain_activate(struct irq_domain *domain, 2821 struct irq_data *d, bool early) 2822 { 2823 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2824 struct its_node *its; 2825 2826 /* If we use the list map, we issue VMAPP on demand... */ 2827 if (its_list_map) 2828 return 0; 2829 2830 /* Map the VPE to the first possible CPU */ 2831 vpe->col_idx = cpumask_first(cpu_online_mask); 2832 2833 list_for_each_entry(its, &its_nodes, entry) { 2834 if (!its->is_v4) 2835 continue; 2836 2837 its_send_vmapp(its, vpe, true); 2838 its_send_vinvall(its, vpe); 2839 } 2840 2841 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 2842 2843 return 0; 2844 } 2845 2846 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain, 2847 struct irq_data *d) 2848 { 2849 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2850 struct its_node *its; 2851 2852 /* 2853 * If we use the list map, we unmap the VPE once no VLPIs are 2854 * associated with the VM. 2855 */ 2856 if (its_list_map) 2857 return; 2858 2859 list_for_each_entry(its, &its_nodes, entry) { 2860 if (!its->is_v4) 2861 continue; 2862 2863 its_send_vmapp(its, vpe, false); 2864 } 2865 } 2866 2867 static const struct irq_domain_ops its_vpe_domain_ops = { 2868 .alloc = its_vpe_irq_domain_alloc, 2869 .free = its_vpe_irq_domain_free, 2870 .activate = its_vpe_irq_domain_activate, 2871 .deactivate = its_vpe_irq_domain_deactivate, 2872 }; 2873 2874 static int its_force_quiescent(void __iomem *base) 2875 { 2876 u32 count = 1000000; /* 1s */ 2877 u32 val; 2878 2879 val = readl_relaxed(base + GITS_CTLR); 2880 /* 2881 * GIC architecture specification requires the ITS to be both 2882 * disabled and quiescent for writes to GITS_BASER<n> or 2883 * GITS_CBASER to not have UNPREDICTABLE results. 2884 */ 2885 if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE)) 2886 return 0; 2887 2888 /* Disable the generation of all interrupts to this ITS */ 2889 val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe); 2890 writel_relaxed(val, base + GITS_CTLR); 2891 2892 /* Poll GITS_CTLR and wait until ITS becomes quiescent */ 2893 while (1) { 2894 val = readl_relaxed(base + GITS_CTLR); 2895 if (val & GITS_CTLR_QUIESCENT) 2896 return 0; 2897 2898 count--; 2899 if (!count) 2900 return -EBUSY; 2901 2902 cpu_relax(); 2903 udelay(1); 2904 } 2905 } 2906 2907 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data) 2908 { 2909 struct its_node *its = data; 2910 2911 /* erratum 22375: only alloc 8MB table size */ 2912 its->device_ids = 0x14; /* 20 bits, 8MB */ 2913 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; 2914 2915 return true; 2916 } 2917 2918 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data) 2919 { 2920 struct its_node *its = data; 2921 2922 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; 2923 2924 return true; 2925 } 2926 2927 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data) 2928 { 2929 struct its_node *its = data; 2930 2931 /* On QDF2400, the size of the ITE is 16Bytes */ 2932 its->ite_size = 16; 2933 2934 return true; 2935 } 2936 2937 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev) 2938 { 2939 struct its_node *its = its_dev->its; 2940 2941 /* 2942 * The Socionext Synquacer SoC has a so-called 'pre-ITS', 2943 * which maps 32-bit writes targeted at a separate window of 2944 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER 2945 * with device ID taken from bits [device_id_bits + 1:2] of 2946 * the window offset. 2947 */ 2948 return its->pre_its_base + (its_dev->device_id << 2); 2949 } 2950 2951 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data) 2952 { 2953 struct its_node *its = data; 2954 u32 pre_its_window[2]; 2955 u32 ids; 2956 2957 if (!fwnode_property_read_u32_array(its->fwnode_handle, 2958 "socionext,synquacer-pre-its", 2959 pre_its_window, 2960 ARRAY_SIZE(pre_its_window))) { 2961 2962 its->pre_its_base = pre_its_window[0]; 2963 its->get_msi_base = its_irq_get_msi_base_pre_its; 2964 2965 ids = ilog2(pre_its_window[1]) - 2; 2966 if (its->device_ids > ids) 2967 its->device_ids = ids; 2968 2969 /* the pre-ITS breaks isolation, so disable MSI remapping */ 2970 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP; 2971 return true; 2972 } 2973 return false; 2974 } 2975 2976 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data) 2977 { 2978 struct its_node *its = data; 2979 2980 /* 2981 * Hip07 insists on using the wrong address for the VLPI 2982 * page. Trick it into doing the right thing... 2983 */ 2984 its->vlpi_redist_offset = SZ_128K; 2985 return true; 2986 } 2987 2988 static const struct gic_quirk its_quirks[] = { 2989 #ifdef CONFIG_CAVIUM_ERRATUM_22375 2990 { 2991 .desc = "ITS: Cavium errata 22375, 24313", 2992 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 2993 .mask = 0xffff0fff, 2994 .init = its_enable_quirk_cavium_22375, 2995 }, 2996 #endif 2997 #ifdef CONFIG_CAVIUM_ERRATUM_23144 2998 { 2999 .desc = "ITS: Cavium erratum 23144", 3000 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 3001 .mask = 0xffff0fff, 3002 .init = its_enable_quirk_cavium_23144, 3003 }, 3004 #endif 3005 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065 3006 { 3007 .desc = "ITS: QDF2400 erratum 0065", 3008 .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */ 3009 .mask = 0xffffffff, 3010 .init = its_enable_quirk_qdf2400_e0065, 3011 }, 3012 #endif 3013 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS 3014 { 3015 /* 3016 * The Socionext Synquacer SoC incorporates ARM's own GIC-500 3017 * implementation, but with a 'pre-ITS' added that requires 3018 * special handling in software. 3019 */ 3020 .desc = "ITS: Socionext Synquacer pre-ITS", 3021 .iidr = 0x0001143b, 3022 .mask = 0xffffffff, 3023 .init = its_enable_quirk_socionext_synquacer, 3024 }, 3025 #endif 3026 #ifdef CONFIG_HISILICON_ERRATUM_161600802 3027 { 3028 .desc = "ITS: Hip07 erratum 161600802", 3029 .iidr = 0x00000004, 3030 .mask = 0xffffffff, 3031 .init = its_enable_quirk_hip07_161600802, 3032 }, 3033 #endif 3034 { 3035 } 3036 }; 3037 3038 static void its_enable_quirks(struct its_node *its) 3039 { 3040 u32 iidr = readl_relaxed(its->base + GITS_IIDR); 3041 3042 gic_enable_quirks(iidr, its_quirks, its); 3043 } 3044 3045 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) 3046 { 3047 struct irq_domain *inner_domain; 3048 struct msi_domain_info *info; 3049 3050 info = kzalloc(sizeof(*info), GFP_KERNEL); 3051 if (!info) 3052 return -ENOMEM; 3053 3054 inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its); 3055 if (!inner_domain) { 3056 kfree(info); 3057 return -ENOMEM; 3058 } 3059 3060 inner_domain->parent = its_parent; 3061 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); 3062 inner_domain->flags |= its->msi_domain_flags; 3063 info->ops = &its_msi_domain_ops; 3064 info->data = its; 3065 inner_domain->host_data = info; 3066 3067 return 0; 3068 } 3069 3070 static int its_init_vpe_domain(void) 3071 { 3072 struct its_node *its; 3073 u32 devid; 3074 int entries; 3075 3076 if (gic_rdists->has_direct_lpi) { 3077 pr_info("ITS: Using DirectLPI for VPE invalidation\n"); 3078 return 0; 3079 } 3080 3081 /* Any ITS will do, even if not v4 */ 3082 its = list_first_entry(&its_nodes, struct its_node, entry); 3083 3084 entries = roundup_pow_of_two(nr_cpu_ids); 3085 vpe_proxy.vpes = kzalloc(sizeof(*vpe_proxy.vpes) * entries, 3086 GFP_KERNEL); 3087 if (!vpe_proxy.vpes) { 3088 pr_err("ITS: Can't allocate GICv4 proxy device array\n"); 3089 return -ENOMEM; 3090 } 3091 3092 /* Use the last possible DevID */ 3093 devid = GENMASK(its->device_ids - 1, 0); 3094 vpe_proxy.dev = its_create_device(its, devid, entries, false); 3095 if (!vpe_proxy.dev) { 3096 kfree(vpe_proxy.vpes); 3097 pr_err("ITS: Can't allocate GICv4 proxy device\n"); 3098 return -ENOMEM; 3099 } 3100 3101 BUG_ON(entries > vpe_proxy.dev->nr_ites); 3102 3103 raw_spin_lock_init(&vpe_proxy.lock); 3104 vpe_proxy.next_victim = 0; 3105 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n", 3106 devid, vpe_proxy.dev->nr_ites); 3107 3108 return 0; 3109 } 3110 3111 static int __init its_compute_its_list_map(struct resource *res, 3112 void __iomem *its_base) 3113 { 3114 int its_number; 3115 u32 ctlr; 3116 3117 /* 3118 * This is assumed to be done early enough that we're 3119 * guaranteed to be single-threaded, hence no 3120 * locking. Should this change, we should address 3121 * this. 3122 */ 3123 its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX); 3124 if (its_number >= GICv4_ITS_LIST_MAX) { 3125 pr_err("ITS@%pa: No ITSList entry available!\n", 3126 &res->start); 3127 return -EINVAL; 3128 } 3129 3130 ctlr = readl_relaxed(its_base + GITS_CTLR); 3131 ctlr &= ~GITS_CTLR_ITS_NUMBER; 3132 ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT; 3133 writel_relaxed(ctlr, its_base + GITS_CTLR); 3134 ctlr = readl_relaxed(its_base + GITS_CTLR); 3135 if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) { 3136 its_number = ctlr & GITS_CTLR_ITS_NUMBER; 3137 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT; 3138 } 3139 3140 if (test_and_set_bit(its_number, &its_list_map)) { 3141 pr_err("ITS@%pa: Duplicate ITSList entry %d\n", 3142 &res->start, its_number); 3143 return -EINVAL; 3144 } 3145 3146 return its_number; 3147 } 3148 3149 static int __init its_probe_one(struct resource *res, 3150 struct fwnode_handle *handle, int numa_node) 3151 { 3152 struct its_node *its; 3153 void __iomem *its_base; 3154 u32 val, ctlr; 3155 u64 baser, tmp, typer; 3156 int err; 3157 3158 its_base = ioremap(res->start, resource_size(res)); 3159 if (!its_base) { 3160 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start); 3161 return -ENOMEM; 3162 } 3163 3164 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK; 3165 if (val != 0x30 && val != 0x40) { 3166 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start); 3167 err = -ENODEV; 3168 goto out_unmap; 3169 } 3170 3171 err = its_force_quiescent(its_base); 3172 if (err) { 3173 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start); 3174 goto out_unmap; 3175 } 3176 3177 pr_info("ITS %pR\n", res); 3178 3179 its = kzalloc(sizeof(*its), GFP_KERNEL); 3180 if (!its) { 3181 err = -ENOMEM; 3182 goto out_unmap; 3183 } 3184 3185 raw_spin_lock_init(&its->lock); 3186 INIT_LIST_HEAD(&its->entry); 3187 INIT_LIST_HEAD(&its->its_device_list); 3188 typer = gic_read_typer(its_base + GITS_TYPER); 3189 its->base = its_base; 3190 its->phys_base = res->start; 3191 its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer); 3192 its->device_ids = GITS_TYPER_DEVBITS(typer); 3193 its->is_v4 = !!(typer & GITS_TYPER_VLPIS); 3194 if (its->is_v4) { 3195 if (!(typer & GITS_TYPER_VMOVP)) { 3196 err = its_compute_its_list_map(res, its_base); 3197 if (err < 0) 3198 goto out_free_its; 3199 3200 its->list_nr = err; 3201 3202 pr_info("ITS@%pa: Using ITS number %d\n", 3203 &res->start, err); 3204 } else { 3205 pr_info("ITS@%pa: Single VMOVP capable\n", &res->start); 3206 } 3207 } 3208 3209 its->numa_node = numa_node; 3210 3211 its->cmd_base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 3212 get_order(ITS_CMD_QUEUE_SZ)); 3213 if (!its->cmd_base) { 3214 err = -ENOMEM; 3215 goto out_free_its; 3216 } 3217 its->cmd_write = its->cmd_base; 3218 its->fwnode_handle = handle; 3219 its->get_msi_base = its_irq_get_msi_base; 3220 its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP; 3221 3222 its_enable_quirks(its); 3223 3224 err = its_alloc_tables(its); 3225 if (err) 3226 goto out_free_cmd; 3227 3228 err = its_alloc_collections(its); 3229 if (err) 3230 goto out_free_tables; 3231 3232 baser = (virt_to_phys(its->cmd_base) | 3233 GITS_CBASER_RaWaWb | 3234 GITS_CBASER_InnerShareable | 3235 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | 3236 GITS_CBASER_VALID); 3237 3238 gits_write_cbaser(baser, its->base + GITS_CBASER); 3239 tmp = gits_read_cbaser(its->base + GITS_CBASER); 3240 3241 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) { 3242 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) { 3243 /* 3244 * The HW reports non-shareable, we must 3245 * remove the cacheability attributes as 3246 * well. 3247 */ 3248 baser &= ~(GITS_CBASER_SHAREABILITY_MASK | 3249 GITS_CBASER_CACHEABILITY_MASK); 3250 baser |= GITS_CBASER_nC; 3251 gits_write_cbaser(baser, its->base + GITS_CBASER); 3252 } 3253 pr_info("ITS: using cache flushing for cmd queue\n"); 3254 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; 3255 } 3256 3257 gits_write_cwriter(0, its->base + GITS_CWRITER); 3258 ctlr = readl_relaxed(its->base + GITS_CTLR); 3259 ctlr |= GITS_CTLR_ENABLE; 3260 if (its->is_v4) 3261 ctlr |= GITS_CTLR_ImDe; 3262 writel_relaxed(ctlr, its->base + GITS_CTLR); 3263 3264 err = its_init_domain(handle, its); 3265 if (err) 3266 goto out_free_tables; 3267 3268 spin_lock(&its_lock); 3269 list_add(&its->entry, &its_nodes); 3270 spin_unlock(&its_lock); 3271 3272 return 0; 3273 3274 out_free_tables: 3275 its_free_tables(its); 3276 out_free_cmd: 3277 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); 3278 out_free_its: 3279 kfree(its); 3280 out_unmap: 3281 iounmap(its_base); 3282 pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err); 3283 return err; 3284 } 3285 3286 static bool gic_rdists_supports_plpis(void) 3287 { 3288 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS); 3289 } 3290 3291 int its_cpu_init(void) 3292 { 3293 if (!list_empty(&its_nodes)) { 3294 if (!gic_rdists_supports_plpis()) { 3295 pr_info("CPU%d: LPIs not supported\n", smp_processor_id()); 3296 return -ENXIO; 3297 } 3298 its_cpu_init_lpis(); 3299 its_cpu_init_collection(); 3300 } 3301 3302 return 0; 3303 } 3304 3305 static const struct of_device_id its_device_id[] = { 3306 { .compatible = "arm,gic-v3-its", }, 3307 {}, 3308 }; 3309 3310 static int __init its_of_probe(struct device_node *node) 3311 { 3312 struct device_node *np; 3313 struct resource res; 3314 3315 for (np = of_find_matching_node(node, its_device_id); np; 3316 np = of_find_matching_node(np, its_device_id)) { 3317 if (!of_property_read_bool(np, "msi-controller")) { 3318 pr_warn("%pOF: no msi-controller property, ITS ignored\n", 3319 np); 3320 continue; 3321 } 3322 3323 if (of_address_to_resource(np, 0, &res)) { 3324 pr_warn("%pOF: no regs?\n", np); 3325 continue; 3326 } 3327 3328 its_probe_one(&res, &np->fwnode, of_node_to_nid(np)); 3329 } 3330 return 0; 3331 } 3332 3333 #ifdef CONFIG_ACPI 3334 3335 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K) 3336 3337 #ifdef CONFIG_ACPI_NUMA 3338 struct its_srat_map { 3339 /* numa node id */ 3340 u32 numa_node; 3341 /* GIC ITS ID */ 3342 u32 its_id; 3343 }; 3344 3345 static struct its_srat_map *its_srat_maps __initdata; 3346 static int its_in_srat __initdata; 3347 3348 static int __init acpi_get_its_numa_node(u32 its_id) 3349 { 3350 int i; 3351 3352 for (i = 0; i < its_in_srat; i++) { 3353 if (its_id == its_srat_maps[i].its_id) 3354 return its_srat_maps[i].numa_node; 3355 } 3356 return NUMA_NO_NODE; 3357 } 3358 3359 static int __init gic_acpi_match_srat_its(struct acpi_subtable_header *header, 3360 const unsigned long end) 3361 { 3362 return 0; 3363 } 3364 3365 static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header, 3366 const unsigned long end) 3367 { 3368 int node; 3369 struct acpi_srat_gic_its_affinity *its_affinity; 3370 3371 its_affinity = (struct acpi_srat_gic_its_affinity *)header; 3372 if (!its_affinity) 3373 return -EINVAL; 3374 3375 if (its_affinity->header.length < sizeof(*its_affinity)) { 3376 pr_err("SRAT: Invalid header length %d in ITS affinity\n", 3377 its_affinity->header.length); 3378 return -EINVAL; 3379 } 3380 3381 node = acpi_map_pxm_to_node(its_affinity->proximity_domain); 3382 3383 if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { 3384 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node); 3385 return 0; 3386 } 3387 3388 its_srat_maps[its_in_srat].numa_node = node; 3389 its_srat_maps[its_in_srat].its_id = its_affinity->its_id; 3390 its_in_srat++; 3391 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n", 3392 its_affinity->proximity_domain, its_affinity->its_id, node); 3393 3394 return 0; 3395 } 3396 3397 static void __init acpi_table_parse_srat_its(void) 3398 { 3399 int count; 3400 3401 count = acpi_table_parse_entries(ACPI_SIG_SRAT, 3402 sizeof(struct acpi_table_srat), 3403 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 3404 gic_acpi_match_srat_its, 0); 3405 if (count <= 0) 3406 return; 3407 3408 its_srat_maps = kmalloc(count * sizeof(struct its_srat_map), 3409 GFP_KERNEL); 3410 if (!its_srat_maps) { 3411 pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n"); 3412 return; 3413 } 3414 3415 acpi_table_parse_entries(ACPI_SIG_SRAT, 3416 sizeof(struct acpi_table_srat), 3417 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 3418 gic_acpi_parse_srat_its, 0); 3419 } 3420 3421 /* free the its_srat_maps after ITS probing */ 3422 static void __init acpi_its_srat_maps_free(void) 3423 { 3424 kfree(its_srat_maps); 3425 } 3426 #else 3427 static void __init acpi_table_parse_srat_its(void) { } 3428 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; } 3429 static void __init acpi_its_srat_maps_free(void) { } 3430 #endif 3431 3432 static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header, 3433 const unsigned long end) 3434 { 3435 struct acpi_madt_generic_translator *its_entry; 3436 struct fwnode_handle *dom_handle; 3437 struct resource res; 3438 int err; 3439 3440 its_entry = (struct acpi_madt_generic_translator *)header; 3441 memset(&res, 0, sizeof(res)); 3442 res.start = its_entry->base_address; 3443 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1; 3444 res.flags = IORESOURCE_MEM; 3445 3446 dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address); 3447 if (!dom_handle) { 3448 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n", 3449 &res.start); 3450 return -ENOMEM; 3451 } 3452 3453 err = iort_register_domain_token(its_entry->translation_id, dom_handle); 3454 if (err) { 3455 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n", 3456 &res.start, its_entry->translation_id); 3457 goto dom_err; 3458 } 3459 3460 err = its_probe_one(&res, dom_handle, 3461 acpi_get_its_numa_node(its_entry->translation_id)); 3462 if (!err) 3463 return 0; 3464 3465 iort_deregister_domain_token(its_entry->translation_id); 3466 dom_err: 3467 irq_domain_free_fwnode(dom_handle); 3468 return err; 3469 } 3470 3471 static void __init its_acpi_probe(void) 3472 { 3473 acpi_table_parse_srat_its(); 3474 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, 3475 gic_acpi_parse_madt_its, 0); 3476 acpi_its_srat_maps_free(); 3477 } 3478 #else 3479 static void __init its_acpi_probe(void) { } 3480 #endif 3481 3482 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, 3483 struct irq_domain *parent_domain) 3484 { 3485 struct device_node *of_node; 3486 struct its_node *its; 3487 bool has_v4 = false; 3488 int err; 3489 3490 its_parent = parent_domain; 3491 of_node = to_of_node(handle); 3492 if (of_node) 3493 its_of_probe(of_node); 3494 else 3495 its_acpi_probe(); 3496 3497 if (list_empty(&its_nodes)) { 3498 pr_warn("ITS: No ITS available, not enabling LPIs\n"); 3499 return -ENXIO; 3500 } 3501 3502 gic_rdists = rdists; 3503 err = its_alloc_lpi_tables(); 3504 if (err) 3505 return err; 3506 3507 list_for_each_entry(its, &its_nodes, entry) 3508 has_v4 |= its->is_v4; 3509 3510 if (has_v4 & rdists->has_vlpis) { 3511 if (its_init_vpe_domain() || 3512 its_init_v4(parent_domain, &its_vpe_domain_ops)) { 3513 rdists->has_vlpis = false; 3514 pr_err("ITS: Disabling GICv4 support\n"); 3515 } 3516 } 3517 3518 return 0; 3519 } 3520