1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VGICv3 MMIO handling functions 4 */ 5 6 #include <linux/bitfield.h> 7 #include <linux/irqchip/arm-gic-v3.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 #include <linux/interrupt.h> 11 #include <kvm/iodev.h> 12 #include <kvm/arm_vgic.h> 13 14 #include <asm/kvm_emulate.h> 15 #include <asm/kvm_arm.h> 16 #include <asm/kvm_mmu.h> 17 18 #include "vgic.h" 19 #include "vgic-mmio.h" 20 21 /* extract @num bytes at @offset bytes offset in data */ 22 unsigned long extract_bytes(u64 data, unsigned int offset, 23 unsigned int num) 24 { 25 return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0); 26 } 27 28 /* allows updates of any half of a 64-bit register (or the whole thing) */ 29 u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len, 30 unsigned long val) 31 { 32 int lower = (offset & 4) * 8; 33 int upper = lower + 8 * len - 1; 34 35 reg &= ~GENMASK_ULL(upper, lower); 36 val &= GENMASK_ULL(len * 8 - 1, 0); 37 38 return reg | ((u64)val << lower); 39 } 40 41 bool vgic_has_its(struct kvm *kvm) 42 { 43 struct vgic_dist *dist = &kvm->arch.vgic; 44 45 if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) 46 return false; 47 48 return dist->has_its; 49 } 50 51 bool vgic_supports_direct_msis(struct kvm *kvm) 52 { 53 return (kvm_vgic_global_state.has_gicv4_1 || 54 (kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm))); 55 } 56 57 /* 58 * The Revision field in the IIDR have the following meanings: 59 * 60 * Revision 2: Interrupt groups are guest-configurable and signaled using 61 * their configured groups. 62 */ 63 64 static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, 65 gpa_t addr, unsigned int len) 66 { 67 struct vgic_dist *vgic = &vcpu->kvm->arch.vgic; 68 u32 value = 0; 69 70 switch (addr & 0x0c) { 71 case GICD_CTLR: 72 if (vgic->enabled) 73 value |= GICD_CTLR_ENABLE_SS_G1; 74 value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS; 75 if (vgic->nassgireq) 76 value |= GICD_CTLR_nASSGIreq; 77 break; 78 case GICD_TYPER: 79 value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS; 80 value = (value >> 5) - 1; 81 if (vgic_has_its(vcpu->kvm)) { 82 value |= (INTERRUPT_ID_BITS_ITS - 1) << 19; 83 value |= GICD_TYPER_LPIS; 84 } else { 85 value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19; 86 } 87 break; 88 case GICD_TYPER2: 89 if (kvm_vgic_global_state.has_gicv4_1) 90 value = GICD_TYPER2_nASSGIcap; 91 break; 92 case GICD_IIDR: 93 value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) | 94 (vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) | 95 (IMPLEMENTER_ARM << GICD_IIDR_IMPLEMENTER_SHIFT); 96 break; 97 default: 98 return 0; 99 } 100 101 return value; 102 } 103 104 static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, 105 gpa_t addr, unsigned int len, 106 unsigned long val) 107 { 108 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 109 110 switch (addr & 0x0c) { 111 case GICD_CTLR: { 112 bool was_enabled, is_hwsgi; 113 114 mutex_lock(&vcpu->kvm->lock); 115 116 was_enabled = dist->enabled; 117 is_hwsgi = dist->nassgireq; 118 119 dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; 120 121 /* Not a GICv4.1? No HW SGIs */ 122 if (!kvm_vgic_global_state.has_gicv4_1) 123 val &= ~GICD_CTLR_nASSGIreq; 124 125 /* Dist stays enabled? nASSGIreq is RO */ 126 if (was_enabled && dist->enabled) { 127 val &= ~GICD_CTLR_nASSGIreq; 128 val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi); 129 } 130 131 /* Switching HW SGIs? */ 132 dist->nassgireq = val & GICD_CTLR_nASSGIreq; 133 if (is_hwsgi != dist->nassgireq) 134 vgic_v4_configure_vsgis(vcpu->kvm); 135 136 if (kvm_vgic_global_state.has_gicv4_1 && 137 was_enabled != dist->enabled) 138 kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4); 139 else if (!was_enabled && dist->enabled) 140 vgic_kick_vcpus(vcpu->kvm); 141 142 mutex_unlock(&vcpu->kvm->lock); 143 break; 144 } 145 case GICD_TYPER: 146 case GICD_TYPER2: 147 case GICD_IIDR: 148 /* This is at best for documentation purposes... */ 149 return; 150 } 151 } 152 153 static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu, 154 gpa_t addr, unsigned int len, 155 unsigned long val) 156 { 157 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 158 159 switch (addr & 0x0c) { 160 case GICD_TYPER2: 161 case GICD_IIDR: 162 if (val != vgic_mmio_read_v3_misc(vcpu, addr, len)) 163 return -EINVAL; 164 return 0; 165 case GICD_CTLR: 166 /* Not a GICv4.1? No HW SGIs */ 167 if (!kvm_vgic_global_state.has_gicv4_1) 168 val &= ~GICD_CTLR_nASSGIreq; 169 170 dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; 171 dist->nassgireq = val & GICD_CTLR_nASSGIreq; 172 return 0; 173 } 174 175 vgic_mmio_write_v3_misc(vcpu, addr, len, val); 176 return 0; 177 } 178 179 static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu, 180 gpa_t addr, unsigned int len) 181 { 182 int intid = VGIC_ADDR_TO_INTID(addr, 64); 183 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid); 184 unsigned long ret = 0; 185 186 if (!irq) 187 return 0; 188 189 /* The upper word is RAZ for us. */ 190 if (!(addr & 4)) 191 ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len); 192 193 vgic_put_irq(vcpu->kvm, irq); 194 return ret; 195 } 196 197 static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu, 198 gpa_t addr, unsigned int len, 199 unsigned long val) 200 { 201 int intid = VGIC_ADDR_TO_INTID(addr, 64); 202 struct vgic_irq *irq; 203 unsigned long flags; 204 205 /* The upper word is WI for us since we don't implement Aff3. */ 206 if (addr & 4) 207 return; 208 209 irq = vgic_get_irq(vcpu->kvm, NULL, intid); 210 211 if (!irq) 212 return; 213 214 raw_spin_lock_irqsave(&irq->irq_lock, flags); 215 216 /* We only care about and preserve Aff0, Aff1 and Aff2. */ 217 irq->mpidr = val & GENMASK(23, 0); 218 irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr); 219 220 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 221 vgic_put_irq(vcpu->kvm, irq); 222 } 223 224 static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu, 225 gpa_t addr, unsigned int len) 226 { 227 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 228 229 return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0; 230 } 231 232 233 static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, 234 gpa_t addr, unsigned int len, 235 unsigned long val) 236 { 237 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 238 bool was_enabled = vgic_cpu->lpis_enabled; 239 240 if (!vgic_has_its(vcpu->kvm)) 241 return; 242 243 vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS; 244 245 if (was_enabled && !vgic_cpu->lpis_enabled) { 246 vgic_flush_pending_lpis(vcpu); 247 vgic_its_invalidate_cache(vcpu->kvm); 248 } 249 250 if (!was_enabled && vgic_cpu->lpis_enabled) 251 vgic_enable_lpis(vcpu); 252 } 253 254 static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu, 255 gpa_t addr, unsigned int len) 256 { 257 unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu); 258 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 259 struct vgic_redist_region *rdreg = vgic_cpu->rdreg; 260 int target_vcpu_id = vcpu->vcpu_id; 261 gpa_t last_rdist_typer = rdreg->base + GICR_TYPER + 262 (rdreg->free_index - 1) * KVM_VGIC_V3_REDIST_SIZE; 263 u64 value; 264 265 value = (u64)(mpidr & GENMASK(23, 0)) << 32; 266 value |= ((target_vcpu_id & 0xffff) << 8); 267 268 if (addr == last_rdist_typer) 269 value |= GICR_TYPER_LAST; 270 if (vgic_has_its(vcpu->kvm)) 271 value |= GICR_TYPER_PLPIS; 272 273 return extract_bytes(value, addr & 7, len); 274 } 275 276 static unsigned long vgic_uaccess_read_v3r_typer(struct kvm_vcpu *vcpu, 277 gpa_t addr, unsigned int len) 278 { 279 unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu); 280 int target_vcpu_id = vcpu->vcpu_id; 281 u64 value; 282 283 value = (u64)(mpidr & GENMASK(23, 0)) << 32; 284 value |= ((target_vcpu_id & 0xffff) << 8); 285 286 if (vgic_has_its(vcpu->kvm)) 287 value |= GICR_TYPER_PLPIS; 288 289 /* reporting of the Last bit is not supported for userspace */ 290 return extract_bytes(value, addr & 7, len); 291 } 292 293 static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu, 294 gpa_t addr, unsigned int len) 295 { 296 return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); 297 } 298 299 static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu, 300 gpa_t addr, unsigned int len) 301 { 302 switch (addr & 0xffff) { 303 case GICD_PIDR2: 304 /* report a GICv3 compliant implementation */ 305 return 0x3b; 306 } 307 308 return 0; 309 } 310 311 static unsigned long vgic_v3_uaccess_read_pending(struct kvm_vcpu *vcpu, 312 gpa_t addr, unsigned int len) 313 { 314 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 315 u32 value = 0; 316 int i; 317 318 /* 319 * pending state of interrupt is latched in pending_latch variable. 320 * Userspace will save and restore pending state and line_level 321 * separately. 322 * Refer to Documentation/virt/kvm/devices/arm-vgic-v3.rst 323 * for handling of ISPENDR and ICPENDR. 324 */ 325 for (i = 0; i < len * 8; i++) { 326 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 327 bool state = irq->pending_latch; 328 329 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 330 int err; 331 332 err = irq_get_irqchip_state(irq->host_irq, 333 IRQCHIP_STATE_PENDING, 334 &state); 335 WARN_ON(err); 336 } 337 338 if (state) 339 value |= (1U << i); 340 341 vgic_put_irq(vcpu->kvm, irq); 342 } 343 344 return value; 345 } 346 347 static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu, 348 gpa_t addr, unsigned int len, 349 unsigned long val) 350 { 351 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 352 int i; 353 unsigned long flags; 354 355 for (i = 0; i < len * 8; i++) { 356 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 357 358 raw_spin_lock_irqsave(&irq->irq_lock, flags); 359 if (test_bit(i, &val)) { 360 /* 361 * pending_latch is set irrespective of irq type 362 * (level or edge) to avoid dependency that VM should 363 * restore irq config before pending info. 364 */ 365 irq->pending_latch = true; 366 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 367 } else { 368 irq->pending_latch = false; 369 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 370 } 371 372 vgic_put_irq(vcpu->kvm, irq); 373 } 374 375 return 0; 376 } 377 378 /* We want to avoid outer shareable. */ 379 u64 vgic_sanitise_shareability(u64 field) 380 { 381 switch (field) { 382 case GIC_BASER_OuterShareable: 383 return GIC_BASER_InnerShareable; 384 default: 385 return field; 386 } 387 } 388 389 /* Avoid any inner non-cacheable mapping. */ 390 u64 vgic_sanitise_inner_cacheability(u64 field) 391 { 392 switch (field) { 393 case GIC_BASER_CACHE_nCnB: 394 case GIC_BASER_CACHE_nC: 395 return GIC_BASER_CACHE_RaWb; 396 default: 397 return field; 398 } 399 } 400 401 /* Non-cacheable or same-as-inner are OK. */ 402 u64 vgic_sanitise_outer_cacheability(u64 field) 403 { 404 switch (field) { 405 case GIC_BASER_CACHE_SameAsInner: 406 case GIC_BASER_CACHE_nC: 407 return field; 408 default: 409 return GIC_BASER_CACHE_SameAsInner; 410 } 411 } 412 413 u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift, 414 u64 (*sanitise_fn)(u64)) 415 { 416 u64 field = (reg & field_mask) >> field_shift; 417 418 field = sanitise_fn(field) << field_shift; 419 return (reg & ~field_mask) | field; 420 } 421 422 #define PROPBASER_RES0_MASK \ 423 (GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5)) 424 #define PENDBASER_RES0_MASK \ 425 (BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) | \ 426 GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0)) 427 428 static u64 vgic_sanitise_pendbaser(u64 reg) 429 { 430 reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK, 431 GICR_PENDBASER_SHAREABILITY_SHIFT, 432 vgic_sanitise_shareability); 433 reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK, 434 GICR_PENDBASER_INNER_CACHEABILITY_SHIFT, 435 vgic_sanitise_inner_cacheability); 436 reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK, 437 GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT, 438 vgic_sanitise_outer_cacheability); 439 440 reg &= ~PENDBASER_RES0_MASK; 441 442 return reg; 443 } 444 445 static u64 vgic_sanitise_propbaser(u64 reg) 446 { 447 reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK, 448 GICR_PROPBASER_SHAREABILITY_SHIFT, 449 vgic_sanitise_shareability); 450 reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK, 451 GICR_PROPBASER_INNER_CACHEABILITY_SHIFT, 452 vgic_sanitise_inner_cacheability); 453 reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK, 454 GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT, 455 vgic_sanitise_outer_cacheability); 456 457 reg &= ~PROPBASER_RES0_MASK; 458 return reg; 459 } 460 461 static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu, 462 gpa_t addr, unsigned int len) 463 { 464 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 465 466 return extract_bytes(dist->propbaser, addr & 7, len); 467 } 468 469 static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu, 470 gpa_t addr, unsigned int len, 471 unsigned long val) 472 { 473 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; 474 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 475 u64 old_propbaser, propbaser; 476 477 /* Storing a value with LPIs already enabled is undefined */ 478 if (vgic_cpu->lpis_enabled) 479 return; 480 481 do { 482 old_propbaser = READ_ONCE(dist->propbaser); 483 propbaser = old_propbaser; 484 propbaser = update_64bit_reg(propbaser, addr & 4, len, val); 485 propbaser = vgic_sanitise_propbaser(propbaser); 486 } while (cmpxchg64(&dist->propbaser, old_propbaser, 487 propbaser) != old_propbaser); 488 } 489 490 static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu, 491 gpa_t addr, unsigned int len) 492 { 493 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 494 u64 value = vgic_cpu->pendbaser; 495 496 value &= ~GICR_PENDBASER_PTZ; 497 498 return extract_bytes(value, addr & 7, len); 499 } 500 501 static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, 502 gpa_t addr, unsigned int len, 503 unsigned long val) 504 { 505 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 506 u64 old_pendbaser, pendbaser; 507 508 /* Storing a value with LPIs already enabled is undefined */ 509 if (vgic_cpu->lpis_enabled) 510 return; 511 512 do { 513 old_pendbaser = READ_ONCE(vgic_cpu->pendbaser); 514 pendbaser = old_pendbaser; 515 pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val); 516 pendbaser = vgic_sanitise_pendbaser(pendbaser); 517 } while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser, 518 pendbaser) != old_pendbaser); 519 } 520 521 /* 522 * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the 523 * redistributors, while SPIs are covered by registers in the distributor 524 * block. Trying to set private IRQs in this block gets ignored. 525 * We take some special care here to fix the calculation of the register 526 * offset. 527 */ 528 #define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, ur, uw, bpi, acc) \ 529 { \ 530 .reg_offset = off, \ 531 .bits_per_irq = bpi, \ 532 .len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ 533 .access_flags = acc, \ 534 .read = vgic_mmio_read_raz, \ 535 .write = vgic_mmio_write_wi, \ 536 }, { \ 537 .reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ 538 .bits_per_irq = bpi, \ 539 .len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8, \ 540 .access_flags = acc, \ 541 .read = rd, \ 542 .write = wr, \ 543 .uaccess_read = ur, \ 544 .uaccess_write = uw, \ 545 } 546 547 static const struct vgic_register_region vgic_v3_dist_registers[] = { 548 REGISTER_DESC_WITH_LENGTH_UACCESS(GICD_CTLR, 549 vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc, 550 NULL, vgic_mmio_uaccess_write_v3_misc, 551 16, VGIC_ACCESS_32bit), 552 REGISTER_DESC_WITH_LENGTH(GICD_STATUSR, 553 vgic_mmio_read_rao, vgic_mmio_write_wi, 4, 554 VGIC_ACCESS_32bit), 555 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR, 556 vgic_mmio_read_group, vgic_mmio_write_group, NULL, NULL, 1, 557 VGIC_ACCESS_32bit), 558 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER, 559 vgic_mmio_read_enable, vgic_mmio_write_senable, 560 NULL, vgic_uaccess_write_senable, 1, 561 VGIC_ACCESS_32bit), 562 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER, 563 vgic_mmio_read_enable, vgic_mmio_write_cenable, 564 NULL, vgic_uaccess_write_cenable, 1, 565 VGIC_ACCESS_32bit), 566 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR, 567 vgic_mmio_read_pending, vgic_mmio_write_spending, 568 vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 1, 569 VGIC_ACCESS_32bit), 570 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR, 571 vgic_mmio_read_pending, vgic_mmio_write_cpending, 572 vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 1, 573 VGIC_ACCESS_32bit), 574 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER, 575 vgic_mmio_read_active, vgic_mmio_write_sactive, 576 vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 1, 577 VGIC_ACCESS_32bit), 578 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER, 579 vgic_mmio_read_active, vgic_mmio_write_cactive, 580 vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 581 1, VGIC_ACCESS_32bit), 582 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR, 583 vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL, 584 8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), 585 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR, 586 vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 8, 587 VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), 588 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR, 589 vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2, 590 VGIC_ACCESS_32bit), 591 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR, 592 vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1, 593 VGIC_ACCESS_32bit), 594 REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER, 595 vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64, 596 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), 597 REGISTER_DESC_WITH_LENGTH(GICD_IDREGS, 598 vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, 599 VGIC_ACCESS_32bit), 600 }; 601 602 static const struct vgic_register_region vgic_v3_rd_registers[] = { 603 /* RD_base registers */ 604 REGISTER_DESC_WITH_LENGTH(GICR_CTLR, 605 vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4, 606 VGIC_ACCESS_32bit), 607 REGISTER_DESC_WITH_LENGTH(GICR_STATUSR, 608 vgic_mmio_read_raz, vgic_mmio_write_wi, 4, 609 VGIC_ACCESS_32bit), 610 REGISTER_DESC_WITH_LENGTH(GICR_IIDR, 611 vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4, 612 VGIC_ACCESS_32bit), 613 REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_TYPER, 614 vgic_mmio_read_v3r_typer, vgic_mmio_write_wi, 615 vgic_uaccess_read_v3r_typer, vgic_mmio_uaccess_write_wi, 8, 616 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), 617 REGISTER_DESC_WITH_LENGTH(GICR_WAKER, 618 vgic_mmio_read_raz, vgic_mmio_write_wi, 4, 619 VGIC_ACCESS_32bit), 620 REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER, 621 vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8, 622 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), 623 REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER, 624 vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8, 625 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), 626 REGISTER_DESC_WITH_LENGTH(GICR_IDREGS, 627 vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, 628 VGIC_ACCESS_32bit), 629 /* SGI_base registers */ 630 REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0, 631 vgic_mmio_read_group, vgic_mmio_write_group, 4, 632 VGIC_ACCESS_32bit), 633 REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISENABLER0, 634 vgic_mmio_read_enable, vgic_mmio_write_senable, 635 NULL, vgic_uaccess_write_senable, 4, 636 VGIC_ACCESS_32bit), 637 REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICENABLER0, 638 vgic_mmio_read_enable, vgic_mmio_write_cenable, 639 NULL, vgic_uaccess_write_cenable, 4, 640 VGIC_ACCESS_32bit), 641 REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0, 642 vgic_mmio_read_pending, vgic_mmio_write_spending, 643 vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4, 644 VGIC_ACCESS_32bit), 645 REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0, 646 vgic_mmio_read_pending, vgic_mmio_write_cpending, 647 vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4, 648 VGIC_ACCESS_32bit), 649 REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0, 650 vgic_mmio_read_active, vgic_mmio_write_sactive, 651 vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 4, 652 VGIC_ACCESS_32bit), 653 REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0, 654 vgic_mmio_read_active, vgic_mmio_write_cactive, 655 vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 4, 656 VGIC_ACCESS_32bit), 657 REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0, 658 vgic_mmio_read_priority, vgic_mmio_write_priority, 32, 659 VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), 660 REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0, 661 vgic_mmio_read_config, vgic_mmio_write_config, 8, 662 VGIC_ACCESS_32bit), 663 REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0, 664 vgic_mmio_read_raz, vgic_mmio_write_wi, 4, 665 VGIC_ACCESS_32bit), 666 REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR, 667 vgic_mmio_read_raz, vgic_mmio_write_wi, 4, 668 VGIC_ACCESS_32bit), 669 }; 670 671 unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev) 672 { 673 dev->regions = vgic_v3_dist_registers; 674 dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers); 675 676 kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops); 677 678 return SZ_64K; 679 } 680 681 /** 682 * vgic_register_redist_iodev - register a single redist iodev 683 * @vcpu: The VCPU to which the redistributor belongs 684 * 685 * Register a KVM iodev for this VCPU's redistributor using the address 686 * provided. 687 * 688 * Return 0 on success, -ERRNO otherwise. 689 */ 690 int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) 691 { 692 struct kvm *kvm = vcpu->kvm; 693 struct vgic_dist *vgic = &kvm->arch.vgic; 694 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 695 struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev; 696 struct vgic_redist_region *rdreg; 697 gpa_t rd_base; 698 int ret; 699 700 if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr)) 701 return 0; 702 703 /* 704 * We may be creating VCPUs before having set the base address for the 705 * redistributor region, in which case we will come back to this 706 * function for all VCPUs when the base address is set. Just return 707 * without doing any work for now. 708 */ 709 rdreg = vgic_v3_rdist_free_slot(&vgic->rd_regions); 710 if (!rdreg) 711 return 0; 712 713 if (!vgic_v3_check_base(kvm)) 714 return -EINVAL; 715 716 vgic_cpu->rdreg = rdreg; 717 718 rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE; 719 720 kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops); 721 rd_dev->base_addr = rd_base; 722 rd_dev->iodev_type = IODEV_REDIST; 723 rd_dev->regions = vgic_v3_rd_registers; 724 rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers); 725 rd_dev->redist_vcpu = vcpu; 726 727 mutex_lock(&kvm->slots_lock); 728 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base, 729 2 * SZ_64K, &rd_dev->dev); 730 mutex_unlock(&kvm->slots_lock); 731 732 if (ret) 733 return ret; 734 735 rdreg->free_index++; 736 return 0; 737 } 738 739 static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu) 740 { 741 struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev; 742 743 kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev); 744 } 745 746 static int vgic_register_all_redist_iodevs(struct kvm *kvm) 747 { 748 struct kvm_vcpu *vcpu; 749 int c, ret = 0; 750 751 kvm_for_each_vcpu(c, vcpu, kvm) { 752 ret = vgic_register_redist_iodev(vcpu); 753 if (ret) 754 break; 755 } 756 757 if (ret) { 758 /* The current c failed, so we start with the previous one. */ 759 mutex_lock(&kvm->slots_lock); 760 for (c--; c >= 0; c--) { 761 vcpu = kvm_get_vcpu(kvm, c); 762 vgic_unregister_redist_iodev(vcpu); 763 } 764 mutex_unlock(&kvm->slots_lock); 765 } 766 767 return ret; 768 } 769 770 /** 771 * vgic_v3_insert_redist_region - Insert a new redistributor region 772 * 773 * Performs various checks before inserting the rdist region in the list. 774 * Those tests depend on whether the size of the rdist region is known 775 * (ie. count != 0). The list is sorted by rdist region index. 776 * 777 * @kvm: kvm handle 778 * @index: redist region index 779 * @base: base of the new rdist region 780 * @count: number of redistributors the region is made of (0 in the old style 781 * single region, whose size is induced from the number of vcpus) 782 * 783 * Return 0 on success, < 0 otherwise 784 */ 785 static int vgic_v3_insert_redist_region(struct kvm *kvm, uint32_t index, 786 gpa_t base, uint32_t count) 787 { 788 struct vgic_dist *d = &kvm->arch.vgic; 789 struct vgic_redist_region *rdreg; 790 struct list_head *rd_regions = &d->rd_regions; 791 size_t size = count * KVM_VGIC_V3_REDIST_SIZE; 792 int ret; 793 794 /* single rdist region already set ?*/ 795 if (!count && !list_empty(rd_regions)) 796 return -EINVAL; 797 798 /* cross the end of memory ? */ 799 if (base + size < base) 800 return -EINVAL; 801 802 if (list_empty(rd_regions)) { 803 if (index != 0) 804 return -EINVAL; 805 } else { 806 rdreg = list_last_entry(rd_regions, 807 struct vgic_redist_region, list); 808 if (index != rdreg->index + 1) 809 return -EINVAL; 810 811 /* Cannot add an explicitly sized regions after legacy region */ 812 if (!rdreg->count) 813 return -EINVAL; 814 } 815 816 /* 817 * For legacy single-region redistributor regions (!count), 818 * check that the redistributor region does not overlap with the 819 * distributor's address space. 820 */ 821 if (!count && !IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) && 822 vgic_dist_overlap(kvm, base, size)) 823 return -EINVAL; 824 825 /* collision with any other rdist region? */ 826 if (vgic_v3_rdist_overlap(kvm, base, size)) 827 return -EINVAL; 828 829 rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL); 830 if (!rdreg) 831 return -ENOMEM; 832 833 rdreg->base = VGIC_ADDR_UNDEF; 834 835 ret = vgic_check_ioaddr(kvm, &rdreg->base, base, SZ_64K); 836 if (ret) 837 goto free; 838 839 rdreg->base = base; 840 rdreg->count = count; 841 rdreg->free_index = 0; 842 rdreg->index = index; 843 844 list_add_tail(&rdreg->list, rd_regions); 845 return 0; 846 free: 847 kfree(rdreg); 848 return ret; 849 } 850 851 int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) 852 { 853 int ret; 854 855 ret = vgic_v3_insert_redist_region(kvm, index, addr, count); 856 if (ret) 857 return ret; 858 859 /* 860 * Register iodevs for each existing VCPU. Adding more VCPUs 861 * afterwards will register the iodevs when needed. 862 */ 863 ret = vgic_register_all_redist_iodevs(kvm); 864 if (ret) 865 return ret; 866 867 return 0; 868 } 869 870 int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) 871 { 872 const struct vgic_register_region *region; 873 struct vgic_io_device iodev; 874 struct vgic_reg_attr reg_attr; 875 struct kvm_vcpu *vcpu; 876 gpa_t addr; 877 int ret; 878 879 ret = vgic_v3_parse_attr(dev, attr, ®_attr); 880 if (ret) 881 return ret; 882 883 vcpu = reg_attr.vcpu; 884 addr = reg_attr.addr; 885 886 switch (attr->group) { 887 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS: 888 iodev.regions = vgic_v3_dist_registers; 889 iodev.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers); 890 iodev.base_addr = 0; 891 break; 892 case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{ 893 iodev.regions = vgic_v3_rd_registers; 894 iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers); 895 iodev.base_addr = 0; 896 break; 897 } 898 case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: { 899 u64 reg, id; 900 901 id = (attr->attr & KVM_DEV_ARM_VGIC_SYSREG_INSTR_MASK); 902 return vgic_v3_has_cpu_sysregs_attr(vcpu, 0, id, ®); 903 } 904 default: 905 return -ENXIO; 906 } 907 908 /* We only support aligned 32-bit accesses. */ 909 if (addr & 3) 910 return -ENXIO; 911 912 region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32)); 913 if (!region) 914 return -ENXIO; 915 916 return 0; 917 } 918 /* 919 * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI 920 * generation register ICC_SGI1R_EL1) with a given VCPU. 921 * If the VCPU's MPIDR matches, return the level0 affinity, otherwise 922 * return -1. 923 */ 924 static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu) 925 { 926 unsigned long affinity; 927 int level0; 928 929 /* 930 * Split the current VCPU's MPIDR into affinity level 0 and the 931 * rest as this is what we have to compare against. 932 */ 933 affinity = kvm_vcpu_get_mpidr_aff(vcpu); 934 level0 = MPIDR_AFFINITY_LEVEL(affinity, 0); 935 affinity &= ~MPIDR_LEVEL_MASK; 936 937 /* bail out if the upper three levels don't match */ 938 if (sgi_aff != affinity) 939 return -1; 940 941 /* Is this VCPU's bit set in the mask ? */ 942 if (!(sgi_cpu_mask & BIT(level0))) 943 return -1; 944 945 return level0; 946 } 947 948 /* 949 * The ICC_SGI* registers encode the affinity differently from the MPIDR, 950 * so provide a wrapper to use the existing defines to isolate a certain 951 * affinity level. 952 */ 953 #define SGI_AFFINITY_LEVEL(reg, level) \ 954 ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \ 955 >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level)) 956 957 /** 958 * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs 959 * @vcpu: The VCPU requesting a SGI 960 * @reg: The value written into ICC_{ASGI1,SGI0,SGI1}R by that VCPU 961 * @allow_group1: Does the sysreg access allow generation of G1 SGIs 962 * 963 * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register. 964 * This will trap in sys_regs.c and call this function. 965 * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the 966 * target processors as well as a bitmask of 16 Aff0 CPUs. 967 * If the interrupt routing mode bit is not set, we iterate over all VCPUs to 968 * check for matching ones. If this bit is set, we signal all, but not the 969 * calling VCPU. 970 */ 971 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1) 972 { 973 struct kvm *kvm = vcpu->kvm; 974 struct kvm_vcpu *c_vcpu; 975 u16 target_cpus; 976 u64 mpidr; 977 int sgi, c; 978 int vcpu_id = vcpu->vcpu_id; 979 bool broadcast; 980 unsigned long flags; 981 982 sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT; 983 broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); 984 target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT; 985 mpidr = SGI_AFFINITY_LEVEL(reg, 3); 986 mpidr |= SGI_AFFINITY_LEVEL(reg, 2); 987 mpidr |= SGI_AFFINITY_LEVEL(reg, 1); 988 989 /* 990 * We iterate over all VCPUs to find the MPIDRs matching the request. 991 * If we have handled one CPU, we clear its bit to detect early 992 * if we are already finished. This avoids iterating through all 993 * VCPUs when most of the times we just signal a single VCPU. 994 */ 995 kvm_for_each_vcpu(c, c_vcpu, kvm) { 996 struct vgic_irq *irq; 997 998 /* Exit early if we have dealt with all requested CPUs */ 999 if (!broadcast && target_cpus == 0) 1000 break; 1001 1002 /* Don't signal the calling VCPU */ 1003 if (broadcast && c == vcpu_id) 1004 continue; 1005 1006 if (!broadcast) { 1007 int level0; 1008 1009 level0 = match_mpidr(mpidr, target_cpus, c_vcpu); 1010 if (level0 == -1) 1011 continue; 1012 1013 /* remove this matching VCPU from the mask */ 1014 target_cpus &= ~BIT(level0); 1015 } 1016 1017 irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi); 1018 1019 raw_spin_lock_irqsave(&irq->irq_lock, flags); 1020 1021 /* 1022 * An access targeting Group0 SGIs can only generate 1023 * those, while an access targeting Group1 SGIs can 1024 * generate interrupts of either group. 1025 */ 1026 if (!irq->group || allow_group1) { 1027 if (!irq->hw) { 1028 irq->pending_latch = true; 1029 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 1030 } else { 1031 /* HW SGI? Ask the GIC to inject it */ 1032 int err; 1033 err = irq_set_irqchip_state(irq->host_irq, 1034 IRQCHIP_STATE_PENDING, 1035 true); 1036 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); 1037 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 1038 } 1039 } else { 1040 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 1041 } 1042 1043 vgic_put_irq(vcpu->kvm, irq); 1044 } 1045 } 1046 1047 int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write, 1048 int offset, u32 *val) 1049 { 1050 struct vgic_io_device dev = { 1051 .regions = vgic_v3_dist_registers, 1052 .nr_regions = ARRAY_SIZE(vgic_v3_dist_registers), 1053 }; 1054 1055 return vgic_uaccess(vcpu, &dev, is_write, offset, val); 1056 } 1057 1058 int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write, 1059 int offset, u32 *val) 1060 { 1061 struct vgic_io_device rd_dev = { 1062 .regions = vgic_v3_rd_registers, 1063 .nr_regions = ARRAY_SIZE(vgic_v3_rd_registers), 1064 }; 1065 1066 return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val); 1067 } 1068 1069 int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write, 1070 u32 intid, u64 *val) 1071 { 1072 if (intid % 32) 1073 return -EINVAL; 1074 1075 if (is_write) 1076 vgic_write_irq_line_level_info(vcpu, intid, *val); 1077 else 1078 *val = vgic_read_irq_line_level_info(vcpu, intid); 1079 1080 return 0; 1081 } 1082