1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015, 2016 ARM Ltd. 4 */ 5 6 #include <linux/irqchip/arm-gic.h> 7 #include <linux/kvm.h> 8 #include <linux/kvm_host.h> 9 #include <kvm/arm_vgic.h> 10 #include <asm/kvm_mmu.h> 11 12 #include "vgic.h" 13 14 static inline void vgic_v2_write_lr(int lr, u32 val) 15 { 16 void __iomem *base = kvm_vgic_global_state.vctrl_base; 17 18 writel_relaxed(val, base + GICH_LR0 + (lr * 4)); 19 } 20 21 void vgic_v2_init_lrs(void) 22 { 23 int i; 24 25 for (i = 0; i < kvm_vgic_global_state.nr_lr; i++) 26 vgic_v2_write_lr(i, 0); 27 } 28 29 void vgic_v2_set_underflow(struct kvm_vcpu *vcpu) 30 { 31 struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2; 32 33 cpuif->vgic_hcr |= GICH_HCR_UIE; 34 } 35 36 static bool lr_signals_eoi_mi(u32 lr_val) 37 { 38 return !(lr_val & GICH_LR_STATE) && (lr_val & GICH_LR_EOI) && 39 !(lr_val & GICH_LR_HW); 40 } 41 42 /* 43 * transfer the content of the LRs back into the corresponding ap_list: 44 * - active bit is transferred as is 45 * - pending bit is 46 * - transferred as is in case of edge sensitive IRQs 47 * - set to the line-level (resample time) for level sensitive IRQs 48 */ 49 void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) 50 { 51 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; 52 struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2; 53 int lr; 54 55 DEBUG_SPINLOCK_BUG_ON(!irqs_disabled()); 56 57 cpuif->vgic_hcr &= ~GICH_HCR_UIE; 58 59 for (lr = 0; lr < vgic_cpu->vgic_v2.used_lrs; lr++) { 60 u32 val = cpuif->vgic_lr[lr]; 61 u32 cpuid, intid = val & GICH_LR_VIRTUALID; 62 struct vgic_irq *irq; 63 64 /* Extract the source vCPU id from the LR */ 65 cpuid = val & GICH_LR_PHYSID_CPUID; 66 cpuid >>= GICH_LR_PHYSID_CPUID_SHIFT; 67 cpuid &= 7; 68 69 /* Notify fds when the guest EOI'ed a level-triggered SPI */ 70 if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid)) 71 kvm_notify_acked_irq(vcpu->kvm, 0, 72 intid - VGIC_NR_PRIVATE_IRQS); 73 74 irq = vgic_get_irq(vcpu->kvm, vcpu, intid); 75 76 raw_spin_lock(&irq->irq_lock); 77 78 /* Always preserve the active bit */ 79 irq->active = !!(val & GICH_LR_ACTIVE_BIT); 80 81 if (irq->active && vgic_irq_is_sgi(intid)) 82 irq->active_source = cpuid; 83 84 /* Edge is the only case where we preserve the pending bit */ 85 if (irq->config == VGIC_CONFIG_EDGE && 86 (val & GICH_LR_PENDING_BIT)) { 87 irq->pending_latch = true; 88 89 if (vgic_irq_is_sgi(intid)) 90 irq->source |= (1 << cpuid); 91 } 92 93 /* 94 * Clear soft pending state when level irqs have been acked. 95 */ 96 if (irq->config == VGIC_CONFIG_LEVEL && !(val & GICH_LR_STATE)) 97 irq->pending_latch = false; 98 99 /* 100 * Level-triggered mapped IRQs are special because we only 101 * observe rising edges as input to the VGIC. 102 * 103 * If the guest never acked the interrupt we have to sample 104 * the physical line and set the line level, because the 105 * device state could have changed or we simply need to 106 * process the still pending interrupt later. 107 * 108 * If this causes us to lower the level, we have to also clear 109 * the physical active state, since we will otherwise never be 110 * told when the interrupt becomes asserted again. 111 */ 112 if (vgic_irq_is_mapped_level(irq) && (val & GICH_LR_PENDING_BIT)) { 113 irq->line_level = vgic_get_phys_line_level(irq); 114 115 if (!irq->line_level) 116 vgic_irq_set_phys_active(irq, false); 117 } 118 119 raw_spin_unlock(&irq->irq_lock); 120 vgic_put_irq(vcpu->kvm, irq); 121 } 122 123 cpuif->used_lrs = 0; 124 } 125 126 /* 127 * Populates the particular LR with the state of a given IRQ: 128 * - for an edge sensitive IRQ the pending state is cleared in struct vgic_irq 129 * - for a level sensitive IRQ the pending state value is unchanged; 130 * it is dictated directly by the input level 131 * 132 * If @irq describes an SGI with multiple sources, we choose the 133 * lowest-numbered source VCPU and clear that bit in the source bitmap. 134 * 135 * The irq_lock must be held by the caller. 136 */ 137 void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr) 138 { 139 u32 val = irq->intid; 140 bool allow_pending = true; 141 142 if (irq->active) { 143 val |= GICH_LR_ACTIVE_BIT; 144 if (vgic_irq_is_sgi(irq->intid)) 145 val |= irq->active_source << GICH_LR_PHYSID_CPUID_SHIFT; 146 if (vgic_irq_is_multi_sgi(irq)) { 147 allow_pending = false; 148 val |= GICH_LR_EOI; 149 } 150 } 151 152 if (irq->group) 153 val |= GICH_LR_GROUP1; 154 155 if (irq->hw) { 156 val |= GICH_LR_HW; 157 val |= irq->hwintid << GICH_LR_PHYSID_CPUID_SHIFT; 158 /* 159 * Never set pending+active on a HW interrupt, as the 160 * pending state is kept at the physical distributor 161 * level. 162 */ 163 if (irq->active) 164 allow_pending = false; 165 } else { 166 if (irq->config == VGIC_CONFIG_LEVEL) { 167 val |= GICH_LR_EOI; 168 169 /* 170 * Software resampling doesn't work very well 171 * if we allow P+A, so let's not do that. 172 */ 173 if (irq->active) 174 allow_pending = false; 175 } 176 } 177 178 if (allow_pending && irq_is_pending(irq)) { 179 val |= GICH_LR_PENDING_BIT; 180 181 if (irq->config == VGIC_CONFIG_EDGE) 182 irq->pending_latch = false; 183 184 if (vgic_irq_is_sgi(irq->intid)) { 185 u32 src = ffs(irq->source); 186 187 if (WARN_RATELIMIT(!src, "No SGI source for INTID %d\n", 188 irq->intid)) 189 return; 190 191 val |= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT; 192 irq->source &= ~(1 << (src - 1)); 193 if (irq->source) { 194 irq->pending_latch = true; 195 val |= GICH_LR_EOI; 196 } 197 } 198 } 199 200 /* 201 * Level-triggered mapped IRQs are special because we only observe 202 * rising edges as input to the VGIC. We therefore lower the line 203 * level here, so that we can take new virtual IRQs. See 204 * vgic_v2_fold_lr_state for more info. 205 */ 206 if (vgic_irq_is_mapped_level(irq) && (val & GICH_LR_PENDING_BIT)) 207 irq->line_level = false; 208 209 /* The GICv2 LR only holds five bits of priority. */ 210 val |= (irq->priority >> 3) << GICH_LR_PRIORITY_SHIFT; 211 212 vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = val; 213 } 214 215 void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr) 216 { 217 vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = 0; 218 } 219 220 void vgic_v2_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp) 221 { 222 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 223 u32 vmcr; 224 225 vmcr = (vmcrp->grpen0 << GICH_VMCR_ENABLE_GRP0_SHIFT) & 226 GICH_VMCR_ENABLE_GRP0_MASK; 227 vmcr |= (vmcrp->grpen1 << GICH_VMCR_ENABLE_GRP1_SHIFT) & 228 GICH_VMCR_ENABLE_GRP1_MASK; 229 vmcr |= (vmcrp->ackctl << GICH_VMCR_ACK_CTL_SHIFT) & 230 GICH_VMCR_ACK_CTL_MASK; 231 vmcr |= (vmcrp->fiqen << GICH_VMCR_FIQ_EN_SHIFT) & 232 GICH_VMCR_FIQ_EN_MASK; 233 vmcr |= (vmcrp->cbpr << GICH_VMCR_CBPR_SHIFT) & 234 GICH_VMCR_CBPR_MASK; 235 vmcr |= (vmcrp->eoim << GICH_VMCR_EOI_MODE_SHIFT) & 236 GICH_VMCR_EOI_MODE_MASK; 237 vmcr |= (vmcrp->abpr << GICH_VMCR_ALIAS_BINPOINT_SHIFT) & 238 GICH_VMCR_ALIAS_BINPOINT_MASK; 239 vmcr |= (vmcrp->bpr << GICH_VMCR_BINPOINT_SHIFT) & 240 GICH_VMCR_BINPOINT_MASK; 241 vmcr |= ((vmcrp->pmr >> GICV_PMR_PRIORITY_SHIFT) << 242 GICH_VMCR_PRIMASK_SHIFT) & GICH_VMCR_PRIMASK_MASK; 243 244 cpu_if->vgic_vmcr = vmcr; 245 } 246 247 void vgic_v2_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp) 248 { 249 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 250 u32 vmcr; 251 252 vmcr = cpu_if->vgic_vmcr; 253 254 vmcrp->grpen0 = (vmcr & GICH_VMCR_ENABLE_GRP0_MASK) >> 255 GICH_VMCR_ENABLE_GRP0_SHIFT; 256 vmcrp->grpen1 = (vmcr & GICH_VMCR_ENABLE_GRP1_MASK) >> 257 GICH_VMCR_ENABLE_GRP1_SHIFT; 258 vmcrp->ackctl = (vmcr & GICH_VMCR_ACK_CTL_MASK) >> 259 GICH_VMCR_ACK_CTL_SHIFT; 260 vmcrp->fiqen = (vmcr & GICH_VMCR_FIQ_EN_MASK) >> 261 GICH_VMCR_FIQ_EN_SHIFT; 262 vmcrp->cbpr = (vmcr & GICH_VMCR_CBPR_MASK) >> 263 GICH_VMCR_CBPR_SHIFT; 264 vmcrp->eoim = (vmcr & GICH_VMCR_EOI_MODE_MASK) >> 265 GICH_VMCR_EOI_MODE_SHIFT; 266 267 vmcrp->abpr = (vmcr & GICH_VMCR_ALIAS_BINPOINT_MASK) >> 268 GICH_VMCR_ALIAS_BINPOINT_SHIFT; 269 vmcrp->bpr = (vmcr & GICH_VMCR_BINPOINT_MASK) >> 270 GICH_VMCR_BINPOINT_SHIFT; 271 vmcrp->pmr = ((vmcr & GICH_VMCR_PRIMASK_MASK) >> 272 GICH_VMCR_PRIMASK_SHIFT) << GICV_PMR_PRIORITY_SHIFT; 273 } 274 275 void vgic_v2_enable(struct kvm_vcpu *vcpu) 276 { 277 /* 278 * By forcing VMCR to zero, the GIC will restore the binary 279 * points to their reset values. Anything else resets to zero 280 * anyway. 281 */ 282 vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = 0; 283 284 /* Get the show on the road... */ 285 vcpu->arch.vgic_cpu.vgic_v2.vgic_hcr = GICH_HCR_EN; 286 } 287 288 /* check for overlapping regions and for regions crossing the end of memory */ 289 static bool vgic_v2_check_base(gpa_t dist_base, gpa_t cpu_base) 290 { 291 if (dist_base + KVM_VGIC_V2_DIST_SIZE < dist_base) 292 return false; 293 if (cpu_base + KVM_VGIC_V2_CPU_SIZE < cpu_base) 294 return false; 295 296 if (dist_base + KVM_VGIC_V2_DIST_SIZE <= cpu_base) 297 return true; 298 if (cpu_base + KVM_VGIC_V2_CPU_SIZE <= dist_base) 299 return true; 300 301 return false; 302 } 303 304 int vgic_v2_map_resources(struct kvm *kvm) 305 { 306 struct vgic_dist *dist = &kvm->arch.vgic; 307 int ret = 0; 308 309 if (vgic_ready(kvm)) 310 goto out; 311 312 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) || 313 IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) { 314 kvm_err("Need to set vgic cpu and dist addresses first\n"); 315 ret = -ENXIO; 316 goto out; 317 } 318 319 if (!vgic_v2_check_base(dist->vgic_dist_base, dist->vgic_cpu_base)) { 320 kvm_err("VGIC CPU and dist frames overlap\n"); 321 ret = -EINVAL; 322 goto out; 323 } 324 325 /* 326 * Initialize the vgic if this hasn't already been done on demand by 327 * accessing the vgic state from userspace. 328 */ 329 ret = vgic_init(kvm); 330 if (ret) { 331 kvm_err("Unable to initialize VGIC dynamic data structures\n"); 332 goto out; 333 } 334 335 ret = vgic_register_dist_iodev(kvm, dist->vgic_dist_base, VGIC_V2); 336 if (ret) { 337 kvm_err("Unable to register VGIC MMIO regions\n"); 338 goto out; 339 } 340 341 if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) { 342 ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base, 343 kvm_vgic_global_state.vcpu_base, 344 KVM_VGIC_V2_CPU_SIZE, true); 345 if (ret) { 346 kvm_err("Unable to remap VGIC CPU to VCPU\n"); 347 goto out; 348 } 349 } 350 351 dist->ready = true; 352 353 out: 354 return ret; 355 } 356 357 DEFINE_STATIC_KEY_FALSE(vgic_v2_cpuif_trap); 358 359 /** 360 * vgic_v2_probe - probe for a VGICv2 compatible interrupt controller 361 * @info: pointer to the GIC description 362 * 363 * Returns 0 if the VGICv2 has been probed successfully, returns an error code 364 * otherwise 365 */ 366 int vgic_v2_probe(const struct gic_kvm_info *info) 367 { 368 int ret; 369 u32 vtr; 370 371 if (!info->vctrl.start) { 372 kvm_err("GICH not present in the firmware table\n"); 373 return -ENXIO; 374 } 375 376 if (!PAGE_ALIGNED(info->vcpu.start) || 377 !PAGE_ALIGNED(resource_size(&info->vcpu))) { 378 kvm_info("GICV region size/alignment is unsafe, using trapping (reduced performance)\n"); 379 380 ret = create_hyp_io_mappings(info->vcpu.start, 381 resource_size(&info->vcpu), 382 &kvm_vgic_global_state.vcpu_base_va, 383 &kvm_vgic_global_state.vcpu_hyp_va); 384 if (ret) { 385 kvm_err("Cannot map GICV into hyp\n"); 386 goto out; 387 } 388 389 static_branch_enable(&vgic_v2_cpuif_trap); 390 } 391 392 ret = create_hyp_io_mappings(info->vctrl.start, 393 resource_size(&info->vctrl), 394 &kvm_vgic_global_state.vctrl_base, 395 &kvm_vgic_global_state.vctrl_hyp); 396 if (ret) { 397 kvm_err("Cannot map VCTRL into hyp\n"); 398 goto out; 399 } 400 401 vtr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VTR); 402 kvm_vgic_global_state.nr_lr = (vtr & 0x3f) + 1; 403 404 ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); 405 if (ret) { 406 kvm_err("Cannot register GICv2 KVM device\n"); 407 goto out; 408 } 409 410 kvm_vgic_global_state.can_emulate_gicv2 = true; 411 kvm_vgic_global_state.vcpu_base = info->vcpu.start; 412 kvm_vgic_global_state.type = VGIC_V2; 413 kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS; 414 415 kvm_debug("vgic-v2@%llx\n", info->vctrl.start); 416 417 return 0; 418 out: 419 if (kvm_vgic_global_state.vctrl_base) 420 iounmap(kvm_vgic_global_state.vctrl_base); 421 if (kvm_vgic_global_state.vcpu_base_va) 422 iounmap(kvm_vgic_global_state.vcpu_base_va); 423 424 return ret; 425 } 426 427 static void save_lrs(struct kvm_vcpu *vcpu, void __iomem *base) 428 { 429 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 430 u64 used_lrs = cpu_if->used_lrs; 431 u64 elrsr; 432 int i; 433 434 elrsr = readl_relaxed(base + GICH_ELRSR0); 435 if (unlikely(used_lrs > 32)) 436 elrsr |= ((u64)readl_relaxed(base + GICH_ELRSR1)) << 32; 437 438 for (i = 0; i < used_lrs; i++) { 439 if (elrsr & (1UL << i)) 440 cpu_if->vgic_lr[i] &= ~GICH_LR_STATE; 441 else 442 cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); 443 444 writel_relaxed(0, base + GICH_LR0 + (i * 4)); 445 } 446 } 447 448 void vgic_v2_save_state(struct kvm_vcpu *vcpu) 449 { 450 void __iomem *base = kvm_vgic_global_state.vctrl_base; 451 u64 used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs; 452 453 if (!base) 454 return; 455 456 if (used_lrs) { 457 save_lrs(vcpu, base); 458 writel_relaxed(0, base + GICH_HCR); 459 } 460 } 461 462 void vgic_v2_restore_state(struct kvm_vcpu *vcpu) 463 { 464 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 465 void __iomem *base = kvm_vgic_global_state.vctrl_base; 466 u64 used_lrs = cpu_if->used_lrs; 467 int i; 468 469 if (!base) 470 return; 471 472 if (used_lrs) { 473 writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR); 474 for (i = 0; i < used_lrs; i++) { 475 writel_relaxed(cpu_if->vgic_lr[i], 476 base + GICH_LR0 + (i * 4)); 477 } 478 } 479 } 480 481 void vgic_v2_load(struct kvm_vcpu *vcpu) 482 { 483 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 484 485 writel_relaxed(cpu_if->vgic_vmcr, 486 kvm_vgic_global_state.vctrl_base + GICH_VMCR); 487 writel_relaxed(cpu_if->vgic_apr, 488 kvm_vgic_global_state.vctrl_base + GICH_APR); 489 } 490 491 void vgic_v2_vmcr_sync(struct kvm_vcpu *vcpu) 492 { 493 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 494 495 cpu_if->vgic_vmcr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VMCR); 496 } 497 498 void vgic_v2_put(struct kvm_vcpu *vcpu) 499 { 500 struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; 501 502 vgic_v2_vmcr_sync(vcpu); 503 cpu_if->vgic_apr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_APR); 504 } 505