1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * irq_comm.c: Common API for in kernel interrupt controller 4 * Copyright (c) 2007, Intel Corporation. 5 * 6 * Authors: 7 * Yaozu (Eddie) Dong <Eddie.dong@intel.com> 8 * 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 */ 11 12 #include <linux/kvm_host.h> 13 #include <linux/slab.h> 14 #include <linux/export.h> 15 #include <linux/rculist.h> 16 17 #include <trace/events/kvm.h> 18 19 #include "irq.h" 20 21 #include "ioapic.h" 22 23 #include "lapic.h" 24 25 #include "hyperv.h" 26 #include "x86.h" 27 #include "xen.h" 28 29 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, 30 struct kvm *kvm, int irq_source_id, int level, 31 bool line_status) 32 { 33 struct kvm_pic *pic = kvm->arch.vpic; 34 return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level); 35 } 36 37 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, 38 struct kvm *kvm, int irq_source_id, int level, 39 bool line_status) 40 { 41 struct kvm_ioapic *ioapic = kvm->arch.vioapic; 42 return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level, 43 line_status); 44 } 45 46 int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, 47 struct kvm_lapic_irq *irq, struct dest_map *dest_map) 48 { 49 int r = -1; 50 struct kvm_vcpu *vcpu, *lowest = NULL; 51 unsigned long i, dest_vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 52 unsigned int dest_vcpus = 0; 53 54 if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map)) 55 return r; 56 57 if (irq->dest_mode == APIC_DEST_PHYSICAL && 58 irq->dest_id == 0xff && kvm_lowest_prio_delivery(irq)) { 59 printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n"); 60 irq->delivery_mode = APIC_DM_FIXED; 61 } 62 63 memset(dest_vcpu_bitmap, 0, sizeof(dest_vcpu_bitmap)); 64 65 kvm_for_each_vcpu(i, vcpu, kvm) { 66 if (!kvm_apic_present(vcpu)) 67 continue; 68 69 if (!kvm_apic_match_dest(vcpu, src, irq->shorthand, 70 irq->dest_id, irq->dest_mode)) 71 continue; 72 73 if (!kvm_lowest_prio_delivery(irq)) { 74 if (r < 0) 75 r = 0; 76 r += kvm_apic_set_irq(vcpu, irq, dest_map); 77 } else if (kvm_apic_sw_enabled(vcpu->arch.apic)) { 78 if (!kvm_vector_hashing_enabled()) { 79 if (!lowest) 80 lowest = vcpu; 81 else if (kvm_apic_compare_prio(vcpu, lowest) < 0) 82 lowest = vcpu; 83 } else { 84 __set_bit(i, dest_vcpu_bitmap); 85 dest_vcpus++; 86 } 87 } 88 } 89 90 if (dest_vcpus != 0) { 91 int idx = kvm_vector_to_index(irq->vector, dest_vcpus, 92 dest_vcpu_bitmap, KVM_MAX_VCPUS); 93 94 lowest = kvm_get_vcpu(kvm, idx); 95 } 96 97 if (lowest) 98 r = kvm_apic_set_irq(lowest, irq, dest_map); 99 100 return r; 101 } 102 103 void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 104 struct kvm_lapic_irq *irq) 105 { 106 struct msi_msg msg = { .address_lo = e->msi.address_lo, 107 .address_hi = e->msi.address_hi, 108 .data = e->msi.data }; 109 110 trace_kvm_msi_set_irq(msg.address_lo | (kvm->arch.x2apic_format ? 111 (u64)msg.address_hi << 32 : 0), msg.data); 112 113 irq->dest_id = x86_msi_msg_get_destid(&msg, kvm->arch.x2apic_format); 114 irq->vector = msg.arch_data.vector; 115 irq->dest_mode = kvm_lapic_irq_dest_mode(msg.arch_addr_lo.dest_mode_logical); 116 irq->trig_mode = msg.arch_data.is_level; 117 irq->delivery_mode = msg.arch_data.delivery_mode << 8; 118 irq->msi_redir_hint = msg.arch_addr_lo.redirect_hint; 119 irq->level = 1; 120 irq->shorthand = APIC_DEST_NOSHORT; 121 } 122 EXPORT_SYMBOL_GPL(kvm_set_msi_irq); 123 124 static inline bool kvm_msi_route_invalid(struct kvm *kvm, 125 struct kvm_kernel_irq_routing_entry *e) 126 { 127 return kvm->arch.x2apic_format && (e->msi.address_hi & 0xff); 128 } 129 130 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, 131 struct kvm *kvm, int irq_source_id, int level, bool line_status) 132 { 133 struct kvm_lapic_irq irq; 134 135 if (kvm_msi_route_invalid(kvm, e)) 136 return -EINVAL; 137 138 if (!level) 139 return -1; 140 141 kvm_set_msi_irq(kvm, e, &irq); 142 143 return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL); 144 } 145 146 147 static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e, 148 struct kvm *kvm, int irq_source_id, int level, 149 bool line_status) 150 { 151 if (!level) 152 return -1; 153 154 return kvm_hv_synic_set_irq(kvm, e->hv_sint.vcpu, e->hv_sint.sint); 155 } 156 157 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 158 struct kvm *kvm, int irq_source_id, int level, 159 bool line_status) 160 { 161 struct kvm_lapic_irq irq; 162 int r; 163 164 switch (e->type) { 165 case KVM_IRQ_ROUTING_HV_SINT: 166 return kvm_hv_set_sint(e, kvm, irq_source_id, level, 167 line_status); 168 169 case KVM_IRQ_ROUTING_MSI: 170 if (kvm_msi_route_invalid(kvm, e)) 171 return -EINVAL; 172 173 kvm_set_msi_irq(kvm, e, &irq); 174 175 if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL)) 176 return r; 177 break; 178 179 #ifdef CONFIG_KVM_XEN 180 case KVM_IRQ_ROUTING_XEN_EVTCHN: 181 if (!level) 182 return -1; 183 184 return kvm_xen_set_evtchn_fast(e, kvm); 185 #endif 186 default: 187 break; 188 } 189 190 return -EWOULDBLOCK; 191 } 192 193 int kvm_request_irq_source_id(struct kvm *kvm) 194 { 195 unsigned long *bitmap = &kvm->arch.irq_sources_bitmap; 196 int irq_source_id; 197 198 mutex_lock(&kvm->irq_lock); 199 irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG); 200 201 if (irq_source_id >= BITS_PER_LONG) { 202 printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n"); 203 irq_source_id = -EFAULT; 204 goto unlock; 205 } 206 207 ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); 208 ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); 209 set_bit(irq_source_id, bitmap); 210 unlock: 211 mutex_unlock(&kvm->irq_lock); 212 213 return irq_source_id; 214 } 215 216 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) 217 { 218 ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); 219 ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); 220 221 mutex_lock(&kvm->irq_lock); 222 if (irq_source_id < 0 || 223 irq_source_id >= BITS_PER_LONG) { 224 printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); 225 goto unlock; 226 } 227 clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); 228 if (!irqchip_kernel(kvm)) 229 goto unlock; 230 231 kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id); 232 kvm_pic_clear_all(kvm->arch.vpic, irq_source_id); 233 unlock: 234 mutex_unlock(&kvm->irq_lock); 235 } 236 237 void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, 238 struct kvm_irq_mask_notifier *kimn) 239 { 240 mutex_lock(&kvm->irq_lock); 241 kimn->irq = irq; 242 hlist_add_head_rcu(&kimn->link, &kvm->arch.mask_notifier_list); 243 mutex_unlock(&kvm->irq_lock); 244 } 245 246 void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, 247 struct kvm_irq_mask_notifier *kimn) 248 { 249 mutex_lock(&kvm->irq_lock); 250 hlist_del_rcu(&kimn->link); 251 mutex_unlock(&kvm->irq_lock); 252 synchronize_srcu(&kvm->irq_srcu); 253 } 254 255 void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, 256 bool mask) 257 { 258 struct kvm_irq_mask_notifier *kimn; 259 int idx, gsi; 260 261 idx = srcu_read_lock(&kvm->irq_srcu); 262 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin); 263 if (gsi != -1) 264 hlist_for_each_entry_rcu(kimn, &kvm->arch.mask_notifier_list, link) 265 if (kimn->irq == gsi) 266 kimn->func(kimn, mask); 267 srcu_read_unlock(&kvm->irq_srcu, idx); 268 } 269 270 bool kvm_arch_can_set_irq_routing(struct kvm *kvm) 271 { 272 return irqchip_in_kernel(kvm); 273 } 274 275 int kvm_set_routing_entry(struct kvm *kvm, 276 struct kvm_kernel_irq_routing_entry *e, 277 const struct kvm_irq_routing_entry *ue) 278 { 279 /* We can't check irqchip_in_kernel() here as some callers are 280 * currently initializing the irqchip. Other callers should therefore 281 * check kvm_arch_can_set_irq_routing() before calling this function. 282 */ 283 switch (ue->type) { 284 case KVM_IRQ_ROUTING_IRQCHIP: 285 if (irqchip_split(kvm)) 286 return -EINVAL; 287 e->irqchip.pin = ue->u.irqchip.pin; 288 switch (ue->u.irqchip.irqchip) { 289 case KVM_IRQCHIP_PIC_SLAVE: 290 e->irqchip.pin += PIC_NUM_PINS / 2; 291 fallthrough; 292 case KVM_IRQCHIP_PIC_MASTER: 293 if (ue->u.irqchip.pin >= PIC_NUM_PINS / 2) 294 return -EINVAL; 295 e->set = kvm_set_pic_irq; 296 break; 297 case KVM_IRQCHIP_IOAPIC: 298 if (ue->u.irqchip.pin >= KVM_IOAPIC_NUM_PINS) 299 return -EINVAL; 300 e->set = kvm_set_ioapic_irq; 301 break; 302 default: 303 return -EINVAL; 304 } 305 e->irqchip.irqchip = ue->u.irqchip.irqchip; 306 break; 307 case KVM_IRQ_ROUTING_MSI: 308 e->set = kvm_set_msi; 309 e->msi.address_lo = ue->u.msi.address_lo; 310 e->msi.address_hi = ue->u.msi.address_hi; 311 e->msi.data = ue->u.msi.data; 312 313 if (kvm_msi_route_invalid(kvm, e)) 314 return -EINVAL; 315 break; 316 case KVM_IRQ_ROUTING_HV_SINT: 317 e->set = kvm_hv_set_sint; 318 e->hv_sint.vcpu = ue->u.hv_sint.vcpu; 319 e->hv_sint.sint = ue->u.hv_sint.sint; 320 break; 321 #ifdef CONFIG_KVM_XEN 322 case KVM_IRQ_ROUTING_XEN_EVTCHN: 323 return kvm_xen_setup_evtchn(kvm, e, ue); 324 #endif 325 default: 326 return -EINVAL; 327 } 328 329 return 0; 330 } 331 332 bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, 333 struct kvm_vcpu **dest_vcpu) 334 { 335 int r = 0; 336 unsigned long i; 337 struct kvm_vcpu *vcpu; 338 339 if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu)) 340 return true; 341 342 kvm_for_each_vcpu(i, vcpu, kvm) { 343 if (!kvm_apic_present(vcpu)) 344 continue; 345 346 if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand, 347 irq->dest_id, irq->dest_mode)) 348 continue; 349 350 if (++r == 2) 351 return false; 352 353 *dest_vcpu = vcpu; 354 } 355 356 return r == 1; 357 } 358 EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu); 359 360 #define IOAPIC_ROUTING_ENTRY(irq) \ 361 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ 362 .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } } 363 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq) 364 365 #define PIC_ROUTING_ENTRY(irq) \ 366 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ 367 .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } } 368 #define ROUTING_ENTRY2(irq) \ 369 IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq) 370 371 static const struct kvm_irq_routing_entry default_routing[] = { 372 ROUTING_ENTRY2(0), ROUTING_ENTRY2(1), 373 ROUTING_ENTRY2(2), ROUTING_ENTRY2(3), 374 ROUTING_ENTRY2(4), ROUTING_ENTRY2(5), 375 ROUTING_ENTRY2(6), ROUTING_ENTRY2(7), 376 ROUTING_ENTRY2(8), ROUTING_ENTRY2(9), 377 ROUTING_ENTRY2(10), ROUTING_ENTRY2(11), 378 ROUTING_ENTRY2(12), ROUTING_ENTRY2(13), 379 ROUTING_ENTRY2(14), ROUTING_ENTRY2(15), 380 ROUTING_ENTRY1(16), ROUTING_ENTRY1(17), 381 ROUTING_ENTRY1(18), ROUTING_ENTRY1(19), 382 ROUTING_ENTRY1(20), ROUTING_ENTRY1(21), 383 ROUTING_ENTRY1(22), ROUTING_ENTRY1(23), 384 }; 385 386 int kvm_setup_default_irq_routing(struct kvm *kvm) 387 { 388 return kvm_set_irq_routing(kvm, default_routing, 389 ARRAY_SIZE(default_routing), 0); 390 } 391 392 static const struct kvm_irq_routing_entry empty_routing[] = {}; 393 394 int kvm_setup_empty_irq_routing(struct kvm *kvm) 395 { 396 return kvm_set_irq_routing(kvm, empty_routing, 0, 0); 397 } 398 399 void kvm_arch_post_irq_routing_update(struct kvm *kvm) 400 { 401 if (!irqchip_split(kvm)) 402 return; 403 kvm_make_scan_ioapic_request(kvm); 404 } 405 406 void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, 407 ulong *ioapic_handled_vectors) 408 { 409 struct kvm *kvm = vcpu->kvm; 410 struct kvm_kernel_irq_routing_entry *entry; 411 struct kvm_irq_routing_table *table; 412 u32 i, nr_ioapic_pins; 413 int idx; 414 415 idx = srcu_read_lock(&kvm->irq_srcu); 416 table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 417 nr_ioapic_pins = min_t(u32, table->nr_rt_entries, 418 kvm->arch.nr_reserved_ioapic_pins); 419 for (i = 0; i < nr_ioapic_pins; ++i) { 420 hlist_for_each_entry(entry, &table->map[i], link) { 421 struct kvm_lapic_irq irq; 422 423 if (entry->type != KVM_IRQ_ROUTING_MSI) 424 continue; 425 426 kvm_set_msi_irq(vcpu->kvm, entry, &irq); 427 428 if (irq.trig_mode && 429 kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT, 430 irq.dest_id, irq.dest_mode)) 431 __set_bit(irq.vector, ioapic_handled_vectors); 432 } 433 } 434 srcu_read_unlock(&kvm->irq_srcu, idx); 435 } 436 437 void kvm_arch_irq_routing_update(struct kvm *kvm) 438 { 439 kvm_hv_irq_routing_update(kvm); 440 } 441