1 /* 2 * irq_comm.c: Common API for in kernel interrupt controller 3 * Copyright (c) 2007, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 16 * Place - Suite 330, Boston, MA 02111-1307 USA. 17 * Authors: 18 * Yaozu (Eddie) Dong <Eddie.dong@intel.com> 19 * 20 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 21 */ 22 23 #include <linux/kvm_host.h> 24 #include <linux/slab.h> 25 #include <linux/export.h> 26 #include <trace/events/kvm.h> 27 28 #include <asm/msidef.h> 29 30 #include "irq.h" 31 32 #include "ioapic.h" 33 34 #include "lapic.h" 35 36 #include "hyperv.h" 37 38 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, 39 struct kvm *kvm, int irq_source_id, int level, 40 bool line_status) 41 { 42 struct kvm_pic *pic = pic_irqchip(kvm); 43 return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level); 44 } 45 46 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, 47 struct kvm *kvm, int irq_source_id, int level, 48 bool line_status) 49 { 50 struct kvm_ioapic *ioapic = kvm->arch.vioapic; 51 return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level, 52 line_status); 53 } 54 55 int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, 56 struct kvm_lapic_irq *irq, unsigned long *dest_map) 57 { 58 int i, r = -1; 59 struct kvm_vcpu *vcpu, *lowest = NULL; 60 61 if (irq->dest_mode == 0 && irq->dest_id == 0xff && 62 kvm_lowest_prio_delivery(irq)) { 63 printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n"); 64 irq->delivery_mode = APIC_DM_FIXED; 65 } 66 67 if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map)) 68 return r; 69 70 kvm_for_each_vcpu(i, vcpu, kvm) { 71 if (!kvm_apic_present(vcpu)) 72 continue; 73 74 if (!kvm_apic_match_dest(vcpu, src, irq->shorthand, 75 irq->dest_id, irq->dest_mode)) 76 continue; 77 78 if (!kvm_lowest_prio_delivery(irq)) { 79 if (r < 0) 80 r = 0; 81 r += kvm_apic_set_irq(vcpu, irq, dest_map); 82 } else if (kvm_lapic_enabled(vcpu)) { 83 if (!lowest) 84 lowest = vcpu; 85 else if (kvm_apic_compare_prio(vcpu, lowest) < 0) 86 lowest = vcpu; 87 } 88 } 89 90 if (lowest) 91 r = kvm_apic_set_irq(lowest, irq, dest_map); 92 93 return r; 94 } 95 96 void kvm_set_msi_irq(struct kvm_kernel_irq_routing_entry *e, 97 struct kvm_lapic_irq *irq) 98 { 99 trace_kvm_msi_set_irq(e->msi.address_lo, e->msi.data); 100 101 irq->dest_id = (e->msi.address_lo & 102 MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT; 103 irq->vector = (e->msi.data & 104 MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT; 105 irq->dest_mode = (1 << MSI_ADDR_DEST_MODE_SHIFT) & e->msi.address_lo; 106 irq->trig_mode = (1 << MSI_DATA_TRIGGER_SHIFT) & e->msi.data; 107 irq->delivery_mode = e->msi.data & 0x700; 108 irq->msi_redir_hint = ((e->msi.address_lo 109 & MSI_ADDR_REDIRECTION_LOWPRI) > 0); 110 irq->level = 1; 111 irq->shorthand = 0; 112 } 113 EXPORT_SYMBOL_GPL(kvm_set_msi_irq); 114 115 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, 116 struct kvm *kvm, int irq_source_id, int level, bool line_status) 117 { 118 struct kvm_lapic_irq irq; 119 120 if (!level) 121 return -1; 122 123 kvm_set_msi_irq(e, &irq); 124 125 return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL); 126 } 127 128 129 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 130 struct kvm *kvm, int irq_source_id, int level, 131 bool line_status) 132 { 133 struct kvm_lapic_irq irq; 134 int r; 135 136 if (unlikely(e->type != KVM_IRQ_ROUTING_MSI)) 137 return -EWOULDBLOCK; 138 139 kvm_set_msi_irq(e, &irq); 140 141 if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL)) 142 return r; 143 else 144 return -EWOULDBLOCK; 145 } 146 147 int kvm_request_irq_source_id(struct kvm *kvm) 148 { 149 unsigned long *bitmap = &kvm->arch.irq_sources_bitmap; 150 int irq_source_id; 151 152 mutex_lock(&kvm->irq_lock); 153 irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG); 154 155 if (irq_source_id >= BITS_PER_LONG) { 156 printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n"); 157 irq_source_id = -EFAULT; 158 goto unlock; 159 } 160 161 ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); 162 ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); 163 set_bit(irq_source_id, bitmap); 164 unlock: 165 mutex_unlock(&kvm->irq_lock); 166 167 return irq_source_id; 168 } 169 170 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) 171 { 172 ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); 173 ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); 174 175 mutex_lock(&kvm->irq_lock); 176 if (irq_source_id < 0 || 177 irq_source_id >= BITS_PER_LONG) { 178 printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); 179 goto unlock; 180 } 181 clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); 182 if (!ioapic_in_kernel(kvm)) 183 goto unlock; 184 185 kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id); 186 kvm_pic_clear_all(pic_irqchip(kvm), irq_source_id); 187 unlock: 188 mutex_unlock(&kvm->irq_lock); 189 } 190 191 void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, 192 struct kvm_irq_mask_notifier *kimn) 193 { 194 mutex_lock(&kvm->irq_lock); 195 kimn->irq = irq; 196 hlist_add_head_rcu(&kimn->link, &kvm->arch.mask_notifier_list); 197 mutex_unlock(&kvm->irq_lock); 198 } 199 200 void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, 201 struct kvm_irq_mask_notifier *kimn) 202 { 203 mutex_lock(&kvm->irq_lock); 204 hlist_del_rcu(&kimn->link); 205 mutex_unlock(&kvm->irq_lock); 206 synchronize_srcu(&kvm->irq_srcu); 207 } 208 209 void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, 210 bool mask) 211 { 212 struct kvm_irq_mask_notifier *kimn; 213 int idx, gsi; 214 215 idx = srcu_read_lock(&kvm->irq_srcu); 216 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin); 217 if (gsi != -1) 218 hlist_for_each_entry_rcu(kimn, &kvm->arch.mask_notifier_list, link) 219 if (kimn->irq == gsi) 220 kimn->func(kimn, mask); 221 srcu_read_unlock(&kvm->irq_srcu, idx); 222 } 223 224 static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e, 225 struct kvm *kvm, int irq_source_id, int level, 226 bool line_status) 227 { 228 if (!level) 229 return -1; 230 231 return kvm_hv_synic_set_irq(kvm, e->hv_sint.vcpu, e->hv_sint.sint); 232 } 233 234 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e, 235 const struct kvm_irq_routing_entry *ue) 236 { 237 int r = -EINVAL; 238 int delta; 239 unsigned max_pin; 240 241 switch (ue->type) { 242 case KVM_IRQ_ROUTING_IRQCHIP: 243 delta = 0; 244 switch (ue->u.irqchip.irqchip) { 245 case KVM_IRQCHIP_PIC_MASTER: 246 e->set = kvm_set_pic_irq; 247 max_pin = PIC_NUM_PINS; 248 break; 249 case KVM_IRQCHIP_PIC_SLAVE: 250 e->set = kvm_set_pic_irq; 251 max_pin = PIC_NUM_PINS; 252 delta = 8; 253 break; 254 case KVM_IRQCHIP_IOAPIC: 255 max_pin = KVM_IOAPIC_NUM_PINS; 256 e->set = kvm_set_ioapic_irq; 257 break; 258 default: 259 goto out; 260 } 261 e->irqchip.irqchip = ue->u.irqchip.irqchip; 262 e->irqchip.pin = ue->u.irqchip.pin + delta; 263 if (e->irqchip.pin >= max_pin) 264 goto out; 265 break; 266 case KVM_IRQ_ROUTING_MSI: 267 e->set = kvm_set_msi; 268 e->msi.address_lo = ue->u.msi.address_lo; 269 e->msi.address_hi = ue->u.msi.address_hi; 270 e->msi.data = ue->u.msi.data; 271 break; 272 case KVM_IRQ_ROUTING_HV_SINT: 273 e->set = kvm_hv_set_sint; 274 e->hv_sint.vcpu = ue->u.hv_sint.vcpu; 275 e->hv_sint.sint = ue->u.hv_sint.sint; 276 break; 277 default: 278 goto out; 279 } 280 281 r = 0; 282 out: 283 return r; 284 } 285 286 bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, 287 struct kvm_vcpu **dest_vcpu) 288 { 289 int i, r = 0; 290 struct kvm_vcpu *vcpu; 291 292 if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu)) 293 return true; 294 295 kvm_for_each_vcpu(i, vcpu, kvm) { 296 if (!kvm_apic_present(vcpu)) 297 continue; 298 299 if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand, 300 irq->dest_id, irq->dest_mode)) 301 continue; 302 303 if (++r == 2) 304 return false; 305 306 *dest_vcpu = vcpu; 307 } 308 309 return r == 1; 310 } 311 EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu); 312 313 #define IOAPIC_ROUTING_ENTRY(irq) \ 314 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ 315 .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } } 316 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq) 317 318 #define PIC_ROUTING_ENTRY(irq) \ 319 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ 320 .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } } 321 #define ROUTING_ENTRY2(irq) \ 322 IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq) 323 324 static const struct kvm_irq_routing_entry default_routing[] = { 325 ROUTING_ENTRY2(0), ROUTING_ENTRY2(1), 326 ROUTING_ENTRY2(2), ROUTING_ENTRY2(3), 327 ROUTING_ENTRY2(4), ROUTING_ENTRY2(5), 328 ROUTING_ENTRY2(6), ROUTING_ENTRY2(7), 329 ROUTING_ENTRY2(8), ROUTING_ENTRY2(9), 330 ROUTING_ENTRY2(10), ROUTING_ENTRY2(11), 331 ROUTING_ENTRY2(12), ROUTING_ENTRY2(13), 332 ROUTING_ENTRY2(14), ROUTING_ENTRY2(15), 333 ROUTING_ENTRY1(16), ROUTING_ENTRY1(17), 334 ROUTING_ENTRY1(18), ROUTING_ENTRY1(19), 335 ROUTING_ENTRY1(20), ROUTING_ENTRY1(21), 336 ROUTING_ENTRY1(22), ROUTING_ENTRY1(23), 337 }; 338 339 int kvm_setup_default_irq_routing(struct kvm *kvm) 340 { 341 return kvm_set_irq_routing(kvm, default_routing, 342 ARRAY_SIZE(default_routing), 0); 343 } 344 345 static const struct kvm_irq_routing_entry empty_routing[] = {}; 346 347 int kvm_setup_empty_irq_routing(struct kvm *kvm) 348 { 349 return kvm_set_irq_routing(kvm, empty_routing, 0, 0); 350 } 351 352 void kvm_arch_post_irq_routing_update(struct kvm *kvm) 353 { 354 if (ioapic_in_kernel(kvm) || !irqchip_in_kernel(kvm)) 355 return; 356 kvm_make_scan_ioapic_request(kvm); 357 } 358 359 void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, 360 ulong *ioapic_handled_vectors) 361 { 362 struct kvm *kvm = vcpu->kvm; 363 struct kvm_kernel_irq_routing_entry *entry; 364 struct kvm_irq_routing_table *table; 365 u32 i, nr_ioapic_pins; 366 int idx; 367 368 /* kvm->irq_routing must be read after clearing 369 * KVM_SCAN_IOAPIC. */ 370 smp_mb(); 371 idx = srcu_read_lock(&kvm->irq_srcu); 372 table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 373 nr_ioapic_pins = min_t(u32, table->nr_rt_entries, 374 kvm->arch.nr_reserved_ioapic_pins); 375 for (i = 0; i < nr_ioapic_pins; ++i) { 376 hlist_for_each_entry(entry, &table->map[i], link) { 377 u32 dest_id, dest_mode; 378 bool level; 379 380 if (entry->type != KVM_IRQ_ROUTING_MSI) 381 continue; 382 dest_id = (entry->msi.address_lo >> 12) & 0xff; 383 dest_mode = (entry->msi.address_lo >> 2) & 0x1; 384 level = entry->msi.data & MSI_DATA_TRIGGER_LEVEL; 385 if (level && kvm_apic_match_dest(vcpu, NULL, 0, 386 dest_id, dest_mode)) { 387 u32 vector = entry->msi.data & 0xff; 388 389 __set_bit(vector, 390 ioapic_handled_vectors); 391 } 392 } 393 } 394 srcu_read_unlock(&kvm->irq_srcu, idx); 395 } 396 397 int kvm_arch_set_irq(struct kvm_kernel_irq_routing_entry *irq, struct kvm *kvm, 398 int irq_source_id, int level, bool line_status) 399 { 400 switch (irq->type) { 401 case KVM_IRQ_ROUTING_HV_SINT: 402 return kvm_hv_set_sint(irq, kvm, irq_source_id, level, 403 line_status); 404 default: 405 return -EWOULDBLOCK; 406 } 407 } 408 409 void kvm_arch_irq_routing_update(struct kvm *kvm) 410 { 411 kvm_hv_irq_routing_update(kvm); 412 } 413