1 /* 2 * irq_comm.c: Common API for in kernel interrupt controller 3 * Copyright (c) 2007, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 16 * Place - Suite 330, Boston, MA 02111-1307 USA. 17 * Authors: 18 * Yaozu (Eddie) Dong <Eddie.dong@intel.com> 19 * 20 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 21 */ 22 23 #include <linux/kvm_host.h> 24 #include <linux/slab.h> 25 #include <linux/export.h> 26 #include <trace/events/kvm.h> 27 28 #include <asm/msidef.h> 29 30 #include "irq.h" 31 32 #include "ioapic.h" 33 34 #include "lapic.h" 35 36 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, 37 struct kvm *kvm, int irq_source_id, int level, 38 bool line_status) 39 { 40 struct kvm_pic *pic = pic_irqchip(kvm); 41 return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level); 42 } 43 44 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, 45 struct kvm *kvm, int irq_source_id, int level, 46 bool line_status) 47 { 48 struct kvm_ioapic *ioapic = kvm->arch.vioapic; 49 return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level, 50 line_status); 51 } 52 53 int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, 54 struct kvm_lapic_irq *irq, unsigned long *dest_map) 55 { 56 int i, r = -1; 57 struct kvm_vcpu *vcpu, *lowest = NULL; 58 59 if (irq->dest_mode == 0 && irq->dest_id == 0xff && 60 kvm_lowest_prio_delivery(irq)) { 61 printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n"); 62 irq->delivery_mode = APIC_DM_FIXED; 63 } 64 65 if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map)) 66 return r; 67 68 kvm_for_each_vcpu(i, vcpu, kvm) { 69 if (!kvm_apic_present(vcpu)) 70 continue; 71 72 if (!kvm_apic_match_dest(vcpu, src, irq->shorthand, 73 irq->dest_id, irq->dest_mode)) 74 continue; 75 76 if (!kvm_lowest_prio_delivery(irq)) { 77 if (r < 0) 78 r = 0; 79 r += kvm_apic_set_irq(vcpu, irq, dest_map); 80 } else if (kvm_lapic_enabled(vcpu)) { 81 if (!lowest) 82 lowest = vcpu; 83 else if (kvm_apic_compare_prio(vcpu, lowest) < 0) 84 lowest = vcpu; 85 } 86 } 87 88 if (lowest) 89 r = kvm_apic_set_irq(lowest, irq, dest_map); 90 91 return r; 92 } 93 94 void kvm_set_msi_irq(struct kvm_kernel_irq_routing_entry *e, 95 struct kvm_lapic_irq *irq) 96 { 97 trace_kvm_msi_set_irq(e->msi.address_lo, e->msi.data); 98 99 irq->dest_id = (e->msi.address_lo & 100 MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT; 101 irq->vector = (e->msi.data & 102 MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT; 103 irq->dest_mode = (1 << MSI_ADDR_DEST_MODE_SHIFT) & e->msi.address_lo; 104 irq->trig_mode = (1 << MSI_DATA_TRIGGER_SHIFT) & e->msi.data; 105 irq->delivery_mode = e->msi.data & 0x700; 106 irq->msi_redir_hint = ((e->msi.address_lo 107 & MSI_ADDR_REDIRECTION_LOWPRI) > 0); 108 irq->level = 1; 109 irq->shorthand = 0; 110 } 111 EXPORT_SYMBOL_GPL(kvm_set_msi_irq); 112 113 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, 114 struct kvm *kvm, int irq_source_id, int level, bool line_status) 115 { 116 struct kvm_lapic_irq irq; 117 118 if (!level) 119 return -1; 120 121 kvm_set_msi_irq(e, &irq); 122 123 return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL); 124 } 125 126 127 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 128 struct kvm *kvm, int irq_source_id, int level, 129 bool line_status) 130 { 131 struct kvm_lapic_irq irq; 132 int r; 133 134 if (unlikely(e->type != KVM_IRQ_ROUTING_MSI)) 135 return -EWOULDBLOCK; 136 137 kvm_set_msi_irq(e, &irq); 138 139 if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL)) 140 return r; 141 else 142 return -EWOULDBLOCK; 143 } 144 145 int kvm_request_irq_source_id(struct kvm *kvm) 146 { 147 unsigned long *bitmap = &kvm->arch.irq_sources_bitmap; 148 int irq_source_id; 149 150 mutex_lock(&kvm->irq_lock); 151 irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG); 152 153 if (irq_source_id >= BITS_PER_LONG) { 154 printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n"); 155 irq_source_id = -EFAULT; 156 goto unlock; 157 } 158 159 ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); 160 ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); 161 set_bit(irq_source_id, bitmap); 162 unlock: 163 mutex_unlock(&kvm->irq_lock); 164 165 return irq_source_id; 166 } 167 168 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) 169 { 170 ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); 171 ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); 172 173 mutex_lock(&kvm->irq_lock); 174 if (irq_source_id < 0 || 175 irq_source_id >= BITS_PER_LONG) { 176 printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); 177 goto unlock; 178 } 179 clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); 180 if (!ioapic_in_kernel(kvm)) 181 goto unlock; 182 183 kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id); 184 kvm_pic_clear_all(pic_irqchip(kvm), irq_source_id); 185 unlock: 186 mutex_unlock(&kvm->irq_lock); 187 } 188 189 void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, 190 struct kvm_irq_mask_notifier *kimn) 191 { 192 mutex_lock(&kvm->irq_lock); 193 kimn->irq = irq; 194 hlist_add_head_rcu(&kimn->link, &kvm->arch.mask_notifier_list); 195 mutex_unlock(&kvm->irq_lock); 196 } 197 198 void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, 199 struct kvm_irq_mask_notifier *kimn) 200 { 201 mutex_lock(&kvm->irq_lock); 202 hlist_del_rcu(&kimn->link); 203 mutex_unlock(&kvm->irq_lock); 204 synchronize_srcu(&kvm->irq_srcu); 205 } 206 207 void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, 208 bool mask) 209 { 210 struct kvm_irq_mask_notifier *kimn; 211 int idx, gsi; 212 213 idx = srcu_read_lock(&kvm->irq_srcu); 214 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin); 215 if (gsi != -1) 216 hlist_for_each_entry_rcu(kimn, &kvm->arch.mask_notifier_list, link) 217 if (kimn->irq == gsi) 218 kimn->func(kimn, mask); 219 srcu_read_unlock(&kvm->irq_srcu, idx); 220 } 221 222 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e, 223 const struct kvm_irq_routing_entry *ue) 224 { 225 int r = -EINVAL; 226 int delta; 227 unsigned max_pin; 228 229 switch (ue->type) { 230 case KVM_IRQ_ROUTING_IRQCHIP: 231 delta = 0; 232 switch (ue->u.irqchip.irqchip) { 233 case KVM_IRQCHIP_PIC_MASTER: 234 e->set = kvm_set_pic_irq; 235 max_pin = PIC_NUM_PINS; 236 break; 237 case KVM_IRQCHIP_PIC_SLAVE: 238 e->set = kvm_set_pic_irq; 239 max_pin = PIC_NUM_PINS; 240 delta = 8; 241 break; 242 case KVM_IRQCHIP_IOAPIC: 243 max_pin = KVM_IOAPIC_NUM_PINS; 244 e->set = kvm_set_ioapic_irq; 245 break; 246 default: 247 goto out; 248 } 249 e->irqchip.irqchip = ue->u.irqchip.irqchip; 250 e->irqchip.pin = ue->u.irqchip.pin + delta; 251 if (e->irqchip.pin >= max_pin) 252 goto out; 253 break; 254 case KVM_IRQ_ROUTING_MSI: 255 e->set = kvm_set_msi; 256 e->msi.address_lo = ue->u.msi.address_lo; 257 e->msi.address_hi = ue->u.msi.address_hi; 258 e->msi.data = ue->u.msi.data; 259 break; 260 default: 261 goto out; 262 } 263 264 r = 0; 265 out: 266 return r; 267 } 268 269 bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, 270 struct kvm_vcpu **dest_vcpu) 271 { 272 int i, r = 0; 273 struct kvm_vcpu *vcpu; 274 275 if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu)) 276 return true; 277 278 kvm_for_each_vcpu(i, vcpu, kvm) { 279 if (!kvm_apic_present(vcpu)) 280 continue; 281 282 if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand, 283 irq->dest_id, irq->dest_mode)) 284 continue; 285 286 if (++r == 2) 287 return false; 288 289 *dest_vcpu = vcpu; 290 } 291 292 return r == 1; 293 } 294 EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu); 295 296 #define IOAPIC_ROUTING_ENTRY(irq) \ 297 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ 298 .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } } 299 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq) 300 301 #define PIC_ROUTING_ENTRY(irq) \ 302 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ 303 .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } } 304 #define ROUTING_ENTRY2(irq) \ 305 IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq) 306 307 static const struct kvm_irq_routing_entry default_routing[] = { 308 ROUTING_ENTRY2(0), ROUTING_ENTRY2(1), 309 ROUTING_ENTRY2(2), ROUTING_ENTRY2(3), 310 ROUTING_ENTRY2(4), ROUTING_ENTRY2(5), 311 ROUTING_ENTRY2(6), ROUTING_ENTRY2(7), 312 ROUTING_ENTRY2(8), ROUTING_ENTRY2(9), 313 ROUTING_ENTRY2(10), ROUTING_ENTRY2(11), 314 ROUTING_ENTRY2(12), ROUTING_ENTRY2(13), 315 ROUTING_ENTRY2(14), ROUTING_ENTRY2(15), 316 ROUTING_ENTRY1(16), ROUTING_ENTRY1(17), 317 ROUTING_ENTRY1(18), ROUTING_ENTRY1(19), 318 ROUTING_ENTRY1(20), ROUTING_ENTRY1(21), 319 ROUTING_ENTRY1(22), ROUTING_ENTRY1(23), 320 }; 321 322 int kvm_setup_default_irq_routing(struct kvm *kvm) 323 { 324 return kvm_set_irq_routing(kvm, default_routing, 325 ARRAY_SIZE(default_routing), 0); 326 } 327 328 static const struct kvm_irq_routing_entry empty_routing[] = {}; 329 330 int kvm_setup_empty_irq_routing(struct kvm *kvm) 331 { 332 return kvm_set_irq_routing(kvm, empty_routing, 0, 0); 333 } 334 335 void kvm_arch_irq_routing_update(struct kvm *kvm) 336 { 337 if (ioapic_in_kernel(kvm) || !irqchip_in_kernel(kvm)) 338 return; 339 kvm_make_scan_ioapic_request(kvm); 340 } 341 342 void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 343 { 344 struct kvm *kvm = vcpu->kvm; 345 struct kvm_kernel_irq_routing_entry *entry; 346 struct kvm_irq_routing_table *table; 347 u32 i, nr_ioapic_pins; 348 int idx; 349 350 /* kvm->irq_routing must be read after clearing 351 * KVM_SCAN_IOAPIC. */ 352 smp_mb(); 353 idx = srcu_read_lock(&kvm->irq_srcu); 354 table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 355 nr_ioapic_pins = min_t(u32, table->nr_rt_entries, 356 kvm->arch.nr_reserved_ioapic_pins); 357 for (i = 0; i < nr_ioapic_pins; ++i) { 358 hlist_for_each_entry(entry, &table->map[i], link) { 359 u32 dest_id, dest_mode; 360 bool level; 361 362 if (entry->type != KVM_IRQ_ROUTING_MSI) 363 continue; 364 dest_id = (entry->msi.address_lo >> 12) & 0xff; 365 dest_mode = (entry->msi.address_lo >> 2) & 0x1; 366 level = entry->msi.data & MSI_DATA_TRIGGER_LEVEL; 367 if (level && kvm_apic_match_dest(vcpu, NULL, 0, 368 dest_id, dest_mode)) { 369 u32 vector = entry->msi.data & 0xff; 370 371 __set_bit(vector, 372 (unsigned long *) eoi_exit_bitmap); 373 } 374 } 375 } 376 srcu_read_unlock(&kvm->irq_srcu, idx); 377 } 378