1 /* 2 * handling kvm guest interrupts 3 * 4 * Copyright IBM Corp. 2008, 2015 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 */ 12 13 #include <linux/interrupt.h> 14 #include <linux/kvm_host.h> 15 #include <linux/hrtimer.h> 16 #include <linux/mmu_context.h> 17 #include <linux/signal.h> 18 #include <linux/slab.h> 19 #include <linux/bitmap.h> 20 #include <linux/vmalloc.h> 21 #include <asm/asm-offsets.h> 22 #include <asm/dis.h> 23 #include <linux/uaccess.h> 24 #include <asm/sclp.h> 25 #include <asm/isc.h> 26 #include <asm/gmap.h> 27 #include <asm/switch_to.h> 28 #include <asm/nmi.h> 29 #include "kvm-s390.h" 30 #include "gaccess.h" 31 #include "trace-s390.h" 32 33 #define PFAULT_INIT 0x0600 34 #define PFAULT_DONE 0x0680 35 #define VIRTIO_PARAM 0x0d00 36 37 /* handle external calls via sigp interpretation facility */ 38 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) 39 { 40 int c, scn; 41 42 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND)) 43 return 0; 44 45 BUG_ON(!kvm_s390_use_sca_entries()); 46 read_lock(&vcpu->kvm->arch.sca_lock); 47 if (vcpu->kvm->arch.use_esca) { 48 struct esca_block *sca = vcpu->kvm->arch.sca; 49 union esca_sigp_ctrl sigp_ctrl = 50 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 51 52 c = sigp_ctrl.c; 53 scn = sigp_ctrl.scn; 54 } else { 55 struct bsca_block *sca = vcpu->kvm->arch.sca; 56 union bsca_sigp_ctrl sigp_ctrl = 57 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 58 59 c = sigp_ctrl.c; 60 scn = sigp_ctrl.scn; 61 } 62 read_unlock(&vcpu->kvm->arch.sca_lock); 63 64 if (src_id) 65 *src_id = scn; 66 67 return c; 68 } 69 70 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) 71 { 72 int expect, rc; 73 74 BUG_ON(!kvm_s390_use_sca_entries()); 75 read_lock(&vcpu->kvm->arch.sca_lock); 76 if (vcpu->kvm->arch.use_esca) { 77 struct esca_block *sca = vcpu->kvm->arch.sca; 78 union esca_sigp_ctrl *sigp_ctrl = 79 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 80 union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 81 82 new_val.scn = src_id; 83 new_val.c = 1; 84 old_val.c = 0; 85 86 expect = old_val.value; 87 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 88 } else { 89 struct bsca_block *sca = vcpu->kvm->arch.sca; 90 union bsca_sigp_ctrl *sigp_ctrl = 91 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 92 union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 93 94 new_val.scn = src_id; 95 new_val.c = 1; 96 old_val.c = 0; 97 98 expect = old_val.value; 99 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 100 } 101 read_unlock(&vcpu->kvm->arch.sca_lock); 102 103 if (rc != expect) { 104 /* another external call is pending */ 105 return -EBUSY; 106 } 107 atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); 108 return 0; 109 } 110 111 static void sca_clear_ext_call(struct kvm_vcpu *vcpu) 112 { 113 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 114 int rc, expect; 115 116 if (!kvm_s390_use_sca_entries()) 117 return; 118 atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags); 119 read_lock(&vcpu->kvm->arch.sca_lock); 120 if (vcpu->kvm->arch.use_esca) { 121 struct esca_block *sca = vcpu->kvm->arch.sca; 122 union esca_sigp_ctrl *sigp_ctrl = 123 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 124 union esca_sigp_ctrl old = *sigp_ctrl; 125 126 expect = old.value; 127 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 128 } else { 129 struct bsca_block *sca = vcpu->kvm->arch.sca; 130 union bsca_sigp_ctrl *sigp_ctrl = 131 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 132 union bsca_sigp_ctrl old = *sigp_ctrl; 133 134 expect = old.value; 135 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 136 } 137 read_unlock(&vcpu->kvm->arch.sca_lock); 138 WARN_ON(rc != expect); /* cannot clear? */ 139 } 140 141 int psw_extint_disabled(struct kvm_vcpu *vcpu) 142 { 143 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 144 } 145 146 static int psw_ioint_disabled(struct kvm_vcpu *vcpu) 147 { 148 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO); 149 } 150 151 static int psw_mchk_disabled(struct kvm_vcpu *vcpu) 152 { 153 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK); 154 } 155 156 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu) 157 { 158 return psw_extint_disabled(vcpu) && 159 psw_ioint_disabled(vcpu) && 160 psw_mchk_disabled(vcpu); 161 } 162 163 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu) 164 { 165 if (psw_extint_disabled(vcpu) || 166 !(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 167 return 0; 168 if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu)) 169 /* No timer interrupts when single stepping */ 170 return 0; 171 return 1; 172 } 173 174 static int ckc_irq_pending(struct kvm_vcpu *vcpu) 175 { 176 if (vcpu->arch.sie_block->ckc >= kvm_s390_get_tod_clock_fast(vcpu->kvm)) 177 return 0; 178 return ckc_interrupts_enabled(vcpu); 179 } 180 181 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu) 182 { 183 return !psw_extint_disabled(vcpu) && 184 (vcpu->arch.sie_block->gcr[0] & 0x400ul); 185 } 186 187 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu) 188 { 189 if (!cpu_timer_interrupts_enabled(vcpu)) 190 return 0; 191 return kvm_s390_get_cpu_timer(vcpu) >> 63; 192 } 193 194 static inline int is_ioirq(unsigned long irq_type) 195 { 196 return ((irq_type >= IRQ_PEND_IO_ISC_0) && 197 (irq_type <= IRQ_PEND_IO_ISC_7)); 198 } 199 200 static uint64_t isc_to_isc_bits(int isc) 201 { 202 return (0x80 >> isc) << 24; 203 } 204 205 static inline u8 int_word_to_isc(u32 int_word) 206 { 207 return (int_word & 0x38000000) >> 27; 208 } 209 210 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) 211 { 212 return vcpu->kvm->arch.float_int.pending_irqs | 213 vcpu->arch.local_int.pending_irqs; 214 } 215 216 static unsigned long disable_iscs(struct kvm_vcpu *vcpu, 217 unsigned long active_mask) 218 { 219 int i; 220 221 for (i = 0; i <= MAX_ISC; i++) 222 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i))) 223 active_mask &= ~(1UL << (IRQ_PEND_IO_ISC_0 + i)); 224 225 return active_mask; 226 } 227 228 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) 229 { 230 unsigned long active_mask; 231 232 active_mask = pending_irqs(vcpu); 233 if (!active_mask) 234 return 0; 235 236 if (psw_extint_disabled(vcpu)) 237 active_mask &= ~IRQ_PEND_EXT_MASK; 238 if (psw_ioint_disabled(vcpu)) 239 active_mask &= ~IRQ_PEND_IO_MASK; 240 else 241 active_mask = disable_iscs(vcpu, active_mask); 242 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 243 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 244 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul)) 245 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask); 246 if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 247 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 248 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul)) 249 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 250 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 251 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask); 252 if (psw_mchk_disabled(vcpu)) 253 active_mask &= ~IRQ_PEND_MCHK_MASK; 254 if (!(vcpu->arch.sie_block->gcr[14] & 255 vcpu->kvm->arch.float_int.mchk.cr14)) 256 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask); 257 258 /* 259 * STOP irqs will never be actively delivered. They are triggered via 260 * intercept requests and cleared when the stop intercept is performed. 261 */ 262 __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask); 263 264 return active_mask; 265 } 266 267 static void __set_cpu_idle(struct kvm_vcpu *vcpu) 268 { 269 atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 270 set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 271 } 272 273 static void __unset_cpu_idle(struct kvm_vcpu *vcpu) 274 { 275 atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 276 clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 277 } 278 279 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) 280 { 281 atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, 282 &vcpu->arch.sie_block->cpuflags); 283 vcpu->arch.sie_block->lctl = 0x0000; 284 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); 285 286 if (guestdbg_enabled(vcpu)) { 287 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 | 288 LCTL_CR10 | LCTL_CR11); 289 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT); 290 } 291 } 292 293 static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag) 294 { 295 atomic_or(flag, &vcpu->arch.sie_block->cpuflags); 296 } 297 298 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) 299 { 300 if (!(pending_irqs(vcpu) & IRQ_PEND_IO_MASK)) 301 return; 302 else if (psw_ioint_disabled(vcpu)) 303 __set_cpuflag(vcpu, CPUSTAT_IO_INT); 304 else 305 vcpu->arch.sie_block->lctl |= LCTL_CR6; 306 } 307 308 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 309 { 310 if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK)) 311 return; 312 if (psw_extint_disabled(vcpu)) 313 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 314 else 315 vcpu->arch.sie_block->lctl |= LCTL_CR0; 316 } 317 318 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu) 319 { 320 if (!(pending_irqs(vcpu) & IRQ_PEND_MCHK_MASK)) 321 return; 322 if (psw_mchk_disabled(vcpu)) 323 vcpu->arch.sie_block->ictl |= ICTL_LPSW; 324 else 325 vcpu->arch.sie_block->lctl |= LCTL_CR14; 326 } 327 328 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu) 329 { 330 if (kvm_s390_is_stop_irq_pending(vcpu)) 331 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 332 } 333 334 /* Set interception request for non-deliverable interrupts */ 335 static void set_intercept_indicators(struct kvm_vcpu *vcpu) 336 { 337 set_intercept_indicators_io(vcpu); 338 set_intercept_indicators_ext(vcpu); 339 set_intercept_indicators_mchk(vcpu); 340 set_intercept_indicators_stop(vcpu); 341 } 342 343 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) 344 { 345 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 346 int rc; 347 348 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 349 0, 0); 350 351 rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, 352 (u16 *)__LC_EXT_INT_CODE); 353 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 354 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 355 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 356 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 357 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 358 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 359 return rc ? -EFAULT : 0; 360 } 361 362 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu) 363 { 364 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 365 int rc; 366 367 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 368 0, 0); 369 370 rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, 371 (u16 __user *)__LC_EXT_INT_CODE); 372 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 373 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 374 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 375 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 376 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 377 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 378 return rc ? -EFAULT : 0; 379 } 380 381 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu) 382 { 383 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 384 struct kvm_s390_ext_info ext; 385 int rc; 386 387 spin_lock(&li->lock); 388 ext = li->irq.ext; 389 clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 390 li->irq.ext.ext_params2 = 0; 391 spin_unlock(&li->lock); 392 393 VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx", 394 ext.ext_params2); 395 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 396 KVM_S390_INT_PFAULT_INIT, 397 0, ext.ext_params2); 398 399 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE); 400 rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR); 401 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 402 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 403 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 404 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 405 rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2); 406 return rc ? -EFAULT : 0; 407 } 408 409 static int __write_machine_check(struct kvm_vcpu *vcpu, 410 struct kvm_s390_mchk_info *mchk) 411 { 412 unsigned long ext_sa_addr; 413 unsigned long lc; 414 freg_t fprs[NUM_FPRS]; 415 union mci mci; 416 int rc; 417 418 mci.val = mchk->mcic; 419 /* take care of lazy register loading */ 420 save_fpu_regs(); 421 save_access_regs(vcpu->run->s.regs.acrs); 422 if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) 423 save_gs_cb(current->thread.gs_cb); 424 425 /* Extended save area */ 426 rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr, 427 sizeof(unsigned long)); 428 /* Only bits 0 through 63-LC are used for address formation */ 429 lc = ext_sa_addr & MCESA_LC_MASK; 430 if (test_kvm_facility(vcpu->kvm, 133)) { 431 switch (lc) { 432 case 0: 433 case 10: 434 ext_sa_addr &= ~0x3ffUL; 435 break; 436 case 11: 437 ext_sa_addr &= ~0x7ffUL; 438 break; 439 case 12: 440 ext_sa_addr &= ~0xfffUL; 441 break; 442 default: 443 ext_sa_addr = 0; 444 break; 445 } 446 } else { 447 ext_sa_addr &= ~0x3ffUL; 448 } 449 450 if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) { 451 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs, 452 512)) 453 mci.vr = 0; 454 } else { 455 mci.vr = 0; 456 } 457 if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133) 458 && (lc == 11 || lc == 12)) { 459 if (write_guest_abs(vcpu, ext_sa_addr + 1024, 460 &vcpu->run->s.regs.gscb, 32)) 461 mci.gs = 0; 462 } else { 463 mci.gs = 0; 464 } 465 466 /* General interruption information */ 467 rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID); 468 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 469 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 470 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 471 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 472 rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE); 473 474 /* Register-save areas */ 475 if (MACHINE_HAS_VX) { 476 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 477 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128); 478 } else { 479 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, 480 vcpu->run->s.regs.fprs, 128); 481 } 482 rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, 483 vcpu->run->s.regs.gprs, 128); 484 rc |= put_guest_lc(vcpu, current->thread.fpu.fpc, 485 (u32 __user *) __LC_FP_CREG_SAVE_AREA); 486 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, 487 (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); 488 rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu), 489 (u64 __user *) __LC_CPU_TIMER_SAVE_AREA); 490 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8, 491 (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA); 492 rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA, 493 &vcpu->run->s.regs.acrs, 64); 494 rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA, 495 &vcpu->arch.sie_block->gcr, 128); 496 497 /* Extended interruption information */ 498 rc |= put_guest_lc(vcpu, mchk->ext_damage_code, 499 (u32 __user *) __LC_EXT_DAMAGE_CODE); 500 rc |= put_guest_lc(vcpu, mchk->failing_storage_address, 501 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 502 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout, 503 sizeof(mchk->fixed_logout)); 504 return rc ? -EFAULT : 0; 505 } 506 507 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 508 { 509 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 510 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 511 struct kvm_s390_mchk_info mchk = {}; 512 int deliver = 0; 513 int rc = 0; 514 515 spin_lock(&fi->lock); 516 spin_lock(&li->lock); 517 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || 518 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { 519 /* 520 * If there was an exigent machine check pending, then any 521 * repressible machine checks that might have been pending 522 * are indicated along with it, so always clear bits for 523 * repressible and exigent interrupts 524 */ 525 mchk = li->irq.mchk; 526 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 527 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 528 memset(&li->irq.mchk, 0, sizeof(mchk)); 529 deliver = 1; 530 } 531 /* 532 * We indicate floating repressible conditions along with 533 * other pending conditions. Channel Report Pending and Channel 534 * Subsystem damage are the only two and and are indicated by 535 * bits in mcic and masked in cr14. 536 */ 537 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 538 mchk.mcic |= fi->mchk.mcic; 539 mchk.cr14 |= fi->mchk.cr14; 540 memset(&fi->mchk, 0, sizeof(mchk)); 541 deliver = 1; 542 } 543 spin_unlock(&li->lock); 544 spin_unlock(&fi->lock); 545 546 if (deliver) { 547 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", 548 mchk.mcic); 549 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 550 KVM_S390_MCHK, 551 mchk.cr14, mchk.mcic); 552 rc = __write_machine_check(vcpu, &mchk); 553 } 554 return rc; 555 } 556 557 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu) 558 { 559 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 560 int rc; 561 562 VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart"); 563 vcpu->stat.deliver_restart_signal++; 564 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 565 566 rc = write_guest_lc(vcpu, 567 offsetof(struct lowcore, restart_old_psw), 568 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 569 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), 570 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 571 clear_bit(IRQ_PEND_RESTART, &li->pending_irqs); 572 return rc ? -EFAULT : 0; 573 } 574 575 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu) 576 { 577 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 578 struct kvm_s390_prefix_info prefix; 579 580 spin_lock(&li->lock); 581 prefix = li->irq.prefix; 582 li->irq.prefix.address = 0; 583 clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 584 spin_unlock(&li->lock); 585 586 vcpu->stat.deliver_prefix_signal++; 587 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 588 KVM_S390_SIGP_SET_PREFIX, 589 prefix.address, 0); 590 591 kvm_s390_set_prefix(vcpu, prefix.address); 592 return 0; 593 } 594 595 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu) 596 { 597 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 598 int rc; 599 int cpu_addr; 600 601 spin_lock(&li->lock); 602 cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS); 603 clear_bit(cpu_addr, li->sigp_emerg_pending); 604 if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS)) 605 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 606 spin_unlock(&li->lock); 607 608 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg"); 609 vcpu->stat.deliver_emergency_signal++; 610 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 611 cpu_addr, 0); 612 613 rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG, 614 (u16 *)__LC_EXT_INT_CODE); 615 rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR); 616 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 617 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 618 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 619 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 620 return rc ? -EFAULT : 0; 621 } 622 623 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) 624 { 625 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 626 struct kvm_s390_extcall_info extcall; 627 int rc; 628 629 spin_lock(&li->lock); 630 extcall = li->irq.extcall; 631 li->irq.extcall.code = 0; 632 clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 633 spin_unlock(&li->lock); 634 635 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call"); 636 vcpu->stat.deliver_external_call++; 637 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 638 KVM_S390_INT_EXTERNAL_CALL, 639 extcall.code, 0); 640 641 rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL, 642 (u16 *)__LC_EXT_INT_CODE); 643 rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR); 644 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 645 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 646 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw, 647 sizeof(psw_t)); 648 return rc ? -EFAULT : 0; 649 } 650 651 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) 652 { 653 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 654 struct kvm_s390_pgm_info pgm_info; 655 int rc = 0, nullifying = false; 656 u16 ilen; 657 658 spin_lock(&li->lock); 659 pgm_info = li->irq.pgm; 660 clear_bit(IRQ_PEND_PROG, &li->pending_irqs); 661 memset(&li->irq.pgm, 0, sizeof(pgm_info)); 662 spin_unlock(&li->lock); 663 664 ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK; 665 VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d", 666 pgm_info.code, ilen); 667 vcpu->stat.deliver_program_int++; 668 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 669 pgm_info.code, 0); 670 671 switch (pgm_info.code & ~PGM_PER) { 672 case PGM_AFX_TRANSLATION: 673 case PGM_ASX_TRANSLATION: 674 case PGM_EX_TRANSLATION: 675 case PGM_LFX_TRANSLATION: 676 case PGM_LSTE_SEQUENCE: 677 case PGM_LSX_TRANSLATION: 678 case PGM_LX_TRANSLATION: 679 case PGM_PRIMARY_AUTHORITY: 680 case PGM_SECONDARY_AUTHORITY: 681 nullifying = true; 682 /* fall through */ 683 case PGM_SPACE_SWITCH: 684 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 685 (u64 *)__LC_TRANS_EXC_CODE); 686 break; 687 case PGM_ALEN_TRANSLATION: 688 case PGM_ALE_SEQUENCE: 689 case PGM_ASTE_INSTANCE: 690 case PGM_ASTE_SEQUENCE: 691 case PGM_ASTE_VALIDITY: 692 case PGM_EXTENDED_AUTHORITY: 693 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 694 (u8 *)__LC_EXC_ACCESS_ID); 695 nullifying = true; 696 break; 697 case PGM_ASCE_TYPE: 698 case PGM_PAGE_TRANSLATION: 699 case PGM_REGION_FIRST_TRANS: 700 case PGM_REGION_SECOND_TRANS: 701 case PGM_REGION_THIRD_TRANS: 702 case PGM_SEGMENT_TRANSLATION: 703 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 704 (u64 *)__LC_TRANS_EXC_CODE); 705 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 706 (u8 *)__LC_EXC_ACCESS_ID); 707 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 708 (u8 *)__LC_OP_ACCESS_ID); 709 nullifying = true; 710 break; 711 case PGM_MONITOR: 712 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 713 (u16 *)__LC_MON_CLASS_NR); 714 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 715 (u64 *)__LC_MON_CODE); 716 break; 717 case PGM_VECTOR_PROCESSING: 718 case PGM_DATA: 719 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 720 (u32 *)__LC_DATA_EXC_CODE); 721 break; 722 case PGM_PROTECTION: 723 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 724 (u64 *)__LC_TRANS_EXC_CODE); 725 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 726 (u8 *)__LC_EXC_ACCESS_ID); 727 break; 728 case PGM_STACK_FULL: 729 case PGM_STACK_EMPTY: 730 case PGM_STACK_SPECIFICATION: 731 case PGM_STACK_TYPE: 732 case PGM_STACK_OPERATION: 733 case PGM_TRACE_TABEL: 734 case PGM_CRYPTO_OPERATION: 735 nullifying = true; 736 break; 737 } 738 739 if (pgm_info.code & PGM_PER) { 740 rc |= put_guest_lc(vcpu, pgm_info.per_code, 741 (u8 *) __LC_PER_CODE); 742 rc |= put_guest_lc(vcpu, pgm_info.per_atmid, 743 (u8 *)__LC_PER_ATMID); 744 rc |= put_guest_lc(vcpu, pgm_info.per_address, 745 (u64 *) __LC_PER_ADDRESS); 746 rc |= put_guest_lc(vcpu, pgm_info.per_access_id, 747 (u8 *) __LC_PER_ACCESS_ID); 748 } 749 750 if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND)) 751 kvm_s390_rewind_psw(vcpu, ilen); 752 753 /* bit 1+2 of the target are the ilc, so we can directly use ilen */ 754 rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC); 755 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea, 756 (u64 *) __LC_LAST_BREAK); 757 rc |= put_guest_lc(vcpu, pgm_info.code, 758 (u16 *)__LC_PGM_INT_CODE); 759 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 760 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 761 rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW, 762 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 763 return rc ? -EFAULT : 0; 764 } 765 766 static int __must_check __deliver_service(struct kvm_vcpu *vcpu) 767 { 768 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 769 struct kvm_s390_ext_info ext; 770 int rc = 0; 771 772 spin_lock(&fi->lock); 773 if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { 774 spin_unlock(&fi->lock); 775 return 0; 776 } 777 ext = fi->srv_signal; 778 memset(&fi->srv_signal, 0, sizeof(ext)); 779 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 780 spin_unlock(&fi->lock); 781 782 VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", 783 ext.ext_params); 784 vcpu->stat.deliver_service_signal++; 785 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 786 ext.ext_params, 0); 787 788 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 789 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 790 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 791 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 792 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 793 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 794 rc |= put_guest_lc(vcpu, ext.ext_params, 795 (u32 *)__LC_EXT_PARAMS); 796 797 return rc ? -EFAULT : 0; 798 } 799 800 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) 801 { 802 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 803 struct kvm_s390_interrupt_info *inti; 804 int rc = 0; 805 806 spin_lock(&fi->lock); 807 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], 808 struct kvm_s390_interrupt_info, 809 list); 810 if (inti) { 811 list_del(&inti->list); 812 fi->counters[FIRQ_CNTR_PFAULT] -= 1; 813 } 814 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) 815 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 816 spin_unlock(&fi->lock); 817 818 if (inti) { 819 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 820 KVM_S390_INT_PFAULT_DONE, 0, 821 inti->ext.ext_params2); 822 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx", 823 inti->ext.ext_params2); 824 825 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 826 (u16 *)__LC_EXT_INT_CODE); 827 rc |= put_guest_lc(vcpu, PFAULT_DONE, 828 (u16 *)__LC_EXT_CPU_ADDR); 829 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 830 &vcpu->arch.sie_block->gpsw, 831 sizeof(psw_t)); 832 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 833 &vcpu->arch.sie_block->gpsw, 834 sizeof(psw_t)); 835 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 836 (u64 *)__LC_EXT_PARAMS2); 837 kfree(inti); 838 } 839 return rc ? -EFAULT : 0; 840 } 841 842 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) 843 { 844 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 845 struct kvm_s390_interrupt_info *inti; 846 int rc = 0; 847 848 spin_lock(&fi->lock); 849 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], 850 struct kvm_s390_interrupt_info, 851 list); 852 if (inti) { 853 VCPU_EVENT(vcpu, 4, 854 "deliver: virtio parm: 0x%x,parm64: 0x%llx", 855 inti->ext.ext_params, inti->ext.ext_params2); 856 vcpu->stat.deliver_virtio_interrupt++; 857 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 858 inti->type, 859 inti->ext.ext_params, 860 inti->ext.ext_params2); 861 list_del(&inti->list); 862 fi->counters[FIRQ_CNTR_VIRTIO] -= 1; 863 } 864 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) 865 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 866 spin_unlock(&fi->lock); 867 868 if (inti) { 869 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 870 (u16 *)__LC_EXT_INT_CODE); 871 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, 872 (u16 *)__LC_EXT_CPU_ADDR); 873 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 874 &vcpu->arch.sie_block->gpsw, 875 sizeof(psw_t)); 876 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 877 &vcpu->arch.sie_block->gpsw, 878 sizeof(psw_t)); 879 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 880 (u32 *)__LC_EXT_PARAMS); 881 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 882 (u64 *)__LC_EXT_PARAMS2); 883 kfree(inti); 884 } 885 return rc ? -EFAULT : 0; 886 } 887 888 static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 889 unsigned long irq_type) 890 { 891 struct list_head *isc_list; 892 struct kvm_s390_float_interrupt *fi; 893 struct kvm_s390_interrupt_info *inti = NULL; 894 int rc = 0; 895 896 fi = &vcpu->kvm->arch.float_int; 897 898 spin_lock(&fi->lock); 899 isc_list = &fi->lists[irq_type - IRQ_PEND_IO_ISC_0]; 900 inti = list_first_entry_or_null(isc_list, 901 struct kvm_s390_interrupt_info, 902 list); 903 if (inti) { 904 if (inti->type & KVM_S390_INT_IO_AI_MASK) 905 VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)"); 906 else 907 VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x", 908 inti->io.subchannel_id >> 8, 909 inti->io.subchannel_id >> 1 & 0x3, 910 inti->io.subchannel_nr); 911 912 vcpu->stat.deliver_io_int++; 913 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 914 inti->type, 915 ((__u32)inti->io.subchannel_id << 16) | 916 inti->io.subchannel_nr, 917 ((__u64)inti->io.io_int_parm << 32) | 918 inti->io.io_int_word); 919 list_del(&inti->list); 920 fi->counters[FIRQ_CNTR_IO] -= 1; 921 } 922 if (list_empty(isc_list)) 923 clear_bit(irq_type, &fi->pending_irqs); 924 spin_unlock(&fi->lock); 925 926 if (inti) { 927 rc = put_guest_lc(vcpu, inti->io.subchannel_id, 928 (u16 *)__LC_SUBCHANNEL_ID); 929 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr, 930 (u16 *)__LC_SUBCHANNEL_NR); 931 rc |= put_guest_lc(vcpu, inti->io.io_int_parm, 932 (u32 *)__LC_IO_INT_PARM); 933 rc |= put_guest_lc(vcpu, inti->io.io_int_word, 934 (u32 *)__LC_IO_INT_WORD); 935 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW, 936 &vcpu->arch.sie_block->gpsw, 937 sizeof(psw_t)); 938 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW, 939 &vcpu->arch.sie_block->gpsw, 940 sizeof(psw_t)); 941 kfree(inti); 942 } 943 944 return rc ? -EFAULT : 0; 945 } 946 947 typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu); 948 949 static const deliver_irq_t deliver_irq_funcs[] = { 950 [IRQ_PEND_MCHK_EX] = __deliver_machine_check, 951 [IRQ_PEND_MCHK_REP] = __deliver_machine_check, 952 [IRQ_PEND_PROG] = __deliver_prog, 953 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal, 954 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call, 955 [IRQ_PEND_EXT_CLOCK_COMP] = __deliver_ckc, 956 [IRQ_PEND_EXT_CPU_TIMER] = __deliver_cpu_timer, 957 [IRQ_PEND_RESTART] = __deliver_restart, 958 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix, 959 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init, 960 [IRQ_PEND_EXT_SERVICE] = __deliver_service, 961 [IRQ_PEND_PFAULT_DONE] = __deliver_pfault_done, 962 [IRQ_PEND_VIRTIO] = __deliver_virtio, 963 }; 964 965 /* Check whether an external call is pending (deliverable or not) */ 966 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 967 { 968 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 969 970 if (!sclp.has_sigpif) 971 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 972 973 return sca_ext_call_pending(vcpu, NULL); 974 } 975 976 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 977 { 978 if (deliverable_irqs(vcpu)) 979 return 1; 980 981 if (kvm_cpu_has_pending_timer(vcpu)) 982 return 1; 983 984 /* external call pending and deliverable */ 985 if (kvm_s390_ext_call_pending(vcpu) && 986 !psw_extint_disabled(vcpu) && 987 (vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 988 return 1; 989 990 if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu)) 991 return 1; 992 return 0; 993 } 994 995 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 996 { 997 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu); 998 } 999 1000 static u64 __calculate_sltime(struct kvm_vcpu *vcpu) 1001 { 1002 u64 now, cputm, sltime = 0; 1003 1004 if (ckc_interrupts_enabled(vcpu)) { 1005 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 1006 sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now); 1007 /* already expired or overflow? */ 1008 if (!sltime || vcpu->arch.sie_block->ckc <= now) 1009 return 0; 1010 if (cpu_timer_interrupts_enabled(vcpu)) { 1011 cputm = kvm_s390_get_cpu_timer(vcpu); 1012 /* already expired? */ 1013 if (cputm >> 63) 1014 return 0; 1015 return min(sltime, tod_to_ns(cputm)); 1016 } 1017 } else if (cpu_timer_interrupts_enabled(vcpu)) { 1018 sltime = kvm_s390_get_cpu_timer(vcpu); 1019 /* already expired? */ 1020 if (sltime >> 63) 1021 return 0; 1022 } 1023 return sltime; 1024 } 1025 1026 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 1027 { 1028 u64 sltime; 1029 1030 vcpu->stat.exit_wait_state++; 1031 1032 /* fast path */ 1033 if (kvm_arch_vcpu_runnable(vcpu)) 1034 return 0; 1035 1036 if (psw_interrupts_disabled(vcpu)) { 1037 VCPU_EVENT(vcpu, 3, "%s", "disabled wait"); 1038 return -EOPNOTSUPP; /* disabled wait */ 1039 } 1040 1041 if (!ckc_interrupts_enabled(vcpu) && 1042 !cpu_timer_interrupts_enabled(vcpu)) { 1043 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer"); 1044 __set_cpu_idle(vcpu); 1045 goto no_timer; 1046 } 1047 1048 sltime = __calculate_sltime(vcpu); 1049 if (!sltime) 1050 return 0; 1051 1052 __set_cpu_idle(vcpu); 1053 hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL); 1054 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime); 1055 no_timer: 1056 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1057 kvm_vcpu_block(vcpu); 1058 __unset_cpu_idle(vcpu); 1059 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1060 1061 hrtimer_cancel(&vcpu->arch.ckc_timer); 1062 return 0; 1063 } 1064 1065 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) 1066 { 1067 /* 1068 * We cannot move this into the if, as the CPU might be already 1069 * in kvm_vcpu_block without having the waitqueue set (polling) 1070 */ 1071 vcpu->valid_wakeup = true; 1072 if (swait_active(&vcpu->wq)) { 1073 /* 1074 * The vcpu gave up the cpu voluntarily, mark it as a good 1075 * yield-candidate. 1076 */ 1077 vcpu->preempted = true; 1078 swake_up(&vcpu->wq); 1079 vcpu->stat.halt_wakeup++; 1080 } 1081 /* 1082 * The VCPU might not be sleeping but is executing the VSIE. Let's 1083 * kick it, so it leaves the SIE to process the request. 1084 */ 1085 kvm_s390_vsie_kick(vcpu); 1086 } 1087 1088 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer) 1089 { 1090 struct kvm_vcpu *vcpu; 1091 u64 sltime; 1092 1093 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer); 1094 sltime = __calculate_sltime(vcpu); 1095 1096 /* 1097 * If the monotonic clock runs faster than the tod clock we might be 1098 * woken up too early and have to go back to sleep to avoid deadlocks. 1099 */ 1100 if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime))) 1101 return HRTIMER_RESTART; 1102 kvm_s390_vcpu_wakeup(vcpu); 1103 return HRTIMER_NORESTART; 1104 } 1105 1106 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) 1107 { 1108 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1109 1110 spin_lock(&li->lock); 1111 li->pending_irqs = 0; 1112 bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS); 1113 memset(&li->irq, 0, sizeof(li->irq)); 1114 spin_unlock(&li->lock); 1115 1116 sca_clear_ext_call(vcpu); 1117 } 1118 1119 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 1120 { 1121 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1122 deliver_irq_t func; 1123 int rc = 0; 1124 unsigned long irq_type; 1125 unsigned long irqs; 1126 1127 __reset_intercept_indicators(vcpu); 1128 1129 /* pending ckc conditions might have been invalidated */ 1130 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1131 if (ckc_irq_pending(vcpu)) 1132 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1133 1134 /* pending cpu timer conditions might have been invalidated */ 1135 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1136 if (cpu_timer_irq_pending(vcpu)) 1137 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1138 1139 while ((irqs = deliverable_irqs(vcpu)) && !rc) { 1140 /* bits are in the order of interrupt priority */ 1141 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT); 1142 if (is_ioirq(irq_type)) { 1143 rc = __deliver_io(vcpu, irq_type); 1144 } else { 1145 func = deliver_irq_funcs[irq_type]; 1146 if (!func) { 1147 WARN_ON_ONCE(func == NULL); 1148 clear_bit(irq_type, &li->pending_irqs); 1149 continue; 1150 } 1151 rc = func(vcpu); 1152 } 1153 } 1154 1155 set_intercept_indicators(vcpu); 1156 1157 return rc; 1158 } 1159 1160 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1161 { 1162 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1163 1164 VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code); 1165 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 1166 irq->u.pgm.code, 0); 1167 1168 if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) { 1169 /* auto detection if no valid ILC was given */ 1170 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK; 1171 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu); 1172 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID; 1173 } 1174 1175 if (irq->u.pgm.code == PGM_PER) { 1176 li->irq.pgm.code |= PGM_PER; 1177 li->irq.pgm.flags = irq->u.pgm.flags; 1178 /* only modify PER related information */ 1179 li->irq.pgm.per_address = irq->u.pgm.per_address; 1180 li->irq.pgm.per_code = irq->u.pgm.per_code; 1181 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid; 1182 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id; 1183 } else if (!(irq->u.pgm.code & PGM_PER)) { 1184 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) | 1185 irq->u.pgm.code; 1186 li->irq.pgm.flags = irq->u.pgm.flags; 1187 /* only modify non-PER information */ 1188 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code; 1189 li->irq.pgm.mon_code = irq->u.pgm.mon_code; 1190 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code; 1191 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr; 1192 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id; 1193 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id; 1194 } else { 1195 li->irq.pgm = irq->u.pgm; 1196 } 1197 set_bit(IRQ_PEND_PROG, &li->pending_irqs); 1198 return 0; 1199 } 1200 1201 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1202 { 1203 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1204 1205 VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx", 1206 irq->u.ext.ext_params2); 1207 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT, 1208 irq->u.ext.ext_params, 1209 irq->u.ext.ext_params2); 1210 1211 li->irq.ext = irq->u.ext; 1212 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 1213 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1214 return 0; 1215 } 1216 1217 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1218 { 1219 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1220 struct kvm_s390_extcall_info *extcall = &li->irq.extcall; 1221 uint16_t src_id = irq->u.extcall.code; 1222 1223 VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u", 1224 src_id); 1225 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL, 1226 src_id, 0); 1227 1228 /* sending vcpu invalid */ 1229 if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL) 1230 return -EINVAL; 1231 1232 if (sclp.has_sigpif) 1233 return sca_inject_ext_call(vcpu, src_id); 1234 1235 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) 1236 return -EBUSY; 1237 *extcall = irq->u.extcall; 1238 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1239 return 0; 1240 } 1241 1242 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1243 { 1244 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1245 struct kvm_s390_prefix_info *prefix = &li->irq.prefix; 1246 1247 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x", 1248 irq->u.prefix.address); 1249 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX, 1250 irq->u.prefix.address, 0); 1251 1252 if (!is_vcpu_stopped(vcpu)) 1253 return -EBUSY; 1254 1255 *prefix = irq->u.prefix; 1256 set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 1257 return 0; 1258 } 1259 1260 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) 1261 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1262 { 1263 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1264 struct kvm_s390_stop_info *stop = &li->irq.stop; 1265 int rc = 0; 1266 1267 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); 1268 1269 if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) 1270 return -EINVAL; 1271 1272 if (is_vcpu_stopped(vcpu)) { 1273 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) 1274 rc = kvm_s390_store_status_unloaded(vcpu, 1275 KVM_S390_STORE_STATUS_NOADDR); 1276 return rc; 1277 } 1278 1279 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) 1280 return -EBUSY; 1281 stop->flags = irq->u.stop.flags; 1282 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 1283 return 0; 1284 } 1285 1286 static int __inject_sigp_restart(struct kvm_vcpu *vcpu, 1287 struct kvm_s390_irq *irq) 1288 { 1289 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1290 1291 VCPU_EVENT(vcpu, 3, "%s", "inject: restart int"); 1292 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 1293 1294 set_bit(IRQ_PEND_RESTART, &li->pending_irqs); 1295 return 0; 1296 } 1297 1298 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, 1299 struct kvm_s390_irq *irq) 1300 { 1301 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1302 1303 VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u", 1304 irq->u.emerg.code); 1305 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 1306 irq->u.emerg.code, 0); 1307 1308 /* sending vcpu invalid */ 1309 if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL) 1310 return -EINVAL; 1311 1312 set_bit(irq->u.emerg.code, li->sigp_emerg_pending); 1313 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 1314 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1315 return 0; 1316 } 1317 1318 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1319 { 1320 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1321 struct kvm_s390_mchk_info *mchk = &li->irq.mchk; 1322 1323 VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx", 1324 irq->u.mchk.mcic); 1325 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0, 1326 irq->u.mchk.mcic); 1327 1328 /* 1329 * Because repressible machine checks can be indicated along with 1330 * exigent machine checks (PoP, Chapter 11, Interruption action) 1331 * we need to combine cr14, mcic and external damage code. 1332 * Failing storage address and the logout area should not be or'ed 1333 * together, we just indicate the last occurrence of the corresponding 1334 * machine check 1335 */ 1336 mchk->cr14 |= irq->u.mchk.cr14; 1337 mchk->mcic |= irq->u.mchk.mcic; 1338 mchk->ext_damage_code |= irq->u.mchk.ext_damage_code; 1339 mchk->failing_storage_address = irq->u.mchk.failing_storage_address; 1340 memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout, 1341 sizeof(mchk->fixed_logout)); 1342 if (mchk->mcic & MCHK_EX_MASK) 1343 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 1344 else if (mchk->mcic & MCHK_REP_MASK) 1345 set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 1346 return 0; 1347 } 1348 1349 static int __inject_ckc(struct kvm_vcpu *vcpu) 1350 { 1351 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1352 1353 VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external"); 1354 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 1355 0, 0); 1356 1357 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1358 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1359 return 0; 1360 } 1361 1362 static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 1363 { 1364 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1365 1366 VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external"); 1367 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 1368 0, 0); 1369 1370 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1371 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1372 return 0; 1373 } 1374 1375 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, 1376 int isc, u32 schid) 1377 { 1378 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1379 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1380 struct kvm_s390_interrupt_info *iter; 1381 u16 id = (schid & 0xffff0000U) >> 16; 1382 u16 nr = schid & 0x0000ffffU; 1383 1384 spin_lock(&fi->lock); 1385 list_for_each_entry(iter, isc_list, list) { 1386 if (schid && (id != iter->io.subchannel_id || 1387 nr != iter->io.subchannel_nr)) 1388 continue; 1389 /* found an appropriate entry */ 1390 list_del_init(&iter->list); 1391 fi->counters[FIRQ_CNTR_IO] -= 1; 1392 if (list_empty(isc_list)) 1393 clear_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs); 1394 spin_unlock(&fi->lock); 1395 return iter; 1396 } 1397 spin_unlock(&fi->lock); 1398 return NULL; 1399 } 1400 1401 /* 1402 * Dequeue and return an I/O interrupt matching any of the interruption 1403 * subclasses as designated by the isc mask in cr6 and the schid (if != 0). 1404 */ 1405 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1406 u64 isc_mask, u32 schid) 1407 { 1408 struct kvm_s390_interrupt_info *inti = NULL; 1409 int isc; 1410 1411 for (isc = 0; isc <= MAX_ISC && !inti; isc++) { 1412 if (isc_mask & isc_to_isc_bits(isc)) 1413 inti = get_io_int(kvm, isc, schid); 1414 } 1415 return inti; 1416 } 1417 1418 #define SCCB_MASK 0xFFFFFFF8 1419 #define SCCB_EVENT_PENDING 0x3 1420 1421 static int __inject_service(struct kvm *kvm, 1422 struct kvm_s390_interrupt_info *inti) 1423 { 1424 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1425 1426 spin_lock(&fi->lock); 1427 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; 1428 /* 1429 * Early versions of the QEMU s390 bios will inject several 1430 * service interrupts after another without handling a 1431 * condition code indicating busy. 1432 * We will silently ignore those superfluous sccb values. 1433 * A future version of QEMU will take care of serialization 1434 * of servc requests 1435 */ 1436 if (fi->srv_signal.ext_params & SCCB_MASK) 1437 goto out; 1438 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; 1439 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 1440 out: 1441 spin_unlock(&fi->lock); 1442 kfree(inti); 1443 return 0; 1444 } 1445 1446 static int __inject_virtio(struct kvm *kvm, 1447 struct kvm_s390_interrupt_info *inti) 1448 { 1449 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1450 1451 spin_lock(&fi->lock); 1452 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { 1453 spin_unlock(&fi->lock); 1454 return -EBUSY; 1455 } 1456 fi->counters[FIRQ_CNTR_VIRTIO] += 1; 1457 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); 1458 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1459 spin_unlock(&fi->lock); 1460 return 0; 1461 } 1462 1463 static int __inject_pfault_done(struct kvm *kvm, 1464 struct kvm_s390_interrupt_info *inti) 1465 { 1466 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1467 1468 spin_lock(&fi->lock); 1469 if (fi->counters[FIRQ_CNTR_PFAULT] >= 1470 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { 1471 spin_unlock(&fi->lock); 1472 return -EBUSY; 1473 } 1474 fi->counters[FIRQ_CNTR_PFAULT] += 1; 1475 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); 1476 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1477 spin_unlock(&fi->lock); 1478 return 0; 1479 } 1480 1481 #define CR_PENDING_SUBCLASS 28 1482 static int __inject_float_mchk(struct kvm *kvm, 1483 struct kvm_s390_interrupt_info *inti) 1484 { 1485 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1486 1487 spin_lock(&fi->lock); 1488 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); 1489 fi->mchk.mcic |= inti->mchk.mcic; 1490 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); 1491 spin_unlock(&fi->lock); 1492 kfree(inti); 1493 return 0; 1494 } 1495 1496 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1497 { 1498 struct kvm_s390_float_interrupt *fi; 1499 struct list_head *list; 1500 int isc; 1501 1502 fi = &kvm->arch.float_int; 1503 spin_lock(&fi->lock); 1504 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { 1505 spin_unlock(&fi->lock); 1506 return -EBUSY; 1507 } 1508 fi->counters[FIRQ_CNTR_IO] += 1; 1509 1510 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1511 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)"); 1512 else 1513 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x", 1514 inti->io.subchannel_id >> 8, 1515 inti->io.subchannel_id >> 1 & 0x3, 1516 inti->io.subchannel_nr); 1517 isc = int_word_to_isc(inti->io.io_int_word); 1518 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1519 list_add_tail(&inti->list, list); 1520 set_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs); 1521 spin_unlock(&fi->lock); 1522 return 0; 1523 } 1524 1525 /* 1526 * Find a destination VCPU for a floating irq and kick it. 1527 */ 1528 static void __floating_irq_kick(struct kvm *kvm, u64 type) 1529 { 1530 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1531 struct kvm_s390_local_interrupt *li; 1532 struct kvm_vcpu *dst_vcpu; 1533 int sigcpu, online_vcpus, nr_tries = 0; 1534 1535 online_vcpus = atomic_read(&kvm->online_vcpus); 1536 if (!online_vcpus) 1537 return; 1538 1539 /* find idle VCPUs first, then round robin */ 1540 sigcpu = find_first_bit(fi->idle_mask, online_vcpus); 1541 if (sigcpu == online_vcpus) { 1542 do { 1543 sigcpu = fi->next_rr_cpu; 1544 fi->next_rr_cpu = (fi->next_rr_cpu + 1) % online_vcpus; 1545 /* avoid endless loops if all vcpus are stopped */ 1546 if (nr_tries++ >= online_vcpus) 1547 return; 1548 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu))); 1549 } 1550 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1551 1552 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1553 li = &dst_vcpu->arch.local_int; 1554 spin_lock(&li->lock); 1555 switch (type) { 1556 case KVM_S390_MCHK: 1557 atomic_or(CPUSTAT_STOP_INT, li->cpuflags); 1558 break; 1559 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1560 atomic_or(CPUSTAT_IO_INT, li->cpuflags); 1561 break; 1562 default: 1563 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1564 break; 1565 } 1566 spin_unlock(&li->lock); 1567 kvm_s390_vcpu_wakeup(dst_vcpu); 1568 } 1569 1570 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1571 { 1572 u64 type = READ_ONCE(inti->type); 1573 int rc; 1574 1575 switch (type) { 1576 case KVM_S390_MCHK: 1577 rc = __inject_float_mchk(kvm, inti); 1578 break; 1579 case KVM_S390_INT_VIRTIO: 1580 rc = __inject_virtio(kvm, inti); 1581 break; 1582 case KVM_S390_INT_SERVICE: 1583 rc = __inject_service(kvm, inti); 1584 break; 1585 case KVM_S390_INT_PFAULT_DONE: 1586 rc = __inject_pfault_done(kvm, inti); 1587 break; 1588 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1589 rc = __inject_io(kvm, inti); 1590 break; 1591 default: 1592 rc = -EINVAL; 1593 } 1594 if (rc) 1595 return rc; 1596 1597 __floating_irq_kick(kvm, type); 1598 return 0; 1599 } 1600 1601 int kvm_s390_inject_vm(struct kvm *kvm, 1602 struct kvm_s390_interrupt *s390int) 1603 { 1604 struct kvm_s390_interrupt_info *inti; 1605 int rc; 1606 1607 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1608 if (!inti) 1609 return -ENOMEM; 1610 1611 inti->type = s390int->type; 1612 switch (inti->type) { 1613 case KVM_S390_INT_VIRTIO: 1614 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx", 1615 s390int->parm, s390int->parm64); 1616 inti->ext.ext_params = s390int->parm; 1617 inti->ext.ext_params2 = s390int->parm64; 1618 break; 1619 case KVM_S390_INT_SERVICE: 1620 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm); 1621 inti->ext.ext_params = s390int->parm; 1622 break; 1623 case KVM_S390_INT_PFAULT_DONE: 1624 inti->ext.ext_params2 = s390int->parm64; 1625 break; 1626 case KVM_S390_MCHK: 1627 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx", 1628 s390int->parm64); 1629 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */ 1630 inti->mchk.mcic = s390int->parm64; 1631 break; 1632 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1633 inti->io.subchannel_id = s390int->parm >> 16; 1634 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu; 1635 inti->io.io_int_parm = s390int->parm64 >> 32; 1636 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; 1637 break; 1638 default: 1639 kfree(inti); 1640 return -EINVAL; 1641 } 1642 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 1643 2); 1644 1645 rc = __inject_vm(kvm, inti); 1646 if (rc) 1647 kfree(inti); 1648 return rc; 1649 } 1650 1651 int kvm_s390_reinject_io_int(struct kvm *kvm, 1652 struct kvm_s390_interrupt_info *inti) 1653 { 1654 return __inject_vm(kvm, inti); 1655 } 1656 1657 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 1658 struct kvm_s390_irq *irq) 1659 { 1660 irq->type = s390int->type; 1661 switch (irq->type) { 1662 case KVM_S390_PROGRAM_INT: 1663 if (s390int->parm & 0xffff0000) 1664 return -EINVAL; 1665 irq->u.pgm.code = s390int->parm; 1666 break; 1667 case KVM_S390_SIGP_SET_PREFIX: 1668 irq->u.prefix.address = s390int->parm; 1669 break; 1670 case KVM_S390_SIGP_STOP: 1671 irq->u.stop.flags = s390int->parm; 1672 break; 1673 case KVM_S390_INT_EXTERNAL_CALL: 1674 if (s390int->parm & 0xffff0000) 1675 return -EINVAL; 1676 irq->u.extcall.code = s390int->parm; 1677 break; 1678 case KVM_S390_INT_EMERGENCY: 1679 if (s390int->parm & 0xffff0000) 1680 return -EINVAL; 1681 irq->u.emerg.code = s390int->parm; 1682 break; 1683 case KVM_S390_MCHK: 1684 irq->u.mchk.mcic = s390int->parm64; 1685 break; 1686 } 1687 return 0; 1688 } 1689 1690 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu) 1691 { 1692 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1693 1694 return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1695 } 1696 1697 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) 1698 { 1699 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1700 1701 spin_lock(&li->lock); 1702 li->irq.stop.flags = 0; 1703 clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1704 spin_unlock(&li->lock); 1705 } 1706 1707 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1708 { 1709 int rc; 1710 1711 switch (irq->type) { 1712 case KVM_S390_PROGRAM_INT: 1713 rc = __inject_prog(vcpu, irq); 1714 break; 1715 case KVM_S390_SIGP_SET_PREFIX: 1716 rc = __inject_set_prefix(vcpu, irq); 1717 break; 1718 case KVM_S390_SIGP_STOP: 1719 rc = __inject_sigp_stop(vcpu, irq); 1720 break; 1721 case KVM_S390_RESTART: 1722 rc = __inject_sigp_restart(vcpu, irq); 1723 break; 1724 case KVM_S390_INT_CLOCK_COMP: 1725 rc = __inject_ckc(vcpu); 1726 break; 1727 case KVM_S390_INT_CPU_TIMER: 1728 rc = __inject_cpu_timer(vcpu); 1729 break; 1730 case KVM_S390_INT_EXTERNAL_CALL: 1731 rc = __inject_extcall(vcpu, irq); 1732 break; 1733 case KVM_S390_INT_EMERGENCY: 1734 rc = __inject_sigp_emergency(vcpu, irq); 1735 break; 1736 case KVM_S390_MCHK: 1737 rc = __inject_mchk(vcpu, irq); 1738 break; 1739 case KVM_S390_INT_PFAULT_INIT: 1740 rc = __inject_pfault_init(vcpu, irq); 1741 break; 1742 case KVM_S390_INT_VIRTIO: 1743 case KVM_S390_INT_SERVICE: 1744 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1745 default: 1746 rc = -EINVAL; 1747 } 1748 1749 return rc; 1750 } 1751 1752 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1753 { 1754 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1755 int rc; 1756 1757 spin_lock(&li->lock); 1758 rc = do_inject_vcpu(vcpu, irq); 1759 spin_unlock(&li->lock); 1760 if (!rc) 1761 kvm_s390_vcpu_wakeup(vcpu); 1762 return rc; 1763 } 1764 1765 static inline void clear_irq_list(struct list_head *_list) 1766 { 1767 struct kvm_s390_interrupt_info *inti, *n; 1768 1769 list_for_each_entry_safe(inti, n, _list, list) { 1770 list_del(&inti->list); 1771 kfree(inti); 1772 } 1773 } 1774 1775 static void inti_to_irq(struct kvm_s390_interrupt_info *inti, 1776 struct kvm_s390_irq *irq) 1777 { 1778 irq->type = inti->type; 1779 switch (inti->type) { 1780 case KVM_S390_INT_PFAULT_INIT: 1781 case KVM_S390_INT_PFAULT_DONE: 1782 case KVM_S390_INT_VIRTIO: 1783 irq->u.ext = inti->ext; 1784 break; 1785 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1786 irq->u.io = inti->io; 1787 break; 1788 } 1789 } 1790 1791 void kvm_s390_clear_float_irqs(struct kvm *kvm) 1792 { 1793 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1794 int i; 1795 1796 spin_lock(&fi->lock); 1797 fi->pending_irqs = 0; 1798 memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); 1799 memset(&fi->mchk, 0, sizeof(fi->mchk)); 1800 for (i = 0; i < FIRQ_LIST_COUNT; i++) 1801 clear_irq_list(&fi->lists[i]); 1802 for (i = 0; i < FIRQ_MAX_COUNT; i++) 1803 fi->counters[i] = 0; 1804 spin_unlock(&fi->lock); 1805 }; 1806 1807 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) 1808 { 1809 struct kvm_s390_interrupt_info *inti; 1810 struct kvm_s390_float_interrupt *fi; 1811 struct kvm_s390_irq *buf; 1812 struct kvm_s390_irq *irq; 1813 int max_irqs; 1814 int ret = 0; 1815 int n = 0; 1816 int i; 1817 1818 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) 1819 return -EINVAL; 1820 1821 /* 1822 * We are already using -ENOMEM to signal 1823 * userspace it may retry with a bigger buffer, 1824 * so we need to use something else for this case 1825 */ 1826 buf = vzalloc(len); 1827 if (!buf) 1828 return -ENOBUFS; 1829 1830 max_irqs = len / sizeof(struct kvm_s390_irq); 1831 1832 fi = &kvm->arch.float_int; 1833 spin_lock(&fi->lock); 1834 for (i = 0; i < FIRQ_LIST_COUNT; i++) { 1835 list_for_each_entry(inti, &fi->lists[i], list) { 1836 if (n == max_irqs) { 1837 /* signal userspace to try again */ 1838 ret = -ENOMEM; 1839 goto out; 1840 } 1841 inti_to_irq(inti, &buf[n]); 1842 n++; 1843 } 1844 } 1845 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) { 1846 if (n == max_irqs) { 1847 /* signal userspace to try again */ 1848 ret = -ENOMEM; 1849 goto out; 1850 } 1851 irq = (struct kvm_s390_irq *) &buf[n]; 1852 irq->type = KVM_S390_INT_SERVICE; 1853 irq->u.ext = fi->srv_signal; 1854 n++; 1855 } 1856 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 1857 if (n == max_irqs) { 1858 /* signal userspace to try again */ 1859 ret = -ENOMEM; 1860 goto out; 1861 } 1862 irq = (struct kvm_s390_irq *) &buf[n]; 1863 irq->type = KVM_S390_MCHK; 1864 irq->u.mchk = fi->mchk; 1865 n++; 1866 } 1867 1868 out: 1869 spin_unlock(&fi->lock); 1870 if (!ret && n > 0) { 1871 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) 1872 ret = -EFAULT; 1873 } 1874 vfree(buf); 1875 1876 return ret < 0 ? ret : n; 1877 } 1878 1879 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1880 { 1881 int r; 1882 1883 switch (attr->group) { 1884 case KVM_DEV_FLIC_GET_ALL_IRQS: 1885 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr, 1886 attr->attr); 1887 break; 1888 default: 1889 r = -EINVAL; 1890 } 1891 1892 return r; 1893 } 1894 1895 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti, 1896 u64 addr) 1897 { 1898 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 1899 void *target = NULL; 1900 void __user *source; 1901 u64 size; 1902 1903 if (get_user(inti->type, (u64 __user *)addr)) 1904 return -EFAULT; 1905 1906 switch (inti->type) { 1907 case KVM_S390_INT_PFAULT_INIT: 1908 case KVM_S390_INT_PFAULT_DONE: 1909 case KVM_S390_INT_VIRTIO: 1910 case KVM_S390_INT_SERVICE: 1911 target = (void *) &inti->ext; 1912 source = &uptr->u.ext; 1913 size = sizeof(inti->ext); 1914 break; 1915 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1916 target = (void *) &inti->io; 1917 source = &uptr->u.io; 1918 size = sizeof(inti->io); 1919 break; 1920 case KVM_S390_MCHK: 1921 target = (void *) &inti->mchk; 1922 source = &uptr->u.mchk; 1923 size = sizeof(inti->mchk); 1924 break; 1925 default: 1926 return -EINVAL; 1927 } 1928 1929 if (copy_from_user(target, source, size)) 1930 return -EFAULT; 1931 1932 return 0; 1933 } 1934 1935 static int enqueue_floating_irq(struct kvm_device *dev, 1936 struct kvm_device_attr *attr) 1937 { 1938 struct kvm_s390_interrupt_info *inti = NULL; 1939 int r = 0; 1940 int len = attr->attr; 1941 1942 if (len % sizeof(struct kvm_s390_irq) != 0) 1943 return -EINVAL; 1944 else if (len > KVM_S390_FLIC_MAX_BUFFER) 1945 return -EINVAL; 1946 1947 while (len >= sizeof(struct kvm_s390_irq)) { 1948 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1949 if (!inti) 1950 return -ENOMEM; 1951 1952 r = copy_irq_from_user(inti, attr->addr); 1953 if (r) { 1954 kfree(inti); 1955 return r; 1956 } 1957 r = __inject_vm(dev->kvm, inti); 1958 if (r) { 1959 kfree(inti); 1960 return r; 1961 } 1962 len -= sizeof(struct kvm_s390_irq); 1963 attr->addr += sizeof(struct kvm_s390_irq); 1964 } 1965 1966 return r; 1967 } 1968 1969 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id) 1970 { 1971 if (id >= MAX_S390_IO_ADAPTERS) 1972 return NULL; 1973 return kvm->arch.adapters[id]; 1974 } 1975 1976 static int register_io_adapter(struct kvm_device *dev, 1977 struct kvm_device_attr *attr) 1978 { 1979 struct s390_io_adapter *adapter; 1980 struct kvm_s390_io_adapter adapter_info; 1981 1982 if (copy_from_user(&adapter_info, 1983 (void __user *)attr->addr, sizeof(adapter_info))) 1984 return -EFAULT; 1985 1986 if ((adapter_info.id >= MAX_S390_IO_ADAPTERS) || 1987 (dev->kvm->arch.adapters[adapter_info.id] != NULL)) 1988 return -EINVAL; 1989 1990 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 1991 if (!adapter) 1992 return -ENOMEM; 1993 1994 INIT_LIST_HEAD(&adapter->maps); 1995 init_rwsem(&adapter->maps_lock); 1996 atomic_set(&adapter->nr_maps, 0); 1997 adapter->id = adapter_info.id; 1998 adapter->isc = adapter_info.isc; 1999 adapter->maskable = adapter_info.maskable; 2000 adapter->masked = false; 2001 adapter->swap = adapter_info.swap; 2002 adapter->suppressible = (adapter_info.flags) & 2003 KVM_S390_ADAPTER_SUPPRESSIBLE; 2004 dev->kvm->arch.adapters[adapter->id] = adapter; 2005 2006 return 0; 2007 } 2008 2009 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) 2010 { 2011 int ret; 2012 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2013 2014 if (!adapter || !adapter->maskable) 2015 return -EINVAL; 2016 ret = adapter->masked; 2017 adapter->masked = masked; 2018 return ret; 2019 } 2020 2021 static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) 2022 { 2023 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2024 struct s390_map_info *map; 2025 int ret; 2026 2027 if (!adapter || !addr) 2028 return -EINVAL; 2029 2030 map = kzalloc(sizeof(*map), GFP_KERNEL); 2031 if (!map) { 2032 ret = -ENOMEM; 2033 goto out; 2034 } 2035 INIT_LIST_HEAD(&map->list); 2036 map->guest_addr = addr; 2037 map->addr = gmap_translate(kvm->arch.gmap, addr); 2038 if (map->addr == -EFAULT) { 2039 ret = -EFAULT; 2040 goto out; 2041 } 2042 ret = get_user_pages_fast(map->addr, 1, 1, &map->page); 2043 if (ret < 0) 2044 goto out; 2045 BUG_ON(ret != 1); 2046 down_write(&adapter->maps_lock); 2047 if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) { 2048 list_add_tail(&map->list, &adapter->maps); 2049 ret = 0; 2050 } else { 2051 put_page(map->page); 2052 ret = -EINVAL; 2053 } 2054 up_write(&adapter->maps_lock); 2055 out: 2056 if (ret) 2057 kfree(map); 2058 return ret; 2059 } 2060 2061 static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) 2062 { 2063 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2064 struct s390_map_info *map, *tmp; 2065 int found = 0; 2066 2067 if (!adapter || !addr) 2068 return -EINVAL; 2069 2070 down_write(&adapter->maps_lock); 2071 list_for_each_entry_safe(map, tmp, &adapter->maps, list) { 2072 if (map->guest_addr == addr) { 2073 found = 1; 2074 atomic_dec(&adapter->nr_maps); 2075 list_del(&map->list); 2076 put_page(map->page); 2077 kfree(map); 2078 break; 2079 } 2080 } 2081 up_write(&adapter->maps_lock); 2082 2083 return found ? 0 : -EINVAL; 2084 } 2085 2086 void kvm_s390_destroy_adapters(struct kvm *kvm) 2087 { 2088 int i; 2089 struct s390_map_info *map, *tmp; 2090 2091 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { 2092 if (!kvm->arch.adapters[i]) 2093 continue; 2094 list_for_each_entry_safe(map, tmp, 2095 &kvm->arch.adapters[i]->maps, list) { 2096 list_del(&map->list); 2097 put_page(map->page); 2098 kfree(map); 2099 } 2100 kfree(kvm->arch.adapters[i]); 2101 } 2102 } 2103 2104 static int modify_io_adapter(struct kvm_device *dev, 2105 struct kvm_device_attr *attr) 2106 { 2107 struct kvm_s390_io_adapter_req req; 2108 struct s390_io_adapter *adapter; 2109 int ret; 2110 2111 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2112 return -EFAULT; 2113 2114 adapter = get_io_adapter(dev->kvm, req.id); 2115 if (!adapter) 2116 return -EINVAL; 2117 switch (req.type) { 2118 case KVM_S390_IO_ADAPTER_MASK: 2119 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask); 2120 if (ret > 0) 2121 ret = 0; 2122 break; 2123 case KVM_S390_IO_ADAPTER_MAP: 2124 ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); 2125 break; 2126 case KVM_S390_IO_ADAPTER_UNMAP: 2127 ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); 2128 break; 2129 default: 2130 ret = -EINVAL; 2131 } 2132 2133 return ret; 2134 } 2135 2136 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr) 2137 2138 { 2139 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 2140 u32 schid; 2141 2142 if (attr->flags) 2143 return -EINVAL; 2144 if (attr->attr != sizeof(schid)) 2145 return -EINVAL; 2146 if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid))) 2147 return -EFAULT; 2148 kfree(kvm_s390_get_io_int(kvm, isc_mask, schid)); 2149 /* 2150 * If userspace is conforming to the architecture, we can have at most 2151 * one pending I/O interrupt per subchannel, so this is effectively a 2152 * clear all. 2153 */ 2154 return 0; 2155 } 2156 2157 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2158 { 2159 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2160 struct kvm_s390_ais_req req; 2161 int ret = 0; 2162 2163 if (!fi->ais_enabled) 2164 return -ENOTSUPP; 2165 2166 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2167 return -EFAULT; 2168 2169 if (req.isc > MAX_ISC) 2170 return -EINVAL; 2171 2172 trace_kvm_s390_modify_ais_mode(req.isc, 2173 (fi->simm & AIS_MODE_MASK(req.isc)) ? 2174 (fi->nimm & AIS_MODE_MASK(req.isc)) ? 2175 2 : KVM_S390_AIS_MODE_SINGLE : 2176 KVM_S390_AIS_MODE_ALL, req.mode); 2177 2178 mutex_lock(&fi->ais_lock); 2179 switch (req.mode) { 2180 case KVM_S390_AIS_MODE_ALL: 2181 fi->simm &= ~AIS_MODE_MASK(req.isc); 2182 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2183 break; 2184 case KVM_S390_AIS_MODE_SINGLE: 2185 fi->simm |= AIS_MODE_MASK(req.isc); 2186 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2187 break; 2188 default: 2189 ret = -EINVAL; 2190 } 2191 mutex_unlock(&fi->ais_lock); 2192 2193 return ret; 2194 } 2195 2196 static int kvm_s390_inject_airq(struct kvm *kvm, 2197 struct s390_io_adapter *adapter) 2198 { 2199 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2200 struct kvm_s390_interrupt s390int = { 2201 .type = KVM_S390_INT_IO(1, 0, 0, 0), 2202 .parm = 0, 2203 .parm64 = (adapter->isc << 27) | 0x80000000, 2204 }; 2205 int ret = 0; 2206 2207 if (!fi->ais_enabled || !adapter->suppressible) 2208 return kvm_s390_inject_vm(kvm, &s390int); 2209 2210 mutex_lock(&fi->ais_lock); 2211 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { 2212 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); 2213 goto out; 2214 } 2215 2216 ret = kvm_s390_inject_vm(kvm, &s390int); 2217 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { 2218 fi->nimm |= AIS_MODE_MASK(adapter->isc); 2219 trace_kvm_s390_modify_ais_mode(adapter->isc, 2220 KVM_S390_AIS_MODE_SINGLE, 2); 2221 } 2222 out: 2223 mutex_unlock(&fi->ais_lock); 2224 return ret; 2225 } 2226 2227 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) 2228 { 2229 unsigned int id = attr->attr; 2230 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2231 2232 if (!adapter) 2233 return -EINVAL; 2234 2235 return kvm_s390_inject_airq(kvm, adapter); 2236 } 2237 2238 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2239 { 2240 int r = 0; 2241 unsigned int i; 2242 struct kvm_vcpu *vcpu; 2243 2244 switch (attr->group) { 2245 case KVM_DEV_FLIC_ENQUEUE: 2246 r = enqueue_floating_irq(dev, attr); 2247 break; 2248 case KVM_DEV_FLIC_CLEAR_IRQS: 2249 kvm_s390_clear_float_irqs(dev->kvm); 2250 break; 2251 case KVM_DEV_FLIC_APF_ENABLE: 2252 dev->kvm->arch.gmap->pfault_enabled = 1; 2253 break; 2254 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2255 dev->kvm->arch.gmap->pfault_enabled = 0; 2256 /* 2257 * Make sure no async faults are in transition when 2258 * clearing the queues. So we don't need to worry 2259 * about late coming workers. 2260 */ 2261 synchronize_srcu(&dev->kvm->srcu); 2262 kvm_for_each_vcpu(i, vcpu, dev->kvm) 2263 kvm_clear_async_pf_completion_queue(vcpu); 2264 break; 2265 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2266 r = register_io_adapter(dev, attr); 2267 break; 2268 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2269 r = modify_io_adapter(dev, attr); 2270 break; 2271 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2272 r = clear_io_irq(dev->kvm, attr); 2273 break; 2274 case KVM_DEV_FLIC_AISM: 2275 r = modify_ais_mode(dev->kvm, attr); 2276 break; 2277 case KVM_DEV_FLIC_AIRQ_INJECT: 2278 r = flic_inject_airq(dev->kvm, attr); 2279 break; 2280 default: 2281 r = -EINVAL; 2282 } 2283 2284 return r; 2285 } 2286 2287 static int flic_has_attr(struct kvm_device *dev, 2288 struct kvm_device_attr *attr) 2289 { 2290 switch (attr->group) { 2291 case KVM_DEV_FLIC_GET_ALL_IRQS: 2292 case KVM_DEV_FLIC_ENQUEUE: 2293 case KVM_DEV_FLIC_CLEAR_IRQS: 2294 case KVM_DEV_FLIC_APF_ENABLE: 2295 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2296 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2297 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2298 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2299 case KVM_DEV_FLIC_AISM: 2300 case KVM_DEV_FLIC_AIRQ_INJECT: 2301 return 0; 2302 } 2303 return -ENXIO; 2304 } 2305 2306 static int flic_create(struct kvm_device *dev, u32 type) 2307 { 2308 if (!dev) 2309 return -EINVAL; 2310 if (dev->kvm->arch.flic) 2311 return -EINVAL; 2312 dev->kvm->arch.flic = dev; 2313 return 0; 2314 } 2315 2316 static void flic_destroy(struct kvm_device *dev) 2317 { 2318 dev->kvm->arch.flic = NULL; 2319 kfree(dev); 2320 } 2321 2322 /* s390 floating irq controller (flic) */ 2323 struct kvm_device_ops kvm_flic_ops = { 2324 .name = "kvm-flic", 2325 .get_attr = flic_get_attr, 2326 .set_attr = flic_set_attr, 2327 .has_attr = flic_has_attr, 2328 .create = flic_create, 2329 .destroy = flic_destroy, 2330 }; 2331 2332 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) 2333 { 2334 unsigned long bit; 2335 2336 bit = bit_nr + (addr % PAGE_SIZE) * 8; 2337 2338 return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; 2339 } 2340 2341 static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, 2342 u64 addr) 2343 { 2344 struct s390_map_info *map; 2345 2346 if (!adapter) 2347 return NULL; 2348 2349 list_for_each_entry(map, &adapter->maps, list) { 2350 if (map->guest_addr == addr) 2351 return map; 2352 } 2353 return NULL; 2354 } 2355 2356 static int adapter_indicators_set(struct kvm *kvm, 2357 struct s390_io_adapter *adapter, 2358 struct kvm_s390_adapter_int *adapter_int) 2359 { 2360 unsigned long bit; 2361 int summary_set, idx; 2362 struct s390_map_info *info; 2363 void *map; 2364 2365 info = get_map_info(adapter, adapter_int->ind_addr); 2366 if (!info) 2367 return -1; 2368 map = page_address(info->page); 2369 bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap); 2370 set_bit(bit, map); 2371 idx = srcu_read_lock(&kvm->srcu); 2372 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2373 set_page_dirty_lock(info->page); 2374 info = get_map_info(adapter, adapter_int->summary_addr); 2375 if (!info) { 2376 srcu_read_unlock(&kvm->srcu, idx); 2377 return -1; 2378 } 2379 map = page_address(info->page); 2380 bit = get_ind_bit(info->addr, adapter_int->summary_offset, 2381 adapter->swap); 2382 summary_set = test_and_set_bit(bit, map); 2383 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2384 set_page_dirty_lock(info->page); 2385 srcu_read_unlock(&kvm->srcu, idx); 2386 return summary_set ? 0 : 1; 2387 } 2388 2389 /* 2390 * < 0 - not injected due to error 2391 * = 0 - coalesced, summary indicator already active 2392 * > 0 - injected interrupt 2393 */ 2394 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, 2395 struct kvm *kvm, int irq_source_id, int level, 2396 bool line_status) 2397 { 2398 int ret; 2399 struct s390_io_adapter *adapter; 2400 2401 /* We're only interested in the 0->1 transition. */ 2402 if (!level) 2403 return 0; 2404 adapter = get_io_adapter(kvm, e->adapter.adapter_id); 2405 if (!adapter) 2406 return -1; 2407 down_read(&adapter->maps_lock); 2408 ret = adapter_indicators_set(kvm, adapter, &e->adapter); 2409 up_read(&adapter->maps_lock); 2410 if ((ret > 0) && !adapter->masked) { 2411 ret = kvm_s390_inject_airq(kvm, adapter); 2412 if (ret == 0) 2413 ret = 1; 2414 } 2415 return ret; 2416 } 2417 2418 int kvm_set_routing_entry(struct kvm *kvm, 2419 struct kvm_kernel_irq_routing_entry *e, 2420 const struct kvm_irq_routing_entry *ue) 2421 { 2422 int ret; 2423 2424 switch (ue->type) { 2425 case KVM_IRQ_ROUTING_S390_ADAPTER: 2426 e->set = set_adapter_int; 2427 e->adapter.summary_addr = ue->u.adapter.summary_addr; 2428 e->adapter.ind_addr = ue->u.adapter.ind_addr; 2429 e->adapter.summary_offset = ue->u.adapter.summary_offset; 2430 e->adapter.ind_offset = ue->u.adapter.ind_offset; 2431 e->adapter.adapter_id = ue->u.adapter.adapter_id; 2432 ret = 0; 2433 break; 2434 default: 2435 ret = -EINVAL; 2436 } 2437 2438 return ret; 2439 } 2440 2441 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 2442 int irq_source_id, int level, bool line_status) 2443 { 2444 return -EINVAL; 2445 } 2446 2447 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) 2448 { 2449 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2450 struct kvm_s390_irq *buf; 2451 int r = 0; 2452 int n; 2453 2454 buf = vmalloc(len); 2455 if (!buf) 2456 return -ENOMEM; 2457 2458 if (copy_from_user((void *) buf, irqstate, len)) { 2459 r = -EFAULT; 2460 goto out_free; 2461 } 2462 2463 /* 2464 * Don't allow setting the interrupt state 2465 * when there are already interrupts pending 2466 */ 2467 spin_lock(&li->lock); 2468 if (li->pending_irqs) { 2469 r = -EBUSY; 2470 goto out_unlock; 2471 } 2472 2473 for (n = 0; n < len / sizeof(*buf); n++) { 2474 r = do_inject_vcpu(vcpu, &buf[n]); 2475 if (r) 2476 break; 2477 } 2478 2479 out_unlock: 2480 spin_unlock(&li->lock); 2481 out_free: 2482 vfree(buf); 2483 2484 return r; 2485 } 2486 2487 static void store_local_irq(struct kvm_s390_local_interrupt *li, 2488 struct kvm_s390_irq *irq, 2489 unsigned long irq_type) 2490 { 2491 switch (irq_type) { 2492 case IRQ_PEND_MCHK_EX: 2493 case IRQ_PEND_MCHK_REP: 2494 irq->type = KVM_S390_MCHK; 2495 irq->u.mchk = li->irq.mchk; 2496 break; 2497 case IRQ_PEND_PROG: 2498 irq->type = KVM_S390_PROGRAM_INT; 2499 irq->u.pgm = li->irq.pgm; 2500 break; 2501 case IRQ_PEND_PFAULT_INIT: 2502 irq->type = KVM_S390_INT_PFAULT_INIT; 2503 irq->u.ext = li->irq.ext; 2504 break; 2505 case IRQ_PEND_EXT_EXTERNAL: 2506 irq->type = KVM_S390_INT_EXTERNAL_CALL; 2507 irq->u.extcall = li->irq.extcall; 2508 break; 2509 case IRQ_PEND_EXT_CLOCK_COMP: 2510 irq->type = KVM_S390_INT_CLOCK_COMP; 2511 break; 2512 case IRQ_PEND_EXT_CPU_TIMER: 2513 irq->type = KVM_S390_INT_CPU_TIMER; 2514 break; 2515 case IRQ_PEND_SIGP_STOP: 2516 irq->type = KVM_S390_SIGP_STOP; 2517 irq->u.stop = li->irq.stop; 2518 break; 2519 case IRQ_PEND_RESTART: 2520 irq->type = KVM_S390_RESTART; 2521 break; 2522 case IRQ_PEND_SET_PREFIX: 2523 irq->type = KVM_S390_SIGP_SET_PREFIX; 2524 irq->u.prefix = li->irq.prefix; 2525 break; 2526 } 2527 } 2528 2529 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) 2530 { 2531 int scn; 2532 unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 2533 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2534 unsigned long pending_irqs; 2535 struct kvm_s390_irq irq; 2536 unsigned long irq_type; 2537 int cpuaddr; 2538 int n = 0; 2539 2540 spin_lock(&li->lock); 2541 pending_irqs = li->pending_irqs; 2542 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending, 2543 sizeof(sigp_emerg_pending)); 2544 spin_unlock(&li->lock); 2545 2546 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) { 2547 memset(&irq, 0, sizeof(irq)); 2548 if (irq_type == IRQ_PEND_EXT_EMERGENCY) 2549 continue; 2550 if (n + sizeof(irq) > len) 2551 return -ENOBUFS; 2552 store_local_irq(&vcpu->arch.local_int, &irq, irq_type); 2553 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2554 return -EFAULT; 2555 n += sizeof(irq); 2556 } 2557 2558 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) { 2559 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) { 2560 memset(&irq, 0, sizeof(irq)); 2561 if (n + sizeof(irq) > len) 2562 return -ENOBUFS; 2563 irq.type = KVM_S390_INT_EMERGENCY; 2564 irq.u.emerg.code = cpuaddr; 2565 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2566 return -EFAULT; 2567 n += sizeof(irq); 2568 } 2569 } 2570 2571 if (sca_ext_call_pending(vcpu, &scn)) { 2572 if (n + sizeof(irq) > len) 2573 return -ENOBUFS; 2574 memset(&irq, 0, sizeof(irq)); 2575 irq.type = KVM_S390_INT_EXTERNAL_CALL; 2576 irq.u.extcall.code = scn; 2577 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2578 return -EFAULT; 2579 n += sizeof(irq); 2580 } 2581 2582 return n; 2583 } 2584