1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * in-kernel handling for sie intercepts 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com> 9 */ 10 11 #include <linux/kvm_host.h> 12 #include <linux/errno.h> 13 #include <linux/pagemap.h> 14 15 #include <asm/asm-offsets.h> 16 #include <asm/irq.h> 17 #include <asm/sysinfo.h> 18 #include <asm/uv.h> 19 20 #include "kvm-s390.h" 21 #include "gaccess.h" 22 #include "trace.h" 23 #include "trace-s390.h" 24 25 u8 kvm_s390_get_ilen(struct kvm_vcpu *vcpu) 26 { 27 struct kvm_s390_sie_block *sie_block = vcpu->arch.sie_block; 28 u8 ilen = 0; 29 30 switch (vcpu->arch.sie_block->icptcode) { 31 case ICPT_INST: 32 case ICPT_INSTPROGI: 33 case ICPT_OPEREXC: 34 case ICPT_PARTEXEC: 35 case ICPT_IOINST: 36 /* instruction only stored for these icptcodes */ 37 ilen = insn_length(vcpu->arch.sie_block->ipa >> 8); 38 /* Use the length of the EXECUTE instruction if necessary */ 39 if (sie_block->icptstatus & 1) { 40 ilen = (sie_block->icptstatus >> 4) & 0x6; 41 if (!ilen) 42 ilen = 4; 43 } 44 break; 45 case ICPT_PROGI: 46 /* bit 1+2 of pgmilc are the ilc, so we directly get ilen */ 47 ilen = vcpu->arch.sie_block->pgmilc & 0x6; 48 break; 49 } 50 return ilen; 51 } 52 53 static int handle_stop(struct kvm_vcpu *vcpu) 54 { 55 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 56 int rc = 0; 57 uint8_t flags, stop_pending; 58 59 vcpu->stat.exit_stop_request++; 60 61 /* delay the stop if any non-stop irq is pending */ 62 if (kvm_s390_vcpu_has_irq(vcpu, 1)) 63 return 0; 64 65 /* avoid races with the injection/SIGP STOP code */ 66 spin_lock(&li->lock); 67 flags = li->irq.stop.flags; 68 stop_pending = kvm_s390_is_stop_irq_pending(vcpu); 69 spin_unlock(&li->lock); 70 71 trace_kvm_s390_stop_request(stop_pending, flags); 72 if (!stop_pending) 73 return 0; 74 75 if (flags & KVM_S390_STOP_FLAG_STORE_STATUS) { 76 rc = kvm_s390_vcpu_store_status(vcpu, 77 KVM_S390_STORE_STATUS_NOADDR); 78 if (rc) 79 return rc; 80 } 81 82 /* 83 * no need to check the return value of vcpu_stop as it can only have 84 * an error for protvirt, but protvirt means user cpu state 85 */ 86 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) 87 kvm_s390_vcpu_stop(vcpu); 88 return -EOPNOTSUPP; 89 } 90 91 static int handle_validity(struct kvm_vcpu *vcpu) 92 { 93 int viwhy = vcpu->arch.sie_block->ipb >> 16; 94 95 vcpu->stat.exit_validity++; 96 trace_kvm_s390_intercept_validity(vcpu, viwhy); 97 KVM_EVENT(3, "validity intercept 0x%x for pid %u (kvm 0x%pK)", viwhy, 98 current->pid, vcpu->kvm); 99 100 /* do not warn on invalid runtime instrumentation mode */ 101 WARN_ONCE(viwhy != 0x44, "kvm: unhandled validity intercept 0x%x\n", 102 viwhy); 103 return -EINVAL; 104 } 105 106 static int handle_instruction(struct kvm_vcpu *vcpu) 107 { 108 vcpu->stat.exit_instruction++; 109 trace_kvm_s390_intercept_instruction(vcpu, 110 vcpu->arch.sie_block->ipa, 111 vcpu->arch.sie_block->ipb); 112 113 switch (vcpu->arch.sie_block->ipa >> 8) { 114 case 0x01: 115 return kvm_s390_handle_01(vcpu); 116 case 0x82: 117 return kvm_s390_handle_lpsw(vcpu); 118 case 0x83: 119 return kvm_s390_handle_diag(vcpu); 120 case 0xaa: 121 return kvm_s390_handle_aa(vcpu); 122 case 0xae: 123 return kvm_s390_handle_sigp(vcpu); 124 case 0xb2: 125 return kvm_s390_handle_b2(vcpu); 126 case 0xb6: 127 return kvm_s390_handle_stctl(vcpu); 128 case 0xb7: 129 return kvm_s390_handle_lctl(vcpu); 130 case 0xb9: 131 return kvm_s390_handle_b9(vcpu); 132 case 0xe3: 133 return kvm_s390_handle_e3(vcpu); 134 case 0xe5: 135 return kvm_s390_handle_e5(vcpu); 136 case 0xeb: 137 return kvm_s390_handle_eb(vcpu); 138 default: 139 return -EOPNOTSUPP; 140 } 141 } 142 143 static int inject_prog_on_prog_intercept(struct kvm_vcpu *vcpu) 144 { 145 struct kvm_s390_pgm_info pgm_info = { 146 .code = vcpu->arch.sie_block->iprcc, 147 /* the PSW has already been rewound */ 148 .flags = KVM_S390_PGM_FLAGS_NO_REWIND, 149 }; 150 151 switch (vcpu->arch.sie_block->iprcc & ~PGM_PER) { 152 case PGM_AFX_TRANSLATION: 153 case PGM_ASX_TRANSLATION: 154 case PGM_EX_TRANSLATION: 155 case PGM_LFX_TRANSLATION: 156 case PGM_LSTE_SEQUENCE: 157 case PGM_LSX_TRANSLATION: 158 case PGM_LX_TRANSLATION: 159 case PGM_PRIMARY_AUTHORITY: 160 case PGM_SECONDARY_AUTHORITY: 161 case PGM_SPACE_SWITCH: 162 pgm_info.trans_exc_code = vcpu->arch.sie_block->tecmc; 163 break; 164 case PGM_ALEN_TRANSLATION: 165 case PGM_ALE_SEQUENCE: 166 case PGM_ASTE_INSTANCE: 167 case PGM_ASTE_SEQUENCE: 168 case PGM_ASTE_VALIDITY: 169 case PGM_EXTENDED_AUTHORITY: 170 pgm_info.exc_access_id = vcpu->arch.sie_block->eai; 171 break; 172 case PGM_ASCE_TYPE: 173 case PGM_PAGE_TRANSLATION: 174 case PGM_REGION_FIRST_TRANS: 175 case PGM_REGION_SECOND_TRANS: 176 case PGM_REGION_THIRD_TRANS: 177 case PGM_SEGMENT_TRANSLATION: 178 pgm_info.trans_exc_code = vcpu->arch.sie_block->tecmc; 179 pgm_info.exc_access_id = vcpu->arch.sie_block->eai; 180 pgm_info.op_access_id = vcpu->arch.sie_block->oai; 181 break; 182 case PGM_MONITOR: 183 pgm_info.mon_class_nr = vcpu->arch.sie_block->mcn; 184 pgm_info.mon_code = vcpu->arch.sie_block->tecmc; 185 break; 186 case PGM_VECTOR_PROCESSING: 187 case PGM_DATA: 188 pgm_info.data_exc_code = vcpu->arch.sie_block->dxc; 189 break; 190 case PGM_PROTECTION: 191 pgm_info.trans_exc_code = vcpu->arch.sie_block->tecmc; 192 pgm_info.exc_access_id = vcpu->arch.sie_block->eai; 193 break; 194 default: 195 break; 196 } 197 198 if (vcpu->arch.sie_block->iprcc & PGM_PER) { 199 pgm_info.per_code = vcpu->arch.sie_block->perc; 200 pgm_info.per_atmid = vcpu->arch.sie_block->peratmid; 201 pgm_info.per_address = vcpu->arch.sie_block->peraddr; 202 pgm_info.per_access_id = vcpu->arch.sie_block->peraid; 203 } 204 return kvm_s390_inject_prog_irq(vcpu, &pgm_info); 205 } 206 207 /* 208 * restore ITDB to program-interruption TDB in guest lowcore 209 * and set TX abort indication if required 210 */ 211 static int handle_itdb(struct kvm_vcpu *vcpu) 212 { 213 struct kvm_s390_itdb *itdb; 214 int rc; 215 216 if (!IS_TE_ENABLED(vcpu) || !IS_ITDB_VALID(vcpu)) 217 return 0; 218 if (current->thread.per_flags & PER_FLAG_NO_TE) 219 return 0; 220 itdb = (struct kvm_s390_itdb *)vcpu->arch.sie_block->itdba; 221 rc = write_guest_lc(vcpu, __LC_PGM_TDB, itdb, sizeof(*itdb)); 222 if (rc) 223 return rc; 224 memset(itdb, 0, sizeof(*itdb)); 225 226 return 0; 227 } 228 229 #define per_event(vcpu) (vcpu->arch.sie_block->iprcc & PGM_PER) 230 231 static int handle_prog(struct kvm_vcpu *vcpu) 232 { 233 psw_t psw; 234 int rc; 235 236 vcpu->stat.exit_program_interruption++; 237 238 /* 239 * Intercept 8 indicates a loop of specification exceptions 240 * for protected guests. 241 */ 242 if (kvm_s390_pv_cpu_is_protected(vcpu)) 243 return -EOPNOTSUPP; 244 245 if (guestdbg_enabled(vcpu) && per_event(vcpu)) { 246 rc = kvm_s390_handle_per_event(vcpu); 247 if (rc) 248 return rc; 249 /* the interrupt might have been filtered out completely */ 250 if (vcpu->arch.sie_block->iprcc == 0) 251 return 0; 252 } 253 254 trace_kvm_s390_intercept_prog(vcpu, vcpu->arch.sie_block->iprcc); 255 if (vcpu->arch.sie_block->iprcc == PGM_SPECIFICATION) { 256 rc = read_guest_lc(vcpu, __LC_PGM_NEW_PSW, &psw, sizeof(psw_t)); 257 if (rc) 258 return rc; 259 /* Avoid endless loops of specification exceptions */ 260 if (!is_valid_psw(&psw)) 261 return -EOPNOTSUPP; 262 } 263 rc = handle_itdb(vcpu); 264 if (rc) 265 return rc; 266 267 return inject_prog_on_prog_intercept(vcpu); 268 } 269 270 /** 271 * handle_external_interrupt - used for external interruption interceptions 272 * @vcpu: virtual cpu 273 * 274 * This interception only occurs if the CPUSTAT_EXT_INT bit was set, or if 275 * the new PSW does not have external interrupts disabled. In the first case, 276 * we've got to deliver the interrupt manually, and in the second case, we 277 * drop to userspace to handle the situation there. 278 */ 279 static int handle_external_interrupt(struct kvm_vcpu *vcpu) 280 { 281 u16 eic = vcpu->arch.sie_block->eic; 282 struct kvm_s390_irq irq; 283 psw_t newpsw; 284 int rc; 285 286 vcpu->stat.exit_external_interrupt++; 287 288 rc = read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &newpsw, sizeof(psw_t)); 289 if (rc) 290 return rc; 291 /* We can not handle clock comparator or timer interrupt with bad PSW */ 292 if ((eic == EXT_IRQ_CLK_COMP || eic == EXT_IRQ_CPU_TIMER) && 293 (newpsw.mask & PSW_MASK_EXT)) 294 return -EOPNOTSUPP; 295 296 switch (eic) { 297 case EXT_IRQ_CLK_COMP: 298 irq.type = KVM_S390_INT_CLOCK_COMP; 299 break; 300 case EXT_IRQ_CPU_TIMER: 301 irq.type = KVM_S390_INT_CPU_TIMER; 302 break; 303 case EXT_IRQ_EXTERNAL_CALL: 304 irq.type = KVM_S390_INT_EXTERNAL_CALL; 305 irq.u.extcall.code = vcpu->arch.sie_block->extcpuaddr; 306 rc = kvm_s390_inject_vcpu(vcpu, &irq); 307 /* ignore if another external call is already pending */ 308 if (rc == -EBUSY) 309 return 0; 310 return rc; 311 default: 312 return -EOPNOTSUPP; 313 } 314 315 return kvm_s390_inject_vcpu(vcpu, &irq); 316 } 317 318 /** 319 * handle_mvpg_pei - Handle MOVE PAGE partial execution interception. 320 * @vcpu: virtual cpu 321 * 322 * This interception can only happen for guests with DAT disabled and 323 * addresses that are currently not mapped in the host. Thus we try to 324 * set up the mappings for the corresponding user pages here (or throw 325 * addressing exceptions in case of illegal guest addresses). 326 */ 327 static int handle_mvpg_pei(struct kvm_vcpu *vcpu) 328 { 329 unsigned long srcaddr, dstaddr; 330 int reg1, reg2, rc; 331 332 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 333 334 /* Make sure that the source is paged-in */ 335 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg2], 336 reg2, &srcaddr, GACC_FETCH); 337 if (rc) 338 return kvm_s390_inject_prog_cond(vcpu, rc); 339 rc = kvm_arch_fault_in_page(vcpu, srcaddr, 0); 340 if (rc != 0) 341 return rc; 342 343 /* Make sure that the destination is paged-in */ 344 rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg1], 345 reg1, &dstaddr, GACC_STORE); 346 if (rc) 347 return kvm_s390_inject_prog_cond(vcpu, rc); 348 rc = kvm_arch_fault_in_page(vcpu, dstaddr, 1); 349 if (rc != 0) 350 return rc; 351 352 kvm_s390_retry_instr(vcpu); 353 354 return 0; 355 } 356 357 static int handle_partial_execution(struct kvm_vcpu *vcpu) 358 { 359 vcpu->stat.exit_pei++; 360 361 if (vcpu->arch.sie_block->ipa == 0xb254) /* MVPG */ 362 return handle_mvpg_pei(vcpu); 363 if (vcpu->arch.sie_block->ipa >> 8 == 0xae) /* SIGP */ 364 return kvm_s390_handle_sigp_pei(vcpu); 365 366 return -EOPNOTSUPP; 367 } 368 369 /* 370 * Handle the sthyi instruction that provides the guest with system 371 * information, like current CPU resources available at each level of 372 * the machine. 373 */ 374 int handle_sthyi(struct kvm_vcpu *vcpu) 375 { 376 int reg1, reg2, r = 0; 377 u64 code, addr, cc = 0, rc = 0; 378 struct sthyi_sctns *sctns = NULL; 379 380 if (!test_kvm_facility(vcpu->kvm, 74)) 381 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 382 383 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 384 code = vcpu->run->s.regs.gprs[reg1]; 385 addr = vcpu->run->s.regs.gprs[reg2]; 386 387 vcpu->stat.instruction_sthyi++; 388 VCPU_EVENT(vcpu, 3, "STHYI: fc: %llu addr: 0x%016llx", code, addr); 389 trace_kvm_s390_handle_sthyi(vcpu, code, addr); 390 391 if (reg1 == reg2 || reg1 & 1 || reg2 & 1) 392 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 393 394 if (code & 0xffff) { 395 cc = 3; 396 rc = 4; 397 goto out; 398 } 399 400 if (!kvm_s390_pv_cpu_is_protected(vcpu) && (addr & ~PAGE_MASK)) 401 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 402 403 sctns = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 404 if (!sctns) 405 return -ENOMEM; 406 407 cc = sthyi_fill(sctns, &rc); 408 409 out: 410 if (!cc) { 411 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 412 memcpy((void *)(sida_origin(vcpu->arch.sie_block)), 413 sctns, PAGE_SIZE); 414 } else { 415 r = write_guest(vcpu, addr, reg2, sctns, PAGE_SIZE); 416 if (r) { 417 free_page((unsigned long)sctns); 418 return kvm_s390_inject_prog_cond(vcpu, r); 419 } 420 } 421 } 422 423 free_page((unsigned long)sctns); 424 vcpu->run->s.regs.gprs[reg2 + 1] = rc; 425 kvm_s390_set_psw_cc(vcpu, cc); 426 return r; 427 } 428 429 static int handle_operexc(struct kvm_vcpu *vcpu) 430 { 431 psw_t oldpsw, newpsw; 432 int rc; 433 434 vcpu->stat.exit_operation_exception++; 435 trace_kvm_s390_handle_operexc(vcpu, vcpu->arch.sie_block->ipa, 436 vcpu->arch.sie_block->ipb); 437 438 if (vcpu->arch.sie_block->ipa == 0xb256) 439 return handle_sthyi(vcpu); 440 441 if (vcpu->arch.sie_block->ipa == 0 && vcpu->kvm->arch.user_instr0) 442 return -EOPNOTSUPP; 443 rc = read_guest_lc(vcpu, __LC_PGM_NEW_PSW, &newpsw, sizeof(psw_t)); 444 if (rc) 445 return rc; 446 /* 447 * Avoid endless loops of operation exceptions, if the pgm new 448 * PSW will cause a new operation exception. 449 * The heuristic checks if the pgm new psw is within 6 bytes before 450 * the faulting psw address (with same DAT, AS settings) and the 451 * new psw is not a wait psw and the fault was not triggered by 452 * problem state. 453 */ 454 oldpsw = vcpu->arch.sie_block->gpsw; 455 if (oldpsw.addr - newpsw.addr <= 6 && 456 !(newpsw.mask & PSW_MASK_WAIT) && 457 !(oldpsw.mask & PSW_MASK_PSTATE) && 458 (newpsw.mask & PSW_MASK_ASC) == (oldpsw.mask & PSW_MASK_ASC) && 459 (newpsw.mask & PSW_MASK_DAT) == (oldpsw.mask & PSW_MASK_DAT)) 460 return -EOPNOTSUPP; 461 462 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 463 } 464 465 static int handle_pv_spx(struct kvm_vcpu *vcpu) 466 { 467 u32 pref = *(u32 *)vcpu->arch.sie_block->sidad; 468 469 kvm_s390_set_prefix(vcpu, pref); 470 trace_kvm_s390_handle_prefix(vcpu, 1, pref); 471 return 0; 472 } 473 474 static int handle_pv_sclp(struct kvm_vcpu *vcpu) 475 { 476 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 477 478 spin_lock(&fi->lock); 479 /* 480 * 2 cases: 481 * a: an sccb answering interrupt was already pending or in flight. 482 * As the sccb value is not known we can simply set some value to 483 * trigger delivery of a saved SCCB. UV will then use its saved 484 * copy of the SCCB value. 485 * b: an error SCCB interrupt needs to be injected so we also inject 486 * a fake SCCB address. Firmware will use the proper one. 487 * This makes sure, that both errors and real sccb returns will only 488 * be delivered after a notification intercept (instruction has 489 * finished) but not after others. 490 */ 491 fi->srv_signal.ext_params |= 0x43000; 492 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 493 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); 494 spin_unlock(&fi->lock); 495 return 0; 496 } 497 498 static int handle_pv_uvc(struct kvm_vcpu *vcpu) 499 { 500 struct uv_cb_share *guest_uvcb = (void *)vcpu->arch.sie_block->sidad; 501 struct uv_cb_cts uvcb = { 502 .header.cmd = UVC_CMD_UNPIN_PAGE_SHARED, 503 .header.len = sizeof(uvcb), 504 .guest_handle = kvm_s390_pv_get_handle(vcpu->kvm), 505 .gaddr = guest_uvcb->paddr, 506 }; 507 int rc; 508 509 if (guest_uvcb->header.cmd != UVC_CMD_REMOVE_SHARED_ACCESS) { 510 WARN_ONCE(1, "Unexpected notification intercept for UVC 0x%x\n", 511 guest_uvcb->header.cmd); 512 return 0; 513 } 514 rc = gmap_make_secure(vcpu->arch.gmap, uvcb.gaddr, &uvcb); 515 /* 516 * If the unpin did not succeed, the guest will exit again for the UVC 517 * and we will retry the unpin. 518 */ 519 if (rc == -EINVAL) 520 return 0; 521 /* 522 * If we got -EAGAIN here, we simply return it. It will eventually 523 * get propagated all the way to userspace, which should then try 524 * again. 525 */ 526 return rc; 527 } 528 529 static int handle_pv_notification(struct kvm_vcpu *vcpu) 530 { 531 if (vcpu->arch.sie_block->ipa == 0xb210) 532 return handle_pv_spx(vcpu); 533 if (vcpu->arch.sie_block->ipa == 0xb220) 534 return handle_pv_sclp(vcpu); 535 if (vcpu->arch.sie_block->ipa == 0xb9a4) 536 return handle_pv_uvc(vcpu); 537 538 return handle_instruction(vcpu); 539 } 540 541 int kvm_handle_sie_intercept(struct kvm_vcpu *vcpu) 542 { 543 int rc, per_rc = 0; 544 545 if (kvm_is_ucontrol(vcpu->kvm)) 546 return -EOPNOTSUPP; 547 548 switch (vcpu->arch.sie_block->icptcode) { 549 case ICPT_EXTREQ: 550 vcpu->stat.exit_external_request++; 551 return 0; 552 case ICPT_IOREQ: 553 vcpu->stat.exit_io_request++; 554 return 0; 555 case ICPT_INST: 556 rc = handle_instruction(vcpu); 557 break; 558 case ICPT_PROGI: 559 return handle_prog(vcpu); 560 case ICPT_EXTINT: 561 return handle_external_interrupt(vcpu); 562 case ICPT_WAIT: 563 return kvm_s390_handle_wait(vcpu); 564 case ICPT_VALIDITY: 565 return handle_validity(vcpu); 566 case ICPT_STOP: 567 return handle_stop(vcpu); 568 case ICPT_OPEREXC: 569 rc = handle_operexc(vcpu); 570 break; 571 case ICPT_PARTEXEC: 572 rc = handle_partial_execution(vcpu); 573 break; 574 case ICPT_KSS: 575 rc = kvm_s390_skey_check_enable(vcpu); 576 break; 577 case ICPT_MCHKREQ: 578 case ICPT_INT_ENABLE: 579 /* 580 * PSW bit 13 or a CR (0, 6, 14) changed and we might 581 * now be able to deliver interrupts. The pre-run code 582 * will take care of this. 583 */ 584 rc = 0; 585 break; 586 case ICPT_PV_INSTR: 587 rc = handle_instruction(vcpu); 588 break; 589 case ICPT_PV_NOTIFY: 590 rc = handle_pv_notification(vcpu); 591 break; 592 case ICPT_PV_PREF: 593 rc = 0; 594 gmap_convert_to_secure(vcpu->arch.gmap, 595 kvm_s390_get_prefix(vcpu)); 596 gmap_convert_to_secure(vcpu->arch.gmap, 597 kvm_s390_get_prefix(vcpu) + PAGE_SIZE); 598 break; 599 default: 600 return -EOPNOTSUPP; 601 } 602 603 /* process PER, also if the instrution is processed in user space */ 604 if (vcpu->arch.sie_block->icptstatus & 0x02 && 605 (!rc || rc == -EOPNOTSUPP)) 606 per_rc = kvm_s390_handle_per_ifetch_icpt(vcpu); 607 return per_rc ? per_rc : rc; 608 } 609