1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * handling privileged instructions 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com> 9 */ 10 11 #include <linux/kvm.h> 12 #include <linux/gfp.h> 13 #include <linux/errno.h> 14 #include <linux/mm_types.h> 15 #include <linux/pgtable.h> 16 #include <linux/io.h> 17 #include <asm/asm-offsets.h> 18 #include <asm/facility.h> 19 #include <asm/current.h> 20 #include <asm/debug.h> 21 #include <asm/ebcdic.h> 22 #include <asm/sysinfo.h> 23 #include <asm/page-states.h> 24 #include <asm/gmap.h> 25 #include <asm/ptrace.h> 26 #include <asm/sclp.h> 27 #include <asm/ap.h> 28 #include "gaccess.h" 29 #include "kvm-s390.h" 30 #include "trace.h" 31 32 static int handle_ri(struct kvm_vcpu *vcpu) 33 { 34 vcpu->stat.instruction_ri++; 35 36 if (test_kvm_facility(vcpu->kvm, 64)) { 37 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)"); 38 vcpu->arch.sie_block->ecb3 |= ECB3_RI; 39 kvm_s390_retry_instr(vcpu); 40 return 0; 41 } else 42 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 43 } 44 45 int kvm_s390_handle_aa(struct kvm_vcpu *vcpu) 46 { 47 if ((vcpu->arch.sie_block->ipa & 0xf) <= 4) 48 return handle_ri(vcpu); 49 else 50 return -EOPNOTSUPP; 51 } 52 53 static int handle_gs(struct kvm_vcpu *vcpu) 54 { 55 vcpu->stat.instruction_gs++; 56 57 if (test_kvm_facility(vcpu->kvm, 133)) { 58 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); 59 preempt_disable(); 60 __ctl_set_bit(2, 4); 61 current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; 62 restore_gs_cb(current->thread.gs_cb); 63 preempt_enable(); 64 vcpu->arch.sie_block->ecb |= ECB_GS; 65 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 66 vcpu->arch.gs_enabled = 1; 67 kvm_s390_retry_instr(vcpu); 68 return 0; 69 } else 70 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 71 } 72 73 int kvm_s390_handle_e3(struct kvm_vcpu *vcpu) 74 { 75 int code = vcpu->arch.sie_block->ipb & 0xff; 76 77 if (code == 0x49 || code == 0x4d) 78 return handle_gs(vcpu); 79 else 80 return -EOPNOTSUPP; 81 } 82 /* Handle SCK (SET CLOCK) interception */ 83 static int handle_set_clock(struct kvm_vcpu *vcpu) 84 { 85 struct kvm_s390_vm_tod_clock gtod = { 0 }; 86 int rc; 87 u8 ar; 88 u64 op2; 89 90 vcpu->stat.instruction_sck++; 91 92 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 93 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 94 95 op2 = kvm_s390_get_base_disp_s(vcpu, &ar); 96 if (op2 & 7) /* Operand must be on a doubleword boundary */ 97 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 98 rc = read_guest(vcpu, op2, ar, >od.tod, sizeof(gtod.tod)); 99 if (rc) 100 return kvm_s390_inject_prog_cond(vcpu, rc); 101 102 VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", gtod.tod); 103 /* 104 * To set the TOD clock the kvm lock must be taken, but the vcpu lock 105 * is already held in handle_set_clock. The usual lock order is the 106 * opposite. As SCK is deprecated and should not be used in several 107 * cases, for example when the multiple epoch facility or TOD clock 108 * steering facility is installed (see Principles of Operation), a 109 * slow path can be used. If the lock can not be taken via try_lock, 110 * the instruction will be retried via -EAGAIN at a later point in 111 * time. 112 */ 113 if (!kvm_s390_try_set_tod_clock(vcpu->kvm, >od)) { 114 kvm_s390_retry_instr(vcpu); 115 return -EAGAIN; 116 } 117 118 kvm_s390_set_psw_cc(vcpu, 0); 119 return 0; 120 } 121 122 static int handle_set_prefix(struct kvm_vcpu *vcpu) 123 { 124 u64 operand2; 125 u32 address; 126 int rc; 127 u8 ar; 128 129 vcpu->stat.instruction_spx++; 130 131 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 132 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 133 134 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 135 136 /* must be word boundary */ 137 if (operand2 & 3) 138 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 139 140 /* get the value */ 141 rc = read_guest(vcpu, operand2, ar, &address, sizeof(address)); 142 if (rc) 143 return kvm_s390_inject_prog_cond(vcpu, rc); 144 145 address &= 0x7fffe000u; 146 147 /* 148 * Make sure the new value is valid memory. We only need to check the 149 * first page, since address is 8k aligned and memory pieces are always 150 * at least 1MB aligned and have at least a size of 1MB. 151 */ 152 if (kvm_is_error_gpa(vcpu->kvm, address)) 153 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 154 155 kvm_s390_set_prefix(vcpu, address); 156 trace_kvm_s390_handle_prefix(vcpu, 1, address); 157 return 0; 158 } 159 160 static int handle_store_prefix(struct kvm_vcpu *vcpu) 161 { 162 u64 operand2; 163 u32 address; 164 int rc; 165 u8 ar; 166 167 vcpu->stat.instruction_stpx++; 168 169 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 170 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 171 172 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 173 174 /* must be word boundary */ 175 if (operand2 & 3) 176 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 177 178 address = kvm_s390_get_prefix(vcpu); 179 180 /* get the value */ 181 rc = write_guest(vcpu, operand2, ar, &address, sizeof(address)); 182 if (rc) 183 return kvm_s390_inject_prog_cond(vcpu, rc); 184 185 VCPU_EVENT(vcpu, 3, "STPX: storing prefix 0x%x into 0x%llx", address, operand2); 186 trace_kvm_s390_handle_prefix(vcpu, 0, address); 187 return 0; 188 } 189 190 static int handle_store_cpu_address(struct kvm_vcpu *vcpu) 191 { 192 u16 vcpu_id = vcpu->vcpu_id; 193 u64 ga; 194 int rc; 195 u8 ar; 196 197 vcpu->stat.instruction_stap++; 198 199 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 200 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 201 202 ga = kvm_s390_get_base_disp_s(vcpu, &ar); 203 204 if (ga & 1) 205 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 206 207 rc = write_guest(vcpu, ga, ar, &vcpu_id, sizeof(vcpu_id)); 208 if (rc) 209 return kvm_s390_inject_prog_cond(vcpu, rc); 210 211 VCPU_EVENT(vcpu, 3, "STAP: storing cpu address (%u) to 0x%llx", vcpu_id, ga); 212 trace_kvm_s390_handle_stap(vcpu, ga); 213 return 0; 214 } 215 216 int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu) 217 { 218 int rc; 219 220 trace_kvm_s390_skey_related_inst(vcpu); 221 /* Already enabled? */ 222 if (vcpu->arch.skey_enabled) 223 return 0; 224 225 rc = s390_enable_skey(); 226 VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc); 227 if (rc) 228 return rc; 229 230 if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS)) 231 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS); 232 if (!vcpu->kvm->arch.use_skf) 233 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 234 else 235 vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE); 236 vcpu->arch.skey_enabled = true; 237 return 0; 238 } 239 240 static int try_handle_skey(struct kvm_vcpu *vcpu) 241 { 242 int rc; 243 244 rc = kvm_s390_skey_check_enable(vcpu); 245 if (rc) 246 return rc; 247 if (vcpu->kvm->arch.use_skf) { 248 /* with storage-key facility, SIE interprets it for us */ 249 kvm_s390_retry_instr(vcpu); 250 VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation"); 251 return -EAGAIN; 252 } 253 return 0; 254 } 255 256 static int handle_iske(struct kvm_vcpu *vcpu) 257 { 258 unsigned long gaddr, vmaddr; 259 unsigned char key; 260 int reg1, reg2; 261 bool unlocked; 262 int rc; 263 264 vcpu->stat.instruction_iske++; 265 266 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 267 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 268 269 rc = try_handle_skey(vcpu); 270 if (rc) 271 return rc != -EAGAIN ? rc : 0; 272 273 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 274 275 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 276 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 277 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 278 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 279 if (kvm_is_error_hva(vmaddr)) 280 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 281 retry: 282 unlocked = false; 283 mmap_read_lock(current->mm); 284 rc = get_guest_storage_key(current->mm, vmaddr, &key); 285 286 if (rc) { 287 rc = fixup_user_fault(current->mm, vmaddr, 288 FAULT_FLAG_WRITE, &unlocked); 289 if (!rc) { 290 mmap_read_unlock(current->mm); 291 goto retry; 292 } 293 } 294 mmap_read_unlock(current->mm); 295 if (rc == -EFAULT) 296 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 297 if (rc < 0) 298 return rc; 299 vcpu->run->s.regs.gprs[reg1] &= ~0xff; 300 vcpu->run->s.regs.gprs[reg1] |= key; 301 return 0; 302 } 303 304 static int handle_rrbe(struct kvm_vcpu *vcpu) 305 { 306 unsigned long vmaddr, gaddr; 307 int reg1, reg2; 308 bool unlocked; 309 int rc; 310 311 vcpu->stat.instruction_rrbe++; 312 313 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 314 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 315 316 rc = try_handle_skey(vcpu); 317 if (rc) 318 return rc != -EAGAIN ? rc : 0; 319 320 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 321 322 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 323 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 324 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 325 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 326 if (kvm_is_error_hva(vmaddr)) 327 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 328 retry: 329 unlocked = false; 330 mmap_read_lock(current->mm); 331 rc = reset_guest_reference_bit(current->mm, vmaddr); 332 if (rc < 0) { 333 rc = fixup_user_fault(current->mm, vmaddr, 334 FAULT_FLAG_WRITE, &unlocked); 335 if (!rc) { 336 mmap_read_unlock(current->mm); 337 goto retry; 338 } 339 } 340 mmap_read_unlock(current->mm); 341 if (rc == -EFAULT) 342 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 343 if (rc < 0) 344 return rc; 345 kvm_s390_set_psw_cc(vcpu, rc); 346 return 0; 347 } 348 349 #define SSKE_NQ 0x8 350 #define SSKE_MR 0x4 351 #define SSKE_MC 0x2 352 #define SSKE_MB 0x1 353 static int handle_sske(struct kvm_vcpu *vcpu) 354 { 355 unsigned char m3 = vcpu->arch.sie_block->ipb >> 28; 356 unsigned long start, end; 357 unsigned char key, oldkey; 358 int reg1, reg2; 359 bool unlocked; 360 int rc; 361 362 vcpu->stat.instruction_sske++; 363 364 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 365 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 366 367 rc = try_handle_skey(vcpu); 368 if (rc) 369 return rc != -EAGAIN ? rc : 0; 370 371 if (!test_kvm_facility(vcpu->kvm, 8)) 372 m3 &= ~SSKE_MB; 373 if (!test_kvm_facility(vcpu->kvm, 10)) 374 m3 &= ~(SSKE_MC | SSKE_MR); 375 if (!test_kvm_facility(vcpu->kvm, 14)) 376 m3 &= ~SSKE_NQ; 377 378 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 379 380 key = vcpu->run->s.regs.gprs[reg1] & 0xfe; 381 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 382 start = kvm_s390_logical_to_effective(vcpu, start); 383 if (m3 & SSKE_MB) { 384 /* start already designates an absolute address */ 385 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 386 } else { 387 start = kvm_s390_real_to_abs(vcpu, start); 388 end = start + PAGE_SIZE; 389 } 390 391 while (start != end) { 392 unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 393 unlocked = false; 394 395 if (kvm_is_error_hva(vmaddr)) 396 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 397 398 mmap_read_lock(current->mm); 399 rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey, 400 m3 & SSKE_NQ, m3 & SSKE_MR, 401 m3 & SSKE_MC); 402 403 if (rc < 0) { 404 rc = fixup_user_fault(current->mm, vmaddr, 405 FAULT_FLAG_WRITE, &unlocked); 406 rc = !rc ? -EAGAIN : rc; 407 } 408 mmap_read_unlock(current->mm); 409 if (rc == -EFAULT) 410 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 411 if (rc == -EAGAIN) 412 continue; 413 if (rc < 0) 414 return rc; 415 start += PAGE_SIZE; 416 } 417 418 if (m3 & (SSKE_MC | SSKE_MR)) { 419 if (m3 & SSKE_MB) { 420 /* skey in reg1 is unpredictable */ 421 kvm_s390_set_psw_cc(vcpu, 3); 422 } else { 423 kvm_s390_set_psw_cc(vcpu, rc); 424 vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL; 425 vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8; 426 } 427 } 428 if (m3 & SSKE_MB) { 429 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) 430 vcpu->run->s.regs.gprs[reg2] &= ~PAGE_MASK; 431 else 432 vcpu->run->s.regs.gprs[reg2] &= ~0xfffff000UL; 433 end = kvm_s390_logical_to_effective(vcpu, end); 434 vcpu->run->s.regs.gprs[reg2] |= end; 435 } 436 return 0; 437 } 438 439 static int handle_ipte_interlock(struct kvm_vcpu *vcpu) 440 { 441 vcpu->stat.instruction_ipte_interlock++; 442 if (psw_bits(vcpu->arch.sie_block->gpsw).pstate) 443 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 444 wait_event(vcpu->kvm->arch.ipte_wq, !ipte_lock_held(vcpu->kvm)); 445 kvm_s390_retry_instr(vcpu); 446 VCPU_EVENT(vcpu, 4, "%s", "retrying ipte interlock operation"); 447 return 0; 448 } 449 450 static int handle_test_block(struct kvm_vcpu *vcpu) 451 { 452 gpa_t addr; 453 int reg2; 454 455 vcpu->stat.instruction_tb++; 456 457 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 458 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 459 460 kvm_s390_get_regs_rre(vcpu, NULL, ®2); 461 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 462 addr = kvm_s390_logical_to_effective(vcpu, addr); 463 if (kvm_s390_check_low_addr_prot_real(vcpu, addr)) 464 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 465 addr = kvm_s390_real_to_abs(vcpu, addr); 466 467 if (kvm_is_error_gpa(vcpu->kvm, addr)) 468 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 469 /* 470 * We don't expect errors on modern systems, and do not care 471 * about storage keys (yet), so let's just clear the page. 472 */ 473 if (kvm_clear_guest(vcpu->kvm, addr, PAGE_SIZE)) 474 return -EFAULT; 475 kvm_s390_set_psw_cc(vcpu, 0); 476 vcpu->run->s.regs.gprs[0] = 0; 477 return 0; 478 } 479 480 static int handle_tpi(struct kvm_vcpu *vcpu) 481 { 482 struct kvm_s390_interrupt_info *inti; 483 unsigned long len; 484 u32 tpi_data[3]; 485 int rc; 486 u64 addr; 487 u8 ar; 488 489 vcpu->stat.instruction_tpi++; 490 491 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 492 if (addr & 3) 493 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 494 495 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0); 496 if (!inti) { 497 kvm_s390_set_psw_cc(vcpu, 0); 498 return 0; 499 } 500 501 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr; 502 tpi_data[1] = inti->io.io_int_parm; 503 tpi_data[2] = inti->io.io_int_word; 504 if (addr) { 505 /* 506 * Store the two-word I/O interruption code into the 507 * provided area. 508 */ 509 len = sizeof(tpi_data) - 4; 510 rc = write_guest(vcpu, addr, ar, &tpi_data, len); 511 if (rc) { 512 rc = kvm_s390_inject_prog_cond(vcpu, rc); 513 goto reinject_interrupt; 514 } 515 } else { 516 /* 517 * Store the three-word I/O interruption code into 518 * the appropriate lowcore area. 519 */ 520 len = sizeof(tpi_data); 521 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) { 522 /* failed writes to the low core are not recoverable */ 523 rc = -EFAULT; 524 goto reinject_interrupt; 525 } 526 } 527 528 /* irq was successfully handed to the guest */ 529 kfree(inti); 530 kvm_s390_set_psw_cc(vcpu, 1); 531 return 0; 532 reinject_interrupt: 533 /* 534 * If we encounter a problem storing the interruption code, the 535 * instruction is suppressed from the guest's view: reinject the 536 * interrupt. 537 */ 538 if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) { 539 kfree(inti); 540 rc = -EFAULT; 541 } 542 /* don't set the cc, a pgm irq was injected or we drop to user space */ 543 return rc ? -EFAULT : 0; 544 } 545 546 static int handle_tsch(struct kvm_vcpu *vcpu) 547 { 548 struct kvm_s390_interrupt_info *inti = NULL; 549 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 550 551 vcpu->stat.instruction_tsch++; 552 553 /* a valid schid has at least one bit set */ 554 if (vcpu->run->s.regs.gprs[1]) 555 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask, 556 vcpu->run->s.regs.gprs[1]); 557 558 /* 559 * Prepare exit to userspace. 560 * We indicate whether we dequeued a pending I/O interrupt 561 * so that userspace can re-inject it if the instruction gets 562 * a program check. While this may re-order the pending I/O 563 * interrupts, this is no problem since the priority is kept 564 * intact. 565 */ 566 vcpu->run->exit_reason = KVM_EXIT_S390_TSCH; 567 vcpu->run->s390_tsch.dequeued = !!inti; 568 if (inti) { 569 vcpu->run->s390_tsch.subchannel_id = inti->io.subchannel_id; 570 vcpu->run->s390_tsch.subchannel_nr = inti->io.subchannel_nr; 571 vcpu->run->s390_tsch.io_int_parm = inti->io.io_int_parm; 572 vcpu->run->s390_tsch.io_int_word = inti->io.io_int_word; 573 } 574 vcpu->run->s390_tsch.ipb = vcpu->arch.sie_block->ipb; 575 kfree(inti); 576 return -EREMOTE; 577 } 578 579 static int handle_io_inst(struct kvm_vcpu *vcpu) 580 { 581 VCPU_EVENT(vcpu, 4, "%s", "I/O instruction"); 582 583 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 584 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 585 586 if (vcpu->kvm->arch.css_support) { 587 /* 588 * Most I/O instructions will be handled by userspace. 589 * Exceptions are tpi and the interrupt portion of tsch. 590 */ 591 if (vcpu->arch.sie_block->ipa == 0xb236) 592 return handle_tpi(vcpu); 593 if (vcpu->arch.sie_block->ipa == 0xb235) 594 return handle_tsch(vcpu); 595 /* Handle in userspace. */ 596 vcpu->stat.instruction_io_other++; 597 return -EOPNOTSUPP; 598 } else { 599 /* 600 * Set condition code 3 to stop the guest from issuing channel 601 * I/O instructions. 602 */ 603 kvm_s390_set_psw_cc(vcpu, 3); 604 return 0; 605 } 606 } 607 608 /* 609 * handle_pqap: Handling pqap interception 610 * @vcpu: the vcpu having issue the pqap instruction 611 * 612 * We now support PQAP/AQIC instructions and we need to correctly 613 * answer the guest even if no dedicated driver's hook is available. 614 * 615 * The intercepting code calls a dedicated callback for this instruction 616 * if a driver did register one in the CRYPTO satellite of the 617 * SIE block. 618 * 619 * If no callback is available, the queues are not available, return this 620 * response code to the caller and set CC to 3. 621 * Else return the response code returned by the callback. 622 */ 623 static int handle_pqap(struct kvm_vcpu *vcpu) 624 { 625 struct ap_queue_status status = {}; 626 crypto_hook pqap_hook; 627 unsigned long reg0; 628 int ret; 629 uint8_t fc; 630 631 /* Verify that the AP instruction are available */ 632 if (!ap_instructions_available()) 633 return -EOPNOTSUPP; 634 /* Verify that the guest is allowed to use AP instructions */ 635 if (!(vcpu->arch.sie_block->eca & ECA_APIE)) 636 return -EOPNOTSUPP; 637 /* 638 * The only possibly intercepted functions when AP instructions are 639 * available for the guest are AQIC and TAPQ with the t bit set 640 * since we do not set IC.3 (FIII) we currently will only intercept 641 * the AQIC function code. 642 * Note: running nested under z/VM can result in intercepts for other 643 * function codes, e.g. PQAP(QCI). We do not support this and bail out. 644 */ 645 reg0 = vcpu->run->s.regs.gprs[0]; 646 fc = (reg0 >> 24) & 0xff; 647 if (fc != 0x03) 648 return -EOPNOTSUPP; 649 650 /* PQAP instruction is allowed for guest kernel only */ 651 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 652 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 653 654 /* Common PQAP instruction specification exceptions */ 655 /* bits 41-47 must all be zeros */ 656 if (reg0 & 0x007f0000UL) 657 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 658 /* APFT not install and T bit set */ 659 if (!test_kvm_facility(vcpu->kvm, 15) && (reg0 & 0x00800000UL)) 660 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 661 /* APXA not installed and APID greater 64 or APQI greater 16 */ 662 if (!(vcpu->kvm->arch.crypto.crycbd & 0x02) && (reg0 & 0x0000c0f0UL)) 663 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 664 665 /* AQIC function code specific exception */ 666 /* facility 65 not present for AQIC function code */ 667 if (!test_kvm_facility(vcpu->kvm, 65)) 668 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 669 670 /* 671 * If the hook callback is registered, there will be a pointer to the 672 * hook function pointer in the kvm_s390_crypto structure. Lock the 673 * owner, retrieve the hook function pointer and call the hook. 674 */ 675 down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 676 if (vcpu->kvm->arch.crypto.pqap_hook) { 677 pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook; 678 ret = pqap_hook(vcpu); 679 if (!ret && vcpu->run->s.regs.gprs[1] & 0x00ff0000) 680 kvm_s390_set_psw_cc(vcpu, 3); 681 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 682 return ret; 683 } 684 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 685 /* 686 * A vfio_driver must register a hook. 687 * No hook means no driver to enable the SIE CRYCB and no queues. 688 * We send this response to the guest. 689 */ 690 status.response_code = 0x01; 691 memcpy(&vcpu->run->s.regs.gprs[1], &status, sizeof(status)); 692 kvm_s390_set_psw_cc(vcpu, 3); 693 return 0; 694 } 695 696 static int handle_stfl(struct kvm_vcpu *vcpu) 697 { 698 int rc; 699 unsigned int fac; 700 701 vcpu->stat.instruction_stfl++; 702 703 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 704 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 705 706 /* 707 * We need to shift the lower 32 facility bits (bit 0-31) from a u64 708 * into a u32 memory representation. They will remain bits 0-31. 709 */ 710 fac = *vcpu->kvm->arch.model.fac_list >> 32; 711 rc = write_guest_lc(vcpu, offsetof(struct lowcore, stfl_fac_list), 712 &fac, sizeof(fac)); 713 if (rc) 714 return rc; 715 VCPU_EVENT(vcpu, 3, "STFL: store facility list 0x%x", fac); 716 trace_kvm_s390_handle_stfl(vcpu, fac); 717 return 0; 718 } 719 720 #define PSW_MASK_ADDR_MODE (PSW_MASK_EA | PSW_MASK_BA) 721 #define PSW_MASK_UNASSIGNED 0xb80800fe7fffffffUL 722 #define PSW_ADDR_24 0x0000000000ffffffUL 723 #define PSW_ADDR_31 0x000000007fffffffUL 724 725 int is_valid_psw(psw_t *psw) 726 { 727 if (psw->mask & PSW_MASK_UNASSIGNED) 728 return 0; 729 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_BA) { 730 if (psw->addr & ~PSW_ADDR_31) 731 return 0; 732 } 733 if (!(psw->mask & PSW_MASK_ADDR_MODE) && (psw->addr & ~PSW_ADDR_24)) 734 return 0; 735 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_EA) 736 return 0; 737 if (psw->addr & 1) 738 return 0; 739 return 1; 740 } 741 742 int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu) 743 { 744 psw_t *gpsw = &vcpu->arch.sie_block->gpsw; 745 psw_compat_t new_psw; 746 u64 addr; 747 int rc; 748 u8 ar; 749 750 vcpu->stat.instruction_lpsw++; 751 752 if (gpsw->mask & PSW_MASK_PSTATE) 753 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 754 755 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 756 if (addr & 7) 757 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 758 759 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 760 if (rc) 761 return kvm_s390_inject_prog_cond(vcpu, rc); 762 if (!(new_psw.mask & PSW32_MASK_BASE)) 763 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 764 gpsw->mask = (new_psw.mask & ~PSW32_MASK_BASE) << 32; 765 gpsw->mask |= new_psw.addr & PSW32_ADDR_AMODE; 766 gpsw->addr = new_psw.addr & ~PSW32_ADDR_AMODE; 767 if (!is_valid_psw(gpsw)) 768 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 769 return 0; 770 } 771 772 static int handle_lpswe(struct kvm_vcpu *vcpu) 773 { 774 psw_t new_psw; 775 u64 addr; 776 int rc; 777 u8 ar; 778 779 vcpu->stat.instruction_lpswe++; 780 781 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 782 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 783 784 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 785 if (addr & 7) 786 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 787 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 788 if (rc) 789 return kvm_s390_inject_prog_cond(vcpu, rc); 790 vcpu->arch.sie_block->gpsw = new_psw; 791 if (!is_valid_psw(&vcpu->arch.sie_block->gpsw)) 792 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 793 return 0; 794 } 795 796 static int handle_lpswey(struct kvm_vcpu *vcpu) 797 { 798 psw_t new_psw; 799 u64 addr; 800 int rc; 801 u8 ar; 802 803 vcpu->stat.instruction_lpswey++; 804 805 if (!test_kvm_facility(vcpu->kvm, 193)) 806 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 807 808 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 809 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 810 811 addr = kvm_s390_get_base_disp_siy(vcpu, &ar); 812 if (addr & 7) 813 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 814 815 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 816 if (rc) 817 return kvm_s390_inject_prog_cond(vcpu, rc); 818 819 vcpu->arch.sie_block->gpsw = new_psw; 820 if (!is_valid_psw(&vcpu->arch.sie_block->gpsw)) 821 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 822 823 return 0; 824 } 825 826 static int handle_stidp(struct kvm_vcpu *vcpu) 827 { 828 u64 stidp_data = vcpu->kvm->arch.model.cpuid; 829 u64 operand2; 830 int rc; 831 u8 ar; 832 833 vcpu->stat.instruction_stidp++; 834 835 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 836 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 837 838 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 839 840 if (operand2 & 7) 841 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 842 843 rc = write_guest(vcpu, operand2, ar, &stidp_data, sizeof(stidp_data)); 844 if (rc) 845 return kvm_s390_inject_prog_cond(vcpu, rc); 846 847 VCPU_EVENT(vcpu, 3, "STIDP: store cpu id 0x%llx", stidp_data); 848 return 0; 849 } 850 851 static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem) 852 { 853 int cpus = 0; 854 int n; 855 856 cpus = atomic_read(&vcpu->kvm->online_vcpus); 857 858 /* deal with other level 3 hypervisors */ 859 if (stsi(mem, 3, 2, 2)) 860 mem->count = 0; 861 if (mem->count < 8) 862 mem->count++; 863 for (n = mem->count - 1; n > 0 ; n--) 864 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0])); 865 866 memset(&mem->vm[0], 0, sizeof(mem->vm[0])); 867 mem->vm[0].cpus_total = cpus; 868 mem->vm[0].cpus_configured = cpus; 869 mem->vm[0].cpus_standby = 0; 870 mem->vm[0].cpus_reserved = 0; 871 mem->vm[0].caf = 1000; 872 memcpy(mem->vm[0].name, "KVMguest", 8); 873 ASCEBC(mem->vm[0].name, 8); 874 memcpy(mem->vm[0].cpi, "KVM/Linux ", 16); 875 ASCEBC(mem->vm[0].cpi, 16); 876 } 877 878 static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, u8 ar, 879 u8 fc, u8 sel1, u16 sel2) 880 { 881 vcpu->run->exit_reason = KVM_EXIT_S390_STSI; 882 vcpu->run->s390_stsi.addr = addr; 883 vcpu->run->s390_stsi.ar = ar; 884 vcpu->run->s390_stsi.fc = fc; 885 vcpu->run->s390_stsi.sel1 = sel1; 886 vcpu->run->s390_stsi.sel2 = sel2; 887 } 888 889 static int handle_stsi(struct kvm_vcpu *vcpu) 890 { 891 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; 892 int sel1 = vcpu->run->s.regs.gprs[0] & 0xff; 893 int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff; 894 unsigned long mem = 0; 895 u64 operand2; 896 int rc = 0; 897 u8 ar; 898 899 vcpu->stat.instruction_stsi++; 900 VCPU_EVENT(vcpu, 3, "STSI: fc: %u sel1: %u sel2: %u", fc, sel1, sel2); 901 902 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 903 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 904 905 /* Bailout forbidden function codes */ 906 if (fc > 3 && fc != 15) 907 goto out_no_data; 908 909 /* 910 * fc 15 is provided only with 911 * - PTF/CPU topology support through facility 15 912 * - KVM_CAP_S390_USER_STSI 913 */ 914 if (fc == 15 && (!test_kvm_facility(vcpu->kvm, 11) || 915 !vcpu->kvm->arch.user_stsi)) 916 goto out_no_data; 917 918 if (vcpu->run->s.regs.gprs[0] & 0x0fffff00 919 || vcpu->run->s.regs.gprs[1] & 0xffff0000) 920 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 921 922 if (fc == 0) { 923 vcpu->run->s.regs.gprs[0] = 3 << 28; 924 kvm_s390_set_psw_cc(vcpu, 0); 925 return 0; 926 } 927 928 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 929 930 if (!kvm_s390_pv_cpu_is_protected(vcpu) && (operand2 & 0xfff)) 931 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 932 933 switch (fc) { 934 case 1: /* same handling for 1 and 2 */ 935 case 2: 936 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 937 if (!mem) 938 goto out_no_data; 939 if (stsi((void *) mem, fc, sel1, sel2)) 940 goto out_no_data; 941 break; 942 case 3: 943 if (sel1 != 2 || sel2 != 2) 944 goto out_no_data; 945 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 946 if (!mem) 947 goto out_no_data; 948 handle_stsi_3_2_2(vcpu, (void *) mem); 949 break; 950 case 15: /* fc 15 is fully handled in userspace */ 951 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 952 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 953 return -EREMOTE; 954 } 955 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 956 memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE); 957 rc = 0; 958 } else { 959 rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); 960 } 961 if (rc) { 962 rc = kvm_s390_inject_prog_cond(vcpu, rc); 963 goto out; 964 } 965 if (vcpu->kvm->arch.user_stsi) { 966 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 967 rc = -EREMOTE; 968 } 969 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 970 free_page(mem); 971 kvm_s390_set_psw_cc(vcpu, 0); 972 vcpu->run->s.regs.gprs[0] = 0; 973 return rc; 974 out_no_data: 975 kvm_s390_set_psw_cc(vcpu, 3); 976 out: 977 free_page(mem); 978 return rc; 979 } 980 981 int kvm_s390_handle_b2(struct kvm_vcpu *vcpu) 982 { 983 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 984 case 0x02: 985 return handle_stidp(vcpu); 986 case 0x04: 987 return handle_set_clock(vcpu); 988 case 0x10: 989 return handle_set_prefix(vcpu); 990 case 0x11: 991 return handle_store_prefix(vcpu); 992 case 0x12: 993 return handle_store_cpu_address(vcpu); 994 case 0x14: 995 return kvm_s390_handle_vsie(vcpu); 996 case 0x21: 997 case 0x50: 998 return handle_ipte_interlock(vcpu); 999 case 0x29: 1000 return handle_iske(vcpu); 1001 case 0x2a: 1002 return handle_rrbe(vcpu); 1003 case 0x2b: 1004 return handle_sske(vcpu); 1005 case 0x2c: 1006 return handle_test_block(vcpu); 1007 case 0x30: 1008 case 0x31: 1009 case 0x32: 1010 case 0x33: 1011 case 0x34: 1012 case 0x35: 1013 case 0x36: 1014 case 0x37: 1015 case 0x38: 1016 case 0x39: 1017 case 0x3a: 1018 case 0x3b: 1019 case 0x3c: 1020 case 0x5f: 1021 case 0x74: 1022 case 0x76: 1023 return handle_io_inst(vcpu); 1024 case 0x56: 1025 return handle_sthyi(vcpu); 1026 case 0x7d: 1027 return handle_stsi(vcpu); 1028 case 0xaf: 1029 return handle_pqap(vcpu); 1030 case 0xb1: 1031 return handle_stfl(vcpu); 1032 case 0xb2: 1033 return handle_lpswe(vcpu); 1034 default: 1035 return -EOPNOTSUPP; 1036 } 1037 } 1038 1039 static int handle_epsw(struct kvm_vcpu *vcpu) 1040 { 1041 int reg1, reg2; 1042 1043 vcpu->stat.instruction_epsw++; 1044 1045 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1046 1047 /* This basically extracts the mask half of the psw. */ 1048 vcpu->run->s.regs.gprs[reg1] &= 0xffffffff00000000UL; 1049 vcpu->run->s.regs.gprs[reg1] |= vcpu->arch.sie_block->gpsw.mask >> 32; 1050 if (reg2) { 1051 vcpu->run->s.regs.gprs[reg2] &= 0xffffffff00000000UL; 1052 vcpu->run->s.regs.gprs[reg2] |= 1053 vcpu->arch.sie_block->gpsw.mask & 0x00000000ffffffffUL; 1054 } 1055 return 0; 1056 } 1057 1058 #define PFMF_RESERVED 0xfffc0101UL 1059 #define PFMF_SK 0x00020000UL 1060 #define PFMF_CF 0x00010000UL 1061 #define PFMF_UI 0x00008000UL 1062 #define PFMF_FSC 0x00007000UL 1063 #define PFMF_NQ 0x00000800UL 1064 #define PFMF_MR 0x00000400UL 1065 #define PFMF_MC 0x00000200UL 1066 #define PFMF_KEY 0x000000feUL 1067 1068 static int handle_pfmf(struct kvm_vcpu *vcpu) 1069 { 1070 bool mr = false, mc = false, nq; 1071 int reg1, reg2; 1072 unsigned long start, end; 1073 unsigned char key; 1074 1075 vcpu->stat.instruction_pfmf++; 1076 1077 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1078 1079 if (!test_kvm_facility(vcpu->kvm, 8)) 1080 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1081 1082 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1083 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1084 1085 if (vcpu->run->s.regs.gprs[reg1] & PFMF_RESERVED) 1086 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1087 1088 /* Only provide non-quiescing support if enabled for the guest */ 1089 if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ && 1090 !test_kvm_facility(vcpu->kvm, 14)) 1091 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1092 1093 /* Only provide conditional-SSKE support if enabled for the guest */ 1094 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK && 1095 test_kvm_facility(vcpu->kvm, 10)) { 1096 mr = vcpu->run->s.regs.gprs[reg1] & PFMF_MR; 1097 mc = vcpu->run->s.regs.gprs[reg1] & PFMF_MC; 1098 } 1099 1100 nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ; 1101 key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY; 1102 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 1103 start = kvm_s390_logical_to_effective(vcpu, start); 1104 1105 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1106 if (kvm_s390_check_low_addr_prot_real(vcpu, start)) 1107 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 1108 } 1109 1110 switch (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1111 case 0x00000000: 1112 /* only 4k frames specify a real address */ 1113 start = kvm_s390_real_to_abs(vcpu, start); 1114 end = (start + PAGE_SIZE) & ~(PAGE_SIZE - 1); 1115 break; 1116 case 0x00001000: 1117 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 1118 break; 1119 case 0x00002000: 1120 /* only support 2G frame size if EDAT2 is available and we are 1121 not in 24-bit addressing mode */ 1122 if (!test_kvm_facility(vcpu->kvm, 78) || 1123 psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_24BIT) 1124 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1125 end = (start + _REGION3_SIZE) & ~(_REGION3_SIZE - 1); 1126 break; 1127 default: 1128 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1129 } 1130 1131 while (start != end) { 1132 unsigned long vmaddr; 1133 bool unlocked = false; 1134 1135 /* Translate guest address to host address */ 1136 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 1137 if (kvm_is_error_hva(vmaddr)) 1138 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1139 1140 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1141 if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE)) 1142 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1143 } 1144 1145 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK) { 1146 int rc = kvm_s390_skey_check_enable(vcpu); 1147 1148 if (rc) 1149 return rc; 1150 mmap_read_lock(current->mm); 1151 rc = cond_set_guest_storage_key(current->mm, vmaddr, 1152 key, NULL, nq, mr, mc); 1153 if (rc < 0) { 1154 rc = fixup_user_fault(current->mm, vmaddr, 1155 FAULT_FLAG_WRITE, &unlocked); 1156 rc = !rc ? -EAGAIN : rc; 1157 } 1158 mmap_read_unlock(current->mm); 1159 if (rc == -EFAULT) 1160 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1161 if (rc == -EAGAIN) 1162 continue; 1163 if (rc < 0) 1164 return rc; 1165 } 1166 start += PAGE_SIZE; 1167 } 1168 if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1169 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) { 1170 vcpu->run->s.regs.gprs[reg2] = end; 1171 } else { 1172 vcpu->run->s.regs.gprs[reg2] &= ~0xffffffffUL; 1173 end = kvm_s390_logical_to_effective(vcpu, end); 1174 vcpu->run->s.regs.gprs[reg2] |= end; 1175 } 1176 } 1177 return 0; 1178 } 1179 1180 /* 1181 * Must be called with relevant read locks held (kvm->mm->mmap_lock, kvm->srcu) 1182 */ 1183 static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) 1184 { 1185 int r1, r2, nappended, entries; 1186 unsigned long gfn, hva, res, pgstev, ptev; 1187 unsigned long *cbrlo; 1188 1189 /* 1190 * We don't need to set SD.FPF.SK to 1 here, because if we have a 1191 * machine check here we either handle it or crash 1192 */ 1193 1194 kvm_s390_get_regs_rre(vcpu, &r1, &r2); 1195 gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT; 1196 hva = gfn_to_hva(vcpu->kvm, gfn); 1197 entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1198 1199 if (kvm_is_error_hva(hva)) 1200 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1201 1202 nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev); 1203 if (nappended < 0) { 1204 res = orc ? 0x10 : 0; 1205 vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */ 1206 return 0; 1207 } 1208 res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22; 1209 /* 1210 * Set the block-content state part of the result. 0 means resident, so 1211 * nothing to do if the page is valid. 2 is for preserved pages 1212 * (non-present and non-zero), and 3 for zero pages (non-present and 1213 * zero). 1214 */ 1215 if (ptev & _PAGE_INVALID) { 1216 res |= 2; 1217 if (pgstev & _PGSTE_GPS_ZERO) 1218 res |= 1; 1219 } 1220 if (pgstev & _PGSTE_GPS_NODAT) 1221 res |= 0x20; 1222 vcpu->run->s.regs.gprs[r1] = res; 1223 /* 1224 * It is possible that all the normal 511 slots were full, in which case 1225 * we will now write in the 512th slot, which is reserved for host use. 1226 * In both cases we let the normal essa handling code process all the 1227 * slots, including the reserved one, if needed. 1228 */ 1229 if (nappended > 0) { 1230 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK); 1231 cbrlo[entries] = gfn << PAGE_SHIFT; 1232 } 1233 1234 if (orc) { 1235 struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn); 1236 1237 /* Increment only if we are really flipping the bit */ 1238 if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms))) 1239 atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages); 1240 } 1241 1242 return nappended; 1243 } 1244 1245 static int handle_essa(struct kvm_vcpu *vcpu) 1246 { 1247 /* entries expected to be 1FF */ 1248 int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1249 unsigned long *cbrlo; 1250 struct gmap *gmap; 1251 int i, orc; 1252 1253 VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries); 1254 gmap = vcpu->arch.gmap; 1255 vcpu->stat.instruction_essa++; 1256 if (!vcpu->kvm->arch.use_cmma) 1257 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1258 1259 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1260 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1261 /* Check for invalid operation request code */ 1262 orc = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; 1263 /* ORCs 0-6 are always valid */ 1264 if (orc > (test_kvm_facility(vcpu->kvm, 147) ? ESSA_SET_STABLE_NODAT 1265 : ESSA_SET_STABLE_IF_RESIDENT)) 1266 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1267 1268 if (!vcpu->kvm->arch.migration_mode) { 1269 /* 1270 * CMMA is enabled in the KVM settings, but is disabled in 1271 * the SIE block and in the mm_context, and we are not doing 1272 * a migration. Enable CMMA in the mm_context. 1273 * Since we need to take a write lock to write to the context 1274 * to avoid races with storage keys handling, we check if the 1275 * value really needs to be written to; if the value is 1276 * already correct, we do nothing and avoid the lock. 1277 */ 1278 if (vcpu->kvm->mm->context.uses_cmm == 0) { 1279 mmap_write_lock(vcpu->kvm->mm); 1280 vcpu->kvm->mm->context.uses_cmm = 1; 1281 mmap_write_unlock(vcpu->kvm->mm); 1282 } 1283 /* 1284 * If we are here, we are supposed to have CMMA enabled in 1285 * the SIE block. Enabling CMMA works on a per-CPU basis, 1286 * while the context use_cmma flag is per process. 1287 * It's possible that the context flag is enabled and the 1288 * SIE flag is not, so we set the flag always; if it was 1289 * already set, nothing changes, otherwise we enable it 1290 * on this CPU too. 1291 */ 1292 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA; 1293 /* Retry the ESSA instruction */ 1294 kvm_s390_retry_instr(vcpu); 1295 } else { 1296 int srcu_idx; 1297 1298 mmap_read_lock(vcpu->kvm->mm); 1299 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1300 i = __do_essa(vcpu, orc); 1301 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 1302 mmap_read_unlock(vcpu->kvm->mm); 1303 if (i < 0) 1304 return i; 1305 /* Account for the possible extra cbrl entry */ 1306 entries += i; 1307 } 1308 vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ 1309 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); 1310 mmap_read_lock(gmap->mm); 1311 for (i = 0; i < entries; ++i) 1312 __gmap_zap(gmap, cbrlo[i]); 1313 mmap_read_unlock(gmap->mm); 1314 return 0; 1315 } 1316 1317 int kvm_s390_handle_b9(struct kvm_vcpu *vcpu) 1318 { 1319 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1320 case 0x8a: 1321 case 0x8e: 1322 case 0x8f: 1323 return handle_ipte_interlock(vcpu); 1324 case 0x8d: 1325 return handle_epsw(vcpu); 1326 case 0xab: 1327 return handle_essa(vcpu); 1328 case 0xaf: 1329 return handle_pfmf(vcpu); 1330 default: 1331 return -EOPNOTSUPP; 1332 } 1333 } 1334 1335 int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu) 1336 { 1337 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1338 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1339 int reg, rc, nr_regs; 1340 u32 ctl_array[16]; 1341 u64 ga; 1342 u8 ar; 1343 1344 vcpu->stat.instruction_lctl++; 1345 1346 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1347 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1348 1349 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1350 1351 if (ga & 3) 1352 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1353 1354 VCPU_EVENT(vcpu, 4, "LCTL: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1355 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga); 1356 1357 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1358 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1359 if (rc) 1360 return kvm_s390_inject_prog_cond(vcpu, rc); 1361 reg = reg1; 1362 nr_regs = 0; 1363 do { 1364 vcpu->arch.sie_block->gcr[reg] &= 0xffffffff00000000ul; 1365 vcpu->arch.sie_block->gcr[reg] |= ctl_array[nr_regs++]; 1366 if (reg == reg3) 1367 break; 1368 reg = (reg + 1) % 16; 1369 } while (1); 1370 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1371 return 0; 1372 } 1373 1374 int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu) 1375 { 1376 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1377 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1378 int reg, rc, nr_regs; 1379 u32 ctl_array[16]; 1380 u64 ga; 1381 u8 ar; 1382 1383 vcpu->stat.instruction_stctl++; 1384 1385 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1386 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1387 1388 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1389 1390 if (ga & 3) 1391 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1392 1393 VCPU_EVENT(vcpu, 4, "STCTL r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1394 trace_kvm_s390_handle_stctl(vcpu, 0, reg1, reg3, ga); 1395 1396 reg = reg1; 1397 nr_regs = 0; 1398 do { 1399 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1400 if (reg == reg3) 1401 break; 1402 reg = (reg + 1) % 16; 1403 } while (1); 1404 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1405 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1406 } 1407 1408 static int handle_lctlg(struct kvm_vcpu *vcpu) 1409 { 1410 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1411 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1412 int reg, rc, nr_regs; 1413 u64 ctl_array[16]; 1414 u64 ga; 1415 u8 ar; 1416 1417 vcpu->stat.instruction_lctlg++; 1418 1419 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1420 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1421 1422 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1423 1424 if (ga & 7) 1425 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1426 1427 VCPU_EVENT(vcpu, 4, "LCTLG: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1428 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga); 1429 1430 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1431 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1432 if (rc) 1433 return kvm_s390_inject_prog_cond(vcpu, rc); 1434 reg = reg1; 1435 nr_regs = 0; 1436 do { 1437 vcpu->arch.sie_block->gcr[reg] = ctl_array[nr_regs++]; 1438 if (reg == reg3) 1439 break; 1440 reg = (reg + 1) % 16; 1441 } while (1); 1442 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1443 return 0; 1444 } 1445 1446 static int handle_stctg(struct kvm_vcpu *vcpu) 1447 { 1448 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1449 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1450 int reg, rc, nr_regs; 1451 u64 ctl_array[16]; 1452 u64 ga; 1453 u8 ar; 1454 1455 vcpu->stat.instruction_stctg++; 1456 1457 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1458 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1459 1460 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1461 1462 if (ga & 7) 1463 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1464 1465 VCPU_EVENT(vcpu, 4, "STCTG r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1466 trace_kvm_s390_handle_stctl(vcpu, 1, reg1, reg3, ga); 1467 1468 reg = reg1; 1469 nr_regs = 0; 1470 do { 1471 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1472 if (reg == reg3) 1473 break; 1474 reg = (reg + 1) % 16; 1475 } while (1); 1476 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1477 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1478 } 1479 1480 int kvm_s390_handle_eb(struct kvm_vcpu *vcpu) 1481 { 1482 switch (vcpu->arch.sie_block->ipb & 0x000000ff) { 1483 case 0x25: 1484 return handle_stctg(vcpu); 1485 case 0x2f: 1486 return handle_lctlg(vcpu); 1487 case 0x60: 1488 case 0x61: 1489 case 0x62: 1490 return handle_ri(vcpu); 1491 case 0x71: 1492 return handle_lpswey(vcpu); 1493 default: 1494 return -EOPNOTSUPP; 1495 } 1496 } 1497 1498 static int handle_tprot(struct kvm_vcpu *vcpu) 1499 { 1500 u64 address, operand2; 1501 unsigned long gpa; 1502 u8 access_key; 1503 bool writable; 1504 int ret, cc; 1505 u8 ar; 1506 1507 vcpu->stat.instruction_tprot++; 1508 1509 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1510 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1511 1512 kvm_s390_get_base_disp_sse(vcpu, &address, &operand2, &ar, NULL); 1513 access_key = (operand2 & 0xf0) >> 4; 1514 1515 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1516 ipte_lock(vcpu->kvm); 1517 1518 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1519 GACC_STORE, access_key); 1520 if (ret == 0) { 1521 gfn_to_hva_prot(vcpu->kvm, gpa_to_gfn(gpa), &writable); 1522 } else if (ret == PGM_PROTECTION) { 1523 writable = false; 1524 /* Write protected? Try again with read-only... */ 1525 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1526 GACC_FETCH, access_key); 1527 } 1528 if (ret >= 0) { 1529 cc = -1; 1530 1531 /* Fetching permitted; storing permitted */ 1532 if (ret == 0 && writable) 1533 cc = 0; 1534 /* Fetching permitted; storing not permitted */ 1535 else if (ret == 0 && !writable) 1536 cc = 1; 1537 /* Fetching not permitted; storing not permitted */ 1538 else if (ret == PGM_PROTECTION) 1539 cc = 2; 1540 /* Translation not available */ 1541 else if (ret != PGM_ADDRESSING && ret != PGM_TRANSLATION_SPEC) 1542 cc = 3; 1543 1544 if (cc != -1) { 1545 kvm_s390_set_psw_cc(vcpu, cc); 1546 ret = 0; 1547 } else { 1548 ret = kvm_s390_inject_program_int(vcpu, ret); 1549 } 1550 } 1551 1552 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1553 ipte_unlock(vcpu->kvm); 1554 return ret; 1555 } 1556 1557 int kvm_s390_handle_e5(struct kvm_vcpu *vcpu) 1558 { 1559 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1560 case 0x01: 1561 return handle_tprot(vcpu); 1562 default: 1563 return -EOPNOTSUPP; 1564 } 1565 } 1566 1567 static int handle_sckpf(struct kvm_vcpu *vcpu) 1568 { 1569 u32 value; 1570 1571 vcpu->stat.instruction_sckpf++; 1572 1573 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1574 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1575 1576 if (vcpu->run->s.regs.gprs[0] & 0x00000000ffff0000) 1577 return kvm_s390_inject_program_int(vcpu, 1578 PGM_SPECIFICATION); 1579 1580 value = vcpu->run->s.regs.gprs[0] & 0x000000000000ffff; 1581 vcpu->arch.sie_block->todpr = value; 1582 1583 return 0; 1584 } 1585 1586 static int handle_ptff(struct kvm_vcpu *vcpu) 1587 { 1588 vcpu->stat.instruction_ptff++; 1589 1590 /* we don't emulate any control instructions yet */ 1591 kvm_s390_set_psw_cc(vcpu, 3); 1592 return 0; 1593 } 1594 1595 int kvm_s390_handle_01(struct kvm_vcpu *vcpu) 1596 { 1597 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1598 case 0x04: 1599 return handle_ptff(vcpu); 1600 case 0x07: 1601 return handle_sckpf(vcpu); 1602 default: 1603 return -EOPNOTSUPP; 1604 } 1605 } 1606