1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * hosting IBM Z kernel virtual machines (s390x) 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com> 9 * Heiko Carstens <heiko.carstens@de.ibm.com> 10 * Christian Ehrhardt <ehrhardt@de.ibm.com> 11 * Jason J. Herne <jjherne@us.ibm.com> 12 */ 13 14 #define KMSG_COMPONENT "kvm-s390" 15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 16 17 #include <linux/compiler.h> 18 #include <linux/err.h> 19 #include <linux/fs.h> 20 #include <linux/hrtimer.h> 21 #include <linux/init.h> 22 #include <linux/kvm.h> 23 #include <linux/kvm_host.h> 24 #include <linux/mman.h> 25 #include <linux/module.h> 26 #include <linux/moduleparam.h> 27 #include <linux/random.h> 28 #include <linux/slab.h> 29 #include <linux/timer.h> 30 #include <linux/vmalloc.h> 31 #include <linux/bitmap.h> 32 #include <linux/sched/signal.h> 33 #include <linux/string.h> 34 35 #include <asm/asm-offsets.h> 36 #include <asm/lowcore.h> 37 #include <asm/stp.h> 38 #include <asm/pgtable.h> 39 #include <asm/gmap.h> 40 #include <asm/nmi.h> 41 #include <asm/switch_to.h> 42 #include <asm/isc.h> 43 #include <asm/sclp.h> 44 #include <asm/cpacf.h> 45 #include <asm/timex.h> 46 #include <asm/ap.h> 47 #include <asm/uv.h> 48 #include "kvm-s390.h" 49 #include "gaccess.h" 50 51 #define CREATE_TRACE_POINTS 52 #include "trace.h" 53 #include "trace-s390.h" 54 55 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */ 56 #define LOCAL_IRQS 32 57 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ 58 (KVM_MAX_VCPUS + LOCAL_IRQS)) 59 60 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 61 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM 62 63 struct kvm_stats_debugfs_item debugfs_entries[] = { 64 { "userspace_handled", VCPU_STAT(exit_userspace) }, 65 { "exit_null", VCPU_STAT(exit_null) }, 66 { "exit_validity", VCPU_STAT(exit_validity) }, 67 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 68 { "exit_external_request", VCPU_STAT(exit_external_request) }, 69 { "exit_io_request", VCPU_STAT(exit_io_request) }, 70 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 71 { "exit_instruction", VCPU_STAT(exit_instruction) }, 72 { "exit_pei", VCPU_STAT(exit_pei) }, 73 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 74 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 75 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) }, 76 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, 77 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, 78 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, 79 { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) }, 80 { "halt_wakeup", VCPU_STAT(halt_wakeup) }, 81 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 82 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 83 { "instruction_stctl", VCPU_STAT(instruction_stctl) }, 84 { "instruction_stctg", VCPU_STAT(instruction_stctg) }, 85 { "deliver_ckc", VCPU_STAT(deliver_ckc) }, 86 { "deliver_cputm", VCPU_STAT(deliver_cputm) }, 87 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 88 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 89 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 90 { "deliver_virtio", VCPU_STAT(deliver_virtio) }, 91 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 92 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 93 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 94 { "deliver_program", VCPU_STAT(deliver_program) }, 95 { "deliver_io", VCPU_STAT(deliver_io) }, 96 { "deliver_machine_check", VCPU_STAT(deliver_machine_check) }, 97 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 98 { "inject_ckc", VCPU_STAT(inject_ckc) }, 99 { "inject_cputm", VCPU_STAT(inject_cputm) }, 100 { "inject_external_call", VCPU_STAT(inject_external_call) }, 101 { "inject_float_mchk", VM_STAT(inject_float_mchk) }, 102 { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) }, 103 { "inject_io", VM_STAT(inject_io) }, 104 { "inject_mchk", VCPU_STAT(inject_mchk) }, 105 { "inject_pfault_done", VM_STAT(inject_pfault_done) }, 106 { "inject_program", VCPU_STAT(inject_program) }, 107 { "inject_restart", VCPU_STAT(inject_restart) }, 108 { "inject_service_signal", VM_STAT(inject_service_signal) }, 109 { "inject_set_prefix", VCPU_STAT(inject_set_prefix) }, 110 { "inject_stop_signal", VCPU_STAT(inject_stop_signal) }, 111 { "inject_pfault_init", VCPU_STAT(inject_pfault_init) }, 112 { "inject_virtio", VM_STAT(inject_virtio) }, 113 { "instruction_epsw", VCPU_STAT(instruction_epsw) }, 114 { "instruction_gs", VCPU_STAT(instruction_gs) }, 115 { "instruction_io_other", VCPU_STAT(instruction_io_other) }, 116 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) }, 117 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) }, 118 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 119 { "instruction_ptff", VCPU_STAT(instruction_ptff) }, 120 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 121 { "instruction_sck", VCPU_STAT(instruction_sck) }, 122 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) }, 123 { "instruction_spx", VCPU_STAT(instruction_spx) }, 124 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 125 { "instruction_stap", VCPU_STAT(instruction_stap) }, 126 { "instruction_iske", VCPU_STAT(instruction_iske) }, 127 { "instruction_ri", VCPU_STAT(instruction_ri) }, 128 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) }, 129 { "instruction_sske", VCPU_STAT(instruction_sske) }, 130 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, 131 { "instruction_essa", VCPU_STAT(instruction_essa) }, 132 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 133 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 134 { "instruction_tb", VCPU_STAT(instruction_tb) }, 135 { "instruction_tpi", VCPU_STAT(instruction_tpi) }, 136 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 137 { "instruction_tsch", VCPU_STAT(instruction_tsch) }, 138 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) }, 139 { "instruction_sie", VCPU_STAT(instruction_sie) }, 140 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 141 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 142 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 143 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 144 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) }, 145 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) }, 146 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 147 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) }, 148 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) }, 149 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) }, 150 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 151 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 152 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 153 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) }, 154 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) }, 155 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) }, 156 { "instruction_diag_10", VCPU_STAT(diagnose_10) }, 157 { "instruction_diag_44", VCPU_STAT(diagnose_44) }, 158 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) }, 159 { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) }, 160 { "instruction_diag_258", VCPU_STAT(diagnose_258) }, 161 { "instruction_diag_308", VCPU_STAT(diagnose_308) }, 162 { "instruction_diag_500", VCPU_STAT(diagnose_500) }, 163 { "instruction_diag_other", VCPU_STAT(diagnose_other) }, 164 { NULL } 165 }; 166 167 struct kvm_s390_tod_clock_ext { 168 __u8 epoch_idx; 169 __u64 tod; 170 __u8 reserved[7]; 171 } __packed; 172 173 /* allow nested virtualization in KVM (if enabled by user space) */ 174 static int nested; 175 module_param(nested, int, S_IRUGO); 176 MODULE_PARM_DESC(nested, "Nested virtualization support"); 177 178 /* allow 1m huge page guest backing, if !nested */ 179 static int hpage; 180 module_param(hpage, int, 0444); 181 MODULE_PARM_DESC(hpage, "1m huge page backing support"); 182 183 /* maximum percentage of steal time for polling. >100 is treated like 100 */ 184 static u8 halt_poll_max_steal = 10; 185 module_param(halt_poll_max_steal, byte, 0644); 186 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling"); 187 188 /* if set to true, the GISA will be initialized and used if available */ 189 static bool use_gisa = true; 190 module_param(use_gisa, bool, 0644); 191 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it."); 192 193 /* 194 * For now we handle at most 16 double words as this is what the s390 base 195 * kernel handles and stores in the prefix page. If we ever need to go beyond 196 * this, this requires changes to code, but the external uapi can stay. 197 */ 198 #define SIZE_INTERNAL 16 199 200 /* 201 * Base feature mask that defines default mask for facilities. Consists of the 202 * defines in FACILITIES_KVM and the non-hypervisor managed bits. 203 */ 204 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM }; 205 /* 206 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL 207 * and defines the facilities that can be enabled via a cpu model. 208 */ 209 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL }; 210 211 static unsigned long kvm_s390_fac_size(void) 212 { 213 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64); 214 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64); 215 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) > 216 sizeof(S390_lowcore.stfle_fac_list)); 217 218 return SIZE_INTERNAL; 219 } 220 221 /* available cpu features supported by kvm */ 222 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); 223 /* available subfunctions indicated via query / "test bit" */ 224 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc; 225 226 static struct gmap_notifier gmap_notifier; 227 static struct gmap_notifier vsie_gmap_notifier; 228 debug_info_t *kvm_s390_dbf; 229 debug_info_t *kvm_s390_dbf_uv; 230 231 /* Section: not file related */ 232 int kvm_arch_hardware_enable(void) 233 { 234 /* every s390 is virtualization enabled ;-) */ 235 return 0; 236 } 237 238 int kvm_arch_check_processor_compat(void *opaque) 239 { 240 return 0; 241 } 242 243 /* forward declarations */ 244 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, 245 unsigned long end); 246 static int sca_switch_to_extended(struct kvm *kvm); 247 248 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta) 249 { 250 u8 delta_idx = 0; 251 252 /* 253 * The TOD jumps by delta, we have to compensate this by adding 254 * -delta to the epoch. 255 */ 256 delta = -delta; 257 258 /* sign-extension - we're adding to signed values below */ 259 if ((s64)delta < 0) 260 delta_idx = -1; 261 262 scb->epoch += delta; 263 if (scb->ecd & ECD_MEF) { 264 scb->epdx += delta_idx; 265 if (scb->epoch < delta) 266 scb->epdx += 1; 267 } 268 } 269 270 /* 271 * This callback is executed during stop_machine(). All CPUs are therefore 272 * temporarily stopped. In order not to change guest behavior, we have to 273 * disable preemption whenever we touch the epoch of kvm and the VCPUs, 274 * so a CPU won't be stopped while calculating with the epoch. 275 */ 276 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val, 277 void *v) 278 { 279 struct kvm *kvm; 280 struct kvm_vcpu *vcpu; 281 int i; 282 unsigned long long *delta = v; 283 284 list_for_each_entry(kvm, &vm_list, vm_list) { 285 kvm_for_each_vcpu(i, vcpu, kvm) { 286 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta); 287 if (i == 0) { 288 kvm->arch.epoch = vcpu->arch.sie_block->epoch; 289 kvm->arch.epdx = vcpu->arch.sie_block->epdx; 290 } 291 if (vcpu->arch.cputm_enabled) 292 vcpu->arch.cputm_start += *delta; 293 if (vcpu->arch.vsie_block) 294 kvm_clock_sync_scb(vcpu->arch.vsie_block, 295 *delta); 296 } 297 } 298 return NOTIFY_OK; 299 } 300 301 static struct notifier_block kvm_clock_notifier = { 302 .notifier_call = kvm_clock_sync, 303 }; 304 305 int kvm_arch_hardware_setup(void *opaque) 306 { 307 gmap_notifier.notifier_call = kvm_gmap_notifier; 308 gmap_register_pte_notifier(&gmap_notifier); 309 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier; 310 gmap_register_pte_notifier(&vsie_gmap_notifier); 311 atomic_notifier_chain_register(&s390_epoch_delta_notifier, 312 &kvm_clock_notifier); 313 return 0; 314 } 315 316 void kvm_arch_hardware_unsetup(void) 317 { 318 gmap_unregister_pte_notifier(&gmap_notifier); 319 gmap_unregister_pte_notifier(&vsie_gmap_notifier); 320 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier, 321 &kvm_clock_notifier); 322 } 323 324 static void allow_cpu_feat(unsigned long nr) 325 { 326 set_bit_inv(nr, kvm_s390_available_cpu_feat); 327 } 328 329 static inline int plo_test_bit(unsigned char nr) 330 { 331 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100; 332 int cc; 333 334 asm volatile( 335 /* Parameter registers are ignored for "test bit" */ 336 " plo 0,0,0,0(0)\n" 337 " ipm %0\n" 338 " srl %0,28\n" 339 : "=d" (cc) 340 : "d" (r0) 341 : "cc"); 342 return cc == 0; 343 } 344 345 static __always_inline void __insn32_query(unsigned int opcode, u8 *query) 346 { 347 register unsigned long r0 asm("0") = 0; /* query function */ 348 register unsigned long r1 asm("1") = (unsigned long) query; 349 350 asm volatile( 351 /* Parameter regs are ignored */ 352 " .insn rrf,%[opc] << 16,2,4,6,0\n" 353 : 354 : "d" (r0), "a" (r1), [opc] "i" (opcode) 355 : "cc", "memory"); 356 } 357 358 #define INSN_SORTL 0xb938 359 #define INSN_DFLTCC 0xb939 360 361 static void kvm_s390_cpu_feat_init(void) 362 { 363 int i; 364 365 for (i = 0; i < 256; ++i) { 366 if (plo_test_bit(i)) 367 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7); 368 } 369 370 if (test_facility(28)) /* TOD-clock steering */ 371 ptff(kvm_s390_available_subfunc.ptff, 372 sizeof(kvm_s390_available_subfunc.ptff), 373 PTFF_QAF); 374 375 if (test_facility(17)) { /* MSA */ 376 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *) 377 kvm_s390_available_subfunc.kmac); 378 __cpacf_query(CPACF_KMC, (cpacf_mask_t *) 379 kvm_s390_available_subfunc.kmc); 380 __cpacf_query(CPACF_KM, (cpacf_mask_t *) 381 kvm_s390_available_subfunc.km); 382 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *) 383 kvm_s390_available_subfunc.kimd); 384 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *) 385 kvm_s390_available_subfunc.klmd); 386 } 387 if (test_facility(76)) /* MSA3 */ 388 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *) 389 kvm_s390_available_subfunc.pckmo); 390 if (test_facility(77)) { /* MSA4 */ 391 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *) 392 kvm_s390_available_subfunc.kmctr); 393 __cpacf_query(CPACF_KMF, (cpacf_mask_t *) 394 kvm_s390_available_subfunc.kmf); 395 __cpacf_query(CPACF_KMO, (cpacf_mask_t *) 396 kvm_s390_available_subfunc.kmo); 397 __cpacf_query(CPACF_PCC, (cpacf_mask_t *) 398 kvm_s390_available_subfunc.pcc); 399 } 400 if (test_facility(57)) /* MSA5 */ 401 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *) 402 kvm_s390_available_subfunc.ppno); 403 404 if (test_facility(146)) /* MSA8 */ 405 __cpacf_query(CPACF_KMA, (cpacf_mask_t *) 406 kvm_s390_available_subfunc.kma); 407 408 if (test_facility(155)) /* MSA9 */ 409 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *) 410 kvm_s390_available_subfunc.kdsa); 411 412 if (test_facility(150)) /* SORTL */ 413 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl); 414 415 if (test_facility(151)) /* DFLTCC */ 416 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc); 417 418 if (MACHINE_HAS_ESOP) 419 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP); 420 /* 421 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow), 422 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing). 423 */ 424 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao || 425 !test_facility(3) || !nested) 426 return; 427 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2); 428 if (sclp.has_64bscao) 429 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); 430 if (sclp.has_siif) 431 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF); 432 if (sclp.has_gpere) 433 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE); 434 if (sclp.has_gsls) 435 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS); 436 if (sclp.has_ib) 437 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB); 438 if (sclp.has_cei) 439 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI); 440 if (sclp.has_ibs) 441 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS); 442 if (sclp.has_kss) 443 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS); 444 /* 445 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make 446 * all skey handling functions read/set the skey from the PGSTE 447 * instead of the real storage key. 448 * 449 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make 450 * pages being detected as preserved although they are resident. 451 * 452 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will 453 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY. 454 * 455 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and 456 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be 457 * correctly shadowed. We can do that for the PGSTE but not for PTE.I. 458 * 459 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We 460 * cannot easily shadow the SCA because of the ipte lock. 461 */ 462 } 463 464 int kvm_arch_init(void *opaque) 465 { 466 int rc = -ENOMEM; 467 468 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long)); 469 if (!kvm_s390_dbf) 470 return -ENOMEM; 471 472 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long)); 473 if (!kvm_s390_dbf_uv) 474 goto out; 475 476 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) || 477 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view)) 478 goto out; 479 480 kvm_s390_cpu_feat_init(); 481 482 /* Register floating interrupt controller interface. */ 483 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC); 484 if (rc) { 485 pr_err("A FLIC registration call failed with rc=%d\n", rc); 486 goto out; 487 } 488 489 rc = kvm_s390_gib_init(GAL_ISC); 490 if (rc) 491 goto out; 492 493 return 0; 494 495 out: 496 kvm_arch_exit(); 497 return rc; 498 } 499 500 void kvm_arch_exit(void) 501 { 502 kvm_s390_gib_destroy(); 503 debug_unregister(kvm_s390_dbf); 504 debug_unregister(kvm_s390_dbf_uv); 505 } 506 507 /* Section: device related */ 508 long kvm_arch_dev_ioctl(struct file *filp, 509 unsigned int ioctl, unsigned long arg) 510 { 511 if (ioctl == KVM_S390_ENABLE_SIE) 512 return s390_enable_sie(); 513 return -EINVAL; 514 } 515 516 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 517 { 518 int r; 519 520 switch (ext) { 521 case KVM_CAP_S390_PSW: 522 case KVM_CAP_S390_GMAP: 523 case KVM_CAP_SYNC_MMU: 524 #ifdef CONFIG_KVM_S390_UCONTROL 525 case KVM_CAP_S390_UCONTROL: 526 #endif 527 case KVM_CAP_ASYNC_PF: 528 case KVM_CAP_SYNC_REGS: 529 case KVM_CAP_ONE_REG: 530 case KVM_CAP_ENABLE_CAP: 531 case KVM_CAP_S390_CSS_SUPPORT: 532 case KVM_CAP_IOEVENTFD: 533 case KVM_CAP_DEVICE_CTRL: 534 case KVM_CAP_S390_IRQCHIP: 535 case KVM_CAP_VM_ATTRIBUTES: 536 case KVM_CAP_MP_STATE: 537 case KVM_CAP_IMMEDIATE_EXIT: 538 case KVM_CAP_S390_INJECT_IRQ: 539 case KVM_CAP_S390_USER_SIGP: 540 case KVM_CAP_S390_USER_STSI: 541 case KVM_CAP_S390_SKEYS: 542 case KVM_CAP_S390_IRQ_STATE: 543 case KVM_CAP_S390_USER_INSTR0: 544 case KVM_CAP_S390_CMMA_MIGRATION: 545 case KVM_CAP_S390_AIS: 546 case KVM_CAP_S390_AIS_MIGRATION: 547 case KVM_CAP_S390_VCPU_RESETS: 548 r = 1; 549 break; 550 case KVM_CAP_S390_HPAGE_1M: 551 r = 0; 552 if (hpage && !kvm_is_ucontrol(kvm)) 553 r = 1; 554 break; 555 case KVM_CAP_S390_MEM_OP: 556 r = MEM_OP_MAX_SIZE; 557 break; 558 case KVM_CAP_NR_VCPUS: 559 case KVM_CAP_MAX_VCPUS: 560 case KVM_CAP_MAX_VCPU_ID: 561 r = KVM_S390_BSCA_CPU_SLOTS; 562 if (!kvm_s390_use_sca_entries()) 563 r = KVM_MAX_VCPUS; 564 else if (sclp.has_esca && sclp.has_64bscao) 565 r = KVM_S390_ESCA_CPU_SLOTS; 566 break; 567 case KVM_CAP_S390_COW: 568 r = MACHINE_HAS_ESOP; 569 break; 570 case KVM_CAP_S390_VECTOR_REGISTERS: 571 r = MACHINE_HAS_VX; 572 break; 573 case KVM_CAP_S390_RI: 574 r = test_facility(64); 575 break; 576 case KVM_CAP_S390_GS: 577 r = test_facility(133); 578 break; 579 case KVM_CAP_S390_BPB: 580 r = test_facility(82); 581 break; 582 case KVM_CAP_S390_PROTECTED: 583 r = is_prot_virt_host(); 584 break; 585 default: 586 r = 0; 587 } 588 return r; 589 } 590 591 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 592 { 593 int i; 594 gfn_t cur_gfn, last_gfn; 595 unsigned long gaddr, vmaddr; 596 struct gmap *gmap = kvm->arch.gmap; 597 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES); 598 599 /* Loop over all guest segments */ 600 cur_gfn = memslot->base_gfn; 601 last_gfn = memslot->base_gfn + memslot->npages; 602 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) { 603 gaddr = gfn_to_gpa(cur_gfn); 604 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn); 605 if (kvm_is_error_hva(vmaddr)) 606 continue; 607 608 bitmap_zero(bitmap, _PAGE_ENTRIES); 609 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr); 610 for (i = 0; i < _PAGE_ENTRIES; i++) { 611 if (test_bit(i, bitmap)) 612 mark_page_dirty(kvm, cur_gfn + i); 613 } 614 615 if (fatal_signal_pending(current)) 616 return; 617 cond_resched(); 618 } 619 } 620 621 /* Section: vm related */ 622 static void sca_del_vcpu(struct kvm_vcpu *vcpu); 623 624 /* 625 * Get (and clear) the dirty memory log for a memory slot. 626 */ 627 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 628 struct kvm_dirty_log *log) 629 { 630 int r; 631 unsigned long n; 632 struct kvm_memory_slot *memslot; 633 int is_dirty; 634 635 if (kvm_is_ucontrol(kvm)) 636 return -EINVAL; 637 638 mutex_lock(&kvm->slots_lock); 639 640 r = -EINVAL; 641 if (log->slot >= KVM_USER_MEM_SLOTS) 642 goto out; 643 644 r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); 645 if (r) 646 goto out; 647 648 /* Clear the dirty log */ 649 if (is_dirty) { 650 n = kvm_dirty_bitmap_bytes(memslot); 651 memset(memslot->dirty_bitmap, 0, n); 652 } 653 r = 0; 654 out: 655 mutex_unlock(&kvm->slots_lock); 656 return r; 657 } 658 659 static void icpt_operexc_on_all_vcpus(struct kvm *kvm) 660 { 661 unsigned int i; 662 struct kvm_vcpu *vcpu; 663 664 kvm_for_each_vcpu(i, vcpu, kvm) { 665 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu); 666 } 667 } 668 669 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) 670 { 671 int r; 672 673 if (cap->flags) 674 return -EINVAL; 675 676 switch (cap->cap) { 677 case KVM_CAP_S390_IRQCHIP: 678 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP"); 679 kvm->arch.use_irqchip = 1; 680 r = 0; 681 break; 682 case KVM_CAP_S390_USER_SIGP: 683 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP"); 684 kvm->arch.user_sigp = 1; 685 r = 0; 686 break; 687 case KVM_CAP_S390_VECTOR_REGISTERS: 688 mutex_lock(&kvm->lock); 689 if (kvm->created_vcpus) { 690 r = -EBUSY; 691 } else if (MACHINE_HAS_VX) { 692 set_kvm_facility(kvm->arch.model.fac_mask, 129); 693 set_kvm_facility(kvm->arch.model.fac_list, 129); 694 if (test_facility(134)) { 695 set_kvm_facility(kvm->arch.model.fac_mask, 134); 696 set_kvm_facility(kvm->arch.model.fac_list, 134); 697 } 698 if (test_facility(135)) { 699 set_kvm_facility(kvm->arch.model.fac_mask, 135); 700 set_kvm_facility(kvm->arch.model.fac_list, 135); 701 } 702 if (test_facility(148)) { 703 set_kvm_facility(kvm->arch.model.fac_mask, 148); 704 set_kvm_facility(kvm->arch.model.fac_list, 148); 705 } 706 if (test_facility(152)) { 707 set_kvm_facility(kvm->arch.model.fac_mask, 152); 708 set_kvm_facility(kvm->arch.model.fac_list, 152); 709 } 710 r = 0; 711 } else 712 r = -EINVAL; 713 mutex_unlock(&kvm->lock); 714 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s", 715 r ? "(not available)" : "(success)"); 716 break; 717 case KVM_CAP_S390_RI: 718 r = -EINVAL; 719 mutex_lock(&kvm->lock); 720 if (kvm->created_vcpus) { 721 r = -EBUSY; 722 } else if (test_facility(64)) { 723 set_kvm_facility(kvm->arch.model.fac_mask, 64); 724 set_kvm_facility(kvm->arch.model.fac_list, 64); 725 r = 0; 726 } 727 mutex_unlock(&kvm->lock); 728 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s", 729 r ? "(not available)" : "(success)"); 730 break; 731 case KVM_CAP_S390_AIS: 732 mutex_lock(&kvm->lock); 733 if (kvm->created_vcpus) { 734 r = -EBUSY; 735 } else { 736 set_kvm_facility(kvm->arch.model.fac_mask, 72); 737 set_kvm_facility(kvm->arch.model.fac_list, 72); 738 r = 0; 739 } 740 mutex_unlock(&kvm->lock); 741 VM_EVENT(kvm, 3, "ENABLE: AIS %s", 742 r ? "(not available)" : "(success)"); 743 break; 744 case KVM_CAP_S390_GS: 745 r = -EINVAL; 746 mutex_lock(&kvm->lock); 747 if (kvm->created_vcpus) { 748 r = -EBUSY; 749 } else if (test_facility(133)) { 750 set_kvm_facility(kvm->arch.model.fac_mask, 133); 751 set_kvm_facility(kvm->arch.model.fac_list, 133); 752 r = 0; 753 } 754 mutex_unlock(&kvm->lock); 755 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s", 756 r ? "(not available)" : "(success)"); 757 break; 758 case KVM_CAP_S390_HPAGE_1M: 759 mutex_lock(&kvm->lock); 760 if (kvm->created_vcpus) 761 r = -EBUSY; 762 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) 763 r = -EINVAL; 764 else { 765 r = 0; 766 down_write(&kvm->mm->mmap_sem); 767 kvm->mm->context.allow_gmap_hpage_1m = 1; 768 up_write(&kvm->mm->mmap_sem); 769 /* 770 * We might have to create fake 4k page 771 * tables. To avoid that the hardware works on 772 * stale PGSTEs, we emulate these instructions. 773 */ 774 kvm->arch.use_skf = 0; 775 kvm->arch.use_pfmfi = 0; 776 } 777 mutex_unlock(&kvm->lock); 778 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s", 779 r ? "(not available)" : "(success)"); 780 break; 781 case KVM_CAP_S390_USER_STSI: 782 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI"); 783 kvm->arch.user_stsi = 1; 784 r = 0; 785 break; 786 case KVM_CAP_S390_USER_INSTR0: 787 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0"); 788 kvm->arch.user_instr0 = 1; 789 icpt_operexc_on_all_vcpus(kvm); 790 r = 0; 791 break; 792 default: 793 r = -EINVAL; 794 break; 795 } 796 return r; 797 } 798 799 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 800 { 801 int ret; 802 803 switch (attr->attr) { 804 case KVM_S390_VM_MEM_LIMIT_SIZE: 805 ret = 0; 806 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes", 807 kvm->arch.mem_limit); 808 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr)) 809 ret = -EFAULT; 810 break; 811 default: 812 ret = -ENXIO; 813 break; 814 } 815 return ret; 816 } 817 818 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 819 { 820 int ret; 821 unsigned int idx; 822 switch (attr->attr) { 823 case KVM_S390_VM_MEM_ENABLE_CMMA: 824 ret = -ENXIO; 825 if (!sclp.has_cmma) 826 break; 827 828 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support"); 829 mutex_lock(&kvm->lock); 830 if (kvm->created_vcpus) 831 ret = -EBUSY; 832 else if (kvm->mm->context.allow_gmap_hpage_1m) 833 ret = -EINVAL; 834 else { 835 kvm->arch.use_cmma = 1; 836 /* Not compatible with cmma. */ 837 kvm->arch.use_pfmfi = 0; 838 ret = 0; 839 } 840 mutex_unlock(&kvm->lock); 841 break; 842 case KVM_S390_VM_MEM_CLR_CMMA: 843 ret = -ENXIO; 844 if (!sclp.has_cmma) 845 break; 846 ret = -EINVAL; 847 if (!kvm->arch.use_cmma) 848 break; 849 850 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states"); 851 mutex_lock(&kvm->lock); 852 idx = srcu_read_lock(&kvm->srcu); 853 s390_reset_cmma(kvm->arch.gmap->mm); 854 srcu_read_unlock(&kvm->srcu, idx); 855 mutex_unlock(&kvm->lock); 856 ret = 0; 857 break; 858 case KVM_S390_VM_MEM_LIMIT_SIZE: { 859 unsigned long new_limit; 860 861 if (kvm_is_ucontrol(kvm)) 862 return -EINVAL; 863 864 if (get_user(new_limit, (u64 __user *)attr->addr)) 865 return -EFAULT; 866 867 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT && 868 new_limit > kvm->arch.mem_limit) 869 return -E2BIG; 870 871 if (!new_limit) 872 return -EINVAL; 873 874 /* gmap_create takes last usable address */ 875 if (new_limit != KVM_S390_NO_MEM_LIMIT) 876 new_limit -= 1; 877 878 ret = -EBUSY; 879 mutex_lock(&kvm->lock); 880 if (!kvm->created_vcpus) { 881 /* gmap_create will round the limit up */ 882 struct gmap *new = gmap_create(current->mm, new_limit); 883 884 if (!new) { 885 ret = -ENOMEM; 886 } else { 887 gmap_remove(kvm->arch.gmap); 888 new->private = kvm; 889 kvm->arch.gmap = new; 890 ret = 0; 891 } 892 } 893 mutex_unlock(&kvm->lock); 894 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit); 895 VM_EVENT(kvm, 3, "New guest asce: 0x%pK", 896 (void *) kvm->arch.gmap->asce); 897 break; 898 } 899 default: 900 ret = -ENXIO; 901 break; 902 } 903 return ret; 904 } 905 906 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu); 907 908 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm) 909 { 910 struct kvm_vcpu *vcpu; 911 int i; 912 913 kvm_s390_vcpu_block_all(kvm); 914 915 kvm_for_each_vcpu(i, vcpu, kvm) { 916 kvm_s390_vcpu_crypto_setup(vcpu); 917 /* recreate the shadow crycb by leaving the VSIE handler */ 918 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu); 919 } 920 921 kvm_s390_vcpu_unblock_all(kvm); 922 } 923 924 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr) 925 { 926 mutex_lock(&kvm->lock); 927 switch (attr->attr) { 928 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 929 if (!test_kvm_facility(kvm, 76)) { 930 mutex_unlock(&kvm->lock); 931 return -EINVAL; 932 } 933 get_random_bytes( 934 kvm->arch.crypto.crycb->aes_wrapping_key_mask, 935 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 936 kvm->arch.crypto.aes_kw = 1; 937 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support"); 938 break; 939 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 940 if (!test_kvm_facility(kvm, 76)) { 941 mutex_unlock(&kvm->lock); 942 return -EINVAL; 943 } 944 get_random_bytes( 945 kvm->arch.crypto.crycb->dea_wrapping_key_mask, 946 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 947 kvm->arch.crypto.dea_kw = 1; 948 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support"); 949 break; 950 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 951 if (!test_kvm_facility(kvm, 76)) { 952 mutex_unlock(&kvm->lock); 953 return -EINVAL; 954 } 955 kvm->arch.crypto.aes_kw = 0; 956 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0, 957 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 958 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support"); 959 break; 960 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 961 if (!test_kvm_facility(kvm, 76)) { 962 mutex_unlock(&kvm->lock); 963 return -EINVAL; 964 } 965 kvm->arch.crypto.dea_kw = 0; 966 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0, 967 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 968 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support"); 969 break; 970 case KVM_S390_VM_CRYPTO_ENABLE_APIE: 971 if (!ap_instructions_available()) { 972 mutex_unlock(&kvm->lock); 973 return -EOPNOTSUPP; 974 } 975 kvm->arch.crypto.apie = 1; 976 break; 977 case KVM_S390_VM_CRYPTO_DISABLE_APIE: 978 if (!ap_instructions_available()) { 979 mutex_unlock(&kvm->lock); 980 return -EOPNOTSUPP; 981 } 982 kvm->arch.crypto.apie = 0; 983 break; 984 default: 985 mutex_unlock(&kvm->lock); 986 return -ENXIO; 987 } 988 989 kvm_s390_vcpu_crypto_reset_all(kvm); 990 mutex_unlock(&kvm->lock); 991 return 0; 992 } 993 994 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) 995 { 996 int cx; 997 struct kvm_vcpu *vcpu; 998 999 kvm_for_each_vcpu(cx, vcpu, kvm) 1000 kvm_s390_sync_request(req, vcpu); 1001 } 1002 1003 /* 1004 * Must be called with kvm->srcu held to avoid races on memslots, and with 1005 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration. 1006 */ 1007 static int kvm_s390_vm_start_migration(struct kvm *kvm) 1008 { 1009 struct kvm_memory_slot *ms; 1010 struct kvm_memslots *slots; 1011 unsigned long ram_pages = 0; 1012 int slotnr; 1013 1014 /* migration mode already enabled */ 1015 if (kvm->arch.migration_mode) 1016 return 0; 1017 slots = kvm_memslots(kvm); 1018 if (!slots || !slots->used_slots) 1019 return -EINVAL; 1020 1021 if (!kvm->arch.use_cmma) { 1022 kvm->arch.migration_mode = 1; 1023 return 0; 1024 } 1025 /* mark all the pages in active slots as dirty */ 1026 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) { 1027 ms = slots->memslots + slotnr; 1028 if (!ms->dirty_bitmap) 1029 return -EINVAL; 1030 /* 1031 * The second half of the bitmap is only used on x86, 1032 * and would be wasted otherwise, so we put it to good 1033 * use here to keep track of the state of the storage 1034 * attributes. 1035 */ 1036 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms)); 1037 ram_pages += ms->npages; 1038 } 1039 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages); 1040 kvm->arch.migration_mode = 1; 1041 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION); 1042 return 0; 1043 } 1044 1045 /* 1046 * Must be called with kvm->slots_lock to avoid races with ourselves and 1047 * kvm_s390_vm_start_migration. 1048 */ 1049 static int kvm_s390_vm_stop_migration(struct kvm *kvm) 1050 { 1051 /* migration mode already disabled */ 1052 if (!kvm->arch.migration_mode) 1053 return 0; 1054 kvm->arch.migration_mode = 0; 1055 if (kvm->arch.use_cmma) 1056 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); 1057 return 0; 1058 } 1059 1060 static int kvm_s390_vm_set_migration(struct kvm *kvm, 1061 struct kvm_device_attr *attr) 1062 { 1063 int res = -ENXIO; 1064 1065 mutex_lock(&kvm->slots_lock); 1066 switch (attr->attr) { 1067 case KVM_S390_VM_MIGRATION_START: 1068 res = kvm_s390_vm_start_migration(kvm); 1069 break; 1070 case KVM_S390_VM_MIGRATION_STOP: 1071 res = kvm_s390_vm_stop_migration(kvm); 1072 break; 1073 default: 1074 break; 1075 } 1076 mutex_unlock(&kvm->slots_lock); 1077 1078 return res; 1079 } 1080 1081 static int kvm_s390_vm_get_migration(struct kvm *kvm, 1082 struct kvm_device_attr *attr) 1083 { 1084 u64 mig = kvm->arch.migration_mode; 1085 1086 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS) 1087 return -ENXIO; 1088 1089 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig))) 1090 return -EFAULT; 1091 return 0; 1092 } 1093 1094 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr) 1095 { 1096 struct kvm_s390_vm_tod_clock gtod; 1097 1098 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod))) 1099 return -EFAULT; 1100 1101 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx) 1102 return -EINVAL; 1103 kvm_s390_set_tod_clock(kvm, >od); 1104 1105 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx", 1106 gtod.epoch_idx, gtod.tod); 1107 1108 return 0; 1109 } 1110 1111 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 1112 { 1113 u8 gtod_high; 1114 1115 if (copy_from_user(>od_high, (void __user *)attr->addr, 1116 sizeof(gtod_high))) 1117 return -EFAULT; 1118 1119 if (gtod_high != 0) 1120 return -EINVAL; 1121 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high); 1122 1123 return 0; 1124 } 1125 1126 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 1127 { 1128 struct kvm_s390_vm_tod_clock gtod = { 0 }; 1129 1130 if (copy_from_user(>od.tod, (void __user *)attr->addr, 1131 sizeof(gtod.tod))) 1132 return -EFAULT; 1133 1134 kvm_s390_set_tod_clock(kvm, >od); 1135 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod); 1136 return 0; 1137 } 1138 1139 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr) 1140 { 1141 int ret; 1142 1143 if (attr->flags) 1144 return -EINVAL; 1145 1146 switch (attr->attr) { 1147 case KVM_S390_VM_TOD_EXT: 1148 ret = kvm_s390_set_tod_ext(kvm, attr); 1149 break; 1150 case KVM_S390_VM_TOD_HIGH: 1151 ret = kvm_s390_set_tod_high(kvm, attr); 1152 break; 1153 case KVM_S390_VM_TOD_LOW: 1154 ret = kvm_s390_set_tod_low(kvm, attr); 1155 break; 1156 default: 1157 ret = -ENXIO; 1158 break; 1159 } 1160 return ret; 1161 } 1162 1163 static void kvm_s390_get_tod_clock(struct kvm *kvm, 1164 struct kvm_s390_vm_tod_clock *gtod) 1165 { 1166 struct kvm_s390_tod_clock_ext htod; 1167 1168 preempt_disable(); 1169 1170 get_tod_clock_ext((char *)&htod); 1171 1172 gtod->tod = htod.tod + kvm->arch.epoch; 1173 gtod->epoch_idx = 0; 1174 if (test_kvm_facility(kvm, 139)) { 1175 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx; 1176 if (gtod->tod < htod.tod) 1177 gtod->epoch_idx += 1; 1178 } 1179 1180 preempt_enable(); 1181 } 1182 1183 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr) 1184 { 1185 struct kvm_s390_vm_tod_clock gtod; 1186 1187 memset(>od, 0, sizeof(gtod)); 1188 kvm_s390_get_tod_clock(kvm, >od); 1189 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod))) 1190 return -EFAULT; 1191 1192 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx", 1193 gtod.epoch_idx, gtod.tod); 1194 return 0; 1195 } 1196 1197 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 1198 { 1199 u8 gtod_high = 0; 1200 1201 if (copy_to_user((void __user *)attr->addr, >od_high, 1202 sizeof(gtod_high))) 1203 return -EFAULT; 1204 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high); 1205 1206 return 0; 1207 } 1208 1209 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 1210 { 1211 u64 gtod; 1212 1213 gtod = kvm_s390_get_tod_clock_fast(kvm); 1214 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod))) 1215 return -EFAULT; 1216 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod); 1217 1218 return 0; 1219 } 1220 1221 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr) 1222 { 1223 int ret; 1224 1225 if (attr->flags) 1226 return -EINVAL; 1227 1228 switch (attr->attr) { 1229 case KVM_S390_VM_TOD_EXT: 1230 ret = kvm_s390_get_tod_ext(kvm, attr); 1231 break; 1232 case KVM_S390_VM_TOD_HIGH: 1233 ret = kvm_s390_get_tod_high(kvm, attr); 1234 break; 1235 case KVM_S390_VM_TOD_LOW: 1236 ret = kvm_s390_get_tod_low(kvm, attr); 1237 break; 1238 default: 1239 ret = -ENXIO; 1240 break; 1241 } 1242 return ret; 1243 } 1244 1245 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) 1246 { 1247 struct kvm_s390_vm_cpu_processor *proc; 1248 u16 lowest_ibc, unblocked_ibc; 1249 int ret = 0; 1250 1251 mutex_lock(&kvm->lock); 1252 if (kvm->created_vcpus) { 1253 ret = -EBUSY; 1254 goto out; 1255 } 1256 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 1257 if (!proc) { 1258 ret = -ENOMEM; 1259 goto out; 1260 } 1261 if (!copy_from_user(proc, (void __user *)attr->addr, 1262 sizeof(*proc))) { 1263 kvm->arch.model.cpuid = proc->cpuid; 1264 lowest_ibc = sclp.ibc >> 16 & 0xfff; 1265 unblocked_ibc = sclp.ibc & 0xfff; 1266 if (lowest_ibc && proc->ibc) { 1267 if (proc->ibc > unblocked_ibc) 1268 kvm->arch.model.ibc = unblocked_ibc; 1269 else if (proc->ibc < lowest_ibc) 1270 kvm->arch.model.ibc = lowest_ibc; 1271 else 1272 kvm->arch.model.ibc = proc->ibc; 1273 } 1274 memcpy(kvm->arch.model.fac_list, proc->fac_list, 1275 S390_ARCH_FAC_LIST_SIZE_BYTE); 1276 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx", 1277 kvm->arch.model.ibc, 1278 kvm->arch.model.cpuid); 1279 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx", 1280 kvm->arch.model.fac_list[0], 1281 kvm->arch.model.fac_list[1], 1282 kvm->arch.model.fac_list[2]); 1283 } else 1284 ret = -EFAULT; 1285 kfree(proc); 1286 out: 1287 mutex_unlock(&kvm->lock); 1288 return ret; 1289 } 1290 1291 static int kvm_s390_set_processor_feat(struct kvm *kvm, 1292 struct kvm_device_attr *attr) 1293 { 1294 struct kvm_s390_vm_cpu_feat data; 1295 1296 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data))) 1297 return -EFAULT; 1298 if (!bitmap_subset((unsigned long *) data.feat, 1299 kvm_s390_available_cpu_feat, 1300 KVM_S390_VM_CPU_FEAT_NR_BITS)) 1301 return -EINVAL; 1302 1303 mutex_lock(&kvm->lock); 1304 if (kvm->created_vcpus) { 1305 mutex_unlock(&kvm->lock); 1306 return -EBUSY; 1307 } 1308 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat, 1309 KVM_S390_VM_CPU_FEAT_NR_BITS); 1310 mutex_unlock(&kvm->lock); 1311 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx", 1312 data.feat[0], 1313 data.feat[1], 1314 data.feat[2]); 1315 return 0; 1316 } 1317 1318 static int kvm_s390_set_processor_subfunc(struct kvm *kvm, 1319 struct kvm_device_attr *attr) 1320 { 1321 mutex_lock(&kvm->lock); 1322 if (kvm->created_vcpus) { 1323 mutex_unlock(&kvm->lock); 1324 return -EBUSY; 1325 } 1326 1327 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr, 1328 sizeof(struct kvm_s390_vm_cpu_subfunc))) { 1329 mutex_unlock(&kvm->lock); 1330 return -EFAULT; 1331 } 1332 mutex_unlock(&kvm->lock); 1333 1334 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1335 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0], 1336 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1], 1337 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2], 1338 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]); 1339 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx", 1340 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0], 1341 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]); 1342 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx", 1343 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0], 1344 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]); 1345 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx", 1346 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0], 1347 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]); 1348 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx", 1349 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0], 1350 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]); 1351 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx", 1352 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0], 1353 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]); 1354 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx", 1355 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0], 1356 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]); 1357 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx", 1358 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0], 1359 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]); 1360 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx", 1361 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0], 1362 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]); 1363 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx", 1364 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0], 1365 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]); 1366 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx", 1367 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0], 1368 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]); 1369 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx", 1370 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0], 1371 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]); 1372 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx", 1373 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0], 1374 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]); 1375 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx", 1376 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0], 1377 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]); 1378 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx", 1379 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0], 1380 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]); 1381 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1382 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0], 1383 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1], 1384 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2], 1385 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]); 1386 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1387 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0], 1388 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1], 1389 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2], 1390 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]); 1391 1392 return 0; 1393 } 1394 1395 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 1396 { 1397 int ret = -ENXIO; 1398 1399 switch (attr->attr) { 1400 case KVM_S390_VM_CPU_PROCESSOR: 1401 ret = kvm_s390_set_processor(kvm, attr); 1402 break; 1403 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 1404 ret = kvm_s390_set_processor_feat(kvm, attr); 1405 break; 1406 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 1407 ret = kvm_s390_set_processor_subfunc(kvm, attr); 1408 break; 1409 } 1410 return ret; 1411 } 1412 1413 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr) 1414 { 1415 struct kvm_s390_vm_cpu_processor *proc; 1416 int ret = 0; 1417 1418 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 1419 if (!proc) { 1420 ret = -ENOMEM; 1421 goto out; 1422 } 1423 proc->cpuid = kvm->arch.model.cpuid; 1424 proc->ibc = kvm->arch.model.ibc; 1425 memcpy(&proc->fac_list, kvm->arch.model.fac_list, 1426 S390_ARCH_FAC_LIST_SIZE_BYTE); 1427 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx", 1428 kvm->arch.model.ibc, 1429 kvm->arch.model.cpuid); 1430 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx", 1431 kvm->arch.model.fac_list[0], 1432 kvm->arch.model.fac_list[1], 1433 kvm->arch.model.fac_list[2]); 1434 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc))) 1435 ret = -EFAULT; 1436 kfree(proc); 1437 out: 1438 return ret; 1439 } 1440 1441 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr) 1442 { 1443 struct kvm_s390_vm_cpu_machine *mach; 1444 int ret = 0; 1445 1446 mach = kzalloc(sizeof(*mach), GFP_KERNEL); 1447 if (!mach) { 1448 ret = -ENOMEM; 1449 goto out; 1450 } 1451 get_cpu_id((struct cpuid *) &mach->cpuid); 1452 mach->ibc = sclp.ibc; 1453 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask, 1454 S390_ARCH_FAC_LIST_SIZE_BYTE); 1455 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list, 1456 sizeof(S390_lowcore.stfle_fac_list)); 1457 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx", 1458 kvm->arch.model.ibc, 1459 kvm->arch.model.cpuid); 1460 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx", 1461 mach->fac_mask[0], 1462 mach->fac_mask[1], 1463 mach->fac_mask[2]); 1464 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx", 1465 mach->fac_list[0], 1466 mach->fac_list[1], 1467 mach->fac_list[2]); 1468 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach))) 1469 ret = -EFAULT; 1470 kfree(mach); 1471 out: 1472 return ret; 1473 } 1474 1475 static int kvm_s390_get_processor_feat(struct kvm *kvm, 1476 struct kvm_device_attr *attr) 1477 { 1478 struct kvm_s390_vm_cpu_feat data; 1479 1480 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat, 1481 KVM_S390_VM_CPU_FEAT_NR_BITS); 1482 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data))) 1483 return -EFAULT; 1484 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx", 1485 data.feat[0], 1486 data.feat[1], 1487 data.feat[2]); 1488 return 0; 1489 } 1490 1491 static int kvm_s390_get_machine_feat(struct kvm *kvm, 1492 struct kvm_device_attr *attr) 1493 { 1494 struct kvm_s390_vm_cpu_feat data; 1495 1496 bitmap_copy((unsigned long *) data.feat, 1497 kvm_s390_available_cpu_feat, 1498 KVM_S390_VM_CPU_FEAT_NR_BITS); 1499 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data))) 1500 return -EFAULT; 1501 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx", 1502 data.feat[0], 1503 data.feat[1], 1504 data.feat[2]); 1505 return 0; 1506 } 1507 1508 static int kvm_s390_get_processor_subfunc(struct kvm *kvm, 1509 struct kvm_device_attr *attr) 1510 { 1511 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs, 1512 sizeof(struct kvm_s390_vm_cpu_subfunc))) 1513 return -EFAULT; 1514 1515 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1516 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0], 1517 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1], 1518 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2], 1519 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]); 1520 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx", 1521 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0], 1522 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]); 1523 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx", 1524 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0], 1525 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]); 1526 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx", 1527 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0], 1528 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]); 1529 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx", 1530 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0], 1531 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]); 1532 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx", 1533 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0], 1534 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]); 1535 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx", 1536 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0], 1537 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]); 1538 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx", 1539 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0], 1540 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]); 1541 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx", 1542 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0], 1543 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]); 1544 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx", 1545 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0], 1546 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]); 1547 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx", 1548 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0], 1549 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]); 1550 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx", 1551 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0], 1552 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]); 1553 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx", 1554 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0], 1555 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]); 1556 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx", 1557 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0], 1558 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]); 1559 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx", 1560 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0], 1561 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]); 1562 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1563 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0], 1564 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1], 1565 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2], 1566 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]); 1567 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1568 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0], 1569 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1], 1570 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2], 1571 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]); 1572 1573 return 0; 1574 } 1575 1576 static int kvm_s390_get_machine_subfunc(struct kvm *kvm, 1577 struct kvm_device_attr *attr) 1578 { 1579 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc, 1580 sizeof(struct kvm_s390_vm_cpu_subfunc))) 1581 return -EFAULT; 1582 1583 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1584 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0], 1585 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1], 1586 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2], 1587 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]); 1588 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx", 1589 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0], 1590 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]); 1591 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx", 1592 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0], 1593 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]); 1594 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx", 1595 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0], 1596 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]); 1597 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx", 1598 ((unsigned long *) &kvm_s390_available_subfunc.km)[0], 1599 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]); 1600 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx", 1601 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0], 1602 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]); 1603 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx", 1604 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0], 1605 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]); 1606 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx", 1607 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0], 1608 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]); 1609 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx", 1610 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0], 1611 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]); 1612 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx", 1613 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0], 1614 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]); 1615 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx", 1616 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0], 1617 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]); 1618 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx", 1619 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0], 1620 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]); 1621 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx", 1622 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0], 1623 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]); 1624 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx", 1625 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0], 1626 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]); 1627 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx", 1628 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0], 1629 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]); 1630 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1631 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0], 1632 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1], 1633 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2], 1634 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]); 1635 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx", 1636 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0], 1637 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1], 1638 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2], 1639 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]); 1640 1641 return 0; 1642 } 1643 1644 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 1645 { 1646 int ret = -ENXIO; 1647 1648 switch (attr->attr) { 1649 case KVM_S390_VM_CPU_PROCESSOR: 1650 ret = kvm_s390_get_processor(kvm, attr); 1651 break; 1652 case KVM_S390_VM_CPU_MACHINE: 1653 ret = kvm_s390_get_machine(kvm, attr); 1654 break; 1655 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 1656 ret = kvm_s390_get_processor_feat(kvm, attr); 1657 break; 1658 case KVM_S390_VM_CPU_MACHINE_FEAT: 1659 ret = kvm_s390_get_machine_feat(kvm, attr); 1660 break; 1661 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 1662 ret = kvm_s390_get_processor_subfunc(kvm, attr); 1663 break; 1664 case KVM_S390_VM_CPU_MACHINE_SUBFUNC: 1665 ret = kvm_s390_get_machine_subfunc(kvm, attr); 1666 break; 1667 } 1668 return ret; 1669 } 1670 1671 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1672 { 1673 int ret; 1674 1675 switch (attr->group) { 1676 case KVM_S390_VM_MEM_CTRL: 1677 ret = kvm_s390_set_mem_control(kvm, attr); 1678 break; 1679 case KVM_S390_VM_TOD: 1680 ret = kvm_s390_set_tod(kvm, attr); 1681 break; 1682 case KVM_S390_VM_CPU_MODEL: 1683 ret = kvm_s390_set_cpu_model(kvm, attr); 1684 break; 1685 case KVM_S390_VM_CRYPTO: 1686 ret = kvm_s390_vm_set_crypto(kvm, attr); 1687 break; 1688 case KVM_S390_VM_MIGRATION: 1689 ret = kvm_s390_vm_set_migration(kvm, attr); 1690 break; 1691 default: 1692 ret = -ENXIO; 1693 break; 1694 } 1695 1696 return ret; 1697 } 1698 1699 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1700 { 1701 int ret; 1702 1703 switch (attr->group) { 1704 case KVM_S390_VM_MEM_CTRL: 1705 ret = kvm_s390_get_mem_control(kvm, attr); 1706 break; 1707 case KVM_S390_VM_TOD: 1708 ret = kvm_s390_get_tod(kvm, attr); 1709 break; 1710 case KVM_S390_VM_CPU_MODEL: 1711 ret = kvm_s390_get_cpu_model(kvm, attr); 1712 break; 1713 case KVM_S390_VM_MIGRATION: 1714 ret = kvm_s390_vm_get_migration(kvm, attr); 1715 break; 1716 default: 1717 ret = -ENXIO; 1718 break; 1719 } 1720 1721 return ret; 1722 } 1723 1724 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1725 { 1726 int ret; 1727 1728 switch (attr->group) { 1729 case KVM_S390_VM_MEM_CTRL: 1730 switch (attr->attr) { 1731 case KVM_S390_VM_MEM_ENABLE_CMMA: 1732 case KVM_S390_VM_MEM_CLR_CMMA: 1733 ret = sclp.has_cmma ? 0 : -ENXIO; 1734 break; 1735 case KVM_S390_VM_MEM_LIMIT_SIZE: 1736 ret = 0; 1737 break; 1738 default: 1739 ret = -ENXIO; 1740 break; 1741 } 1742 break; 1743 case KVM_S390_VM_TOD: 1744 switch (attr->attr) { 1745 case KVM_S390_VM_TOD_LOW: 1746 case KVM_S390_VM_TOD_HIGH: 1747 ret = 0; 1748 break; 1749 default: 1750 ret = -ENXIO; 1751 break; 1752 } 1753 break; 1754 case KVM_S390_VM_CPU_MODEL: 1755 switch (attr->attr) { 1756 case KVM_S390_VM_CPU_PROCESSOR: 1757 case KVM_S390_VM_CPU_MACHINE: 1758 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 1759 case KVM_S390_VM_CPU_MACHINE_FEAT: 1760 case KVM_S390_VM_CPU_MACHINE_SUBFUNC: 1761 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 1762 ret = 0; 1763 break; 1764 default: 1765 ret = -ENXIO; 1766 break; 1767 } 1768 break; 1769 case KVM_S390_VM_CRYPTO: 1770 switch (attr->attr) { 1771 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 1772 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 1773 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 1774 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 1775 ret = 0; 1776 break; 1777 case KVM_S390_VM_CRYPTO_ENABLE_APIE: 1778 case KVM_S390_VM_CRYPTO_DISABLE_APIE: 1779 ret = ap_instructions_available() ? 0 : -ENXIO; 1780 break; 1781 default: 1782 ret = -ENXIO; 1783 break; 1784 } 1785 break; 1786 case KVM_S390_VM_MIGRATION: 1787 ret = 0; 1788 break; 1789 default: 1790 ret = -ENXIO; 1791 break; 1792 } 1793 1794 return ret; 1795 } 1796 1797 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) 1798 { 1799 uint8_t *keys; 1800 uint64_t hva; 1801 int srcu_idx, i, r = 0; 1802 1803 if (args->flags != 0) 1804 return -EINVAL; 1805 1806 /* Is this guest using storage keys? */ 1807 if (!mm_uses_skeys(current->mm)) 1808 return KVM_S390_GET_SKEYS_NONE; 1809 1810 /* Enforce sane limit on memory allocation */ 1811 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) 1812 return -EINVAL; 1813 1814 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL); 1815 if (!keys) 1816 return -ENOMEM; 1817 1818 down_read(¤t->mm->mmap_sem); 1819 srcu_idx = srcu_read_lock(&kvm->srcu); 1820 for (i = 0; i < args->count; i++) { 1821 hva = gfn_to_hva(kvm, args->start_gfn + i); 1822 if (kvm_is_error_hva(hva)) { 1823 r = -EFAULT; 1824 break; 1825 } 1826 1827 r = get_guest_storage_key(current->mm, hva, &keys[i]); 1828 if (r) 1829 break; 1830 } 1831 srcu_read_unlock(&kvm->srcu, srcu_idx); 1832 up_read(¤t->mm->mmap_sem); 1833 1834 if (!r) { 1835 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys, 1836 sizeof(uint8_t) * args->count); 1837 if (r) 1838 r = -EFAULT; 1839 } 1840 1841 kvfree(keys); 1842 return r; 1843 } 1844 1845 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) 1846 { 1847 uint8_t *keys; 1848 uint64_t hva; 1849 int srcu_idx, i, r = 0; 1850 bool unlocked; 1851 1852 if (args->flags != 0) 1853 return -EINVAL; 1854 1855 /* Enforce sane limit on memory allocation */ 1856 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) 1857 return -EINVAL; 1858 1859 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL); 1860 if (!keys) 1861 return -ENOMEM; 1862 1863 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr, 1864 sizeof(uint8_t) * args->count); 1865 if (r) { 1866 r = -EFAULT; 1867 goto out; 1868 } 1869 1870 /* Enable storage key handling for the guest */ 1871 r = s390_enable_skey(); 1872 if (r) 1873 goto out; 1874 1875 i = 0; 1876 down_read(¤t->mm->mmap_sem); 1877 srcu_idx = srcu_read_lock(&kvm->srcu); 1878 while (i < args->count) { 1879 unlocked = false; 1880 hva = gfn_to_hva(kvm, args->start_gfn + i); 1881 if (kvm_is_error_hva(hva)) { 1882 r = -EFAULT; 1883 break; 1884 } 1885 1886 /* Lowest order bit is reserved */ 1887 if (keys[i] & 0x01) { 1888 r = -EINVAL; 1889 break; 1890 } 1891 1892 r = set_guest_storage_key(current->mm, hva, keys[i], 0); 1893 if (r) { 1894 r = fixup_user_fault(current, current->mm, hva, 1895 FAULT_FLAG_WRITE, &unlocked); 1896 if (r) 1897 break; 1898 } 1899 if (!r) 1900 i++; 1901 } 1902 srcu_read_unlock(&kvm->srcu, srcu_idx); 1903 up_read(¤t->mm->mmap_sem); 1904 out: 1905 kvfree(keys); 1906 return r; 1907 } 1908 1909 /* 1910 * Base address and length must be sent at the start of each block, therefore 1911 * it's cheaper to send some clean data, as long as it's less than the size of 1912 * two longs. 1913 */ 1914 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *)) 1915 /* for consistency */ 1916 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX) 1917 1918 /* 1919 * Similar to gfn_to_memslot, but returns the index of a memslot also when the 1920 * address falls in a hole. In that case the index of one of the memslots 1921 * bordering the hole is returned. 1922 */ 1923 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn) 1924 { 1925 int start = 0, end = slots->used_slots; 1926 int slot = atomic_read(&slots->lru_slot); 1927 struct kvm_memory_slot *memslots = slots->memslots; 1928 1929 if (gfn >= memslots[slot].base_gfn && 1930 gfn < memslots[slot].base_gfn + memslots[slot].npages) 1931 return slot; 1932 1933 while (start < end) { 1934 slot = start + (end - start) / 2; 1935 1936 if (gfn >= memslots[slot].base_gfn) 1937 end = slot; 1938 else 1939 start = slot + 1; 1940 } 1941 1942 if (gfn >= memslots[start].base_gfn && 1943 gfn < memslots[start].base_gfn + memslots[start].npages) { 1944 atomic_set(&slots->lru_slot, start); 1945 } 1946 1947 return start; 1948 } 1949 1950 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args, 1951 u8 *res, unsigned long bufsize) 1952 { 1953 unsigned long pgstev, hva, cur_gfn = args->start_gfn; 1954 1955 args->count = 0; 1956 while (args->count < bufsize) { 1957 hva = gfn_to_hva(kvm, cur_gfn); 1958 /* 1959 * We return an error if the first value was invalid, but we 1960 * return successfully if at least one value was copied. 1961 */ 1962 if (kvm_is_error_hva(hva)) 1963 return args->count ? 0 : -EFAULT; 1964 if (get_pgste(kvm->mm, hva, &pgstev) < 0) 1965 pgstev = 0; 1966 res[args->count++] = (pgstev >> 24) & 0x43; 1967 cur_gfn++; 1968 } 1969 1970 return 0; 1971 } 1972 1973 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots, 1974 unsigned long cur_gfn) 1975 { 1976 int slotidx = gfn_to_memslot_approx(slots, cur_gfn); 1977 struct kvm_memory_slot *ms = slots->memslots + slotidx; 1978 unsigned long ofs = cur_gfn - ms->base_gfn; 1979 1980 if (ms->base_gfn + ms->npages <= cur_gfn) { 1981 slotidx--; 1982 /* If we are above the highest slot, wrap around */ 1983 if (slotidx < 0) 1984 slotidx = slots->used_slots - 1; 1985 1986 ms = slots->memslots + slotidx; 1987 ofs = 0; 1988 } 1989 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs); 1990 while ((slotidx > 0) && (ofs >= ms->npages)) { 1991 slotidx--; 1992 ms = slots->memslots + slotidx; 1993 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0); 1994 } 1995 return ms->base_gfn + ofs; 1996 } 1997 1998 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args, 1999 u8 *res, unsigned long bufsize) 2000 { 2001 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev; 2002 struct kvm_memslots *slots = kvm_memslots(kvm); 2003 struct kvm_memory_slot *ms; 2004 2005 if (unlikely(!slots->used_slots)) 2006 return 0; 2007 2008 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn); 2009 ms = gfn_to_memslot(kvm, cur_gfn); 2010 args->count = 0; 2011 args->start_gfn = cur_gfn; 2012 if (!ms) 2013 return 0; 2014 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1); 2015 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages; 2016 2017 while (args->count < bufsize) { 2018 hva = gfn_to_hva(kvm, cur_gfn); 2019 if (kvm_is_error_hva(hva)) 2020 return 0; 2021 /* Decrement only if we actually flipped the bit to 0 */ 2022 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms))) 2023 atomic64_dec(&kvm->arch.cmma_dirty_pages); 2024 if (get_pgste(kvm->mm, hva, &pgstev) < 0) 2025 pgstev = 0; 2026 /* Save the value */ 2027 res[args->count++] = (pgstev >> 24) & 0x43; 2028 /* If the next bit is too far away, stop. */ 2029 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE) 2030 return 0; 2031 /* If we reached the previous "next", find the next one */ 2032 if (cur_gfn == next_gfn) 2033 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1); 2034 /* Reached the end of memory or of the buffer, stop */ 2035 if ((next_gfn >= mem_end) || 2036 (next_gfn - args->start_gfn >= bufsize)) 2037 return 0; 2038 cur_gfn++; 2039 /* Reached the end of the current memslot, take the next one. */ 2040 if (cur_gfn - ms->base_gfn >= ms->npages) { 2041 ms = gfn_to_memslot(kvm, cur_gfn); 2042 if (!ms) 2043 return 0; 2044 } 2045 } 2046 return 0; 2047 } 2048 2049 /* 2050 * This function searches for the next page with dirty CMMA attributes, and 2051 * saves the attributes in the buffer up to either the end of the buffer or 2052 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found; 2053 * no trailing clean bytes are saved. 2054 * In case no dirty bits were found, or if CMMA was not enabled or used, the 2055 * output buffer will indicate 0 as length. 2056 */ 2057 static int kvm_s390_get_cmma_bits(struct kvm *kvm, 2058 struct kvm_s390_cmma_log *args) 2059 { 2060 unsigned long bufsize; 2061 int srcu_idx, peek, ret; 2062 u8 *values; 2063 2064 if (!kvm->arch.use_cmma) 2065 return -ENXIO; 2066 /* Invalid/unsupported flags were specified */ 2067 if (args->flags & ~KVM_S390_CMMA_PEEK) 2068 return -EINVAL; 2069 /* Migration mode query, and we are not doing a migration */ 2070 peek = !!(args->flags & KVM_S390_CMMA_PEEK); 2071 if (!peek && !kvm->arch.migration_mode) 2072 return -EINVAL; 2073 /* CMMA is disabled or was not used, or the buffer has length zero */ 2074 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX); 2075 if (!bufsize || !kvm->mm->context.uses_cmm) { 2076 memset(args, 0, sizeof(*args)); 2077 return 0; 2078 } 2079 /* We are not peeking, and there are no dirty pages */ 2080 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) { 2081 memset(args, 0, sizeof(*args)); 2082 return 0; 2083 } 2084 2085 values = vmalloc(bufsize); 2086 if (!values) 2087 return -ENOMEM; 2088 2089 down_read(&kvm->mm->mmap_sem); 2090 srcu_idx = srcu_read_lock(&kvm->srcu); 2091 if (peek) 2092 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize); 2093 else 2094 ret = kvm_s390_get_cmma(kvm, args, values, bufsize); 2095 srcu_read_unlock(&kvm->srcu, srcu_idx); 2096 up_read(&kvm->mm->mmap_sem); 2097 2098 if (kvm->arch.migration_mode) 2099 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages); 2100 else 2101 args->remaining = 0; 2102 2103 if (copy_to_user((void __user *)args->values, values, args->count)) 2104 ret = -EFAULT; 2105 2106 vfree(values); 2107 return ret; 2108 } 2109 2110 /* 2111 * This function sets the CMMA attributes for the given pages. If the input 2112 * buffer has zero length, no action is taken, otherwise the attributes are 2113 * set and the mm->context.uses_cmm flag is set. 2114 */ 2115 static int kvm_s390_set_cmma_bits(struct kvm *kvm, 2116 const struct kvm_s390_cmma_log *args) 2117 { 2118 unsigned long hva, mask, pgstev, i; 2119 uint8_t *bits; 2120 int srcu_idx, r = 0; 2121 2122 mask = args->mask; 2123 2124 if (!kvm->arch.use_cmma) 2125 return -ENXIO; 2126 /* invalid/unsupported flags */ 2127 if (args->flags != 0) 2128 return -EINVAL; 2129 /* Enforce sane limit on memory allocation */ 2130 if (args->count > KVM_S390_CMMA_SIZE_MAX) 2131 return -EINVAL; 2132 /* Nothing to do */ 2133 if (args->count == 0) 2134 return 0; 2135 2136 bits = vmalloc(array_size(sizeof(*bits), args->count)); 2137 if (!bits) 2138 return -ENOMEM; 2139 2140 r = copy_from_user(bits, (void __user *)args->values, args->count); 2141 if (r) { 2142 r = -EFAULT; 2143 goto out; 2144 } 2145 2146 down_read(&kvm->mm->mmap_sem); 2147 srcu_idx = srcu_read_lock(&kvm->srcu); 2148 for (i = 0; i < args->count; i++) { 2149 hva = gfn_to_hva(kvm, args->start_gfn + i); 2150 if (kvm_is_error_hva(hva)) { 2151 r = -EFAULT; 2152 break; 2153 } 2154 2155 pgstev = bits[i]; 2156 pgstev = pgstev << 24; 2157 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT; 2158 set_pgste_bits(kvm->mm, hva, mask, pgstev); 2159 } 2160 srcu_read_unlock(&kvm->srcu, srcu_idx); 2161 up_read(&kvm->mm->mmap_sem); 2162 2163 if (!kvm->mm->context.uses_cmm) { 2164 down_write(&kvm->mm->mmap_sem); 2165 kvm->mm->context.uses_cmm = 1; 2166 up_write(&kvm->mm->mmap_sem); 2167 } 2168 out: 2169 vfree(bits); 2170 return r; 2171 } 2172 2173 static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp) 2174 { 2175 struct kvm_vcpu *vcpu; 2176 u16 rc, rrc; 2177 int ret = 0; 2178 int i; 2179 2180 /* 2181 * We ignore failures and try to destroy as many CPUs as possible. 2182 * At the same time we must not free the assigned resources when 2183 * this fails, as the ultravisor has still access to that memory. 2184 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak 2185 * behind. 2186 * We want to return the first failure rc and rrc, though. 2187 */ 2188 kvm_for_each_vcpu(i, vcpu, kvm) { 2189 mutex_lock(&vcpu->mutex); 2190 if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) { 2191 *rcp = rc; 2192 *rrcp = rrc; 2193 ret = -EIO; 2194 } 2195 mutex_unlock(&vcpu->mutex); 2196 } 2197 return ret; 2198 } 2199 2200 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc) 2201 { 2202 int i, r = 0; 2203 u16 dummy; 2204 2205 struct kvm_vcpu *vcpu; 2206 2207 kvm_for_each_vcpu(i, vcpu, kvm) { 2208 mutex_lock(&vcpu->mutex); 2209 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc); 2210 mutex_unlock(&vcpu->mutex); 2211 if (r) 2212 break; 2213 } 2214 if (r) 2215 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy); 2216 return r; 2217 } 2218 2219 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) 2220 { 2221 int r = 0; 2222 u16 dummy; 2223 void __user *argp = (void __user *)cmd->data; 2224 2225 switch (cmd->cmd) { 2226 case KVM_PV_ENABLE: { 2227 r = -EINVAL; 2228 if (kvm_s390_pv_is_protected(kvm)) 2229 break; 2230 2231 /* 2232 * FMT 4 SIE needs esca. As we never switch back to bsca from 2233 * esca, we need no cleanup in the error cases below 2234 */ 2235 r = sca_switch_to_extended(kvm); 2236 if (r) 2237 break; 2238 2239 down_write(¤t->mm->mmap_sem); 2240 r = gmap_mark_unmergeable(); 2241 up_write(¤t->mm->mmap_sem); 2242 if (r) 2243 break; 2244 2245 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc); 2246 if (r) 2247 break; 2248 2249 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc); 2250 if (r) 2251 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy); 2252 2253 /* we need to block service interrupts from now on */ 2254 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs); 2255 break; 2256 } 2257 case KVM_PV_DISABLE: { 2258 r = -EINVAL; 2259 if (!kvm_s390_pv_is_protected(kvm)) 2260 break; 2261 2262 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc); 2263 /* 2264 * If a CPU could not be destroyed, destroy VM will also fail. 2265 * There is no point in trying to destroy it. Instead return 2266 * the rc and rrc from the first CPU that failed destroying. 2267 */ 2268 if (r) 2269 break; 2270 r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc); 2271 2272 /* no need to block service interrupts any more */ 2273 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs); 2274 break; 2275 } 2276 case KVM_PV_SET_SEC_PARMS: { 2277 struct kvm_s390_pv_sec_parm parms = {}; 2278 void *hdr; 2279 2280 r = -EINVAL; 2281 if (!kvm_s390_pv_is_protected(kvm)) 2282 break; 2283 2284 r = -EFAULT; 2285 if (copy_from_user(&parms, argp, sizeof(parms))) 2286 break; 2287 2288 /* Currently restricted to 8KB */ 2289 r = -EINVAL; 2290 if (parms.length > PAGE_SIZE * 2) 2291 break; 2292 2293 r = -ENOMEM; 2294 hdr = vmalloc(parms.length); 2295 if (!hdr) 2296 break; 2297 2298 r = -EFAULT; 2299 if (!copy_from_user(hdr, (void __user *)parms.origin, 2300 parms.length)) 2301 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length, 2302 &cmd->rc, &cmd->rrc); 2303 2304 vfree(hdr); 2305 break; 2306 } 2307 case KVM_PV_UNPACK: { 2308 struct kvm_s390_pv_unp unp = {}; 2309 2310 r = -EINVAL; 2311 if (!kvm_s390_pv_is_protected(kvm)) 2312 break; 2313 2314 r = -EFAULT; 2315 if (copy_from_user(&unp, argp, sizeof(unp))) 2316 break; 2317 2318 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak, 2319 &cmd->rc, &cmd->rrc); 2320 break; 2321 } 2322 case KVM_PV_VERIFY: { 2323 r = -EINVAL; 2324 if (!kvm_s390_pv_is_protected(kvm)) 2325 break; 2326 2327 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), 2328 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc); 2329 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc, 2330 cmd->rrc); 2331 break; 2332 } 2333 case KVM_PV_PREP_RESET: { 2334 r = -EINVAL; 2335 if (!kvm_s390_pv_is_protected(kvm)) 2336 break; 2337 2338 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), 2339 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc); 2340 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x", 2341 cmd->rc, cmd->rrc); 2342 break; 2343 } 2344 case KVM_PV_UNSHARE_ALL: { 2345 r = -EINVAL; 2346 if (!kvm_s390_pv_is_protected(kvm)) 2347 break; 2348 2349 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), 2350 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc); 2351 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x", 2352 cmd->rc, cmd->rrc); 2353 break; 2354 } 2355 default: 2356 r = -ENOTTY; 2357 } 2358 return r; 2359 } 2360 2361 long kvm_arch_vm_ioctl(struct file *filp, 2362 unsigned int ioctl, unsigned long arg) 2363 { 2364 struct kvm *kvm = filp->private_data; 2365 void __user *argp = (void __user *)arg; 2366 struct kvm_device_attr attr; 2367 int r; 2368 2369 switch (ioctl) { 2370 case KVM_S390_INTERRUPT: { 2371 struct kvm_s390_interrupt s390int; 2372 2373 r = -EFAULT; 2374 if (copy_from_user(&s390int, argp, sizeof(s390int))) 2375 break; 2376 r = kvm_s390_inject_vm(kvm, &s390int); 2377 break; 2378 } 2379 case KVM_CREATE_IRQCHIP: { 2380 struct kvm_irq_routing_entry routing; 2381 2382 r = -EINVAL; 2383 if (kvm->arch.use_irqchip) { 2384 /* Set up dummy routing. */ 2385 memset(&routing, 0, sizeof(routing)); 2386 r = kvm_set_irq_routing(kvm, &routing, 0, 0); 2387 } 2388 break; 2389 } 2390 case KVM_SET_DEVICE_ATTR: { 2391 r = -EFAULT; 2392 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2393 break; 2394 r = kvm_s390_vm_set_attr(kvm, &attr); 2395 break; 2396 } 2397 case KVM_GET_DEVICE_ATTR: { 2398 r = -EFAULT; 2399 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2400 break; 2401 r = kvm_s390_vm_get_attr(kvm, &attr); 2402 break; 2403 } 2404 case KVM_HAS_DEVICE_ATTR: { 2405 r = -EFAULT; 2406 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 2407 break; 2408 r = kvm_s390_vm_has_attr(kvm, &attr); 2409 break; 2410 } 2411 case KVM_S390_GET_SKEYS: { 2412 struct kvm_s390_skeys args; 2413 2414 r = -EFAULT; 2415 if (copy_from_user(&args, argp, 2416 sizeof(struct kvm_s390_skeys))) 2417 break; 2418 r = kvm_s390_get_skeys(kvm, &args); 2419 break; 2420 } 2421 case KVM_S390_SET_SKEYS: { 2422 struct kvm_s390_skeys args; 2423 2424 r = -EFAULT; 2425 if (copy_from_user(&args, argp, 2426 sizeof(struct kvm_s390_skeys))) 2427 break; 2428 r = kvm_s390_set_skeys(kvm, &args); 2429 break; 2430 } 2431 case KVM_S390_GET_CMMA_BITS: { 2432 struct kvm_s390_cmma_log args; 2433 2434 r = -EFAULT; 2435 if (copy_from_user(&args, argp, sizeof(args))) 2436 break; 2437 mutex_lock(&kvm->slots_lock); 2438 r = kvm_s390_get_cmma_bits(kvm, &args); 2439 mutex_unlock(&kvm->slots_lock); 2440 if (!r) { 2441 r = copy_to_user(argp, &args, sizeof(args)); 2442 if (r) 2443 r = -EFAULT; 2444 } 2445 break; 2446 } 2447 case KVM_S390_SET_CMMA_BITS: { 2448 struct kvm_s390_cmma_log args; 2449 2450 r = -EFAULT; 2451 if (copy_from_user(&args, argp, sizeof(args))) 2452 break; 2453 mutex_lock(&kvm->slots_lock); 2454 r = kvm_s390_set_cmma_bits(kvm, &args); 2455 mutex_unlock(&kvm->slots_lock); 2456 break; 2457 } 2458 case KVM_S390_PV_COMMAND: { 2459 struct kvm_pv_cmd args; 2460 2461 /* protvirt means user sigp */ 2462 kvm->arch.user_cpu_state_ctrl = 1; 2463 r = 0; 2464 if (!is_prot_virt_host()) { 2465 r = -EINVAL; 2466 break; 2467 } 2468 if (copy_from_user(&args, argp, sizeof(args))) { 2469 r = -EFAULT; 2470 break; 2471 } 2472 if (args.flags) { 2473 r = -EINVAL; 2474 break; 2475 } 2476 mutex_lock(&kvm->lock); 2477 r = kvm_s390_handle_pv(kvm, &args); 2478 mutex_unlock(&kvm->lock); 2479 if (copy_to_user(argp, &args, sizeof(args))) { 2480 r = -EFAULT; 2481 break; 2482 } 2483 break; 2484 } 2485 default: 2486 r = -ENOTTY; 2487 } 2488 2489 return r; 2490 } 2491 2492 static int kvm_s390_apxa_installed(void) 2493 { 2494 struct ap_config_info info; 2495 2496 if (ap_instructions_available()) { 2497 if (ap_qci(&info) == 0) 2498 return info.apxa; 2499 } 2500 2501 return 0; 2502 } 2503 2504 /* 2505 * The format of the crypto control block (CRYCB) is specified in the 3 low 2506 * order bits of the CRYCB designation (CRYCBD) field as follows: 2507 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the 2508 * AP extended addressing (APXA) facility are installed. 2509 * Format 1: The APXA facility is not installed but the MSAX3 facility is. 2510 * Format 2: Both the APXA and MSAX3 facilities are installed 2511 */ 2512 static void kvm_s390_set_crycb_format(struct kvm *kvm) 2513 { 2514 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb; 2515 2516 /* Clear the CRYCB format bits - i.e., set format 0 by default */ 2517 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK); 2518 2519 /* Check whether MSAX3 is installed */ 2520 if (!test_kvm_facility(kvm, 76)) 2521 return; 2522 2523 if (kvm_s390_apxa_installed()) 2524 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2; 2525 else 2526 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1; 2527 } 2528 2529 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, 2530 unsigned long *aqm, unsigned long *adm) 2531 { 2532 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb; 2533 2534 mutex_lock(&kvm->lock); 2535 kvm_s390_vcpu_block_all(kvm); 2536 2537 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) { 2538 case CRYCB_FORMAT2: /* APCB1 use 256 bits */ 2539 memcpy(crycb->apcb1.apm, apm, 32); 2540 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx", 2541 apm[0], apm[1], apm[2], apm[3]); 2542 memcpy(crycb->apcb1.aqm, aqm, 32); 2543 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx", 2544 aqm[0], aqm[1], aqm[2], aqm[3]); 2545 memcpy(crycb->apcb1.adm, adm, 32); 2546 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx", 2547 adm[0], adm[1], adm[2], adm[3]); 2548 break; 2549 case CRYCB_FORMAT1: 2550 case CRYCB_FORMAT0: /* Fall through both use APCB0 */ 2551 memcpy(crycb->apcb0.apm, apm, 8); 2552 memcpy(crycb->apcb0.aqm, aqm, 2); 2553 memcpy(crycb->apcb0.adm, adm, 2); 2554 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x", 2555 apm[0], *((unsigned short *)aqm), 2556 *((unsigned short *)adm)); 2557 break; 2558 default: /* Can not happen */ 2559 break; 2560 } 2561 2562 /* recreate the shadow crycb for each vcpu */ 2563 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART); 2564 kvm_s390_vcpu_unblock_all(kvm); 2565 mutex_unlock(&kvm->lock); 2566 } 2567 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks); 2568 2569 void kvm_arch_crypto_clear_masks(struct kvm *kvm) 2570 { 2571 mutex_lock(&kvm->lock); 2572 kvm_s390_vcpu_block_all(kvm); 2573 2574 memset(&kvm->arch.crypto.crycb->apcb0, 0, 2575 sizeof(kvm->arch.crypto.crycb->apcb0)); 2576 memset(&kvm->arch.crypto.crycb->apcb1, 0, 2577 sizeof(kvm->arch.crypto.crycb->apcb1)); 2578 2579 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:"); 2580 /* recreate the shadow crycb for each vcpu */ 2581 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART); 2582 kvm_s390_vcpu_unblock_all(kvm); 2583 mutex_unlock(&kvm->lock); 2584 } 2585 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks); 2586 2587 static u64 kvm_s390_get_initial_cpuid(void) 2588 { 2589 struct cpuid cpuid; 2590 2591 get_cpu_id(&cpuid); 2592 cpuid.version = 0xff; 2593 return *((u64 *) &cpuid); 2594 } 2595 2596 static void kvm_s390_crypto_init(struct kvm *kvm) 2597 { 2598 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb; 2599 kvm_s390_set_crycb_format(kvm); 2600 2601 if (!test_kvm_facility(kvm, 76)) 2602 return; 2603 2604 /* Enable AES/DEA protected key functions by default */ 2605 kvm->arch.crypto.aes_kw = 1; 2606 kvm->arch.crypto.dea_kw = 1; 2607 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 2608 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 2609 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 2610 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 2611 } 2612 2613 static void sca_dispose(struct kvm *kvm) 2614 { 2615 if (kvm->arch.use_esca) 2616 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block)); 2617 else 2618 free_page((unsigned long)(kvm->arch.sca)); 2619 kvm->arch.sca = NULL; 2620 } 2621 2622 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 2623 { 2624 gfp_t alloc_flags = GFP_KERNEL; 2625 int i, rc; 2626 char debug_name[16]; 2627 static unsigned long sca_offset; 2628 2629 rc = -EINVAL; 2630 #ifdef CONFIG_KVM_S390_UCONTROL 2631 if (type & ~KVM_VM_S390_UCONTROL) 2632 goto out_err; 2633 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 2634 goto out_err; 2635 #else 2636 if (type) 2637 goto out_err; 2638 #endif 2639 2640 rc = s390_enable_sie(); 2641 if (rc) 2642 goto out_err; 2643 2644 rc = -ENOMEM; 2645 2646 if (!sclp.has_64bscao) 2647 alloc_flags |= GFP_DMA; 2648 rwlock_init(&kvm->arch.sca_lock); 2649 /* start with basic SCA */ 2650 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags); 2651 if (!kvm->arch.sca) 2652 goto out_err; 2653 mutex_lock(&kvm_lock); 2654 sca_offset += 16; 2655 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE) 2656 sca_offset = 0; 2657 kvm->arch.sca = (struct bsca_block *) 2658 ((char *) kvm->arch.sca + sca_offset); 2659 mutex_unlock(&kvm_lock); 2660 2661 sprintf(debug_name, "kvm-%u", current->pid); 2662 2663 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long)); 2664 if (!kvm->arch.dbf) 2665 goto out_err; 2666 2667 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096); 2668 kvm->arch.sie_page2 = 2669 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 2670 if (!kvm->arch.sie_page2) 2671 goto out_err; 2672 2673 kvm->arch.sie_page2->kvm = kvm; 2674 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list; 2675 2676 for (i = 0; i < kvm_s390_fac_size(); i++) { 2677 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] & 2678 (kvm_s390_fac_base[i] | 2679 kvm_s390_fac_ext[i]); 2680 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] & 2681 kvm_s390_fac_base[i]; 2682 } 2683 kvm->arch.model.subfuncs = kvm_s390_available_subfunc; 2684 2685 /* we are always in czam mode - even on pre z14 machines */ 2686 set_kvm_facility(kvm->arch.model.fac_mask, 138); 2687 set_kvm_facility(kvm->arch.model.fac_list, 138); 2688 /* we emulate STHYI in kvm */ 2689 set_kvm_facility(kvm->arch.model.fac_mask, 74); 2690 set_kvm_facility(kvm->arch.model.fac_list, 74); 2691 if (MACHINE_HAS_TLB_GUEST) { 2692 set_kvm_facility(kvm->arch.model.fac_mask, 147); 2693 set_kvm_facility(kvm->arch.model.fac_list, 147); 2694 } 2695 2696 if (css_general_characteristics.aiv && test_facility(65)) 2697 set_kvm_facility(kvm->arch.model.fac_mask, 65); 2698 2699 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid(); 2700 kvm->arch.model.ibc = sclp.ibc & 0x0fff; 2701 2702 kvm_s390_crypto_init(kvm); 2703 2704 mutex_init(&kvm->arch.float_int.ais_lock); 2705 spin_lock_init(&kvm->arch.float_int.lock); 2706 for (i = 0; i < FIRQ_LIST_COUNT; i++) 2707 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]); 2708 init_waitqueue_head(&kvm->arch.ipte_wq); 2709 mutex_init(&kvm->arch.ipte_mutex); 2710 2711 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 2712 VM_EVENT(kvm, 3, "vm created with type %lu", type); 2713 2714 if (type & KVM_VM_S390_UCONTROL) { 2715 kvm->arch.gmap = NULL; 2716 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT; 2717 } else { 2718 if (sclp.hamax == U64_MAX) 2719 kvm->arch.mem_limit = TASK_SIZE_MAX; 2720 else 2721 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX, 2722 sclp.hamax + 1); 2723 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1); 2724 if (!kvm->arch.gmap) 2725 goto out_err; 2726 kvm->arch.gmap->private = kvm; 2727 kvm->arch.gmap->pfault_enabled = 0; 2728 } 2729 2730 kvm->arch.use_pfmfi = sclp.has_pfmfi; 2731 kvm->arch.use_skf = sclp.has_skey; 2732 spin_lock_init(&kvm->arch.start_stop_lock); 2733 kvm_s390_vsie_init(kvm); 2734 if (use_gisa) 2735 kvm_s390_gisa_init(kvm); 2736 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid); 2737 2738 return 0; 2739 out_err: 2740 free_page((unsigned long)kvm->arch.sie_page2); 2741 debug_unregister(kvm->arch.dbf); 2742 sca_dispose(kvm); 2743 KVM_EVENT(3, "creation of vm failed: %d", rc); 2744 return rc; 2745 } 2746 2747 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 2748 { 2749 u16 rc, rrc; 2750 2751 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 2752 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 2753 kvm_s390_clear_local_irqs(vcpu); 2754 kvm_clear_async_pf_completion_queue(vcpu); 2755 if (!kvm_is_ucontrol(vcpu->kvm)) 2756 sca_del_vcpu(vcpu); 2757 2758 if (kvm_is_ucontrol(vcpu->kvm)) 2759 gmap_remove(vcpu->arch.gmap); 2760 2761 if (vcpu->kvm->arch.use_cmma) 2762 kvm_s390_vcpu_unsetup_cmma(vcpu); 2763 /* We can not hold the vcpu mutex here, we are already dying */ 2764 if (kvm_s390_pv_cpu_get_handle(vcpu)) 2765 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc); 2766 free_page((unsigned long)(vcpu->arch.sie_block)); 2767 } 2768 2769 static void kvm_free_vcpus(struct kvm *kvm) 2770 { 2771 unsigned int i; 2772 struct kvm_vcpu *vcpu; 2773 2774 kvm_for_each_vcpu(i, vcpu, kvm) 2775 kvm_vcpu_destroy(vcpu); 2776 2777 mutex_lock(&kvm->lock); 2778 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 2779 kvm->vcpus[i] = NULL; 2780 2781 atomic_set(&kvm->online_vcpus, 0); 2782 mutex_unlock(&kvm->lock); 2783 } 2784 2785 void kvm_arch_destroy_vm(struct kvm *kvm) 2786 { 2787 u16 rc, rrc; 2788 2789 kvm_free_vcpus(kvm); 2790 sca_dispose(kvm); 2791 kvm_s390_gisa_destroy(kvm); 2792 /* 2793 * We are already at the end of life and kvm->lock is not taken. 2794 * This is ok as the file descriptor is closed by now and nobody 2795 * can mess with the pv state. To avoid lockdep_assert_held from 2796 * complaining we do not use kvm_s390_pv_is_protected. 2797 */ 2798 if (kvm_s390_pv_get_handle(kvm)) 2799 kvm_s390_pv_deinit_vm(kvm, &rc, &rrc); 2800 debug_unregister(kvm->arch.dbf); 2801 free_page((unsigned long)kvm->arch.sie_page2); 2802 if (!kvm_is_ucontrol(kvm)) 2803 gmap_remove(kvm->arch.gmap); 2804 kvm_s390_destroy_adapters(kvm); 2805 kvm_s390_clear_float_irqs(kvm); 2806 kvm_s390_vsie_destroy(kvm); 2807 KVM_EVENT(3, "vm 0x%pK destroyed", kvm); 2808 } 2809 2810 /* Section: vcpu related */ 2811 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu) 2812 { 2813 vcpu->arch.gmap = gmap_create(current->mm, -1UL); 2814 if (!vcpu->arch.gmap) 2815 return -ENOMEM; 2816 vcpu->arch.gmap->private = vcpu->kvm; 2817 2818 return 0; 2819 } 2820 2821 static void sca_del_vcpu(struct kvm_vcpu *vcpu) 2822 { 2823 if (!kvm_s390_use_sca_entries()) 2824 return; 2825 read_lock(&vcpu->kvm->arch.sca_lock); 2826 if (vcpu->kvm->arch.use_esca) { 2827 struct esca_block *sca = vcpu->kvm->arch.sca; 2828 2829 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn); 2830 sca->cpu[vcpu->vcpu_id].sda = 0; 2831 } else { 2832 struct bsca_block *sca = vcpu->kvm->arch.sca; 2833 2834 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn); 2835 sca->cpu[vcpu->vcpu_id].sda = 0; 2836 } 2837 read_unlock(&vcpu->kvm->arch.sca_lock); 2838 } 2839 2840 static void sca_add_vcpu(struct kvm_vcpu *vcpu) 2841 { 2842 if (!kvm_s390_use_sca_entries()) { 2843 struct bsca_block *sca = vcpu->kvm->arch.sca; 2844 2845 /* we still need the basic sca for the ipte control */ 2846 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 2847 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca; 2848 return; 2849 } 2850 read_lock(&vcpu->kvm->arch.sca_lock); 2851 if (vcpu->kvm->arch.use_esca) { 2852 struct esca_block *sca = vcpu->kvm->arch.sca; 2853 2854 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block; 2855 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 2856 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU; 2857 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA; 2858 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn); 2859 } else { 2860 struct bsca_block *sca = vcpu->kvm->arch.sca; 2861 2862 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block; 2863 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 2864 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca; 2865 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn); 2866 } 2867 read_unlock(&vcpu->kvm->arch.sca_lock); 2868 } 2869 2870 /* Basic SCA to Extended SCA data copy routines */ 2871 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s) 2872 { 2873 d->sda = s->sda; 2874 d->sigp_ctrl.c = s->sigp_ctrl.c; 2875 d->sigp_ctrl.scn = s->sigp_ctrl.scn; 2876 } 2877 2878 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s) 2879 { 2880 int i; 2881 2882 d->ipte_control = s->ipte_control; 2883 d->mcn[0] = s->mcn; 2884 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++) 2885 sca_copy_entry(&d->cpu[i], &s->cpu[i]); 2886 } 2887 2888 static int sca_switch_to_extended(struct kvm *kvm) 2889 { 2890 struct bsca_block *old_sca = kvm->arch.sca; 2891 struct esca_block *new_sca; 2892 struct kvm_vcpu *vcpu; 2893 unsigned int vcpu_idx; 2894 u32 scaol, scaoh; 2895 2896 if (kvm->arch.use_esca) 2897 return 0; 2898 2899 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO); 2900 if (!new_sca) 2901 return -ENOMEM; 2902 2903 scaoh = (u32)((u64)(new_sca) >> 32); 2904 scaol = (u32)(u64)(new_sca) & ~0x3fU; 2905 2906 kvm_s390_vcpu_block_all(kvm); 2907 write_lock(&kvm->arch.sca_lock); 2908 2909 sca_copy_b_to_e(new_sca, old_sca); 2910 2911 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) { 2912 vcpu->arch.sie_block->scaoh = scaoh; 2913 vcpu->arch.sie_block->scaol = scaol; 2914 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA; 2915 } 2916 kvm->arch.sca = new_sca; 2917 kvm->arch.use_esca = 1; 2918 2919 write_unlock(&kvm->arch.sca_lock); 2920 kvm_s390_vcpu_unblock_all(kvm); 2921 2922 free_page((unsigned long)old_sca); 2923 2924 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)", 2925 old_sca, kvm->arch.sca); 2926 return 0; 2927 } 2928 2929 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id) 2930 { 2931 int rc; 2932 2933 if (!kvm_s390_use_sca_entries()) { 2934 if (id < KVM_MAX_VCPUS) 2935 return true; 2936 return false; 2937 } 2938 if (id < KVM_S390_BSCA_CPU_SLOTS) 2939 return true; 2940 if (!sclp.has_esca || !sclp.has_64bscao) 2941 return false; 2942 2943 mutex_lock(&kvm->lock); 2944 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm); 2945 mutex_unlock(&kvm->lock); 2946 2947 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS; 2948 } 2949 2950 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2951 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2952 { 2953 WARN_ON_ONCE(vcpu->arch.cputm_start != 0); 2954 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 2955 vcpu->arch.cputm_start = get_tod_clock_fast(); 2956 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 2957 } 2958 2959 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2960 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2961 { 2962 WARN_ON_ONCE(vcpu->arch.cputm_start == 0); 2963 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 2964 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start; 2965 vcpu->arch.cputm_start = 0; 2966 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 2967 } 2968 2969 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2970 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2971 { 2972 WARN_ON_ONCE(vcpu->arch.cputm_enabled); 2973 vcpu->arch.cputm_enabled = true; 2974 __start_cpu_timer_accounting(vcpu); 2975 } 2976 2977 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2978 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2979 { 2980 WARN_ON_ONCE(!vcpu->arch.cputm_enabled); 2981 __stop_cpu_timer_accounting(vcpu); 2982 vcpu->arch.cputm_enabled = false; 2983 } 2984 2985 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2986 { 2987 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 2988 __enable_cpu_timer_accounting(vcpu); 2989 preempt_enable(); 2990 } 2991 2992 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2993 { 2994 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 2995 __disable_cpu_timer_accounting(vcpu); 2996 preempt_enable(); 2997 } 2998 2999 /* set the cpu timer - may only be called from the VCPU thread itself */ 3000 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm) 3001 { 3002 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 3003 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 3004 if (vcpu->arch.cputm_enabled) 3005 vcpu->arch.cputm_start = get_tod_clock_fast(); 3006 vcpu->arch.sie_block->cputm = cputm; 3007 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 3008 preempt_enable(); 3009 } 3010 3011 /* update and get the cpu timer - can also be called from other VCPU threads */ 3012 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu) 3013 { 3014 unsigned int seq; 3015 __u64 value; 3016 3017 if (unlikely(!vcpu->arch.cputm_enabled)) 3018 return vcpu->arch.sie_block->cputm; 3019 3020 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 3021 do { 3022 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount); 3023 /* 3024 * If the writer would ever execute a read in the critical 3025 * section, e.g. in irq context, we have a deadlock. 3026 */ 3027 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu); 3028 value = vcpu->arch.sie_block->cputm; 3029 /* if cputm_start is 0, accounting is being started/stopped */ 3030 if (likely(vcpu->arch.cputm_start)) 3031 value -= get_tod_clock_fast() - vcpu->arch.cputm_start; 3032 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1)); 3033 preempt_enable(); 3034 return value; 3035 } 3036 3037 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 3038 { 3039 3040 gmap_enable(vcpu->arch.enabled_gmap); 3041 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING); 3042 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 3043 __start_cpu_timer_accounting(vcpu); 3044 vcpu->cpu = cpu; 3045 } 3046 3047 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 3048 { 3049 vcpu->cpu = -1; 3050 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 3051 __stop_cpu_timer_accounting(vcpu); 3052 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING); 3053 vcpu->arch.enabled_gmap = gmap_get_enabled(); 3054 gmap_disable(vcpu->arch.enabled_gmap); 3055 3056 } 3057 3058 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 3059 { 3060 mutex_lock(&vcpu->kvm->lock); 3061 preempt_disable(); 3062 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch; 3063 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx; 3064 preempt_enable(); 3065 mutex_unlock(&vcpu->kvm->lock); 3066 if (!kvm_is_ucontrol(vcpu->kvm)) { 3067 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 3068 sca_add_vcpu(vcpu); 3069 } 3070 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0) 3071 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; 3072 /* make vcpu_load load the right gmap on the first trigger */ 3073 vcpu->arch.enabled_gmap = vcpu->arch.gmap; 3074 } 3075 3076 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr) 3077 { 3078 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) && 3079 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo)) 3080 return true; 3081 return false; 3082 } 3083 3084 static bool kvm_has_pckmo_ecc(struct kvm *kvm) 3085 { 3086 /* At least one ECC subfunction must be present */ 3087 return kvm_has_pckmo_subfunc(kvm, 32) || 3088 kvm_has_pckmo_subfunc(kvm, 33) || 3089 kvm_has_pckmo_subfunc(kvm, 34) || 3090 kvm_has_pckmo_subfunc(kvm, 40) || 3091 kvm_has_pckmo_subfunc(kvm, 41); 3092 3093 } 3094 3095 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) 3096 { 3097 /* 3098 * If the AP instructions are not being interpreted and the MSAX3 3099 * facility is not configured for the guest, there is nothing to set up. 3100 */ 3101 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76)) 3102 return; 3103 3104 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd; 3105 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA); 3106 vcpu->arch.sie_block->eca &= ~ECA_APIE; 3107 vcpu->arch.sie_block->ecd &= ~ECD_ECC; 3108 3109 if (vcpu->kvm->arch.crypto.apie) 3110 vcpu->arch.sie_block->eca |= ECA_APIE; 3111 3112 /* Set up protected key support */ 3113 if (vcpu->kvm->arch.crypto.aes_kw) { 3114 vcpu->arch.sie_block->ecb3 |= ECB3_AES; 3115 /* ecc is also wrapped with AES key */ 3116 if (kvm_has_pckmo_ecc(vcpu->kvm)) 3117 vcpu->arch.sie_block->ecd |= ECD_ECC; 3118 } 3119 3120 if (vcpu->kvm->arch.crypto.dea_kw) 3121 vcpu->arch.sie_block->ecb3 |= ECB3_DEA; 3122 } 3123 3124 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) 3125 { 3126 free_page(vcpu->arch.sie_block->cbrlo); 3127 vcpu->arch.sie_block->cbrlo = 0; 3128 } 3129 3130 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) 3131 { 3132 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL); 3133 if (!vcpu->arch.sie_block->cbrlo) 3134 return -ENOMEM; 3135 return 0; 3136 } 3137 3138 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu) 3139 { 3140 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model; 3141 3142 vcpu->arch.sie_block->ibc = model->ibc; 3143 if (test_kvm_facility(vcpu->kvm, 7)) 3144 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list; 3145 } 3146 3147 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu) 3148 { 3149 int rc = 0; 3150 u16 uvrc, uvrrc; 3151 3152 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 3153 CPUSTAT_SM | 3154 CPUSTAT_STOPPED); 3155 3156 if (test_kvm_facility(vcpu->kvm, 78)) 3157 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2); 3158 else if (test_kvm_facility(vcpu->kvm, 8)) 3159 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED); 3160 3161 kvm_s390_vcpu_setup_model(vcpu); 3162 3163 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */ 3164 if (MACHINE_HAS_ESOP) 3165 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT; 3166 if (test_kvm_facility(vcpu->kvm, 9)) 3167 vcpu->arch.sie_block->ecb |= ECB_SRSI; 3168 if (test_kvm_facility(vcpu->kvm, 73)) 3169 vcpu->arch.sie_block->ecb |= ECB_TE; 3170 3171 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi) 3172 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI; 3173 if (test_kvm_facility(vcpu->kvm, 130)) 3174 vcpu->arch.sie_block->ecb2 |= ECB2_IEP; 3175 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI; 3176 if (sclp.has_cei) 3177 vcpu->arch.sie_block->eca |= ECA_CEI; 3178 if (sclp.has_ib) 3179 vcpu->arch.sie_block->eca |= ECA_IB; 3180 if (sclp.has_siif) 3181 vcpu->arch.sie_block->eca |= ECA_SII; 3182 if (sclp.has_sigpif) 3183 vcpu->arch.sie_block->eca |= ECA_SIGPI; 3184 if (test_kvm_facility(vcpu->kvm, 129)) { 3185 vcpu->arch.sie_block->eca |= ECA_VX; 3186 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 3187 } 3188 if (test_kvm_facility(vcpu->kvm, 139)) 3189 vcpu->arch.sie_block->ecd |= ECD_MEF; 3190 if (test_kvm_facility(vcpu->kvm, 156)) 3191 vcpu->arch.sie_block->ecd |= ECD_ETOKENF; 3192 if (vcpu->arch.sie_block->gd) { 3193 vcpu->arch.sie_block->eca |= ECA_AIV; 3194 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u", 3195 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id); 3196 } 3197 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx) 3198 | SDNXC; 3199 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb; 3200 3201 if (sclp.has_kss) 3202 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS); 3203 else 3204 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 3205 3206 if (vcpu->kvm->arch.use_cmma) { 3207 rc = kvm_s390_vcpu_setup_cmma(vcpu); 3208 if (rc) 3209 return rc; 3210 } 3211 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3212 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 3213 3214 vcpu->arch.sie_block->hpid = HPID_KVM; 3215 3216 kvm_s390_vcpu_crypto_setup(vcpu); 3217 3218 mutex_lock(&vcpu->kvm->lock); 3219 if (kvm_s390_pv_is_protected(vcpu->kvm)) { 3220 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc); 3221 if (rc) 3222 kvm_s390_vcpu_unsetup_cmma(vcpu); 3223 } 3224 mutex_unlock(&vcpu->kvm->lock); 3225 3226 return rc; 3227 } 3228 3229 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 3230 { 3231 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id)) 3232 return -EINVAL; 3233 return 0; 3234 } 3235 3236 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 3237 { 3238 struct sie_page *sie_page; 3239 int rc; 3240 3241 BUILD_BUG_ON(sizeof(struct sie_page) != 4096); 3242 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL); 3243 if (!sie_page) 3244 return -ENOMEM; 3245 3246 vcpu->arch.sie_block = &sie_page->sie_block; 3247 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 3248 3249 /* the real guest size will always be smaller than msl */ 3250 vcpu->arch.sie_block->mso = 0; 3251 vcpu->arch.sie_block->msl = sclp.hamax; 3252 3253 vcpu->arch.sie_block->icpua = vcpu->vcpu_id; 3254 spin_lock_init(&vcpu->arch.local_int.lock); 3255 vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin; 3256 if (vcpu->arch.sie_block->gd && sclp.has_gisaf) 3257 vcpu->arch.sie_block->gd |= GISA_FORMAT1; 3258 seqcount_init(&vcpu->arch.cputm_seqcount); 3259 3260 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 3261 kvm_clear_async_pf_completion_queue(vcpu); 3262 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 3263 KVM_SYNC_GPRS | 3264 KVM_SYNC_ACRS | 3265 KVM_SYNC_CRS | 3266 KVM_SYNC_ARCH0 | 3267 KVM_SYNC_PFAULT; 3268 kvm_s390_set_prefix(vcpu, 0); 3269 if (test_kvm_facility(vcpu->kvm, 64)) 3270 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB; 3271 if (test_kvm_facility(vcpu->kvm, 82)) 3272 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC; 3273 if (test_kvm_facility(vcpu->kvm, 133)) 3274 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB; 3275 if (test_kvm_facility(vcpu->kvm, 156)) 3276 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN; 3277 /* fprs can be synchronized via vrs, even if the guest has no vx. With 3278 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format. 3279 */ 3280 if (MACHINE_HAS_VX) 3281 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS; 3282 else 3283 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS; 3284 3285 if (kvm_is_ucontrol(vcpu->kvm)) { 3286 rc = __kvm_ucontrol_vcpu_init(vcpu); 3287 if (rc) 3288 goto out_free_sie_block; 3289 } 3290 3291 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", 3292 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block); 3293 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block); 3294 3295 rc = kvm_s390_vcpu_setup(vcpu); 3296 if (rc) 3297 goto out_ucontrol_uninit; 3298 return 0; 3299 3300 out_ucontrol_uninit: 3301 if (kvm_is_ucontrol(vcpu->kvm)) 3302 gmap_remove(vcpu->arch.gmap); 3303 out_free_sie_block: 3304 free_page((unsigned long)(vcpu->arch.sie_block)); 3305 return rc; 3306 } 3307 3308 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 3309 { 3310 return kvm_s390_vcpu_has_irq(vcpu, 0); 3311 } 3312 3313 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 3314 { 3315 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE); 3316 } 3317 3318 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu) 3319 { 3320 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 3321 exit_sie(vcpu); 3322 } 3323 3324 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu) 3325 { 3326 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 3327 } 3328 3329 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu) 3330 { 3331 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20); 3332 exit_sie(vcpu); 3333 } 3334 3335 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu) 3336 { 3337 return atomic_read(&vcpu->arch.sie_block->prog20) & 3338 (PROG_BLOCK_SIE | PROG_REQUEST); 3339 } 3340 3341 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu) 3342 { 3343 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20); 3344 } 3345 3346 /* 3347 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running. 3348 * If the CPU is not running (e.g. waiting as idle) the function will 3349 * return immediately. */ 3350 void exit_sie(struct kvm_vcpu *vcpu) 3351 { 3352 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); 3353 kvm_s390_vsie_kick(vcpu); 3354 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 3355 cpu_relax(); 3356 } 3357 3358 /* Kick a guest cpu out of SIE to process a request synchronously */ 3359 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu) 3360 { 3361 kvm_make_request(req, vcpu); 3362 kvm_s390_vcpu_request(vcpu); 3363 } 3364 3365 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, 3366 unsigned long end) 3367 { 3368 struct kvm *kvm = gmap->private; 3369 struct kvm_vcpu *vcpu; 3370 unsigned long prefix; 3371 int i; 3372 3373 if (gmap_is_shadow(gmap)) 3374 return; 3375 if (start >= 1UL << 31) 3376 /* We are only interested in prefix pages */ 3377 return; 3378 kvm_for_each_vcpu(i, vcpu, kvm) { 3379 /* match against both prefix pages */ 3380 prefix = kvm_s390_get_prefix(vcpu); 3381 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) { 3382 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx", 3383 start, end); 3384 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu); 3385 } 3386 } 3387 } 3388 3389 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) 3390 { 3391 /* do not poll with more than halt_poll_max_steal percent of steal time */ 3392 if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >= 3393 halt_poll_max_steal) { 3394 vcpu->stat.halt_no_poll_steal++; 3395 return true; 3396 } 3397 return false; 3398 } 3399 3400 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 3401 { 3402 /* kvm common code refers to this, but never calls it */ 3403 BUG(); 3404 return 0; 3405 } 3406 3407 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 3408 struct kvm_one_reg *reg) 3409 { 3410 int r = -EINVAL; 3411 3412 switch (reg->id) { 3413 case KVM_REG_S390_TODPR: 3414 r = put_user(vcpu->arch.sie_block->todpr, 3415 (u32 __user *)reg->addr); 3416 break; 3417 case KVM_REG_S390_EPOCHDIFF: 3418 r = put_user(vcpu->arch.sie_block->epoch, 3419 (u64 __user *)reg->addr); 3420 break; 3421 case KVM_REG_S390_CPU_TIMER: 3422 r = put_user(kvm_s390_get_cpu_timer(vcpu), 3423 (u64 __user *)reg->addr); 3424 break; 3425 case KVM_REG_S390_CLOCK_COMP: 3426 r = put_user(vcpu->arch.sie_block->ckc, 3427 (u64 __user *)reg->addr); 3428 break; 3429 case KVM_REG_S390_PFTOKEN: 3430 r = put_user(vcpu->arch.pfault_token, 3431 (u64 __user *)reg->addr); 3432 break; 3433 case KVM_REG_S390_PFCOMPARE: 3434 r = put_user(vcpu->arch.pfault_compare, 3435 (u64 __user *)reg->addr); 3436 break; 3437 case KVM_REG_S390_PFSELECT: 3438 r = put_user(vcpu->arch.pfault_select, 3439 (u64 __user *)reg->addr); 3440 break; 3441 case KVM_REG_S390_PP: 3442 r = put_user(vcpu->arch.sie_block->pp, 3443 (u64 __user *)reg->addr); 3444 break; 3445 case KVM_REG_S390_GBEA: 3446 r = put_user(vcpu->arch.sie_block->gbea, 3447 (u64 __user *)reg->addr); 3448 break; 3449 default: 3450 break; 3451 } 3452 3453 return r; 3454 } 3455 3456 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 3457 struct kvm_one_reg *reg) 3458 { 3459 int r = -EINVAL; 3460 __u64 val; 3461 3462 switch (reg->id) { 3463 case KVM_REG_S390_TODPR: 3464 r = get_user(vcpu->arch.sie_block->todpr, 3465 (u32 __user *)reg->addr); 3466 break; 3467 case KVM_REG_S390_EPOCHDIFF: 3468 r = get_user(vcpu->arch.sie_block->epoch, 3469 (u64 __user *)reg->addr); 3470 break; 3471 case KVM_REG_S390_CPU_TIMER: 3472 r = get_user(val, (u64 __user *)reg->addr); 3473 if (!r) 3474 kvm_s390_set_cpu_timer(vcpu, val); 3475 break; 3476 case KVM_REG_S390_CLOCK_COMP: 3477 r = get_user(vcpu->arch.sie_block->ckc, 3478 (u64 __user *)reg->addr); 3479 break; 3480 case KVM_REG_S390_PFTOKEN: 3481 r = get_user(vcpu->arch.pfault_token, 3482 (u64 __user *)reg->addr); 3483 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 3484 kvm_clear_async_pf_completion_queue(vcpu); 3485 break; 3486 case KVM_REG_S390_PFCOMPARE: 3487 r = get_user(vcpu->arch.pfault_compare, 3488 (u64 __user *)reg->addr); 3489 break; 3490 case KVM_REG_S390_PFSELECT: 3491 r = get_user(vcpu->arch.pfault_select, 3492 (u64 __user *)reg->addr); 3493 break; 3494 case KVM_REG_S390_PP: 3495 r = get_user(vcpu->arch.sie_block->pp, 3496 (u64 __user *)reg->addr); 3497 break; 3498 case KVM_REG_S390_GBEA: 3499 r = get_user(vcpu->arch.sie_block->gbea, 3500 (u64 __user *)reg->addr); 3501 break; 3502 default: 3503 break; 3504 } 3505 3506 return r; 3507 } 3508 3509 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu) 3510 { 3511 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI; 3512 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 3513 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb)); 3514 3515 kvm_clear_async_pf_completion_queue(vcpu); 3516 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) 3517 kvm_s390_vcpu_stop(vcpu); 3518 kvm_s390_clear_local_irqs(vcpu); 3519 } 3520 3521 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 3522 { 3523 /* Initial reset is a superset of the normal reset */ 3524 kvm_arch_vcpu_ioctl_normal_reset(vcpu); 3525 3526 /* 3527 * This equals initial cpu reset in pop, but we don't switch to ESA. 3528 * We do not only reset the internal data, but also ... 3529 */ 3530 vcpu->arch.sie_block->gpsw.mask = 0; 3531 vcpu->arch.sie_block->gpsw.addr = 0; 3532 kvm_s390_set_prefix(vcpu, 0); 3533 kvm_s390_set_cpu_timer(vcpu, 0); 3534 vcpu->arch.sie_block->ckc = 0; 3535 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr)); 3536 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK; 3537 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK; 3538 3539 /* ... the data in sync regs */ 3540 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs)); 3541 vcpu->run->s.regs.ckc = 0; 3542 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK; 3543 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK; 3544 vcpu->run->psw_addr = 0; 3545 vcpu->run->psw_mask = 0; 3546 vcpu->run->s.regs.todpr = 0; 3547 vcpu->run->s.regs.cputm = 0; 3548 vcpu->run->s.regs.ckc = 0; 3549 vcpu->run->s.regs.pp = 0; 3550 vcpu->run->s.regs.gbea = 1; 3551 vcpu->run->s.regs.fpc = 0; 3552 /* 3553 * Do not reset these registers in the protected case, as some of 3554 * them are overlayed and they are not accessible in this case 3555 * anyway. 3556 */ 3557 if (!kvm_s390_pv_cpu_is_protected(vcpu)) { 3558 vcpu->arch.sie_block->gbea = 1; 3559 vcpu->arch.sie_block->pp = 0; 3560 vcpu->arch.sie_block->fpf &= ~FPF_BPBC; 3561 vcpu->arch.sie_block->todpr = 0; 3562 } 3563 } 3564 3565 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu) 3566 { 3567 struct kvm_sync_regs *regs = &vcpu->run->s.regs; 3568 3569 /* Clear reset is a superset of the initial reset */ 3570 kvm_arch_vcpu_ioctl_initial_reset(vcpu); 3571 3572 memset(®s->gprs, 0, sizeof(regs->gprs)); 3573 memset(®s->vrs, 0, sizeof(regs->vrs)); 3574 memset(®s->acrs, 0, sizeof(regs->acrs)); 3575 memset(®s->gscb, 0, sizeof(regs->gscb)); 3576 3577 regs->etoken = 0; 3578 regs->etoken_extension = 0; 3579 } 3580 3581 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 3582 { 3583 vcpu_load(vcpu); 3584 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 3585 vcpu_put(vcpu); 3586 return 0; 3587 } 3588 3589 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 3590 { 3591 vcpu_load(vcpu); 3592 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 3593 vcpu_put(vcpu); 3594 return 0; 3595 } 3596 3597 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 3598 struct kvm_sregs *sregs) 3599 { 3600 vcpu_load(vcpu); 3601 3602 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 3603 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 3604 3605 vcpu_put(vcpu); 3606 return 0; 3607 } 3608 3609 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 3610 struct kvm_sregs *sregs) 3611 { 3612 vcpu_load(vcpu); 3613 3614 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 3615 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 3616 3617 vcpu_put(vcpu); 3618 return 0; 3619 } 3620 3621 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 3622 { 3623 int ret = 0; 3624 3625 vcpu_load(vcpu); 3626 3627 if (test_fp_ctl(fpu->fpc)) { 3628 ret = -EINVAL; 3629 goto out; 3630 } 3631 vcpu->run->s.regs.fpc = fpu->fpc; 3632 if (MACHINE_HAS_VX) 3633 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs, 3634 (freg_t *) fpu->fprs); 3635 else 3636 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs)); 3637 3638 out: 3639 vcpu_put(vcpu); 3640 return ret; 3641 } 3642 3643 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 3644 { 3645 vcpu_load(vcpu); 3646 3647 /* make sure we have the latest values */ 3648 save_fpu_regs(); 3649 if (MACHINE_HAS_VX) 3650 convert_vx_to_fp((freg_t *) fpu->fprs, 3651 (__vector128 *) vcpu->run->s.regs.vrs); 3652 else 3653 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs)); 3654 fpu->fpc = vcpu->run->s.regs.fpc; 3655 3656 vcpu_put(vcpu); 3657 return 0; 3658 } 3659 3660 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 3661 { 3662 int rc = 0; 3663 3664 if (!is_vcpu_stopped(vcpu)) 3665 rc = -EBUSY; 3666 else { 3667 vcpu->run->psw_mask = psw.mask; 3668 vcpu->run->psw_addr = psw.addr; 3669 } 3670 return rc; 3671 } 3672 3673 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 3674 struct kvm_translation *tr) 3675 { 3676 return -EINVAL; /* not implemented yet */ 3677 } 3678 3679 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \ 3680 KVM_GUESTDBG_USE_HW_BP | \ 3681 KVM_GUESTDBG_ENABLE) 3682 3683 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 3684 struct kvm_guest_debug *dbg) 3685 { 3686 int rc = 0; 3687 3688 vcpu_load(vcpu); 3689 3690 vcpu->guest_debug = 0; 3691 kvm_s390_clear_bp_data(vcpu); 3692 3693 if (dbg->control & ~VALID_GUESTDBG_FLAGS) { 3694 rc = -EINVAL; 3695 goto out; 3696 } 3697 if (!sclp.has_gpere) { 3698 rc = -EINVAL; 3699 goto out; 3700 } 3701 3702 if (dbg->control & KVM_GUESTDBG_ENABLE) { 3703 vcpu->guest_debug = dbg->control; 3704 /* enforce guest PER */ 3705 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P); 3706 3707 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) 3708 rc = kvm_s390_import_bp_data(vcpu, dbg); 3709 } else { 3710 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P); 3711 vcpu->arch.guestdbg.last_bp = 0; 3712 } 3713 3714 if (rc) { 3715 vcpu->guest_debug = 0; 3716 kvm_s390_clear_bp_data(vcpu); 3717 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P); 3718 } 3719 3720 out: 3721 vcpu_put(vcpu); 3722 return rc; 3723 } 3724 3725 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 3726 struct kvm_mp_state *mp_state) 3727 { 3728 int ret; 3729 3730 vcpu_load(vcpu); 3731 3732 /* CHECK_STOP and LOAD are not supported yet */ 3733 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED : 3734 KVM_MP_STATE_OPERATING; 3735 3736 vcpu_put(vcpu); 3737 return ret; 3738 } 3739 3740 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 3741 struct kvm_mp_state *mp_state) 3742 { 3743 int rc = 0; 3744 3745 vcpu_load(vcpu); 3746 3747 /* user space knows about this interface - let it control the state */ 3748 vcpu->kvm->arch.user_cpu_state_ctrl = 1; 3749 3750 switch (mp_state->mp_state) { 3751 case KVM_MP_STATE_STOPPED: 3752 rc = kvm_s390_vcpu_stop(vcpu); 3753 break; 3754 case KVM_MP_STATE_OPERATING: 3755 rc = kvm_s390_vcpu_start(vcpu); 3756 break; 3757 case KVM_MP_STATE_LOAD: 3758 if (!kvm_s390_pv_cpu_is_protected(vcpu)) { 3759 rc = -ENXIO; 3760 break; 3761 } 3762 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD); 3763 break; 3764 case KVM_MP_STATE_CHECK_STOP: 3765 fallthrough; /* CHECK_STOP and LOAD are not supported yet */ 3766 default: 3767 rc = -ENXIO; 3768 } 3769 3770 vcpu_put(vcpu); 3771 return rc; 3772 } 3773 3774 static bool ibs_enabled(struct kvm_vcpu *vcpu) 3775 { 3776 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS); 3777 } 3778 3779 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 3780 { 3781 retry: 3782 kvm_s390_vcpu_request_handled(vcpu); 3783 if (!kvm_request_pending(vcpu)) 3784 return 0; 3785 /* 3786 * We use MMU_RELOAD just to re-arm the ipte notifier for the 3787 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock. 3788 * This ensures that the ipte instruction for this request has 3789 * already finished. We might race against a second unmapper that 3790 * wants to set the blocking bit. Lets just retry the request loop. 3791 */ 3792 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 3793 int rc; 3794 rc = gmap_mprotect_notify(vcpu->arch.gmap, 3795 kvm_s390_get_prefix(vcpu), 3796 PAGE_SIZE * 2, PROT_WRITE); 3797 if (rc) { 3798 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 3799 return rc; 3800 } 3801 goto retry; 3802 } 3803 3804 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 3805 vcpu->arch.sie_block->ihcpu = 0xffff; 3806 goto retry; 3807 } 3808 3809 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { 3810 if (!ibs_enabled(vcpu)) { 3811 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); 3812 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS); 3813 } 3814 goto retry; 3815 } 3816 3817 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { 3818 if (ibs_enabled(vcpu)) { 3819 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); 3820 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS); 3821 } 3822 goto retry; 3823 } 3824 3825 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) { 3826 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; 3827 goto retry; 3828 } 3829 3830 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) { 3831 /* 3832 * Disable CMM virtualization; we will emulate the ESSA 3833 * instruction manually, in order to provide additional 3834 * functionalities needed for live migration. 3835 */ 3836 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA; 3837 goto retry; 3838 } 3839 3840 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) { 3841 /* 3842 * Re-enable CMM virtualization if CMMA is available and 3843 * CMM has been used. 3844 */ 3845 if ((vcpu->kvm->arch.use_cmma) && 3846 (vcpu->kvm->mm->context.uses_cmm)) 3847 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA; 3848 goto retry; 3849 } 3850 3851 /* nothing to do, just clear the request */ 3852 kvm_clear_request(KVM_REQ_UNHALT, vcpu); 3853 /* we left the vsie handler, nothing to do, just clear the request */ 3854 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu); 3855 3856 return 0; 3857 } 3858 3859 void kvm_s390_set_tod_clock(struct kvm *kvm, 3860 const struct kvm_s390_vm_tod_clock *gtod) 3861 { 3862 struct kvm_vcpu *vcpu; 3863 struct kvm_s390_tod_clock_ext htod; 3864 int i; 3865 3866 mutex_lock(&kvm->lock); 3867 preempt_disable(); 3868 3869 get_tod_clock_ext((char *)&htod); 3870 3871 kvm->arch.epoch = gtod->tod - htod.tod; 3872 kvm->arch.epdx = 0; 3873 if (test_kvm_facility(kvm, 139)) { 3874 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx; 3875 if (kvm->arch.epoch > gtod->tod) 3876 kvm->arch.epdx -= 1; 3877 } 3878 3879 kvm_s390_vcpu_block_all(kvm); 3880 kvm_for_each_vcpu(i, vcpu, kvm) { 3881 vcpu->arch.sie_block->epoch = kvm->arch.epoch; 3882 vcpu->arch.sie_block->epdx = kvm->arch.epdx; 3883 } 3884 3885 kvm_s390_vcpu_unblock_all(kvm); 3886 preempt_enable(); 3887 mutex_unlock(&kvm->lock); 3888 } 3889 3890 /** 3891 * kvm_arch_fault_in_page - fault-in guest page if necessary 3892 * @vcpu: The corresponding virtual cpu 3893 * @gpa: Guest physical address 3894 * @writable: Whether the page should be writable or not 3895 * 3896 * Make sure that a guest page has been faulted-in on the host. 3897 * 3898 * Return: Zero on success, negative error code otherwise. 3899 */ 3900 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable) 3901 { 3902 return gmap_fault(vcpu->arch.gmap, gpa, 3903 writable ? FAULT_FLAG_WRITE : 0); 3904 } 3905 3906 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, 3907 unsigned long token) 3908 { 3909 struct kvm_s390_interrupt inti; 3910 struct kvm_s390_irq irq; 3911 3912 if (start_token) { 3913 irq.u.ext.ext_params2 = token; 3914 irq.type = KVM_S390_INT_PFAULT_INIT; 3915 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); 3916 } else { 3917 inti.type = KVM_S390_INT_PFAULT_DONE; 3918 inti.parm64 = token; 3919 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); 3920 } 3921 } 3922 3923 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 3924 struct kvm_async_pf *work) 3925 { 3926 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token); 3927 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token); 3928 } 3929 3930 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 3931 struct kvm_async_pf *work) 3932 { 3933 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token); 3934 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token); 3935 } 3936 3937 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 3938 struct kvm_async_pf *work) 3939 { 3940 /* s390 will always inject the page directly */ 3941 } 3942 3943 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 3944 { 3945 /* 3946 * s390 will always inject the page directly, 3947 * but we still want check_async_completion to cleanup 3948 */ 3949 return true; 3950 } 3951 3952 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) 3953 { 3954 hva_t hva; 3955 struct kvm_arch_async_pf arch; 3956 int rc; 3957 3958 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 3959 return 0; 3960 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) != 3961 vcpu->arch.pfault_compare) 3962 return 0; 3963 if (psw_extint_disabled(vcpu)) 3964 return 0; 3965 if (kvm_s390_vcpu_has_irq(vcpu, 0)) 3966 return 0; 3967 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) 3968 return 0; 3969 if (!vcpu->arch.gmap->pfault_enabled) 3970 return 0; 3971 3972 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr)); 3973 hva += current->thread.gmap_addr & ~PAGE_MASK; 3974 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8)) 3975 return 0; 3976 3977 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch); 3978 return rc; 3979 } 3980 3981 static int vcpu_pre_run(struct kvm_vcpu *vcpu) 3982 { 3983 int rc, cpuflags; 3984 3985 /* 3986 * On s390 notifications for arriving pages will be delivered directly 3987 * to the guest but the house keeping for completed pfaults is 3988 * handled outside the worker. 3989 */ 3990 kvm_check_async_pf_completion(vcpu); 3991 3992 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14]; 3993 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15]; 3994 3995 if (need_resched()) 3996 schedule(); 3997 3998 if (test_cpu_flag(CIF_MCCK_PENDING)) 3999 s390_handle_mcck(); 4000 4001 if (!kvm_is_ucontrol(vcpu->kvm)) { 4002 rc = kvm_s390_deliver_pending_interrupts(vcpu); 4003 if (rc) 4004 return rc; 4005 } 4006 4007 rc = kvm_s390_handle_requests(vcpu); 4008 if (rc) 4009 return rc; 4010 4011 if (guestdbg_enabled(vcpu)) { 4012 kvm_s390_backup_guest_per_regs(vcpu); 4013 kvm_s390_patch_guest_per_regs(vcpu); 4014 } 4015 4016 clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask); 4017 4018 vcpu->arch.sie_block->icptcode = 0; 4019 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags); 4020 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags); 4021 trace_kvm_s390_sie_enter(vcpu, cpuflags); 4022 4023 return 0; 4024 } 4025 4026 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu) 4027 { 4028 struct kvm_s390_pgm_info pgm_info = { 4029 .code = PGM_ADDRESSING, 4030 }; 4031 u8 opcode, ilen; 4032 int rc; 4033 4034 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 4035 trace_kvm_s390_sie_fault(vcpu); 4036 4037 /* 4038 * We want to inject an addressing exception, which is defined as a 4039 * suppressing or terminating exception. However, since we came here 4040 * by a DAT access exception, the PSW still points to the faulting 4041 * instruction since DAT exceptions are nullifying. So we've got 4042 * to look up the current opcode to get the length of the instruction 4043 * to be able to forward the PSW. 4044 */ 4045 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1); 4046 ilen = insn_length(opcode); 4047 if (rc < 0) { 4048 return rc; 4049 } else if (rc) { 4050 /* Instruction-Fetching Exceptions - we can't detect the ilen. 4051 * Forward by arbitrary ilc, injection will take care of 4052 * nullification if necessary. 4053 */ 4054 pgm_info = vcpu->arch.pgm; 4055 ilen = 4; 4056 } 4057 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID; 4058 kvm_s390_forward_psw(vcpu, ilen); 4059 return kvm_s390_inject_prog_irq(vcpu, &pgm_info); 4060 } 4061 4062 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 4063 { 4064 struct mcck_volatile_info *mcck_info; 4065 struct sie_page *sie_page; 4066 4067 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 4068 vcpu->arch.sie_block->icptcode); 4069 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 4070 4071 if (guestdbg_enabled(vcpu)) 4072 kvm_s390_restore_guest_per_regs(vcpu); 4073 4074 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14; 4075 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; 4076 4077 if (exit_reason == -EINTR) { 4078 VCPU_EVENT(vcpu, 3, "%s", "machine check"); 4079 sie_page = container_of(vcpu->arch.sie_block, 4080 struct sie_page, sie_block); 4081 mcck_info = &sie_page->mcck_info; 4082 kvm_s390_reinject_machine_check(vcpu, mcck_info); 4083 return 0; 4084 } 4085 4086 if (vcpu->arch.sie_block->icptcode > 0) { 4087 int rc = kvm_handle_sie_intercept(vcpu); 4088 4089 if (rc != -EOPNOTSUPP) 4090 return rc; 4091 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC; 4092 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 4093 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 4094 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 4095 return -EREMOTE; 4096 } else if (exit_reason != -EFAULT) { 4097 vcpu->stat.exit_null++; 4098 return 0; 4099 } else if (kvm_is_ucontrol(vcpu->kvm)) { 4100 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; 4101 vcpu->run->s390_ucontrol.trans_exc_code = 4102 current->thread.gmap_addr; 4103 vcpu->run->s390_ucontrol.pgm_code = 0x10; 4104 return -EREMOTE; 4105 } else if (current->thread.gmap_pfault) { 4106 trace_kvm_s390_major_guest_pfault(vcpu); 4107 current->thread.gmap_pfault = 0; 4108 if (kvm_arch_setup_async_pf(vcpu)) 4109 return 0; 4110 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1); 4111 } 4112 return vcpu_post_run_fault_in_sie(vcpu); 4113 } 4114 4115 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK) 4116 static int __vcpu_run(struct kvm_vcpu *vcpu) 4117 { 4118 int rc, exit_reason; 4119 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block; 4120 4121 /* 4122 * We try to hold kvm->srcu during most of vcpu_run (except when run- 4123 * ning the guest), so that memslots (and other stuff) are protected 4124 */ 4125 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 4126 4127 do { 4128 rc = vcpu_pre_run(vcpu); 4129 if (rc) 4130 break; 4131 4132 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 4133 /* 4134 * As PF_VCPU will be used in fault handler, between 4135 * guest_enter and guest_exit should be no uaccess. 4136 */ 4137 local_irq_disable(); 4138 guest_enter_irqoff(); 4139 __disable_cpu_timer_accounting(vcpu); 4140 local_irq_enable(); 4141 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 4142 memcpy(sie_page->pv_grregs, 4143 vcpu->run->s.regs.gprs, 4144 sizeof(sie_page->pv_grregs)); 4145 } 4146 exit_reason = sie64a(vcpu->arch.sie_block, 4147 vcpu->run->s.regs.gprs); 4148 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 4149 memcpy(vcpu->run->s.regs.gprs, 4150 sie_page->pv_grregs, 4151 sizeof(sie_page->pv_grregs)); 4152 /* 4153 * We're not allowed to inject interrupts on intercepts 4154 * that leave the guest state in an "in-between" state 4155 * where the next SIE entry will do a continuation. 4156 * Fence interrupts in our "internal" PSW. 4157 */ 4158 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR || 4159 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) { 4160 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK; 4161 } 4162 } 4163 local_irq_disable(); 4164 __enable_cpu_timer_accounting(vcpu); 4165 guest_exit_irqoff(); 4166 local_irq_enable(); 4167 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 4168 4169 rc = vcpu_post_run(vcpu, exit_reason); 4170 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc); 4171 4172 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 4173 return rc; 4174 } 4175 4176 static void sync_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 4177 { 4178 struct runtime_instr_cb *riccb; 4179 struct gs_cb *gscb; 4180 4181 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb; 4182 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb; 4183 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 4184 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 4185 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) { 4186 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr; 4187 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp; 4188 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea; 4189 } 4190 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) { 4191 vcpu->arch.pfault_token = kvm_run->s.regs.pft; 4192 vcpu->arch.pfault_select = kvm_run->s.regs.pfs; 4193 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc; 4194 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 4195 kvm_clear_async_pf_completion_queue(vcpu); 4196 } 4197 /* 4198 * If userspace sets the riccb (e.g. after migration) to a valid state, 4199 * we should enable RI here instead of doing the lazy enablement. 4200 */ 4201 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) && 4202 test_kvm_facility(vcpu->kvm, 64) && 4203 riccb->v && 4204 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) { 4205 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)"); 4206 vcpu->arch.sie_block->ecb3 |= ECB3_RI; 4207 } 4208 /* 4209 * If userspace sets the gscb (e.g. after migration) to non-zero, 4210 * we should enable GS here instead of doing the lazy enablement. 4211 */ 4212 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) && 4213 test_kvm_facility(vcpu->kvm, 133) && 4214 gscb->gssm && 4215 !vcpu->arch.gs_enabled) { 4216 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)"); 4217 vcpu->arch.sie_block->ecb |= ECB_GS; 4218 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 4219 vcpu->arch.gs_enabled = 1; 4220 } 4221 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) && 4222 test_kvm_facility(vcpu->kvm, 82)) { 4223 vcpu->arch.sie_block->fpf &= ~FPF_BPBC; 4224 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0; 4225 } 4226 if (MACHINE_HAS_GS) { 4227 preempt_disable(); 4228 __ctl_set_bit(2, 4); 4229 if (current->thread.gs_cb) { 4230 vcpu->arch.host_gscb = current->thread.gs_cb; 4231 save_gs_cb(vcpu->arch.host_gscb); 4232 } 4233 if (vcpu->arch.gs_enabled) { 4234 current->thread.gs_cb = (struct gs_cb *) 4235 &vcpu->run->s.regs.gscb; 4236 restore_gs_cb(current->thread.gs_cb); 4237 } 4238 preempt_enable(); 4239 } 4240 /* SIE will load etoken directly from SDNX and therefore kvm_run */ 4241 } 4242 4243 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 4244 { 4245 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) 4246 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 4247 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 4248 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 4249 /* some control register changes require a tlb flush */ 4250 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 4251 } 4252 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) { 4253 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm); 4254 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc; 4255 } 4256 save_access_regs(vcpu->arch.host_acrs); 4257 restore_access_regs(vcpu->run->s.regs.acrs); 4258 /* save host (userspace) fprs/vrs */ 4259 save_fpu_regs(); 4260 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; 4261 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; 4262 if (MACHINE_HAS_VX) 4263 current->thread.fpu.regs = vcpu->run->s.regs.vrs; 4264 else 4265 current->thread.fpu.regs = vcpu->run->s.regs.fprs; 4266 current->thread.fpu.fpc = vcpu->run->s.regs.fpc; 4267 if (test_fp_ctl(current->thread.fpu.fpc)) 4268 /* User space provided an invalid FPC, let's clear it */ 4269 current->thread.fpu.fpc = 0; 4270 4271 /* Sync fmt2 only data */ 4272 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) { 4273 sync_regs_fmt2(vcpu, kvm_run); 4274 } else { 4275 /* 4276 * In several places we have to modify our internal view to 4277 * not do things that are disallowed by the ultravisor. For 4278 * example we must not inject interrupts after specific exits 4279 * (e.g. 112 prefix page not secure). We do this by turning 4280 * off the machine check, external and I/O interrupt bits 4281 * of our PSW copy. To avoid getting validity intercepts, we 4282 * do only accept the condition code from userspace. 4283 */ 4284 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC; 4285 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask & 4286 PSW_MASK_CC; 4287 } 4288 4289 kvm_run->kvm_dirty_regs = 0; 4290 } 4291 4292 static void store_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 4293 { 4294 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr; 4295 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp; 4296 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea; 4297 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC; 4298 if (MACHINE_HAS_GS) { 4299 __ctl_set_bit(2, 4); 4300 if (vcpu->arch.gs_enabled) 4301 save_gs_cb(current->thread.gs_cb); 4302 preempt_disable(); 4303 current->thread.gs_cb = vcpu->arch.host_gscb; 4304 restore_gs_cb(vcpu->arch.host_gscb); 4305 preempt_enable(); 4306 if (!vcpu->arch.host_gscb) 4307 __ctl_clear_bit(2, 4); 4308 vcpu->arch.host_gscb = NULL; 4309 } 4310 /* SIE will save etoken directly into SDNX and therefore kvm_run */ 4311 } 4312 4313 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 4314 { 4315 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 4316 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 4317 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); 4318 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 4319 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu); 4320 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc; 4321 kvm_run->s.regs.pft = vcpu->arch.pfault_token; 4322 kvm_run->s.regs.pfs = vcpu->arch.pfault_select; 4323 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; 4324 save_access_regs(vcpu->run->s.regs.acrs); 4325 restore_access_regs(vcpu->arch.host_acrs); 4326 /* Save guest register state */ 4327 save_fpu_regs(); 4328 vcpu->run->s.regs.fpc = current->thread.fpu.fpc; 4329 /* Restore will be done lazily at return */ 4330 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; 4331 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; 4332 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) 4333 store_regs_fmt2(vcpu, kvm_run); 4334 } 4335 4336 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 4337 { 4338 int rc; 4339 4340 if (kvm_run->immediate_exit) 4341 return -EINTR; 4342 4343 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS || 4344 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS) 4345 return -EINVAL; 4346 4347 vcpu_load(vcpu); 4348 4349 if (guestdbg_exit_pending(vcpu)) { 4350 kvm_s390_prepare_debug_exit(vcpu); 4351 rc = 0; 4352 goto out; 4353 } 4354 4355 kvm_sigset_activate(vcpu); 4356 4357 /* 4358 * no need to check the return value of vcpu_start as it can only have 4359 * an error for protvirt, but protvirt means user cpu state 4360 */ 4361 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) { 4362 kvm_s390_vcpu_start(vcpu); 4363 } else if (is_vcpu_stopped(vcpu)) { 4364 pr_err_ratelimited("can't run stopped vcpu %d\n", 4365 vcpu->vcpu_id); 4366 rc = -EINVAL; 4367 goto out; 4368 } 4369 4370 sync_regs(vcpu, kvm_run); 4371 enable_cpu_timer_accounting(vcpu); 4372 4373 might_fault(); 4374 rc = __vcpu_run(vcpu); 4375 4376 if (signal_pending(current) && !rc) { 4377 kvm_run->exit_reason = KVM_EXIT_INTR; 4378 rc = -EINTR; 4379 } 4380 4381 if (guestdbg_exit_pending(vcpu) && !rc) { 4382 kvm_s390_prepare_debug_exit(vcpu); 4383 rc = 0; 4384 } 4385 4386 if (rc == -EREMOTE) { 4387 /* userspace support is needed, kvm_run has been prepared */ 4388 rc = 0; 4389 } 4390 4391 disable_cpu_timer_accounting(vcpu); 4392 store_regs(vcpu, kvm_run); 4393 4394 kvm_sigset_deactivate(vcpu); 4395 4396 vcpu->stat.exit_userspace++; 4397 out: 4398 vcpu_put(vcpu); 4399 return rc; 4400 } 4401 4402 /* 4403 * store status at address 4404 * we use have two special cases: 4405 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 4406 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 4407 */ 4408 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa) 4409 { 4410 unsigned char archmode = 1; 4411 freg_t fprs[NUM_FPRS]; 4412 unsigned int px; 4413 u64 clkcomp, cputm; 4414 int rc; 4415 4416 px = kvm_s390_get_prefix(vcpu); 4417 if (gpa == KVM_S390_STORE_STATUS_NOADDR) { 4418 if (write_guest_abs(vcpu, 163, &archmode, 1)) 4419 return -EFAULT; 4420 gpa = 0; 4421 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) { 4422 if (write_guest_real(vcpu, 163, &archmode, 1)) 4423 return -EFAULT; 4424 gpa = px; 4425 } else 4426 gpa -= __LC_FPREGS_SAVE_AREA; 4427 4428 /* manually convert vector registers if necessary */ 4429 if (MACHINE_HAS_VX) { 4430 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 4431 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA, 4432 fprs, 128); 4433 } else { 4434 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA, 4435 vcpu->run->s.regs.fprs, 128); 4436 } 4437 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA, 4438 vcpu->run->s.regs.gprs, 128); 4439 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA, 4440 &vcpu->arch.sie_block->gpsw, 16); 4441 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA, 4442 &px, 4); 4443 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA, 4444 &vcpu->run->s.regs.fpc, 4); 4445 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA, 4446 &vcpu->arch.sie_block->todpr, 4); 4447 cputm = kvm_s390_get_cpu_timer(vcpu); 4448 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA, 4449 &cputm, 8); 4450 clkcomp = vcpu->arch.sie_block->ckc >> 8; 4451 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA, 4452 &clkcomp, 8); 4453 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA, 4454 &vcpu->run->s.regs.acrs, 64); 4455 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA, 4456 &vcpu->arch.sie_block->gcr, 128); 4457 return rc ? -EFAULT : 0; 4458 } 4459 4460 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 4461 { 4462 /* 4463 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 4464 * switch in the run ioctl. Let's update our copies before we save 4465 * it into the save area 4466 */ 4467 save_fpu_regs(); 4468 vcpu->run->s.regs.fpc = current->thread.fpu.fpc; 4469 save_access_regs(vcpu->run->s.regs.acrs); 4470 4471 return kvm_s390_store_status_unloaded(vcpu, addr); 4472 } 4473 4474 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 4475 { 4476 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 4477 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu); 4478 } 4479 4480 static void __disable_ibs_on_all_vcpus(struct kvm *kvm) 4481 { 4482 unsigned int i; 4483 struct kvm_vcpu *vcpu; 4484 4485 kvm_for_each_vcpu(i, vcpu, kvm) { 4486 __disable_ibs_on_vcpu(vcpu); 4487 } 4488 } 4489 4490 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 4491 { 4492 if (!sclp.has_ibs) 4493 return; 4494 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu); 4495 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu); 4496 } 4497 4498 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) 4499 { 4500 int i, online_vcpus, r = 0, started_vcpus = 0; 4501 4502 if (!is_vcpu_stopped(vcpu)) 4503 return 0; 4504 4505 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1); 4506 /* Only one cpu at a time may enter/leave the STOPPED state. */ 4507 spin_lock(&vcpu->kvm->arch.start_stop_lock); 4508 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 4509 4510 /* Let's tell the UV that we want to change into the operating state */ 4511 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 4512 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR); 4513 if (r) { 4514 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 4515 return r; 4516 } 4517 } 4518 4519 for (i = 0; i < online_vcpus; i++) { 4520 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) 4521 started_vcpus++; 4522 } 4523 4524 if (started_vcpus == 0) { 4525 /* we're the only active VCPU -> speed it up */ 4526 __enable_ibs_on_vcpu(vcpu); 4527 } else if (started_vcpus == 1) { 4528 /* 4529 * As we are starting a second VCPU, we have to disable 4530 * the IBS facility on all VCPUs to remove potentially 4531 * oustanding ENABLE requests. 4532 */ 4533 __disable_ibs_on_all_vcpus(vcpu->kvm); 4534 } 4535 4536 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED); 4537 /* 4538 * The real PSW might have changed due to a RESTART interpreted by the 4539 * ultravisor. We block all interrupts and let the next sie exit 4540 * refresh our view. 4541 */ 4542 if (kvm_s390_pv_cpu_is_protected(vcpu)) 4543 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK; 4544 /* 4545 * Another VCPU might have used IBS while we were offline. 4546 * Let's play safe and flush the VCPU at startup. 4547 */ 4548 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 4549 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 4550 return 0; 4551 } 4552 4553 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) 4554 { 4555 int i, online_vcpus, r = 0, started_vcpus = 0; 4556 struct kvm_vcpu *started_vcpu = NULL; 4557 4558 if (is_vcpu_stopped(vcpu)) 4559 return 0; 4560 4561 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0); 4562 /* Only one cpu at a time may enter/leave the STOPPED state. */ 4563 spin_lock(&vcpu->kvm->arch.start_stop_lock); 4564 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 4565 4566 /* Let's tell the UV that we want to change into the stopped state */ 4567 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 4568 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP); 4569 if (r) { 4570 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 4571 return r; 4572 } 4573 } 4574 4575 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ 4576 kvm_s390_clear_stop_irq(vcpu); 4577 4578 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED); 4579 __disable_ibs_on_vcpu(vcpu); 4580 4581 for (i = 0; i < online_vcpus; i++) { 4582 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) { 4583 started_vcpus++; 4584 started_vcpu = vcpu->kvm->vcpus[i]; 4585 } 4586 } 4587 4588 if (started_vcpus == 1) { 4589 /* 4590 * As we only have one VCPU left, we want to enable the 4591 * IBS facility for that VCPU to speed it up. 4592 */ 4593 __enable_ibs_on_vcpu(started_vcpu); 4594 } 4595 4596 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 4597 return 0; 4598 } 4599 4600 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 4601 struct kvm_enable_cap *cap) 4602 { 4603 int r; 4604 4605 if (cap->flags) 4606 return -EINVAL; 4607 4608 switch (cap->cap) { 4609 case KVM_CAP_S390_CSS_SUPPORT: 4610 if (!vcpu->kvm->arch.css_support) { 4611 vcpu->kvm->arch.css_support = 1; 4612 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support"); 4613 trace_kvm_s390_enable_css(vcpu->kvm); 4614 } 4615 r = 0; 4616 break; 4617 default: 4618 r = -EINVAL; 4619 break; 4620 } 4621 return r; 4622 } 4623 4624 static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu, 4625 struct kvm_s390_mem_op *mop) 4626 { 4627 void __user *uaddr = (void __user *)mop->buf; 4628 int r = 0; 4629 4630 if (mop->flags || !mop->size) 4631 return -EINVAL; 4632 if (mop->size + mop->sida_offset < mop->size) 4633 return -EINVAL; 4634 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block)) 4635 return -E2BIG; 4636 4637 switch (mop->op) { 4638 case KVM_S390_MEMOP_SIDA_READ: 4639 if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) + 4640 mop->sida_offset), mop->size)) 4641 r = -EFAULT; 4642 4643 break; 4644 case KVM_S390_MEMOP_SIDA_WRITE: 4645 if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) + 4646 mop->sida_offset), uaddr, mop->size)) 4647 r = -EFAULT; 4648 break; 4649 } 4650 return r; 4651 } 4652 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, 4653 struct kvm_s390_mem_op *mop) 4654 { 4655 void __user *uaddr = (void __user *)mop->buf; 4656 void *tmpbuf = NULL; 4657 int r = 0; 4658 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION 4659 | KVM_S390_MEMOP_F_CHECK_ONLY; 4660 4661 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size) 4662 return -EINVAL; 4663 4664 if (mop->size > MEM_OP_MAX_SIZE) 4665 return -E2BIG; 4666 4667 if (kvm_s390_pv_cpu_is_protected(vcpu)) 4668 return -EINVAL; 4669 4670 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) { 4671 tmpbuf = vmalloc(mop->size); 4672 if (!tmpbuf) 4673 return -ENOMEM; 4674 } 4675 4676 switch (mop->op) { 4677 case KVM_S390_MEMOP_LOGICAL_READ: 4678 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { 4679 r = check_gva_range(vcpu, mop->gaddr, mop->ar, 4680 mop->size, GACC_FETCH); 4681 break; 4682 } 4683 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size); 4684 if (r == 0) { 4685 if (copy_to_user(uaddr, tmpbuf, mop->size)) 4686 r = -EFAULT; 4687 } 4688 break; 4689 case KVM_S390_MEMOP_LOGICAL_WRITE: 4690 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { 4691 r = check_gva_range(vcpu, mop->gaddr, mop->ar, 4692 mop->size, GACC_STORE); 4693 break; 4694 } 4695 if (copy_from_user(tmpbuf, uaddr, mop->size)) { 4696 r = -EFAULT; 4697 break; 4698 } 4699 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size); 4700 break; 4701 } 4702 4703 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0) 4704 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 4705 4706 vfree(tmpbuf); 4707 return r; 4708 } 4709 4710 static long kvm_s390_guest_memsida_op(struct kvm_vcpu *vcpu, 4711 struct kvm_s390_mem_op *mop) 4712 { 4713 int r, srcu_idx; 4714 4715 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 4716 4717 switch (mop->op) { 4718 case KVM_S390_MEMOP_LOGICAL_READ: 4719 case KVM_S390_MEMOP_LOGICAL_WRITE: 4720 r = kvm_s390_guest_mem_op(vcpu, mop); 4721 break; 4722 case KVM_S390_MEMOP_SIDA_READ: 4723 case KVM_S390_MEMOP_SIDA_WRITE: 4724 /* we are locked against sida going away by the vcpu->mutex */ 4725 r = kvm_s390_guest_sida_op(vcpu, mop); 4726 break; 4727 default: 4728 r = -EINVAL; 4729 } 4730 4731 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 4732 return r; 4733 } 4734 4735 long kvm_arch_vcpu_async_ioctl(struct file *filp, 4736 unsigned int ioctl, unsigned long arg) 4737 { 4738 struct kvm_vcpu *vcpu = filp->private_data; 4739 void __user *argp = (void __user *)arg; 4740 4741 switch (ioctl) { 4742 case KVM_S390_IRQ: { 4743 struct kvm_s390_irq s390irq; 4744 4745 if (copy_from_user(&s390irq, argp, sizeof(s390irq))) 4746 return -EFAULT; 4747 return kvm_s390_inject_vcpu(vcpu, &s390irq); 4748 } 4749 case KVM_S390_INTERRUPT: { 4750 struct kvm_s390_interrupt s390int; 4751 struct kvm_s390_irq s390irq = {}; 4752 4753 if (copy_from_user(&s390int, argp, sizeof(s390int))) 4754 return -EFAULT; 4755 if (s390int_to_s390irq(&s390int, &s390irq)) 4756 return -EINVAL; 4757 return kvm_s390_inject_vcpu(vcpu, &s390irq); 4758 } 4759 } 4760 return -ENOIOCTLCMD; 4761 } 4762 4763 long kvm_arch_vcpu_ioctl(struct file *filp, 4764 unsigned int ioctl, unsigned long arg) 4765 { 4766 struct kvm_vcpu *vcpu = filp->private_data; 4767 void __user *argp = (void __user *)arg; 4768 int idx; 4769 long r; 4770 u16 rc, rrc; 4771 4772 vcpu_load(vcpu); 4773 4774 switch (ioctl) { 4775 case KVM_S390_STORE_STATUS: 4776 idx = srcu_read_lock(&vcpu->kvm->srcu); 4777 r = kvm_s390_store_status_unloaded(vcpu, arg); 4778 srcu_read_unlock(&vcpu->kvm->srcu, idx); 4779 break; 4780 case KVM_S390_SET_INITIAL_PSW: { 4781 psw_t psw; 4782 4783 r = -EFAULT; 4784 if (copy_from_user(&psw, argp, sizeof(psw))) 4785 break; 4786 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 4787 break; 4788 } 4789 case KVM_S390_CLEAR_RESET: 4790 r = 0; 4791 kvm_arch_vcpu_ioctl_clear_reset(vcpu); 4792 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 4793 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), 4794 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc); 4795 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x", 4796 rc, rrc); 4797 } 4798 break; 4799 case KVM_S390_INITIAL_RESET: 4800 r = 0; 4801 kvm_arch_vcpu_ioctl_initial_reset(vcpu); 4802 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 4803 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), 4804 UVC_CMD_CPU_RESET_INITIAL, 4805 &rc, &rrc); 4806 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x", 4807 rc, rrc); 4808 } 4809 break; 4810 case KVM_S390_NORMAL_RESET: 4811 r = 0; 4812 kvm_arch_vcpu_ioctl_normal_reset(vcpu); 4813 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 4814 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), 4815 UVC_CMD_CPU_RESET, &rc, &rrc); 4816 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x", 4817 rc, rrc); 4818 } 4819 break; 4820 case KVM_SET_ONE_REG: 4821 case KVM_GET_ONE_REG: { 4822 struct kvm_one_reg reg; 4823 r = -EINVAL; 4824 if (kvm_s390_pv_cpu_is_protected(vcpu)) 4825 break; 4826 r = -EFAULT; 4827 if (copy_from_user(®, argp, sizeof(reg))) 4828 break; 4829 if (ioctl == KVM_SET_ONE_REG) 4830 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 4831 else 4832 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 4833 break; 4834 } 4835 #ifdef CONFIG_KVM_S390_UCONTROL 4836 case KVM_S390_UCAS_MAP: { 4837 struct kvm_s390_ucas_mapping ucasmap; 4838 4839 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 4840 r = -EFAULT; 4841 break; 4842 } 4843 4844 if (!kvm_is_ucontrol(vcpu->kvm)) { 4845 r = -EINVAL; 4846 break; 4847 } 4848 4849 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 4850 ucasmap.vcpu_addr, ucasmap.length); 4851 break; 4852 } 4853 case KVM_S390_UCAS_UNMAP: { 4854 struct kvm_s390_ucas_mapping ucasmap; 4855 4856 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 4857 r = -EFAULT; 4858 break; 4859 } 4860 4861 if (!kvm_is_ucontrol(vcpu->kvm)) { 4862 r = -EINVAL; 4863 break; 4864 } 4865 4866 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 4867 ucasmap.length); 4868 break; 4869 } 4870 #endif 4871 case KVM_S390_VCPU_FAULT: { 4872 r = gmap_fault(vcpu->arch.gmap, arg, 0); 4873 break; 4874 } 4875 case KVM_ENABLE_CAP: 4876 { 4877 struct kvm_enable_cap cap; 4878 r = -EFAULT; 4879 if (copy_from_user(&cap, argp, sizeof(cap))) 4880 break; 4881 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 4882 break; 4883 } 4884 case KVM_S390_MEM_OP: { 4885 struct kvm_s390_mem_op mem_op; 4886 4887 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0) 4888 r = kvm_s390_guest_memsida_op(vcpu, &mem_op); 4889 else 4890 r = -EFAULT; 4891 break; 4892 } 4893 case KVM_S390_SET_IRQ_STATE: { 4894 struct kvm_s390_irq_state irq_state; 4895 4896 r = -EFAULT; 4897 if (copy_from_user(&irq_state, argp, sizeof(irq_state))) 4898 break; 4899 if (irq_state.len > VCPU_IRQS_MAX_BUF || 4900 irq_state.len == 0 || 4901 irq_state.len % sizeof(struct kvm_s390_irq) > 0) { 4902 r = -EINVAL; 4903 break; 4904 } 4905 /* do not use irq_state.flags, it will break old QEMUs */ 4906 r = kvm_s390_set_irq_state(vcpu, 4907 (void __user *) irq_state.buf, 4908 irq_state.len); 4909 break; 4910 } 4911 case KVM_S390_GET_IRQ_STATE: { 4912 struct kvm_s390_irq_state irq_state; 4913 4914 r = -EFAULT; 4915 if (copy_from_user(&irq_state, argp, sizeof(irq_state))) 4916 break; 4917 if (irq_state.len == 0) { 4918 r = -EINVAL; 4919 break; 4920 } 4921 /* do not use irq_state.flags, it will break old QEMUs */ 4922 r = kvm_s390_get_irq_state(vcpu, 4923 (__u8 __user *) irq_state.buf, 4924 irq_state.len); 4925 break; 4926 } 4927 default: 4928 r = -ENOTTY; 4929 } 4930 4931 vcpu_put(vcpu); 4932 return r; 4933 } 4934 4935 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 4936 { 4937 #ifdef CONFIG_KVM_S390_UCONTROL 4938 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 4939 && (kvm_is_ucontrol(vcpu->kvm))) { 4940 vmf->page = virt_to_page(vcpu->arch.sie_block); 4941 get_page(vmf->page); 4942 return 0; 4943 } 4944 #endif 4945 return VM_FAULT_SIGBUS; 4946 } 4947 4948 /* Section: memory related */ 4949 int kvm_arch_prepare_memory_region(struct kvm *kvm, 4950 struct kvm_memory_slot *memslot, 4951 const struct kvm_userspace_memory_region *mem, 4952 enum kvm_mr_change change) 4953 { 4954 /* A few sanity checks. We can have memory slots which have to be 4955 located/ended at a segment boundary (1MB). The memory in userland is 4956 ok to be fragmented into various different vmas. It is okay to mmap() 4957 and munmap() stuff in this slot after doing this call at any time */ 4958 4959 if (mem->userspace_addr & 0xffffful) 4960 return -EINVAL; 4961 4962 if (mem->memory_size & 0xffffful) 4963 return -EINVAL; 4964 4965 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit) 4966 return -EINVAL; 4967 4968 /* When we are protected, we should not change the memory slots */ 4969 if (kvm_s390_pv_get_handle(kvm)) 4970 return -EINVAL; 4971 return 0; 4972 } 4973 4974 void kvm_arch_commit_memory_region(struct kvm *kvm, 4975 const struct kvm_userspace_memory_region *mem, 4976 struct kvm_memory_slot *old, 4977 const struct kvm_memory_slot *new, 4978 enum kvm_mr_change change) 4979 { 4980 int rc = 0; 4981 4982 switch (change) { 4983 case KVM_MR_DELETE: 4984 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE, 4985 old->npages * PAGE_SIZE); 4986 break; 4987 case KVM_MR_MOVE: 4988 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE, 4989 old->npages * PAGE_SIZE); 4990 if (rc) 4991 break; 4992 fallthrough; 4993 case KVM_MR_CREATE: 4994 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 4995 mem->guest_phys_addr, mem->memory_size); 4996 break; 4997 case KVM_MR_FLAGS_ONLY: 4998 break; 4999 default: 5000 WARN(1, "Unknown KVM MR CHANGE: %d\n", change); 5001 } 5002 if (rc) 5003 pr_warn("failed to commit memory region\n"); 5004 return; 5005 } 5006 5007 static inline unsigned long nonhyp_mask(int i) 5008 { 5009 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30; 5010 5011 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4); 5012 } 5013 5014 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) 5015 { 5016 vcpu->valid_wakeup = false; 5017 } 5018 5019 static int __init kvm_s390_init(void) 5020 { 5021 int i; 5022 5023 if (!sclp.has_sief2) { 5024 pr_info("SIE is not available\n"); 5025 return -ENODEV; 5026 } 5027 5028 if (nested && hpage) { 5029 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n"); 5030 return -EINVAL; 5031 } 5032 5033 for (i = 0; i < 16; i++) 5034 kvm_s390_fac_base[i] |= 5035 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i); 5036 5037 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 5038 } 5039 5040 static void __exit kvm_s390_exit(void) 5041 { 5042 kvm_exit(); 5043 } 5044 5045 module_init(kvm_s390_init); 5046 module_exit(kvm_s390_exit); 5047 5048 /* 5049 * Enable autoloading of the kvm module. 5050 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 5051 * since x86 takes a different approach. 5052 */ 5053 #include <linux/miscdevice.h> 5054 MODULE_ALIAS_MISCDEV(KVM_MINOR); 5055 MODULE_ALIAS("devname:kvm"); 5056