1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Hosting Protected Virtual Machines 4 * 5 * Copyright IBM Corp. 2019, 2020 6 * Author(s): Janosch Frank <frankja@linux.ibm.com> 7 */ 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 #include <linux/minmax.h> 11 #include <linux/pagemap.h> 12 #include <linux/sched/signal.h> 13 #include <asm/gmap.h> 14 #include <asm/uv.h> 15 #include <asm/mman.h> 16 #include <linux/pagewalk.h> 17 #include <linux/sched/mm.h> 18 #include <linux/mmu_notifier.h> 19 #include "kvm-s390.h" 20 21 static void kvm_s390_clear_pv_state(struct kvm *kvm) 22 { 23 kvm->arch.pv.handle = 0; 24 kvm->arch.pv.guest_len = 0; 25 kvm->arch.pv.stor_base = 0; 26 kvm->arch.pv.stor_var = NULL; 27 } 28 29 int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) 30 { 31 int cc; 32 33 if (!kvm_s390_pv_cpu_get_handle(vcpu)) 34 return 0; 35 36 cc = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), UVC_CMD_DESTROY_SEC_CPU, rc, rrc); 37 38 KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT DESTROY VCPU %d: rc %x rrc %x", 39 vcpu->vcpu_id, *rc, *rrc); 40 WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc); 41 42 /* Intended memory leak for something that should never happen. */ 43 if (!cc) 44 free_pages(vcpu->arch.pv.stor_base, 45 get_order(uv_info.guest_cpu_stor_len)); 46 47 free_page(sida_origin(vcpu->arch.sie_block)); 48 vcpu->arch.sie_block->pv_handle_cpu = 0; 49 vcpu->arch.sie_block->pv_handle_config = 0; 50 memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); 51 vcpu->arch.sie_block->sdf = 0; 52 /* 53 * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). 54 * Use the reset value of gbea to avoid leaking the kernel pointer of 55 * the just freed sida. 56 */ 57 vcpu->arch.sie_block->gbea = 1; 58 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 59 60 return cc ? EIO : 0; 61 } 62 63 int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) 64 { 65 struct uv_cb_csc uvcb = { 66 .header.cmd = UVC_CMD_CREATE_SEC_CPU, 67 .header.len = sizeof(uvcb), 68 }; 69 int cc; 70 71 if (kvm_s390_pv_cpu_get_handle(vcpu)) 72 return -EINVAL; 73 74 vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, 75 get_order(uv_info.guest_cpu_stor_len)); 76 if (!vcpu->arch.pv.stor_base) 77 return -ENOMEM; 78 79 /* Input */ 80 uvcb.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm); 81 uvcb.num = vcpu->arch.sie_block->icpua; 82 uvcb.state_origin = (u64)vcpu->arch.sie_block; 83 uvcb.stor_origin = (u64)vcpu->arch.pv.stor_base; 84 85 /* Alloc Secure Instruction Data Area Designation */ 86 vcpu->arch.sie_block->sidad = __get_free_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 87 if (!vcpu->arch.sie_block->sidad) { 88 free_pages(vcpu->arch.pv.stor_base, 89 get_order(uv_info.guest_cpu_stor_len)); 90 return -ENOMEM; 91 } 92 93 cc = uv_call(0, (u64)&uvcb); 94 *rc = uvcb.header.rc; 95 *rrc = uvcb.header.rrc; 96 KVM_UV_EVENT(vcpu->kvm, 3, 97 "PROTVIRT CREATE VCPU: cpu %d handle %llx rc %x rrc %x", 98 vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc, 99 uvcb.header.rrc); 100 101 if (cc) { 102 u16 dummy; 103 104 kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy); 105 return -EIO; 106 } 107 108 /* Output */ 109 vcpu->arch.pv.handle = uvcb.cpu_handle; 110 vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle; 111 vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_get_handle(vcpu->kvm); 112 vcpu->arch.sie_block->sdf = 2; 113 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 114 return 0; 115 } 116 117 /* only free resources when the destroy was successful */ 118 static void kvm_s390_pv_dealloc_vm(struct kvm *kvm) 119 { 120 vfree(kvm->arch.pv.stor_var); 121 free_pages(kvm->arch.pv.stor_base, 122 get_order(uv_info.guest_base_stor_len)); 123 kvm_s390_clear_pv_state(kvm); 124 } 125 126 static int kvm_s390_pv_alloc_vm(struct kvm *kvm) 127 { 128 unsigned long base = uv_info.guest_base_stor_len; 129 unsigned long virt = uv_info.guest_virt_var_stor_len; 130 unsigned long npages = 0, vlen = 0; 131 132 kvm->arch.pv.stor_var = NULL; 133 kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, get_order(base)); 134 if (!kvm->arch.pv.stor_base) 135 return -ENOMEM; 136 137 /* 138 * Calculate current guest storage for allocation of the 139 * variable storage, which is based on the length in MB. 140 * 141 * Slots are sorted by GFN 142 */ 143 mutex_lock(&kvm->slots_lock); 144 npages = kvm_s390_get_gfn_end(kvm_memslots(kvm)); 145 mutex_unlock(&kvm->slots_lock); 146 147 kvm->arch.pv.guest_len = npages * PAGE_SIZE; 148 149 /* Allocate variable storage */ 150 vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE); 151 vlen += uv_info.guest_virt_base_stor_len; 152 kvm->arch.pv.stor_var = vzalloc(vlen); 153 if (!kvm->arch.pv.stor_var) 154 goto out_err; 155 return 0; 156 157 out_err: 158 kvm_s390_pv_dealloc_vm(kvm); 159 return -ENOMEM; 160 } 161 162 /* this should not fail, but if it does, we must not free the donated memory */ 163 int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc) 164 { 165 int cc; 166 167 cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), 168 UVC_CMD_DESTROY_SEC_CONF, rc, rrc); 169 WRITE_ONCE(kvm->arch.gmap->guest_handle, 0); 170 /* 171 * if the mm still has a mapping, make all its pages accessible 172 * before destroying the guest 173 */ 174 if (mmget_not_zero(kvm->mm)) { 175 s390_uv_destroy_range(kvm->mm, 0, TASK_SIZE); 176 mmput(kvm->mm); 177 } 178 179 if (!cc) { 180 atomic_dec(&kvm->mm->context.protected_count); 181 kvm_s390_pv_dealloc_vm(kvm); 182 } else { 183 /* Intended memory leak on "impossible" error */ 184 s390_replace_asce(kvm->arch.gmap); 185 } 186 KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x", *rc, *rrc); 187 WARN_ONCE(cc, "protvirt destroy vm failed rc %x rrc %x", *rc, *rrc); 188 189 return cc ? -EIO : 0; 190 } 191 192 static void kvm_s390_pv_mmu_notifier_release(struct mmu_notifier *subscription, 193 struct mm_struct *mm) 194 { 195 struct kvm *kvm = container_of(subscription, struct kvm, arch.pv.mmu_notifier); 196 u16 dummy; 197 198 /* 199 * No locking is needed since this is the last thread of the last user of this 200 * struct mm. 201 * When the struct kvm gets deinitialized, this notifier is also 202 * unregistered. This means that if this notifier runs, then the 203 * struct kvm is still valid. 204 */ 205 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy); 206 } 207 208 static const struct mmu_notifier_ops kvm_s390_pv_mmu_notifier_ops = { 209 .release = kvm_s390_pv_mmu_notifier_release, 210 }; 211 212 int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) 213 { 214 struct uv_cb_cgc uvcb = { 215 .header.cmd = UVC_CMD_CREATE_SEC_CONF, 216 .header.len = sizeof(uvcb) 217 }; 218 int cc, ret; 219 u16 dummy; 220 221 ret = kvm_s390_pv_alloc_vm(kvm); 222 if (ret) 223 return ret; 224 225 /* Inputs */ 226 uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */ 227 uvcb.guest_stor_len = kvm->arch.pv.guest_len; 228 uvcb.guest_asce = kvm->arch.gmap->asce; 229 uvcb.guest_sca = (unsigned long)kvm->arch.sca; 230 uvcb.conf_base_stor_origin = (u64)kvm->arch.pv.stor_base; 231 uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var; 232 233 cc = uv_call_sched(0, (u64)&uvcb); 234 *rc = uvcb.header.rc; 235 *rrc = uvcb.header.rrc; 236 KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x", 237 uvcb.guest_handle, uvcb.guest_stor_len, *rc, *rrc); 238 239 /* Outputs */ 240 kvm->arch.pv.handle = uvcb.guest_handle; 241 242 atomic_inc(&kvm->mm->context.protected_count); 243 if (cc) { 244 if (uvcb.header.rc & UVC_RC_NEED_DESTROY) { 245 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy); 246 } else { 247 atomic_dec(&kvm->mm->context.protected_count); 248 kvm_s390_pv_dealloc_vm(kvm); 249 } 250 return -EIO; 251 } 252 kvm->arch.gmap->guest_handle = uvcb.guest_handle; 253 /* Add the notifier only once. No races because we hold kvm->lock */ 254 if (kvm->arch.pv.mmu_notifier.ops != &kvm_s390_pv_mmu_notifier_ops) { 255 kvm->arch.pv.mmu_notifier.ops = &kvm_s390_pv_mmu_notifier_ops; 256 mmu_notifier_register(&kvm->arch.pv.mmu_notifier, kvm->mm); 257 } 258 return 0; 259 } 260 261 int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc, 262 u16 *rrc) 263 { 264 struct uv_cb_ssc uvcb = { 265 .header.cmd = UVC_CMD_SET_SEC_CONF_PARAMS, 266 .header.len = sizeof(uvcb), 267 .sec_header_origin = (u64)hdr, 268 .sec_header_len = length, 269 .guest_handle = kvm_s390_pv_get_handle(kvm), 270 }; 271 int cc = uv_call(0, (u64)&uvcb); 272 273 *rc = uvcb.header.rc; 274 *rrc = uvcb.header.rrc; 275 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x", 276 *rc, *rrc); 277 return cc ? -EINVAL : 0; 278 } 279 280 static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak, 281 u64 offset, u16 *rc, u16 *rrc) 282 { 283 struct uv_cb_unp uvcb = { 284 .header.cmd = UVC_CMD_UNPACK_IMG, 285 .header.len = sizeof(uvcb), 286 .guest_handle = kvm_s390_pv_get_handle(kvm), 287 .gaddr = addr, 288 .tweak[0] = tweak, 289 .tweak[1] = offset, 290 }; 291 int ret = gmap_make_secure(kvm->arch.gmap, addr, &uvcb); 292 293 *rc = uvcb.header.rc; 294 *rrc = uvcb.header.rrc; 295 296 if (ret && ret != -EAGAIN) 297 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x", 298 uvcb.gaddr, *rc, *rrc); 299 return ret; 300 } 301 302 int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size, 303 unsigned long tweak, u16 *rc, u16 *rrc) 304 { 305 u64 offset = 0; 306 int ret = 0; 307 308 if (addr & ~PAGE_MASK || !size || size & ~PAGE_MASK) 309 return -EINVAL; 310 311 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx", 312 addr, size); 313 314 while (offset < size) { 315 ret = unpack_one(kvm, addr, tweak, offset, rc, rrc); 316 if (ret == -EAGAIN) { 317 cond_resched(); 318 if (fatal_signal_pending(current)) 319 break; 320 continue; 321 } 322 if (ret) 323 break; 324 addr += PAGE_SIZE; 325 offset += PAGE_SIZE; 326 } 327 if (!ret) 328 KVM_UV_EVENT(kvm, 3, "%s", "PROTVIRT VM UNPACK: successful"); 329 return ret; 330 } 331 332 int kvm_s390_pv_set_cpu_state(struct kvm_vcpu *vcpu, u8 state) 333 { 334 struct uv_cb_cpu_set_state uvcb = { 335 .header.cmd = UVC_CMD_CPU_SET_STATE, 336 .header.len = sizeof(uvcb), 337 .cpu_handle = kvm_s390_pv_cpu_get_handle(vcpu), 338 .state = state, 339 }; 340 int cc; 341 342 cc = uv_call(0, (u64)&uvcb); 343 KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT SET CPU %d STATE %d rc %x rrc %x", 344 vcpu->vcpu_id, state, uvcb.header.rc, uvcb.header.rrc); 345 if (cc) 346 return -EINVAL; 347 return 0; 348 } 349 350 int kvm_s390_pv_dump_cpu(struct kvm_vcpu *vcpu, void *buff, u16 *rc, u16 *rrc) 351 { 352 struct uv_cb_dump_cpu uvcb = { 353 .header.cmd = UVC_CMD_DUMP_CPU, 354 .header.len = sizeof(uvcb), 355 .cpu_handle = vcpu->arch.pv.handle, 356 .dump_area_origin = (u64)buff, 357 }; 358 int cc; 359 360 cc = uv_call_sched(0, (u64)&uvcb); 361 *rc = uvcb.header.rc; 362 *rrc = uvcb.header.rrc; 363 return cc; 364 } 365 366 /* Size of the cache for the storage state dump data. 1MB for now */ 367 #define DUMP_BUFF_LEN HPAGE_SIZE 368 369 /** 370 * kvm_s390_pv_dump_stor_state 371 * 372 * @kvm: pointer to the guest's KVM struct 373 * @buff_user: Userspace pointer where we will write the results to 374 * @gaddr: Starting absolute guest address for which the storage state 375 * is requested. 376 * @buff_user_len: Length of the buff_user buffer 377 * @rc: Pointer to where the uvcb return code is stored 378 * @rrc: Pointer to where the uvcb return reason code is stored 379 * 380 * Stores buff_len bytes of tweak component values to buff_user 381 * starting with the 1MB block specified by the absolute guest address 382 * (gaddr). The gaddr pointer will be updated with the last address 383 * for which data was written when returning to userspace. buff_user 384 * might be written to even if an error rc is returned. For instance 385 * if we encounter a fault after writing the first page of data. 386 * 387 * Context: kvm->lock needs to be held 388 * 389 * Return: 390 * 0 on success 391 * -ENOMEM if allocating the cache fails 392 * -EINVAL if gaddr is not aligned to 1MB 393 * -EINVAL if buff_user_len is not aligned to uv_info.conf_dump_storage_state_len 394 * -EINVAL if the UV call fails, rc and rrc will be set in this case 395 * -EFAULT if copying the result to buff_user failed 396 */ 397 int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user, 398 u64 *gaddr, u64 buff_user_len, u16 *rc, u16 *rrc) 399 { 400 struct uv_cb_dump_stor_state uvcb = { 401 .header.cmd = UVC_CMD_DUMP_CONF_STOR_STATE, 402 .header.len = sizeof(uvcb), 403 .config_handle = kvm->arch.pv.handle, 404 .gaddr = *gaddr, 405 .dump_area_origin = 0, 406 }; 407 const u64 increment_len = uv_info.conf_dump_storage_state_len; 408 size_t buff_kvm_size; 409 size_t size_done = 0; 410 u8 *buff_kvm = NULL; 411 int cc, ret; 412 413 ret = -EINVAL; 414 /* UV call processes 1MB guest storage chunks at a time */ 415 if (!IS_ALIGNED(*gaddr, HPAGE_SIZE)) 416 goto out; 417 418 /* 419 * We provide the storage state for 1MB chunks of guest 420 * storage. The buffer will need to be aligned to 421 * conf_dump_storage_state_len so we don't end on a partial 422 * chunk. 423 */ 424 if (!buff_user_len || 425 !IS_ALIGNED(buff_user_len, increment_len)) 426 goto out; 427 428 /* 429 * Allocate a buffer from which we will later copy to the user 430 * process. We don't want userspace to dictate our buffer size 431 * so we limit it to DUMP_BUFF_LEN. 432 */ 433 ret = -ENOMEM; 434 buff_kvm_size = min_t(u64, buff_user_len, DUMP_BUFF_LEN); 435 buff_kvm = vzalloc(buff_kvm_size); 436 if (!buff_kvm) 437 goto out; 438 439 ret = 0; 440 uvcb.dump_area_origin = (u64)buff_kvm; 441 /* We will loop until the user buffer is filled or an error occurs */ 442 do { 443 /* Get 1MB worth of guest storage state data */ 444 cc = uv_call_sched(0, (u64)&uvcb); 445 446 /* All or nothing */ 447 if (cc) { 448 ret = -EINVAL; 449 break; 450 } 451 452 size_done += increment_len; 453 uvcb.dump_area_origin += increment_len; 454 buff_user_len -= increment_len; 455 uvcb.gaddr += HPAGE_SIZE; 456 457 /* KVM Buffer full, time to copy to the process */ 458 if (!buff_user_len || size_done == DUMP_BUFF_LEN) { 459 if (copy_to_user(buff_user, buff_kvm, size_done)) { 460 ret = -EFAULT; 461 break; 462 } 463 464 buff_user += size_done; 465 size_done = 0; 466 uvcb.dump_area_origin = (u64)buff_kvm; 467 } 468 } while (buff_user_len); 469 470 /* Report back where we ended dumping */ 471 *gaddr = uvcb.gaddr; 472 473 /* Lets only log errors, we don't want to spam */ 474 out: 475 if (ret) 476 KVM_UV_EVENT(kvm, 3, 477 "PROTVIRT DUMP STORAGE STATE: addr %llx ret %d, uvcb rc %x rrc %x", 478 uvcb.gaddr, ret, uvcb.header.rc, uvcb.header.rrc); 479 *rc = uvcb.header.rc; 480 *rrc = uvcb.header.rrc; 481 vfree(buff_kvm); 482 483 return ret; 484 } 485 486 /** 487 * kvm_s390_pv_dump_complete 488 * 489 * @kvm: pointer to the guest's KVM struct 490 * @buff_user: Userspace pointer where we will write the results to 491 * @rc: Pointer to where the uvcb return code is stored 492 * @rrc: Pointer to where the uvcb return reason code is stored 493 * 494 * Completes the dumping operation and writes the completion data to 495 * user space. 496 * 497 * Context: kvm->lock needs to be held 498 * 499 * Return: 500 * 0 on success 501 * -ENOMEM if allocating the completion buffer fails 502 * -EINVAL if the UV call fails, rc and rrc will be set in this case 503 * -EFAULT if copying the result to buff_user failed 504 */ 505 int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user, 506 u16 *rc, u16 *rrc) 507 { 508 struct uv_cb_dump_complete complete = { 509 .header.len = sizeof(complete), 510 .header.cmd = UVC_CMD_DUMP_COMPLETE, 511 .config_handle = kvm_s390_pv_get_handle(kvm), 512 }; 513 u64 *compl_data; 514 int ret; 515 516 /* Allocate dump area */ 517 compl_data = vzalloc(uv_info.conf_dump_finalize_len); 518 if (!compl_data) 519 return -ENOMEM; 520 complete.dump_area_origin = (u64)compl_data; 521 522 ret = uv_call_sched(0, (u64)&complete); 523 *rc = complete.header.rc; 524 *rrc = complete.header.rrc; 525 KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP COMPLETE: rc %x rrc %x", 526 complete.header.rc, complete.header.rrc); 527 528 if (!ret) { 529 /* 530 * kvm_s390_pv_dealloc_vm() will also (mem)set 531 * this to false on a reboot or other destroy 532 * operation for this vm. 533 */ 534 kvm->arch.pv.dumping = false; 535 kvm_s390_vcpu_unblock_all(kvm); 536 ret = copy_to_user(buff_user, compl_data, uv_info.conf_dump_finalize_len); 537 if (ret) 538 ret = -EFAULT; 539 } 540 vfree(compl_data); 541 /* If the UVC returned an error, translate it to -EINVAL */ 542 if (ret > 0) 543 ret = -EINVAL; 544 return ret; 545 } 546