1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Hosting Protected Virtual Machines 4 * 5 * Copyright IBM Corp. 2019, 2020 6 * Author(s): Janosch Frank <frankja@linux.ibm.com> 7 */ 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 #include <linux/minmax.h> 11 #include <linux/pagemap.h> 12 #include <linux/sched/signal.h> 13 #include <asm/gmap.h> 14 #include <asm/uv.h> 15 #include <asm/mman.h> 16 #include "kvm-s390.h" 17 18 int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) 19 { 20 int cc; 21 22 if (!kvm_s390_pv_cpu_get_handle(vcpu)) 23 return 0; 24 25 cc = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), UVC_CMD_DESTROY_SEC_CPU, rc, rrc); 26 27 KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT DESTROY VCPU %d: rc %x rrc %x", 28 vcpu->vcpu_id, *rc, *rrc); 29 WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc); 30 31 /* Intended memory leak for something that should never happen. */ 32 if (!cc) 33 free_pages(vcpu->arch.pv.stor_base, 34 get_order(uv_info.guest_cpu_stor_len)); 35 36 free_page(sida_origin(vcpu->arch.sie_block)); 37 vcpu->arch.sie_block->pv_handle_cpu = 0; 38 vcpu->arch.sie_block->pv_handle_config = 0; 39 memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); 40 vcpu->arch.sie_block->sdf = 0; 41 /* 42 * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). 43 * Use the reset value of gbea to avoid leaking the kernel pointer of 44 * the just freed sida. 45 */ 46 vcpu->arch.sie_block->gbea = 1; 47 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 48 49 return cc ? EIO : 0; 50 } 51 52 int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) 53 { 54 struct uv_cb_csc uvcb = { 55 .header.cmd = UVC_CMD_CREATE_SEC_CPU, 56 .header.len = sizeof(uvcb), 57 }; 58 int cc; 59 60 if (kvm_s390_pv_cpu_get_handle(vcpu)) 61 return -EINVAL; 62 63 vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, 64 get_order(uv_info.guest_cpu_stor_len)); 65 if (!vcpu->arch.pv.stor_base) 66 return -ENOMEM; 67 68 /* Input */ 69 uvcb.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm); 70 uvcb.num = vcpu->arch.sie_block->icpua; 71 uvcb.state_origin = (u64)vcpu->arch.sie_block; 72 uvcb.stor_origin = (u64)vcpu->arch.pv.stor_base; 73 74 /* Alloc Secure Instruction Data Area Designation */ 75 vcpu->arch.sie_block->sidad = __get_free_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 76 if (!vcpu->arch.sie_block->sidad) { 77 free_pages(vcpu->arch.pv.stor_base, 78 get_order(uv_info.guest_cpu_stor_len)); 79 return -ENOMEM; 80 } 81 82 cc = uv_call(0, (u64)&uvcb); 83 *rc = uvcb.header.rc; 84 *rrc = uvcb.header.rrc; 85 KVM_UV_EVENT(vcpu->kvm, 3, 86 "PROTVIRT CREATE VCPU: cpu %d handle %llx rc %x rrc %x", 87 vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc, 88 uvcb.header.rrc); 89 90 if (cc) { 91 u16 dummy; 92 93 kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy); 94 return -EIO; 95 } 96 97 /* Output */ 98 vcpu->arch.pv.handle = uvcb.cpu_handle; 99 vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle; 100 vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_get_handle(vcpu->kvm); 101 vcpu->arch.sie_block->sdf = 2; 102 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 103 return 0; 104 } 105 106 /* only free resources when the destroy was successful */ 107 static void kvm_s390_pv_dealloc_vm(struct kvm *kvm) 108 { 109 vfree(kvm->arch.pv.stor_var); 110 free_pages(kvm->arch.pv.stor_base, 111 get_order(uv_info.guest_base_stor_len)); 112 memset(&kvm->arch.pv, 0, sizeof(kvm->arch.pv)); 113 } 114 115 static int kvm_s390_pv_alloc_vm(struct kvm *kvm) 116 { 117 unsigned long base = uv_info.guest_base_stor_len; 118 unsigned long virt = uv_info.guest_virt_var_stor_len; 119 unsigned long npages = 0, vlen = 0; 120 121 kvm->arch.pv.stor_var = NULL; 122 kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, get_order(base)); 123 if (!kvm->arch.pv.stor_base) 124 return -ENOMEM; 125 126 /* 127 * Calculate current guest storage for allocation of the 128 * variable storage, which is based on the length in MB. 129 * 130 * Slots are sorted by GFN 131 */ 132 mutex_lock(&kvm->slots_lock); 133 npages = kvm_s390_get_gfn_end(kvm_memslots(kvm)); 134 mutex_unlock(&kvm->slots_lock); 135 136 kvm->arch.pv.guest_len = npages * PAGE_SIZE; 137 138 /* Allocate variable storage */ 139 vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE); 140 vlen += uv_info.guest_virt_base_stor_len; 141 kvm->arch.pv.stor_var = vzalloc(vlen); 142 if (!kvm->arch.pv.stor_var) 143 goto out_err; 144 return 0; 145 146 out_err: 147 kvm_s390_pv_dealloc_vm(kvm); 148 return -ENOMEM; 149 } 150 151 /* this should not fail, but if it does, we must not free the donated memory */ 152 int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc) 153 { 154 int cc; 155 156 /* make all pages accessible before destroying the guest */ 157 s390_reset_acc(kvm->mm); 158 159 cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), 160 UVC_CMD_DESTROY_SEC_CONF, rc, rrc); 161 WRITE_ONCE(kvm->arch.gmap->guest_handle, 0); 162 atomic_set(&kvm->mm->context.is_protected, 0); 163 KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x", *rc, *rrc); 164 WARN_ONCE(cc, "protvirt destroy vm failed rc %x rrc %x", *rc, *rrc); 165 /* Intended memory leak on "impossible" error */ 166 if (!cc) { 167 kvm_s390_pv_dealloc_vm(kvm); 168 return 0; 169 } 170 s390_replace_asce(kvm->arch.gmap); 171 return -EIO; 172 } 173 174 int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) 175 { 176 struct uv_cb_cgc uvcb = { 177 .header.cmd = UVC_CMD_CREATE_SEC_CONF, 178 .header.len = sizeof(uvcb) 179 }; 180 int cc, ret; 181 u16 dummy; 182 183 ret = kvm_s390_pv_alloc_vm(kvm); 184 if (ret) 185 return ret; 186 187 /* Inputs */ 188 uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */ 189 uvcb.guest_stor_len = kvm->arch.pv.guest_len; 190 uvcb.guest_asce = kvm->arch.gmap->asce; 191 uvcb.guest_sca = (unsigned long)kvm->arch.sca; 192 uvcb.conf_base_stor_origin = (u64)kvm->arch.pv.stor_base; 193 uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var; 194 195 cc = uv_call_sched(0, (u64)&uvcb); 196 *rc = uvcb.header.rc; 197 *rrc = uvcb.header.rrc; 198 KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x", 199 uvcb.guest_handle, uvcb.guest_stor_len, *rc, *rrc); 200 201 /* Outputs */ 202 kvm->arch.pv.handle = uvcb.guest_handle; 203 204 if (cc) { 205 if (uvcb.header.rc & UVC_RC_NEED_DESTROY) 206 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy); 207 else 208 kvm_s390_pv_dealloc_vm(kvm); 209 return -EIO; 210 } 211 kvm->arch.gmap->guest_handle = uvcb.guest_handle; 212 return 0; 213 } 214 215 int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc, 216 u16 *rrc) 217 { 218 struct uv_cb_ssc uvcb = { 219 .header.cmd = UVC_CMD_SET_SEC_CONF_PARAMS, 220 .header.len = sizeof(uvcb), 221 .sec_header_origin = (u64)hdr, 222 .sec_header_len = length, 223 .guest_handle = kvm_s390_pv_get_handle(kvm), 224 }; 225 int cc = uv_call(0, (u64)&uvcb); 226 227 *rc = uvcb.header.rc; 228 *rrc = uvcb.header.rrc; 229 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x", 230 *rc, *rrc); 231 if (!cc) 232 atomic_set(&kvm->mm->context.is_protected, 1); 233 return cc ? -EINVAL : 0; 234 } 235 236 static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak, 237 u64 offset, u16 *rc, u16 *rrc) 238 { 239 struct uv_cb_unp uvcb = { 240 .header.cmd = UVC_CMD_UNPACK_IMG, 241 .header.len = sizeof(uvcb), 242 .guest_handle = kvm_s390_pv_get_handle(kvm), 243 .gaddr = addr, 244 .tweak[0] = tweak, 245 .tweak[1] = offset, 246 }; 247 int ret = gmap_make_secure(kvm->arch.gmap, addr, &uvcb); 248 249 *rc = uvcb.header.rc; 250 *rrc = uvcb.header.rrc; 251 252 if (ret && ret != -EAGAIN) 253 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x", 254 uvcb.gaddr, *rc, *rrc); 255 return ret; 256 } 257 258 int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size, 259 unsigned long tweak, u16 *rc, u16 *rrc) 260 { 261 u64 offset = 0; 262 int ret = 0; 263 264 if (addr & ~PAGE_MASK || !size || size & ~PAGE_MASK) 265 return -EINVAL; 266 267 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx", 268 addr, size); 269 270 while (offset < size) { 271 ret = unpack_one(kvm, addr, tweak, offset, rc, rrc); 272 if (ret == -EAGAIN) { 273 cond_resched(); 274 if (fatal_signal_pending(current)) 275 break; 276 continue; 277 } 278 if (ret) 279 break; 280 addr += PAGE_SIZE; 281 offset += PAGE_SIZE; 282 } 283 if (!ret) 284 KVM_UV_EVENT(kvm, 3, "%s", "PROTVIRT VM UNPACK: successful"); 285 return ret; 286 } 287 288 int kvm_s390_pv_set_cpu_state(struct kvm_vcpu *vcpu, u8 state) 289 { 290 struct uv_cb_cpu_set_state uvcb = { 291 .header.cmd = UVC_CMD_CPU_SET_STATE, 292 .header.len = sizeof(uvcb), 293 .cpu_handle = kvm_s390_pv_cpu_get_handle(vcpu), 294 .state = state, 295 }; 296 int cc; 297 298 cc = uv_call(0, (u64)&uvcb); 299 KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT SET CPU %d STATE %d rc %x rrc %x", 300 vcpu->vcpu_id, state, uvcb.header.rc, uvcb.header.rrc); 301 if (cc) 302 return -EINVAL; 303 return 0; 304 } 305 306 int kvm_s390_pv_dump_cpu(struct kvm_vcpu *vcpu, void *buff, u16 *rc, u16 *rrc) 307 { 308 struct uv_cb_dump_cpu uvcb = { 309 .header.cmd = UVC_CMD_DUMP_CPU, 310 .header.len = sizeof(uvcb), 311 .cpu_handle = vcpu->arch.pv.handle, 312 .dump_area_origin = (u64)buff, 313 }; 314 int cc; 315 316 cc = uv_call_sched(0, (u64)&uvcb); 317 *rc = uvcb.header.rc; 318 *rrc = uvcb.header.rrc; 319 return cc; 320 } 321 322 /* Size of the cache for the storage state dump data. 1MB for now */ 323 #define DUMP_BUFF_LEN HPAGE_SIZE 324 325 /** 326 * kvm_s390_pv_dump_stor_state 327 * 328 * @kvm: pointer to the guest's KVM struct 329 * @buff_user: Userspace pointer where we will write the results to 330 * @gaddr: Starting absolute guest address for which the storage state 331 * is requested. 332 * @buff_user_len: Length of the buff_user buffer 333 * @rc: Pointer to where the uvcb return code is stored 334 * @rrc: Pointer to where the uvcb return reason code is stored 335 * 336 * Stores buff_len bytes of tweak component values to buff_user 337 * starting with the 1MB block specified by the absolute guest address 338 * (gaddr). The gaddr pointer will be updated with the last address 339 * for which data was written when returning to userspace. buff_user 340 * might be written to even if an error rc is returned. For instance 341 * if we encounter a fault after writing the first page of data. 342 * 343 * Context: kvm->lock needs to be held 344 * 345 * Return: 346 * 0 on success 347 * -ENOMEM if allocating the cache fails 348 * -EINVAL if gaddr is not aligned to 1MB 349 * -EINVAL if buff_user_len is not aligned to uv_info.conf_dump_storage_state_len 350 * -EINVAL if the UV call fails, rc and rrc will be set in this case 351 * -EFAULT if copying the result to buff_user failed 352 */ 353 int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user, 354 u64 *gaddr, u64 buff_user_len, u16 *rc, u16 *rrc) 355 { 356 struct uv_cb_dump_stor_state uvcb = { 357 .header.cmd = UVC_CMD_DUMP_CONF_STOR_STATE, 358 .header.len = sizeof(uvcb), 359 .config_handle = kvm->arch.pv.handle, 360 .gaddr = *gaddr, 361 .dump_area_origin = 0, 362 }; 363 const u64 increment_len = uv_info.conf_dump_storage_state_len; 364 size_t buff_kvm_size; 365 size_t size_done = 0; 366 u8 *buff_kvm = NULL; 367 int cc, ret; 368 369 ret = -EINVAL; 370 /* UV call processes 1MB guest storage chunks at a time */ 371 if (!IS_ALIGNED(*gaddr, HPAGE_SIZE)) 372 goto out; 373 374 /* 375 * We provide the storage state for 1MB chunks of guest 376 * storage. The buffer will need to be aligned to 377 * conf_dump_storage_state_len so we don't end on a partial 378 * chunk. 379 */ 380 if (!buff_user_len || 381 !IS_ALIGNED(buff_user_len, increment_len)) 382 goto out; 383 384 /* 385 * Allocate a buffer from which we will later copy to the user 386 * process. We don't want userspace to dictate our buffer size 387 * so we limit it to DUMP_BUFF_LEN. 388 */ 389 ret = -ENOMEM; 390 buff_kvm_size = min_t(u64, buff_user_len, DUMP_BUFF_LEN); 391 buff_kvm = vzalloc(buff_kvm_size); 392 if (!buff_kvm) 393 goto out; 394 395 ret = 0; 396 uvcb.dump_area_origin = (u64)buff_kvm; 397 /* We will loop until the user buffer is filled or an error occurs */ 398 do { 399 /* Get 1MB worth of guest storage state data */ 400 cc = uv_call_sched(0, (u64)&uvcb); 401 402 /* All or nothing */ 403 if (cc) { 404 ret = -EINVAL; 405 break; 406 } 407 408 size_done += increment_len; 409 uvcb.dump_area_origin += increment_len; 410 buff_user_len -= increment_len; 411 uvcb.gaddr += HPAGE_SIZE; 412 413 /* KVM Buffer full, time to copy to the process */ 414 if (!buff_user_len || size_done == DUMP_BUFF_LEN) { 415 if (copy_to_user(buff_user, buff_kvm, size_done)) { 416 ret = -EFAULT; 417 break; 418 } 419 420 buff_user += size_done; 421 size_done = 0; 422 uvcb.dump_area_origin = (u64)buff_kvm; 423 } 424 } while (buff_user_len); 425 426 /* Report back where we ended dumping */ 427 *gaddr = uvcb.gaddr; 428 429 /* Lets only log errors, we don't want to spam */ 430 out: 431 if (ret) 432 KVM_UV_EVENT(kvm, 3, 433 "PROTVIRT DUMP STORAGE STATE: addr %llx ret %d, uvcb rc %x rrc %x", 434 uvcb.gaddr, ret, uvcb.header.rc, uvcb.header.rrc); 435 *rc = uvcb.header.rc; 436 *rrc = uvcb.header.rrc; 437 vfree(buff_kvm); 438 439 return ret; 440 } 441 442 /** 443 * kvm_s390_pv_dump_complete 444 * 445 * @kvm: pointer to the guest's KVM struct 446 * @buff_user: Userspace pointer where we will write the results to 447 * @rc: Pointer to where the uvcb return code is stored 448 * @rrc: Pointer to where the uvcb return reason code is stored 449 * 450 * Completes the dumping operation and writes the completion data to 451 * user space. 452 * 453 * Context: kvm->lock needs to be held 454 * 455 * Return: 456 * 0 on success 457 * -ENOMEM if allocating the completion buffer fails 458 * -EINVAL if the UV call fails, rc and rrc will be set in this case 459 * -EFAULT if copying the result to buff_user failed 460 */ 461 int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user, 462 u16 *rc, u16 *rrc) 463 { 464 struct uv_cb_dump_complete complete = { 465 .header.len = sizeof(complete), 466 .header.cmd = UVC_CMD_DUMP_COMPLETE, 467 .config_handle = kvm_s390_pv_get_handle(kvm), 468 }; 469 u64 *compl_data; 470 int ret; 471 472 /* Allocate dump area */ 473 compl_data = vzalloc(uv_info.conf_dump_finalize_len); 474 if (!compl_data) 475 return -ENOMEM; 476 complete.dump_area_origin = (u64)compl_data; 477 478 ret = uv_call_sched(0, (u64)&complete); 479 *rc = complete.header.rc; 480 *rrc = complete.header.rrc; 481 KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP COMPLETE: rc %x rrc %x", 482 complete.header.rc, complete.header.rrc); 483 484 if (!ret) { 485 /* 486 * kvm_s390_pv_dealloc_vm() will also (mem)set 487 * this to false on a reboot or other destroy 488 * operation for this vm. 489 */ 490 kvm->arch.pv.dumping = false; 491 kvm_s390_vcpu_unblock_all(kvm); 492 ret = copy_to_user(buff_user, compl_data, uv_info.conf_dump_finalize_len); 493 if (ret) 494 ret = -EFAULT; 495 } 496 vfree(compl_data); 497 /* If the UVC returned an error, translate it to -EINVAL */ 498 if (ret > 0) 499 ret = -EINVAL; 500 return ret; 501 } 502