1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Hosting Protected Virtual Machines 4 * 5 * Copyright IBM Corp. 2019, 2020 6 * Author(s): Janosch Frank <frankja@linux.ibm.com> 7 */ 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 #include <linux/minmax.h> 11 #include <linux/pagemap.h> 12 #include <linux/sched/signal.h> 13 #include <asm/gmap.h> 14 #include <asm/uv.h> 15 #include <asm/mman.h> 16 #include "kvm-s390.h" 17 18 int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) 19 { 20 int cc; 21 22 if (!kvm_s390_pv_cpu_get_handle(vcpu)) 23 return 0; 24 25 cc = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), UVC_CMD_DESTROY_SEC_CPU, rc, rrc); 26 27 KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT DESTROY VCPU %d: rc %x rrc %x", 28 vcpu->vcpu_id, *rc, *rrc); 29 WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc); 30 31 /* Intended memory leak for something that should never happen. */ 32 if (!cc) 33 free_pages(vcpu->arch.pv.stor_base, 34 get_order(uv_info.guest_cpu_stor_len)); 35 36 free_page(sida_origin(vcpu->arch.sie_block)); 37 vcpu->arch.sie_block->pv_handle_cpu = 0; 38 vcpu->arch.sie_block->pv_handle_config = 0; 39 memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); 40 vcpu->arch.sie_block->sdf = 0; 41 /* 42 * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). 43 * Use the reset value of gbea to avoid leaking the kernel pointer of 44 * the just freed sida. 45 */ 46 vcpu->arch.sie_block->gbea = 1; 47 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 48 49 return cc ? EIO : 0; 50 } 51 52 int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) 53 { 54 struct uv_cb_csc uvcb = { 55 .header.cmd = UVC_CMD_CREATE_SEC_CPU, 56 .header.len = sizeof(uvcb), 57 }; 58 int cc; 59 60 if (kvm_s390_pv_cpu_get_handle(vcpu)) 61 return -EINVAL; 62 63 vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, 64 get_order(uv_info.guest_cpu_stor_len)); 65 if (!vcpu->arch.pv.stor_base) 66 return -ENOMEM; 67 68 /* Input */ 69 uvcb.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm); 70 uvcb.num = vcpu->arch.sie_block->icpua; 71 uvcb.state_origin = (u64)vcpu->arch.sie_block; 72 uvcb.stor_origin = (u64)vcpu->arch.pv.stor_base; 73 74 /* Alloc Secure Instruction Data Area Designation */ 75 vcpu->arch.sie_block->sidad = __get_free_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 76 if (!vcpu->arch.sie_block->sidad) { 77 free_pages(vcpu->arch.pv.stor_base, 78 get_order(uv_info.guest_cpu_stor_len)); 79 return -ENOMEM; 80 } 81 82 cc = uv_call(0, (u64)&uvcb); 83 *rc = uvcb.header.rc; 84 *rrc = uvcb.header.rrc; 85 KVM_UV_EVENT(vcpu->kvm, 3, 86 "PROTVIRT CREATE VCPU: cpu %d handle %llx rc %x rrc %x", 87 vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc, 88 uvcb.header.rrc); 89 90 if (cc) { 91 u16 dummy; 92 93 kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy); 94 return -EIO; 95 } 96 97 /* Output */ 98 vcpu->arch.pv.handle = uvcb.cpu_handle; 99 vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle; 100 vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_get_handle(vcpu->kvm); 101 vcpu->arch.sie_block->sdf = 2; 102 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 103 return 0; 104 } 105 106 /* only free resources when the destroy was successful */ 107 static void kvm_s390_pv_dealloc_vm(struct kvm *kvm) 108 { 109 vfree(kvm->arch.pv.stor_var); 110 free_pages(kvm->arch.pv.stor_base, 111 get_order(uv_info.guest_base_stor_len)); 112 memset(&kvm->arch.pv, 0, sizeof(kvm->arch.pv)); 113 } 114 115 static int kvm_s390_pv_alloc_vm(struct kvm *kvm) 116 { 117 unsigned long base = uv_info.guest_base_stor_len; 118 unsigned long virt = uv_info.guest_virt_var_stor_len; 119 unsigned long npages = 0, vlen = 0; 120 121 kvm->arch.pv.stor_var = NULL; 122 kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, get_order(base)); 123 if (!kvm->arch.pv.stor_base) 124 return -ENOMEM; 125 126 /* 127 * Calculate current guest storage for allocation of the 128 * variable storage, which is based on the length in MB. 129 * 130 * Slots are sorted by GFN 131 */ 132 mutex_lock(&kvm->slots_lock); 133 npages = kvm_s390_get_gfn_end(kvm_memslots(kvm)); 134 mutex_unlock(&kvm->slots_lock); 135 136 kvm->arch.pv.guest_len = npages * PAGE_SIZE; 137 138 /* Allocate variable storage */ 139 vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE); 140 vlen += uv_info.guest_virt_base_stor_len; 141 kvm->arch.pv.stor_var = vzalloc(vlen); 142 if (!kvm->arch.pv.stor_var) 143 goto out_err; 144 return 0; 145 146 out_err: 147 kvm_s390_pv_dealloc_vm(kvm); 148 return -ENOMEM; 149 } 150 151 /* this should not fail, but if it does, we must not free the donated memory */ 152 int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc) 153 { 154 int cc; 155 156 /* make all pages accessible before destroying the guest */ 157 s390_reset_acc(kvm->mm); 158 159 cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), 160 UVC_CMD_DESTROY_SEC_CONF, rc, rrc); 161 WRITE_ONCE(kvm->arch.gmap->guest_handle, 0); 162 atomic_set(&kvm->mm->context.is_protected, 0); 163 KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x", *rc, *rrc); 164 WARN_ONCE(cc, "protvirt destroy vm failed rc %x rrc %x", *rc, *rrc); 165 /* Inteded memory leak on "impossible" error */ 166 if (!cc) 167 kvm_s390_pv_dealloc_vm(kvm); 168 return cc ? -EIO : 0; 169 } 170 171 int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) 172 { 173 struct uv_cb_cgc uvcb = { 174 .header.cmd = UVC_CMD_CREATE_SEC_CONF, 175 .header.len = sizeof(uvcb) 176 }; 177 int cc, ret; 178 u16 dummy; 179 180 ret = kvm_s390_pv_alloc_vm(kvm); 181 if (ret) 182 return ret; 183 184 /* Inputs */ 185 uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */ 186 uvcb.guest_stor_len = kvm->arch.pv.guest_len; 187 uvcb.guest_asce = kvm->arch.gmap->asce; 188 uvcb.guest_sca = (unsigned long)kvm->arch.sca; 189 uvcb.conf_base_stor_origin = (u64)kvm->arch.pv.stor_base; 190 uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var; 191 192 cc = uv_call_sched(0, (u64)&uvcb); 193 *rc = uvcb.header.rc; 194 *rrc = uvcb.header.rrc; 195 KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x", 196 uvcb.guest_handle, uvcb.guest_stor_len, *rc, *rrc); 197 198 /* Outputs */ 199 kvm->arch.pv.handle = uvcb.guest_handle; 200 201 if (cc) { 202 if (uvcb.header.rc & UVC_RC_NEED_DESTROY) 203 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy); 204 else 205 kvm_s390_pv_dealloc_vm(kvm); 206 return -EIO; 207 } 208 kvm->arch.gmap->guest_handle = uvcb.guest_handle; 209 return 0; 210 } 211 212 int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc, 213 u16 *rrc) 214 { 215 struct uv_cb_ssc uvcb = { 216 .header.cmd = UVC_CMD_SET_SEC_CONF_PARAMS, 217 .header.len = sizeof(uvcb), 218 .sec_header_origin = (u64)hdr, 219 .sec_header_len = length, 220 .guest_handle = kvm_s390_pv_get_handle(kvm), 221 }; 222 int cc = uv_call(0, (u64)&uvcb); 223 224 *rc = uvcb.header.rc; 225 *rrc = uvcb.header.rrc; 226 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x", 227 *rc, *rrc); 228 if (!cc) 229 atomic_set(&kvm->mm->context.is_protected, 1); 230 return cc ? -EINVAL : 0; 231 } 232 233 static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak, 234 u64 offset, u16 *rc, u16 *rrc) 235 { 236 struct uv_cb_unp uvcb = { 237 .header.cmd = UVC_CMD_UNPACK_IMG, 238 .header.len = sizeof(uvcb), 239 .guest_handle = kvm_s390_pv_get_handle(kvm), 240 .gaddr = addr, 241 .tweak[0] = tweak, 242 .tweak[1] = offset, 243 }; 244 int ret = gmap_make_secure(kvm->arch.gmap, addr, &uvcb); 245 246 *rc = uvcb.header.rc; 247 *rrc = uvcb.header.rrc; 248 249 if (ret && ret != -EAGAIN) 250 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x", 251 uvcb.gaddr, *rc, *rrc); 252 return ret; 253 } 254 255 int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size, 256 unsigned long tweak, u16 *rc, u16 *rrc) 257 { 258 u64 offset = 0; 259 int ret = 0; 260 261 if (addr & ~PAGE_MASK || !size || size & ~PAGE_MASK) 262 return -EINVAL; 263 264 KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx", 265 addr, size); 266 267 while (offset < size) { 268 ret = unpack_one(kvm, addr, tweak, offset, rc, rrc); 269 if (ret == -EAGAIN) { 270 cond_resched(); 271 if (fatal_signal_pending(current)) 272 break; 273 continue; 274 } 275 if (ret) 276 break; 277 addr += PAGE_SIZE; 278 offset += PAGE_SIZE; 279 } 280 if (!ret) 281 KVM_UV_EVENT(kvm, 3, "%s", "PROTVIRT VM UNPACK: successful"); 282 return ret; 283 } 284 285 int kvm_s390_pv_set_cpu_state(struct kvm_vcpu *vcpu, u8 state) 286 { 287 struct uv_cb_cpu_set_state uvcb = { 288 .header.cmd = UVC_CMD_CPU_SET_STATE, 289 .header.len = sizeof(uvcb), 290 .cpu_handle = kvm_s390_pv_cpu_get_handle(vcpu), 291 .state = state, 292 }; 293 int cc; 294 295 cc = uv_call(0, (u64)&uvcb); 296 KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT SET CPU %d STATE %d rc %x rrc %x", 297 vcpu->vcpu_id, state, uvcb.header.rc, uvcb.header.rrc); 298 if (cc) 299 return -EINVAL; 300 return 0; 301 } 302 303 /* Size of the cache for the storage state dump data. 1MB for now */ 304 #define DUMP_BUFF_LEN HPAGE_SIZE 305 306 /** 307 * kvm_s390_pv_dump_stor_state 308 * 309 * @kvm: pointer to the guest's KVM struct 310 * @buff_user: Userspace pointer where we will write the results to 311 * @gaddr: Starting absolute guest address for which the storage state 312 * is requested. 313 * @buff_user_len: Length of the buff_user buffer 314 * @rc: Pointer to where the uvcb return code is stored 315 * @rrc: Pointer to where the uvcb return reason code is stored 316 * 317 * Stores buff_len bytes of tweak component values to buff_user 318 * starting with the 1MB block specified by the absolute guest address 319 * (gaddr). The gaddr pointer will be updated with the last address 320 * for which data was written when returning to userspace. buff_user 321 * might be written to even if an error rc is returned. For instance 322 * if we encounter a fault after writing the first page of data. 323 * 324 * Context: kvm->lock needs to be held 325 * 326 * Return: 327 * 0 on success 328 * -ENOMEM if allocating the cache fails 329 * -EINVAL if gaddr is not aligned to 1MB 330 * -EINVAL if buff_user_len is not aligned to uv_info.conf_dump_storage_state_len 331 * -EINVAL if the UV call fails, rc and rrc will be set in this case 332 * -EFAULT if copying the result to buff_user failed 333 */ 334 int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user, 335 u64 *gaddr, u64 buff_user_len, u16 *rc, u16 *rrc) 336 { 337 struct uv_cb_dump_stor_state uvcb = { 338 .header.cmd = UVC_CMD_DUMP_CONF_STOR_STATE, 339 .header.len = sizeof(uvcb), 340 .config_handle = kvm->arch.pv.handle, 341 .gaddr = *gaddr, 342 .dump_area_origin = 0, 343 }; 344 const u64 increment_len = uv_info.conf_dump_storage_state_len; 345 size_t buff_kvm_size; 346 size_t size_done = 0; 347 u8 *buff_kvm = NULL; 348 int cc, ret; 349 350 ret = -EINVAL; 351 /* UV call processes 1MB guest storage chunks at a time */ 352 if (!IS_ALIGNED(*gaddr, HPAGE_SIZE)) 353 goto out; 354 355 /* 356 * We provide the storage state for 1MB chunks of guest 357 * storage. The buffer will need to be aligned to 358 * conf_dump_storage_state_len so we don't end on a partial 359 * chunk. 360 */ 361 if (!buff_user_len || 362 !IS_ALIGNED(buff_user_len, increment_len)) 363 goto out; 364 365 /* 366 * Allocate a buffer from which we will later copy to the user 367 * process. We don't want userspace to dictate our buffer size 368 * so we limit it to DUMP_BUFF_LEN. 369 */ 370 ret = -ENOMEM; 371 buff_kvm_size = min_t(u64, buff_user_len, DUMP_BUFF_LEN); 372 buff_kvm = vzalloc(buff_kvm_size); 373 if (!buff_kvm) 374 goto out; 375 376 ret = 0; 377 uvcb.dump_area_origin = (u64)buff_kvm; 378 /* We will loop until the user buffer is filled or an error occurs */ 379 do { 380 /* Get 1MB worth of guest storage state data */ 381 cc = uv_call_sched(0, (u64)&uvcb); 382 383 /* All or nothing */ 384 if (cc) { 385 ret = -EINVAL; 386 break; 387 } 388 389 size_done += increment_len; 390 uvcb.dump_area_origin += increment_len; 391 buff_user_len -= increment_len; 392 uvcb.gaddr += HPAGE_SIZE; 393 394 /* KVM Buffer full, time to copy to the process */ 395 if (!buff_user_len || size_done == DUMP_BUFF_LEN) { 396 if (copy_to_user(buff_user, buff_kvm, size_done)) { 397 ret = -EFAULT; 398 break; 399 } 400 401 buff_user += size_done; 402 size_done = 0; 403 uvcb.dump_area_origin = (u64)buff_kvm; 404 } 405 } while (buff_user_len); 406 407 /* Report back where we ended dumping */ 408 *gaddr = uvcb.gaddr; 409 410 /* Lets only log errors, we don't want to spam */ 411 out: 412 if (ret) 413 KVM_UV_EVENT(kvm, 3, 414 "PROTVIRT DUMP STORAGE STATE: addr %llx ret %d, uvcb rc %x rrc %x", 415 uvcb.gaddr, ret, uvcb.header.rc, uvcb.header.rrc); 416 *rc = uvcb.header.rc; 417 *rrc = uvcb.header.rrc; 418 vfree(buff_kvm); 419 420 return ret; 421 } 422 423 /** 424 * kvm_s390_pv_dump_complete 425 * 426 * @kvm: pointer to the guest's KVM struct 427 * @buff_user: Userspace pointer where we will write the results to 428 * @rc: Pointer to where the uvcb return code is stored 429 * @rrc: Pointer to where the uvcb return reason code is stored 430 * 431 * Completes the dumping operation and writes the completion data to 432 * user space. 433 * 434 * Context: kvm->lock needs to be held 435 * 436 * Return: 437 * 0 on success 438 * -ENOMEM if allocating the completion buffer fails 439 * -EINVAL if the UV call fails, rc and rrc will be set in this case 440 * -EFAULT if copying the result to buff_user failed 441 */ 442 int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user, 443 u16 *rc, u16 *rrc) 444 { 445 struct uv_cb_dump_complete complete = { 446 .header.len = sizeof(complete), 447 .header.cmd = UVC_CMD_DUMP_COMPLETE, 448 .config_handle = kvm_s390_pv_get_handle(kvm), 449 }; 450 u64 *compl_data; 451 int ret; 452 453 /* Allocate dump area */ 454 compl_data = vzalloc(uv_info.conf_dump_finalize_len); 455 if (!compl_data) 456 return -ENOMEM; 457 complete.dump_area_origin = (u64)compl_data; 458 459 ret = uv_call_sched(0, (u64)&complete); 460 *rc = complete.header.rc; 461 *rrc = complete.header.rrc; 462 KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP COMPLETE: rc %x rrc %x", 463 complete.header.rc, complete.header.rrc); 464 465 if (!ret) { 466 /* 467 * kvm_s390_pv_dealloc_vm() will also (mem)set 468 * this to false on a reboot or other destroy 469 * operation for this vm. 470 */ 471 kvm->arch.pv.dumping = false; 472 kvm_s390_vcpu_unblock_all(kvm); 473 ret = copy_to_user(buff_user, compl_data, uv_info.conf_dump_finalize_len); 474 if (ret) 475 ret = -EFAULT; 476 } 477 vfree(compl_data); 478 /* If the UVC returned an error, translate it to -EINVAL */ 479 if (ret > 0) 480 ret = -EINVAL; 481 return ret; 482 } 483