1 /* 2 * Copyright (C) 2010 SUSE Linux Products GmbH. All rights reserved. 3 * 4 * Authors: 5 * Alexander Graf <agraf@suse.de> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License, version 2, as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/init.h> 23 #include <linux/export.h> 24 #include <linux/kvm_para.h> 25 #include <linux/slab.h> 26 #include <linux/of.h> 27 28 #include <asm/reg.h> 29 #include <asm/sections.h> 30 #include <asm/cacheflush.h> 31 #include <asm/disassemble.h> 32 33 #define KVM_MAGIC_PAGE (-4096L) 34 #define magic_var(x) KVM_MAGIC_PAGE + offsetof(struct kvm_vcpu_arch_shared, x) 35 36 #define KVM_INST_LWZ 0x80000000 37 #define KVM_INST_STW 0x90000000 38 #define KVM_INST_LD 0xe8000000 39 #define KVM_INST_STD 0xf8000000 40 #define KVM_INST_NOP 0x60000000 41 #define KVM_INST_B 0x48000000 42 #define KVM_INST_B_MASK 0x03ffffff 43 #define KVM_INST_B_MAX 0x01ffffff 44 45 #define KVM_MASK_RT 0x03e00000 46 #define KVM_RT_30 0x03c00000 47 #define KVM_MASK_RB 0x0000f800 48 #define KVM_INST_MFMSR 0x7c0000a6 49 #define KVM_INST_MFSPR_SPRG0 0x7c1042a6 50 #define KVM_INST_MFSPR_SPRG1 0x7c1142a6 51 #define KVM_INST_MFSPR_SPRG2 0x7c1242a6 52 #define KVM_INST_MFSPR_SPRG3 0x7c1342a6 53 #define KVM_INST_MFSPR_SRR0 0x7c1a02a6 54 #define KVM_INST_MFSPR_SRR1 0x7c1b02a6 55 #define KVM_INST_MFSPR_DAR 0x7c1302a6 56 #define KVM_INST_MFSPR_DSISR 0x7c1202a6 57 58 #define KVM_INST_MTSPR_SPRG0 0x7c1043a6 59 #define KVM_INST_MTSPR_SPRG1 0x7c1143a6 60 #define KVM_INST_MTSPR_SPRG2 0x7c1243a6 61 #define KVM_INST_MTSPR_SPRG3 0x7c1343a6 62 #define KVM_INST_MTSPR_SRR0 0x7c1a03a6 63 #define KVM_INST_MTSPR_SRR1 0x7c1b03a6 64 #define KVM_INST_MTSPR_DAR 0x7c1303a6 65 #define KVM_INST_MTSPR_DSISR 0x7c1203a6 66 67 #define KVM_INST_TLBSYNC 0x7c00046c 68 #define KVM_INST_MTMSRD_L0 0x7c000164 69 #define KVM_INST_MTMSRD_L1 0x7c010164 70 #define KVM_INST_MTMSR 0x7c000124 71 72 #define KVM_INST_WRTEEI_0 0x7c000146 73 #define KVM_INST_WRTEEI_1 0x7c008146 74 75 #define KVM_INST_MTSRIN 0x7c0001e4 76 77 static bool kvm_patching_worked = true; 78 static char kvm_tmp[1024 * 1024]; 79 static int kvm_tmp_index; 80 81 static inline void kvm_patch_ins(u32 *inst, u32 new_inst) 82 { 83 *inst = new_inst; 84 flush_icache_range((ulong)inst, (ulong)inst + 4); 85 } 86 87 static void kvm_patch_ins_ll(u32 *inst, long addr, u32 rt) 88 { 89 #ifdef CONFIG_64BIT 90 kvm_patch_ins(inst, KVM_INST_LD | rt | (addr & 0x0000fffc)); 91 #else 92 kvm_patch_ins(inst, KVM_INST_LWZ | rt | (addr & 0x0000fffc)); 93 #endif 94 } 95 96 static void kvm_patch_ins_ld(u32 *inst, long addr, u32 rt) 97 { 98 #ifdef CONFIG_64BIT 99 kvm_patch_ins(inst, KVM_INST_LD | rt | (addr & 0x0000fffc)); 100 #else 101 kvm_patch_ins(inst, KVM_INST_LWZ | rt | ((addr + 4) & 0x0000fffc)); 102 #endif 103 } 104 105 static void kvm_patch_ins_lwz(u32 *inst, long addr, u32 rt) 106 { 107 kvm_patch_ins(inst, KVM_INST_LWZ | rt | (addr & 0x0000ffff)); 108 } 109 110 static void kvm_patch_ins_std(u32 *inst, long addr, u32 rt) 111 { 112 #ifdef CONFIG_64BIT 113 kvm_patch_ins(inst, KVM_INST_STD | rt | (addr & 0x0000fffc)); 114 #else 115 kvm_patch_ins(inst, KVM_INST_STW | rt | ((addr + 4) & 0x0000fffc)); 116 #endif 117 } 118 119 static void kvm_patch_ins_stw(u32 *inst, long addr, u32 rt) 120 { 121 kvm_patch_ins(inst, KVM_INST_STW | rt | (addr & 0x0000fffc)); 122 } 123 124 static void kvm_patch_ins_nop(u32 *inst) 125 { 126 kvm_patch_ins(inst, KVM_INST_NOP); 127 } 128 129 static void kvm_patch_ins_b(u32 *inst, int addr) 130 { 131 #if defined(CONFIG_RELOCATABLE) && defined(CONFIG_PPC_BOOK3S) 132 /* On relocatable kernels interrupts handlers and our code 133 can be in different regions, so we don't patch them */ 134 135 if ((ulong)inst < (ulong)&__end_interrupts) 136 return; 137 #endif 138 139 kvm_patch_ins(inst, KVM_INST_B | (addr & KVM_INST_B_MASK)); 140 } 141 142 static u32 *kvm_alloc(int len) 143 { 144 u32 *p; 145 146 if ((kvm_tmp_index + len) > ARRAY_SIZE(kvm_tmp)) { 147 printk(KERN_ERR "KVM: No more space (%d + %d)\n", 148 kvm_tmp_index, len); 149 kvm_patching_worked = false; 150 return NULL; 151 } 152 153 p = (void*)&kvm_tmp[kvm_tmp_index]; 154 kvm_tmp_index += len; 155 156 return p; 157 } 158 159 extern u32 kvm_emulate_mtmsrd_branch_offs; 160 extern u32 kvm_emulate_mtmsrd_reg_offs; 161 extern u32 kvm_emulate_mtmsrd_orig_ins_offs; 162 extern u32 kvm_emulate_mtmsrd_len; 163 extern u32 kvm_emulate_mtmsrd[]; 164 165 static void kvm_patch_ins_mtmsrd(u32 *inst, u32 rt) 166 { 167 u32 *p; 168 int distance_start; 169 int distance_end; 170 ulong next_inst; 171 172 p = kvm_alloc(kvm_emulate_mtmsrd_len * 4); 173 if (!p) 174 return; 175 176 /* Find out where we are and put everything there */ 177 distance_start = (ulong)p - (ulong)inst; 178 next_inst = ((ulong)inst + 4); 179 distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsrd_branch_offs]; 180 181 /* Make sure we only write valid b instructions */ 182 if (distance_start > KVM_INST_B_MAX) { 183 kvm_patching_worked = false; 184 return; 185 } 186 187 /* Modify the chunk to fit the invocation */ 188 memcpy(p, kvm_emulate_mtmsrd, kvm_emulate_mtmsrd_len * 4); 189 p[kvm_emulate_mtmsrd_branch_offs] |= distance_end & KVM_INST_B_MASK; 190 switch (get_rt(rt)) { 191 case 30: 192 kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs], 193 magic_var(scratch2), KVM_RT_30); 194 break; 195 case 31: 196 kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs], 197 magic_var(scratch1), KVM_RT_30); 198 break; 199 default: 200 p[kvm_emulate_mtmsrd_reg_offs] |= rt; 201 break; 202 } 203 204 p[kvm_emulate_mtmsrd_orig_ins_offs] = *inst; 205 flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsrd_len * 4); 206 207 /* Patch the invocation */ 208 kvm_patch_ins_b(inst, distance_start); 209 } 210 211 extern u32 kvm_emulate_mtmsr_branch_offs; 212 extern u32 kvm_emulate_mtmsr_reg1_offs; 213 extern u32 kvm_emulate_mtmsr_reg2_offs; 214 extern u32 kvm_emulate_mtmsr_orig_ins_offs; 215 extern u32 kvm_emulate_mtmsr_len; 216 extern u32 kvm_emulate_mtmsr[]; 217 218 static void kvm_patch_ins_mtmsr(u32 *inst, u32 rt) 219 { 220 u32 *p; 221 int distance_start; 222 int distance_end; 223 ulong next_inst; 224 225 p = kvm_alloc(kvm_emulate_mtmsr_len * 4); 226 if (!p) 227 return; 228 229 /* Find out where we are and put everything there */ 230 distance_start = (ulong)p - (ulong)inst; 231 next_inst = ((ulong)inst + 4); 232 distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsr_branch_offs]; 233 234 /* Make sure we only write valid b instructions */ 235 if (distance_start > KVM_INST_B_MAX) { 236 kvm_patching_worked = false; 237 return; 238 } 239 240 /* Modify the chunk to fit the invocation */ 241 memcpy(p, kvm_emulate_mtmsr, kvm_emulate_mtmsr_len * 4); 242 p[kvm_emulate_mtmsr_branch_offs] |= distance_end & KVM_INST_B_MASK; 243 244 /* Make clobbered registers work too */ 245 switch (get_rt(rt)) { 246 case 30: 247 kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs], 248 magic_var(scratch2), KVM_RT_30); 249 kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs], 250 magic_var(scratch2), KVM_RT_30); 251 break; 252 case 31: 253 kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs], 254 magic_var(scratch1), KVM_RT_30); 255 kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs], 256 magic_var(scratch1), KVM_RT_30); 257 break; 258 default: 259 p[kvm_emulate_mtmsr_reg1_offs] |= rt; 260 p[kvm_emulate_mtmsr_reg2_offs] |= rt; 261 break; 262 } 263 264 p[kvm_emulate_mtmsr_orig_ins_offs] = *inst; 265 flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsr_len * 4); 266 267 /* Patch the invocation */ 268 kvm_patch_ins_b(inst, distance_start); 269 } 270 271 #ifdef CONFIG_BOOKE 272 273 extern u32 kvm_emulate_wrteei_branch_offs; 274 extern u32 kvm_emulate_wrteei_ee_offs; 275 extern u32 kvm_emulate_wrteei_len; 276 extern u32 kvm_emulate_wrteei[]; 277 278 static void kvm_patch_ins_wrteei(u32 *inst) 279 { 280 u32 *p; 281 int distance_start; 282 int distance_end; 283 ulong next_inst; 284 285 p = kvm_alloc(kvm_emulate_wrteei_len * 4); 286 if (!p) 287 return; 288 289 /* Find out where we are and put everything there */ 290 distance_start = (ulong)p - (ulong)inst; 291 next_inst = ((ulong)inst + 4); 292 distance_end = next_inst - (ulong)&p[kvm_emulate_wrteei_branch_offs]; 293 294 /* Make sure we only write valid b instructions */ 295 if (distance_start > KVM_INST_B_MAX) { 296 kvm_patching_worked = false; 297 return; 298 } 299 300 /* Modify the chunk to fit the invocation */ 301 memcpy(p, kvm_emulate_wrteei, kvm_emulate_wrteei_len * 4); 302 p[kvm_emulate_wrteei_branch_offs] |= distance_end & KVM_INST_B_MASK; 303 p[kvm_emulate_wrteei_ee_offs] |= (*inst & MSR_EE); 304 flush_icache_range((ulong)p, (ulong)p + kvm_emulate_wrteei_len * 4); 305 306 /* Patch the invocation */ 307 kvm_patch_ins_b(inst, distance_start); 308 } 309 310 #endif 311 312 #ifdef CONFIG_PPC_BOOK3S_32 313 314 extern u32 kvm_emulate_mtsrin_branch_offs; 315 extern u32 kvm_emulate_mtsrin_reg1_offs; 316 extern u32 kvm_emulate_mtsrin_reg2_offs; 317 extern u32 kvm_emulate_mtsrin_orig_ins_offs; 318 extern u32 kvm_emulate_mtsrin_len; 319 extern u32 kvm_emulate_mtsrin[]; 320 321 static void kvm_patch_ins_mtsrin(u32 *inst, u32 rt, u32 rb) 322 { 323 u32 *p; 324 int distance_start; 325 int distance_end; 326 ulong next_inst; 327 328 p = kvm_alloc(kvm_emulate_mtsrin_len * 4); 329 if (!p) 330 return; 331 332 /* Find out where we are and put everything there */ 333 distance_start = (ulong)p - (ulong)inst; 334 next_inst = ((ulong)inst + 4); 335 distance_end = next_inst - (ulong)&p[kvm_emulate_mtsrin_branch_offs]; 336 337 /* Make sure we only write valid b instructions */ 338 if (distance_start > KVM_INST_B_MAX) { 339 kvm_patching_worked = false; 340 return; 341 } 342 343 /* Modify the chunk to fit the invocation */ 344 memcpy(p, kvm_emulate_mtsrin, kvm_emulate_mtsrin_len * 4); 345 p[kvm_emulate_mtsrin_branch_offs] |= distance_end & KVM_INST_B_MASK; 346 p[kvm_emulate_mtsrin_reg1_offs] |= (rb << 10); 347 p[kvm_emulate_mtsrin_reg2_offs] |= rt; 348 p[kvm_emulate_mtsrin_orig_ins_offs] = *inst; 349 flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtsrin_len * 4); 350 351 /* Patch the invocation */ 352 kvm_patch_ins_b(inst, distance_start); 353 } 354 355 #endif 356 357 static void kvm_map_magic_page(void *data) 358 { 359 u32 *features = data; 360 361 ulong in[8]; 362 ulong out[8]; 363 364 in[0] = KVM_MAGIC_PAGE; 365 in[1] = KVM_MAGIC_PAGE; 366 367 kvm_hypercall(in, out, HC_VENDOR_KVM | KVM_HC_PPC_MAP_MAGIC_PAGE); 368 369 *features = out[0]; 370 } 371 372 static void kvm_check_ins(u32 *inst, u32 features) 373 { 374 u32 _inst = *inst; 375 u32 inst_no_rt = _inst & ~KVM_MASK_RT; 376 u32 inst_rt = _inst & KVM_MASK_RT; 377 378 switch (inst_no_rt) { 379 /* Loads */ 380 case KVM_INST_MFMSR: 381 kvm_patch_ins_ld(inst, magic_var(msr), inst_rt); 382 break; 383 case KVM_INST_MFSPR_SPRG0: 384 kvm_patch_ins_ld(inst, magic_var(sprg0), inst_rt); 385 break; 386 case KVM_INST_MFSPR_SPRG1: 387 kvm_patch_ins_ld(inst, magic_var(sprg1), inst_rt); 388 break; 389 case KVM_INST_MFSPR_SPRG2: 390 kvm_patch_ins_ld(inst, magic_var(sprg2), inst_rt); 391 break; 392 case KVM_INST_MFSPR_SPRG3: 393 kvm_patch_ins_ld(inst, magic_var(sprg3), inst_rt); 394 break; 395 case KVM_INST_MFSPR_SRR0: 396 kvm_patch_ins_ld(inst, magic_var(srr0), inst_rt); 397 break; 398 case KVM_INST_MFSPR_SRR1: 399 kvm_patch_ins_ld(inst, magic_var(srr1), inst_rt); 400 break; 401 case KVM_INST_MFSPR_DAR: 402 kvm_patch_ins_ld(inst, magic_var(dar), inst_rt); 403 break; 404 case KVM_INST_MFSPR_DSISR: 405 kvm_patch_ins_lwz(inst, magic_var(dsisr), inst_rt); 406 break; 407 408 /* Stores */ 409 case KVM_INST_MTSPR_SPRG0: 410 kvm_patch_ins_std(inst, magic_var(sprg0), inst_rt); 411 break; 412 case KVM_INST_MTSPR_SPRG1: 413 kvm_patch_ins_std(inst, magic_var(sprg1), inst_rt); 414 break; 415 case KVM_INST_MTSPR_SPRG2: 416 kvm_patch_ins_std(inst, magic_var(sprg2), inst_rt); 417 break; 418 case KVM_INST_MTSPR_SPRG3: 419 kvm_patch_ins_std(inst, magic_var(sprg3), inst_rt); 420 break; 421 case KVM_INST_MTSPR_SRR0: 422 kvm_patch_ins_std(inst, magic_var(srr0), inst_rt); 423 break; 424 case KVM_INST_MTSPR_SRR1: 425 kvm_patch_ins_std(inst, magic_var(srr1), inst_rt); 426 break; 427 case KVM_INST_MTSPR_DAR: 428 kvm_patch_ins_std(inst, magic_var(dar), inst_rt); 429 break; 430 case KVM_INST_MTSPR_DSISR: 431 kvm_patch_ins_stw(inst, magic_var(dsisr), inst_rt); 432 break; 433 434 /* Nops */ 435 case KVM_INST_TLBSYNC: 436 kvm_patch_ins_nop(inst); 437 break; 438 439 /* Rewrites */ 440 case KVM_INST_MTMSRD_L1: 441 kvm_patch_ins_mtmsrd(inst, inst_rt); 442 break; 443 case KVM_INST_MTMSR: 444 case KVM_INST_MTMSRD_L0: 445 kvm_patch_ins_mtmsr(inst, inst_rt); 446 break; 447 } 448 449 switch (inst_no_rt & ~KVM_MASK_RB) { 450 #ifdef CONFIG_PPC_BOOK3S_32 451 case KVM_INST_MTSRIN: 452 if (features & KVM_MAGIC_FEAT_SR) { 453 u32 inst_rb = _inst & KVM_MASK_RB; 454 kvm_patch_ins_mtsrin(inst, inst_rt, inst_rb); 455 } 456 break; 457 break; 458 #endif 459 } 460 461 switch (_inst) { 462 #ifdef CONFIG_BOOKE 463 case KVM_INST_WRTEEI_0: 464 case KVM_INST_WRTEEI_1: 465 kvm_patch_ins_wrteei(inst); 466 break; 467 #endif 468 } 469 } 470 471 static void kvm_use_magic_page(void) 472 { 473 u32 *p; 474 u32 *start, *end; 475 u32 tmp; 476 u32 features; 477 478 /* Tell the host to map the magic page to -4096 on all CPUs */ 479 on_each_cpu(kvm_map_magic_page, &features, 1); 480 481 /* Quick self-test to see if the mapping works */ 482 if (__get_user(tmp, (u32*)KVM_MAGIC_PAGE)) { 483 kvm_patching_worked = false; 484 return; 485 } 486 487 /* Now loop through all code and find instructions */ 488 start = (void*)_stext; 489 end = (void*)_etext; 490 491 for (p = start; p < end; p++) 492 kvm_check_ins(p, features); 493 494 printk(KERN_INFO "KVM: Live patching for a fast VM %s\n", 495 kvm_patching_worked ? "worked" : "failed"); 496 } 497 498 unsigned long kvm_hypercall(unsigned long *in, 499 unsigned long *out, 500 unsigned long nr) 501 { 502 unsigned long register r0 asm("r0"); 503 unsigned long register r3 asm("r3") = in[0]; 504 unsigned long register r4 asm("r4") = in[1]; 505 unsigned long register r5 asm("r5") = in[2]; 506 unsigned long register r6 asm("r6") = in[3]; 507 unsigned long register r7 asm("r7") = in[4]; 508 unsigned long register r8 asm("r8") = in[5]; 509 unsigned long register r9 asm("r9") = in[6]; 510 unsigned long register r10 asm("r10") = in[7]; 511 unsigned long register r11 asm("r11") = nr; 512 unsigned long register r12 asm("r12"); 513 514 asm volatile("bl kvm_hypercall_start" 515 : "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6), 516 "=r"(r7), "=r"(r8), "=r"(r9), "=r"(r10), "=r"(r11), 517 "=r"(r12) 518 : "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "r"(r8), 519 "r"(r9), "r"(r10), "r"(r11) 520 : "memory", "cc", "xer", "ctr", "lr"); 521 522 out[0] = r4; 523 out[1] = r5; 524 out[2] = r6; 525 out[3] = r7; 526 out[4] = r8; 527 out[5] = r9; 528 out[6] = r10; 529 out[7] = r11; 530 531 return r3; 532 } 533 EXPORT_SYMBOL_GPL(kvm_hypercall); 534 535 static int kvm_para_setup(void) 536 { 537 extern u32 kvm_hypercall_start; 538 struct device_node *hyper_node; 539 u32 *insts; 540 int len, i; 541 542 hyper_node = of_find_node_by_path("/hypervisor"); 543 if (!hyper_node) 544 return -1; 545 546 insts = (u32*)of_get_property(hyper_node, "hcall-instructions", &len); 547 if (len % 4) 548 return -1; 549 if (len > (4 * 4)) 550 return -1; 551 552 for (i = 0; i < (len / 4); i++) 553 kvm_patch_ins(&(&kvm_hypercall_start)[i], insts[i]); 554 555 return 0; 556 } 557 558 static __init void kvm_free_tmp(void) 559 { 560 unsigned long start, end; 561 562 start = (ulong)&kvm_tmp[kvm_tmp_index + (PAGE_SIZE - 1)] & PAGE_MASK; 563 end = (ulong)&kvm_tmp[ARRAY_SIZE(kvm_tmp)] & PAGE_MASK; 564 565 /* Free the tmp space we don't need */ 566 for (; start < end; start += PAGE_SIZE) { 567 ClearPageReserved(virt_to_page(start)); 568 init_page_count(virt_to_page(start)); 569 free_page(start); 570 totalram_pages++; 571 } 572 } 573 574 static int __init kvm_guest_init(void) 575 { 576 if (!kvm_para_available()) 577 goto free_tmp; 578 579 if (kvm_para_setup()) 580 goto free_tmp; 581 582 if (kvm_para_has_feature(KVM_FEATURE_MAGIC_PAGE)) 583 kvm_use_magic_page(); 584 585 #ifdef CONFIG_PPC_BOOK3S_64 586 /* Enable napping */ 587 powersave_nap = 1; 588 #endif 589 590 free_tmp: 591 kvm_free_tmp(); 592 593 return 0; 594 } 595 596 postcore_initcall(kvm_guest_init); 597