1 /* 2 * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved. 3 * 4 * Author: Yu Liu, yu.liu@freescale.com 5 * Scott Wood, scottwood@freescale.com 6 * Ashish Kalra, ashish.kalra@freescale.com 7 * Varun Sethi, varun.sethi@freescale.com 8 * Alexander Graf, agraf@suse.de 9 * 10 * Description: 11 * This file is based on arch/powerpc/kvm/44x_tlb.c, 12 * by Hollis Blanchard <hollisb@us.ibm.com>. 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the terms of the GNU General Public License, version 2, as 16 * published by the Free Software Foundation. 17 */ 18 19 #include <linux/kernel.h> 20 #include <linux/types.h> 21 #include <linux/slab.h> 22 #include <linux/string.h> 23 #include <linux/kvm.h> 24 #include <linux/kvm_host.h> 25 #include <linux/highmem.h> 26 #include <linux/log2.h> 27 #include <linux/uaccess.h> 28 #include <linux/sched.h> 29 #include <linux/rwsem.h> 30 #include <linux/vmalloc.h> 31 #include <linux/hugetlb.h> 32 #include <asm/kvm_ppc.h> 33 34 #include "e500.h" 35 #include "trace.h" 36 #include "timing.h" 37 #include "e500_mmu_host.h" 38 39 static inline unsigned int gtlb0_get_next_victim( 40 struct kvmppc_vcpu_e500 *vcpu_e500) 41 { 42 unsigned int victim; 43 44 victim = vcpu_e500->gtlb_nv[0]++; 45 if (unlikely(vcpu_e500->gtlb_nv[0] >= vcpu_e500->gtlb_params[0].ways)) 46 vcpu_e500->gtlb_nv[0] = 0; 47 48 return victim; 49 } 50 51 static int tlb0_set_base(gva_t addr, int sets, int ways) 52 { 53 int set_base; 54 55 set_base = (addr >> PAGE_SHIFT) & (sets - 1); 56 set_base *= ways; 57 58 return set_base; 59 } 60 61 static int gtlb0_set_base(struct kvmppc_vcpu_e500 *vcpu_e500, gva_t addr) 62 { 63 return tlb0_set_base(addr, vcpu_e500->gtlb_params[0].sets, 64 vcpu_e500->gtlb_params[0].ways); 65 } 66 67 static unsigned int get_tlb_esel(struct kvm_vcpu *vcpu, int tlbsel) 68 { 69 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 70 int esel = get_tlb_esel_bit(vcpu); 71 72 if (tlbsel == 0) { 73 esel &= vcpu_e500->gtlb_params[0].ways - 1; 74 esel += gtlb0_set_base(vcpu_e500, vcpu->arch.shared->mas2); 75 } else { 76 esel &= vcpu_e500->gtlb_params[tlbsel].entries - 1; 77 } 78 79 return esel; 80 } 81 82 /* Search the guest TLB for a matching entry. */ 83 static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500, 84 gva_t eaddr, int tlbsel, unsigned int pid, int as) 85 { 86 int size = vcpu_e500->gtlb_params[tlbsel].entries; 87 unsigned int set_base, offset; 88 int i; 89 90 if (tlbsel == 0) { 91 set_base = gtlb0_set_base(vcpu_e500, eaddr); 92 size = vcpu_e500->gtlb_params[0].ways; 93 } else { 94 if (eaddr < vcpu_e500->tlb1_min_eaddr || 95 eaddr > vcpu_e500->tlb1_max_eaddr) 96 return -1; 97 set_base = 0; 98 } 99 100 offset = vcpu_e500->gtlb_offset[tlbsel]; 101 102 for (i = 0; i < size; i++) { 103 struct kvm_book3e_206_tlb_entry *tlbe = 104 &vcpu_e500->gtlb_arch[offset + set_base + i]; 105 unsigned int tid; 106 107 if (eaddr < get_tlb_eaddr(tlbe)) 108 continue; 109 110 if (eaddr > get_tlb_end(tlbe)) 111 continue; 112 113 tid = get_tlb_tid(tlbe); 114 if (tid && (tid != pid)) 115 continue; 116 117 if (!get_tlb_v(tlbe)) 118 continue; 119 120 if (get_tlb_ts(tlbe) != as && as != -1) 121 continue; 122 123 return set_base + i; 124 } 125 126 return -1; 127 } 128 129 static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu, 130 unsigned int eaddr, int as) 131 { 132 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 133 unsigned int victim, tsized; 134 int tlbsel; 135 136 /* since we only have two TLBs, only lower bit is used. */ 137 tlbsel = (vcpu->arch.shared->mas4 >> 28) & 0x1; 138 victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0; 139 tsized = (vcpu->arch.shared->mas4 >> 7) & 0x1f; 140 141 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim) 142 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 143 vcpu->arch.shared->mas1 = MAS1_VALID | (as ? MAS1_TS : 0) 144 | MAS1_TID(get_tlbmiss_tid(vcpu)) 145 | MAS1_TSIZE(tsized); 146 vcpu->arch.shared->mas2 = (eaddr & MAS2_EPN) 147 | (vcpu->arch.shared->mas4 & MAS2_ATTRIB_MASK); 148 vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3; 149 vcpu->arch.shared->mas6 = (vcpu->arch.shared->mas6 & MAS6_SPID1) 150 | (get_cur_pid(vcpu) << 16) 151 | (as ? MAS6_SAS : 0); 152 } 153 154 static void kvmppc_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500) 155 { 156 int size = vcpu_e500->gtlb_params[1].entries; 157 unsigned int offset; 158 gva_t eaddr; 159 int i; 160 161 vcpu_e500->tlb1_min_eaddr = ~0UL; 162 vcpu_e500->tlb1_max_eaddr = 0; 163 offset = vcpu_e500->gtlb_offset[1]; 164 165 for (i = 0; i < size; i++) { 166 struct kvm_book3e_206_tlb_entry *tlbe = 167 &vcpu_e500->gtlb_arch[offset + i]; 168 169 if (!get_tlb_v(tlbe)) 170 continue; 171 172 eaddr = get_tlb_eaddr(tlbe); 173 vcpu_e500->tlb1_min_eaddr = 174 min(vcpu_e500->tlb1_min_eaddr, eaddr); 175 176 eaddr = get_tlb_end(tlbe); 177 vcpu_e500->tlb1_max_eaddr = 178 max(vcpu_e500->tlb1_max_eaddr, eaddr); 179 } 180 } 181 182 static int kvmppc_need_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500, 183 struct kvm_book3e_206_tlb_entry *gtlbe) 184 { 185 unsigned long start, end, size; 186 187 size = get_tlb_bytes(gtlbe); 188 start = get_tlb_eaddr(gtlbe) & ~(size - 1); 189 end = start + size - 1; 190 191 return vcpu_e500->tlb1_min_eaddr == start || 192 vcpu_e500->tlb1_max_eaddr == end; 193 } 194 195 /* This function is supposed to be called for a adding a new valid tlb entry */ 196 static void kvmppc_set_tlb1map_range(struct kvm_vcpu *vcpu, 197 struct kvm_book3e_206_tlb_entry *gtlbe) 198 { 199 unsigned long start, end, size; 200 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 201 202 if (!get_tlb_v(gtlbe)) 203 return; 204 205 size = get_tlb_bytes(gtlbe); 206 start = get_tlb_eaddr(gtlbe) & ~(size - 1); 207 end = start + size - 1; 208 209 vcpu_e500->tlb1_min_eaddr = min(vcpu_e500->tlb1_min_eaddr, start); 210 vcpu_e500->tlb1_max_eaddr = max(vcpu_e500->tlb1_max_eaddr, end); 211 } 212 213 static inline int kvmppc_e500_gtlbe_invalidate( 214 struct kvmppc_vcpu_e500 *vcpu_e500, 215 int tlbsel, int esel) 216 { 217 struct kvm_book3e_206_tlb_entry *gtlbe = 218 get_entry(vcpu_e500, tlbsel, esel); 219 220 if (unlikely(get_tlb_iprot(gtlbe))) 221 return -1; 222 223 if (tlbsel == 1 && kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe)) 224 kvmppc_recalc_tlb1map_range(vcpu_e500); 225 226 gtlbe->mas1 = 0; 227 228 return 0; 229 } 230 231 int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value) 232 { 233 int esel; 234 235 if (value & MMUCSR0_TLB0FI) 236 for (esel = 0; esel < vcpu_e500->gtlb_params[0].entries; esel++) 237 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel); 238 if (value & MMUCSR0_TLB1FI) 239 for (esel = 0; esel < vcpu_e500->gtlb_params[1].entries; esel++) 240 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel); 241 242 /* Invalidate all host shadow mappings */ 243 kvmppc_core_flush_tlb(&vcpu_e500->vcpu); 244 245 return EMULATE_DONE; 246 } 247 248 int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, gva_t ea) 249 { 250 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 251 unsigned int ia; 252 int esel, tlbsel; 253 254 ia = (ea >> 2) & 0x1; 255 256 /* since we only have two TLBs, only lower bit is used. */ 257 tlbsel = (ea >> 3) & 0x1; 258 259 if (ia) { 260 /* invalidate all entries */ 261 for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries; 262 esel++) 263 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 264 } else { 265 ea &= 0xfffff000; 266 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, 267 get_cur_pid(vcpu), -1); 268 if (esel >= 0) 269 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 270 } 271 272 /* Invalidate all host shadow mappings */ 273 kvmppc_core_flush_tlb(&vcpu_e500->vcpu); 274 275 return EMULATE_DONE; 276 } 277 278 static void tlbilx_all(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel, 279 int pid, int type) 280 { 281 struct kvm_book3e_206_tlb_entry *tlbe; 282 int tid, esel; 283 284 /* invalidate all entries */ 285 for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries; esel++) { 286 tlbe = get_entry(vcpu_e500, tlbsel, esel); 287 tid = get_tlb_tid(tlbe); 288 if (type == 0 || tid == pid) { 289 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel); 290 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 291 } 292 } 293 } 294 295 static void tlbilx_one(struct kvmppc_vcpu_e500 *vcpu_e500, int pid, 296 gva_t ea) 297 { 298 int tlbsel, esel; 299 300 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 301 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, -1); 302 if (esel >= 0) { 303 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel); 304 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 305 break; 306 } 307 } 308 } 309 310 int kvmppc_e500_emul_tlbilx(struct kvm_vcpu *vcpu, int type, gva_t ea) 311 { 312 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 313 int pid = get_cur_spid(vcpu); 314 315 if (type == 0 || type == 1) { 316 tlbilx_all(vcpu_e500, 0, pid, type); 317 tlbilx_all(vcpu_e500, 1, pid, type); 318 } else if (type == 3) { 319 tlbilx_one(vcpu_e500, pid, ea); 320 } 321 322 return EMULATE_DONE; 323 } 324 325 int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu) 326 { 327 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 328 int tlbsel, esel; 329 struct kvm_book3e_206_tlb_entry *gtlbe; 330 331 tlbsel = get_tlb_tlbsel(vcpu); 332 esel = get_tlb_esel(vcpu, tlbsel); 333 334 gtlbe = get_entry(vcpu_e500, tlbsel, esel); 335 vcpu->arch.shared->mas0 &= ~MAS0_NV(~0); 336 vcpu->arch.shared->mas0 |= MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 337 vcpu->arch.shared->mas1 = gtlbe->mas1; 338 vcpu->arch.shared->mas2 = gtlbe->mas2; 339 vcpu->arch.shared->mas7_3 = gtlbe->mas7_3; 340 341 return EMULATE_DONE; 342 } 343 344 int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, gva_t ea) 345 { 346 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 347 int as = !!get_cur_sas(vcpu); 348 unsigned int pid = get_cur_spid(vcpu); 349 int esel, tlbsel; 350 struct kvm_book3e_206_tlb_entry *gtlbe = NULL; 351 352 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 353 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as); 354 if (esel >= 0) { 355 gtlbe = get_entry(vcpu_e500, tlbsel, esel); 356 break; 357 } 358 } 359 360 if (gtlbe) { 361 esel &= vcpu_e500->gtlb_params[tlbsel].ways - 1; 362 363 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel) 364 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 365 vcpu->arch.shared->mas1 = gtlbe->mas1; 366 vcpu->arch.shared->mas2 = gtlbe->mas2; 367 vcpu->arch.shared->mas7_3 = gtlbe->mas7_3; 368 } else { 369 int victim; 370 371 /* since we only have two TLBs, only lower bit is used. */ 372 tlbsel = vcpu->arch.shared->mas4 >> 28 & 0x1; 373 victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0; 374 375 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) 376 | MAS0_ESEL(victim) 377 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 378 vcpu->arch.shared->mas1 = 379 (vcpu->arch.shared->mas6 & MAS6_SPID0) 380 | (vcpu->arch.shared->mas6 & (MAS6_SAS ? MAS1_TS : 0)) 381 | (vcpu->arch.shared->mas4 & MAS4_TSIZED(~0)); 382 vcpu->arch.shared->mas2 &= MAS2_EPN; 383 vcpu->arch.shared->mas2 |= vcpu->arch.shared->mas4 & 384 MAS2_ATTRIB_MASK; 385 vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | 386 MAS3_U2 | MAS3_U3; 387 } 388 389 kvmppc_set_exit_type(vcpu, EMULATED_TLBSX_EXITS); 390 return EMULATE_DONE; 391 } 392 393 int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu) 394 { 395 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 396 struct kvm_book3e_206_tlb_entry *gtlbe; 397 int tlbsel, esel; 398 int recal = 0; 399 400 tlbsel = get_tlb_tlbsel(vcpu); 401 esel = get_tlb_esel(vcpu, tlbsel); 402 403 gtlbe = get_entry(vcpu_e500, tlbsel, esel); 404 405 if (get_tlb_v(gtlbe)) { 406 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel); 407 if ((tlbsel == 1) && 408 kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe)) 409 recal = 1; 410 } 411 412 gtlbe->mas1 = vcpu->arch.shared->mas1; 413 gtlbe->mas2 = vcpu->arch.shared->mas2; 414 if (!(vcpu->arch.shared->msr & MSR_CM)) 415 gtlbe->mas2 &= 0xffffffffUL; 416 gtlbe->mas7_3 = vcpu->arch.shared->mas7_3; 417 418 trace_kvm_booke206_gtlb_write(vcpu->arch.shared->mas0, gtlbe->mas1, 419 gtlbe->mas2, gtlbe->mas7_3); 420 421 if (tlbsel == 1) { 422 /* 423 * If a valid tlb1 entry is overwritten then recalculate the 424 * min/max TLB1 map address range otherwise no need to look 425 * in tlb1 array. 426 */ 427 if (recal) 428 kvmppc_recalc_tlb1map_range(vcpu_e500); 429 else 430 kvmppc_set_tlb1map_range(vcpu, gtlbe); 431 } 432 433 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */ 434 if (tlbe_is_host_safe(vcpu, gtlbe)) { 435 u64 eaddr = get_tlb_eaddr(gtlbe); 436 u64 raddr = get_tlb_raddr(gtlbe); 437 438 if (tlbsel == 0) { 439 gtlbe->mas1 &= ~MAS1_TSIZE(~0); 440 gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K); 441 } 442 443 /* Premap the faulting page */ 444 kvmppc_mmu_map(vcpu, eaddr, raddr, index_of(tlbsel, esel)); 445 } 446 447 kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS); 448 return EMULATE_DONE; 449 } 450 451 static int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu, 452 gva_t eaddr, unsigned int pid, int as) 453 { 454 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 455 int esel, tlbsel; 456 457 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 458 esel = kvmppc_e500_tlb_index(vcpu_e500, eaddr, tlbsel, pid, as); 459 if (esel >= 0) 460 return index_of(tlbsel, esel); 461 } 462 463 return -1; 464 } 465 466 /* 'linear_address' is actually an encoding of AS|PID|EADDR . */ 467 int kvmppc_core_vcpu_translate(struct kvm_vcpu *vcpu, 468 struct kvm_translation *tr) 469 { 470 int index; 471 gva_t eaddr; 472 u8 pid; 473 u8 as; 474 475 eaddr = tr->linear_address; 476 pid = (tr->linear_address >> 32) & 0xff; 477 as = (tr->linear_address >> 40) & 0x1; 478 479 index = kvmppc_e500_tlb_search(vcpu, eaddr, pid, as); 480 if (index < 0) { 481 tr->valid = 0; 482 return 0; 483 } 484 485 tr->physical_address = kvmppc_mmu_xlate(vcpu, index, eaddr); 486 /* XXX what does "writeable" and "usermode" even mean? */ 487 tr->valid = 1; 488 489 return 0; 490 } 491 492 493 int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) 494 { 495 unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS); 496 497 return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as); 498 } 499 500 int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) 501 { 502 unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS); 503 504 return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as); 505 } 506 507 void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu) 508 { 509 unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS); 510 511 kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.pc, as); 512 } 513 514 void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu) 515 { 516 unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS); 517 518 kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.fault_dear, as); 519 } 520 521 gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index, 522 gva_t eaddr) 523 { 524 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 525 struct kvm_book3e_206_tlb_entry *gtlbe; 526 u64 pgmask; 527 528 gtlbe = get_entry(vcpu_e500, tlbsel_of(index), esel_of(index)); 529 pgmask = get_tlb_bytes(gtlbe) - 1; 530 531 return get_tlb_raddr(gtlbe) | (eaddr & pgmask); 532 } 533 534 void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) 535 { 536 } 537 538 /*****************************************/ 539 540 static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500) 541 { 542 int i; 543 544 kvmppc_core_flush_tlb(&vcpu_e500->vcpu); 545 kfree(vcpu_e500->g2h_tlb1_map); 546 kfree(vcpu_e500->gtlb_priv[0]); 547 kfree(vcpu_e500->gtlb_priv[1]); 548 549 if (vcpu_e500->shared_tlb_pages) { 550 vfree((void *)(round_down((uintptr_t)vcpu_e500->gtlb_arch, 551 PAGE_SIZE))); 552 553 for (i = 0; i < vcpu_e500->num_shared_tlb_pages; i++) { 554 set_page_dirty_lock(vcpu_e500->shared_tlb_pages[i]); 555 put_page(vcpu_e500->shared_tlb_pages[i]); 556 } 557 558 vcpu_e500->num_shared_tlb_pages = 0; 559 560 kfree(vcpu_e500->shared_tlb_pages); 561 vcpu_e500->shared_tlb_pages = NULL; 562 } else { 563 kfree(vcpu_e500->gtlb_arch); 564 } 565 566 vcpu_e500->gtlb_arch = NULL; 567 } 568 569 void kvmppc_get_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 570 { 571 sregs->u.e.mas0 = vcpu->arch.shared->mas0; 572 sregs->u.e.mas1 = vcpu->arch.shared->mas1; 573 sregs->u.e.mas2 = vcpu->arch.shared->mas2; 574 sregs->u.e.mas7_3 = vcpu->arch.shared->mas7_3; 575 sregs->u.e.mas4 = vcpu->arch.shared->mas4; 576 sregs->u.e.mas6 = vcpu->arch.shared->mas6; 577 578 sregs->u.e.mmucfg = vcpu->arch.mmucfg; 579 sregs->u.e.tlbcfg[0] = vcpu->arch.tlbcfg[0]; 580 sregs->u.e.tlbcfg[1] = vcpu->arch.tlbcfg[1]; 581 sregs->u.e.tlbcfg[2] = 0; 582 sregs->u.e.tlbcfg[3] = 0; 583 } 584 585 int kvmppc_set_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 586 { 587 if (sregs->u.e.features & KVM_SREGS_E_ARCH206_MMU) { 588 vcpu->arch.shared->mas0 = sregs->u.e.mas0; 589 vcpu->arch.shared->mas1 = sregs->u.e.mas1; 590 vcpu->arch.shared->mas2 = sregs->u.e.mas2; 591 vcpu->arch.shared->mas7_3 = sregs->u.e.mas7_3; 592 vcpu->arch.shared->mas4 = sregs->u.e.mas4; 593 vcpu->arch.shared->mas6 = sregs->u.e.mas6; 594 } 595 596 return 0; 597 } 598 599 int kvmppc_get_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id, 600 union kvmppc_one_reg *val) 601 { 602 int r = 0; 603 long int i; 604 605 switch (id) { 606 case KVM_REG_PPC_MAS0: 607 *val = get_reg_val(id, vcpu->arch.shared->mas0); 608 break; 609 case KVM_REG_PPC_MAS1: 610 *val = get_reg_val(id, vcpu->arch.shared->mas1); 611 break; 612 case KVM_REG_PPC_MAS2: 613 *val = get_reg_val(id, vcpu->arch.shared->mas2); 614 break; 615 case KVM_REG_PPC_MAS7_3: 616 *val = get_reg_val(id, vcpu->arch.shared->mas7_3); 617 break; 618 case KVM_REG_PPC_MAS4: 619 *val = get_reg_val(id, vcpu->arch.shared->mas4); 620 break; 621 case KVM_REG_PPC_MAS6: 622 *val = get_reg_val(id, vcpu->arch.shared->mas6); 623 break; 624 case KVM_REG_PPC_MMUCFG: 625 *val = get_reg_val(id, vcpu->arch.mmucfg); 626 break; 627 case KVM_REG_PPC_EPTCFG: 628 *val = get_reg_val(id, vcpu->arch.eptcfg); 629 break; 630 case KVM_REG_PPC_TLB0CFG: 631 case KVM_REG_PPC_TLB1CFG: 632 case KVM_REG_PPC_TLB2CFG: 633 case KVM_REG_PPC_TLB3CFG: 634 i = id - KVM_REG_PPC_TLB0CFG; 635 *val = get_reg_val(id, vcpu->arch.tlbcfg[i]); 636 break; 637 case KVM_REG_PPC_TLB0PS: 638 case KVM_REG_PPC_TLB1PS: 639 case KVM_REG_PPC_TLB2PS: 640 case KVM_REG_PPC_TLB3PS: 641 i = id - KVM_REG_PPC_TLB0PS; 642 *val = get_reg_val(id, vcpu->arch.tlbps[i]); 643 break; 644 default: 645 r = -EINVAL; 646 break; 647 } 648 649 return r; 650 } 651 652 int kvmppc_set_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id, 653 union kvmppc_one_reg *val) 654 { 655 int r = 0; 656 long int i; 657 658 switch (id) { 659 case KVM_REG_PPC_MAS0: 660 vcpu->arch.shared->mas0 = set_reg_val(id, *val); 661 break; 662 case KVM_REG_PPC_MAS1: 663 vcpu->arch.shared->mas1 = set_reg_val(id, *val); 664 break; 665 case KVM_REG_PPC_MAS2: 666 vcpu->arch.shared->mas2 = set_reg_val(id, *val); 667 break; 668 case KVM_REG_PPC_MAS7_3: 669 vcpu->arch.shared->mas7_3 = set_reg_val(id, *val); 670 break; 671 case KVM_REG_PPC_MAS4: 672 vcpu->arch.shared->mas4 = set_reg_val(id, *val); 673 break; 674 case KVM_REG_PPC_MAS6: 675 vcpu->arch.shared->mas6 = set_reg_val(id, *val); 676 break; 677 /* Only allow MMU registers to be set to the config supported by KVM */ 678 case KVM_REG_PPC_MMUCFG: { 679 u32 reg = set_reg_val(id, *val); 680 if (reg != vcpu->arch.mmucfg) 681 r = -EINVAL; 682 break; 683 } 684 case KVM_REG_PPC_EPTCFG: { 685 u32 reg = set_reg_val(id, *val); 686 if (reg != vcpu->arch.eptcfg) 687 r = -EINVAL; 688 break; 689 } 690 case KVM_REG_PPC_TLB0CFG: 691 case KVM_REG_PPC_TLB1CFG: 692 case KVM_REG_PPC_TLB2CFG: 693 case KVM_REG_PPC_TLB3CFG: { 694 /* MMU geometry (N_ENTRY/ASSOC) can be set only using SW_TLB */ 695 u32 reg = set_reg_val(id, *val); 696 i = id - KVM_REG_PPC_TLB0CFG; 697 if (reg != vcpu->arch.tlbcfg[i]) 698 r = -EINVAL; 699 break; 700 } 701 case KVM_REG_PPC_TLB0PS: 702 case KVM_REG_PPC_TLB1PS: 703 case KVM_REG_PPC_TLB2PS: 704 case KVM_REG_PPC_TLB3PS: { 705 u32 reg = set_reg_val(id, *val); 706 i = id - KVM_REG_PPC_TLB0PS; 707 if (reg != vcpu->arch.tlbps[i]) 708 r = -EINVAL; 709 break; 710 } 711 default: 712 r = -EINVAL; 713 break; 714 } 715 716 return r; 717 } 718 719 static int vcpu_mmu_geometry_update(struct kvm_vcpu *vcpu, 720 struct kvm_book3e_206_tlb_params *params) 721 { 722 vcpu->arch.tlbcfg[0] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 723 if (params->tlb_sizes[0] <= 2048) 724 vcpu->arch.tlbcfg[0] |= params->tlb_sizes[0]; 725 vcpu->arch.tlbcfg[0] |= params->tlb_ways[0] << TLBnCFG_ASSOC_SHIFT; 726 727 vcpu->arch.tlbcfg[1] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 728 vcpu->arch.tlbcfg[1] |= params->tlb_sizes[1]; 729 vcpu->arch.tlbcfg[1] |= params->tlb_ways[1] << TLBnCFG_ASSOC_SHIFT; 730 return 0; 731 } 732 733 int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu, 734 struct kvm_config_tlb *cfg) 735 { 736 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 737 struct kvm_book3e_206_tlb_params params; 738 char *virt; 739 struct page **pages; 740 struct tlbe_priv *privs[2] = {}; 741 u64 *g2h_bitmap = NULL; 742 size_t array_len; 743 u32 sets; 744 int num_pages, ret, i; 745 746 if (cfg->mmu_type != KVM_MMU_FSL_BOOKE_NOHV) 747 return -EINVAL; 748 749 if (copy_from_user(¶ms, (void __user *)(uintptr_t)cfg->params, 750 sizeof(params))) 751 return -EFAULT; 752 753 if (params.tlb_sizes[1] > 64) 754 return -EINVAL; 755 if (params.tlb_ways[1] != params.tlb_sizes[1]) 756 return -EINVAL; 757 if (params.tlb_sizes[2] != 0 || params.tlb_sizes[3] != 0) 758 return -EINVAL; 759 if (params.tlb_ways[2] != 0 || params.tlb_ways[3] != 0) 760 return -EINVAL; 761 762 if (!is_power_of_2(params.tlb_ways[0])) 763 return -EINVAL; 764 765 sets = params.tlb_sizes[0] >> ilog2(params.tlb_ways[0]); 766 if (!is_power_of_2(sets)) 767 return -EINVAL; 768 769 array_len = params.tlb_sizes[0] + params.tlb_sizes[1]; 770 array_len *= sizeof(struct kvm_book3e_206_tlb_entry); 771 772 if (cfg->array_len < array_len) 773 return -EINVAL; 774 775 num_pages = DIV_ROUND_UP(cfg->array + array_len - 1, PAGE_SIZE) - 776 cfg->array / PAGE_SIZE; 777 pages = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL); 778 if (!pages) 779 return -ENOMEM; 780 781 ret = get_user_pages_fast(cfg->array, num_pages, 1, pages); 782 if (ret < 0) 783 goto err_pages; 784 785 if (ret != num_pages) { 786 num_pages = ret; 787 ret = -EFAULT; 788 goto err_put_page; 789 } 790 791 virt = vmap(pages, num_pages, VM_MAP, PAGE_KERNEL); 792 if (!virt) { 793 ret = -ENOMEM; 794 goto err_put_page; 795 } 796 797 privs[0] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[0], 798 GFP_KERNEL); 799 privs[1] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[1], 800 GFP_KERNEL); 801 802 if (!privs[0] || !privs[1]) { 803 ret = -ENOMEM; 804 goto err_privs; 805 } 806 807 g2h_bitmap = kzalloc(sizeof(u64) * params.tlb_sizes[1], 808 GFP_KERNEL); 809 if (!g2h_bitmap) { 810 ret = -ENOMEM; 811 goto err_privs; 812 } 813 814 free_gtlb(vcpu_e500); 815 816 vcpu_e500->gtlb_priv[0] = privs[0]; 817 vcpu_e500->gtlb_priv[1] = privs[1]; 818 vcpu_e500->g2h_tlb1_map = g2h_bitmap; 819 820 vcpu_e500->gtlb_arch = (struct kvm_book3e_206_tlb_entry *) 821 (virt + (cfg->array & (PAGE_SIZE - 1))); 822 823 vcpu_e500->gtlb_params[0].entries = params.tlb_sizes[0]; 824 vcpu_e500->gtlb_params[1].entries = params.tlb_sizes[1]; 825 826 vcpu_e500->gtlb_offset[0] = 0; 827 vcpu_e500->gtlb_offset[1] = params.tlb_sizes[0]; 828 829 /* Update vcpu's MMU geometry based on SW_TLB input */ 830 vcpu_mmu_geometry_update(vcpu, ¶ms); 831 832 vcpu_e500->shared_tlb_pages = pages; 833 vcpu_e500->num_shared_tlb_pages = num_pages; 834 835 vcpu_e500->gtlb_params[0].ways = params.tlb_ways[0]; 836 vcpu_e500->gtlb_params[0].sets = sets; 837 838 vcpu_e500->gtlb_params[1].ways = params.tlb_sizes[1]; 839 vcpu_e500->gtlb_params[1].sets = 1; 840 841 kvmppc_recalc_tlb1map_range(vcpu_e500); 842 return 0; 843 844 err_privs: 845 kfree(privs[0]); 846 kfree(privs[1]); 847 848 err_put_page: 849 for (i = 0; i < num_pages; i++) 850 put_page(pages[i]); 851 852 err_pages: 853 kfree(pages); 854 return ret; 855 } 856 857 int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu, 858 struct kvm_dirty_tlb *dirty) 859 { 860 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 861 kvmppc_recalc_tlb1map_range(vcpu_e500); 862 kvmppc_core_flush_tlb(vcpu); 863 return 0; 864 } 865 866 /* Vcpu's MMU default configuration */ 867 static int vcpu_mmu_init(struct kvm_vcpu *vcpu, 868 struct kvmppc_e500_tlb_params *params) 869 { 870 /* Initialize RASIZE, PIDSIZE, NTLBS and MAVN fields with host values*/ 871 vcpu->arch.mmucfg = mfspr(SPRN_MMUCFG) & ~MMUCFG_LPIDSIZE; 872 873 /* Initialize TLBnCFG fields with host values and SW_TLB geometry*/ 874 vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) & 875 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 876 vcpu->arch.tlbcfg[0] |= params[0].entries; 877 vcpu->arch.tlbcfg[0] |= params[0].ways << TLBnCFG_ASSOC_SHIFT; 878 879 vcpu->arch.tlbcfg[1] = mfspr(SPRN_TLB1CFG) & 880 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 881 vcpu->arch.tlbcfg[1] |= params[1].entries; 882 vcpu->arch.tlbcfg[1] |= params[1].ways << TLBnCFG_ASSOC_SHIFT; 883 884 if (has_feature(vcpu, VCPU_FTR_MMU_V2)) { 885 vcpu->arch.tlbps[0] = mfspr(SPRN_TLB0PS); 886 vcpu->arch.tlbps[1] = mfspr(SPRN_TLB1PS); 887 888 vcpu->arch.mmucfg &= ~MMUCFG_LRAT; 889 890 /* Guest mmu emulation currently doesn't handle E.PT */ 891 vcpu->arch.eptcfg = 0; 892 vcpu->arch.tlbcfg[0] &= ~TLBnCFG_PT; 893 vcpu->arch.tlbcfg[1] &= ~TLBnCFG_IND; 894 } 895 896 return 0; 897 } 898 899 int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500) 900 { 901 struct kvm_vcpu *vcpu = &vcpu_e500->vcpu; 902 int entry_size = sizeof(struct kvm_book3e_206_tlb_entry); 903 int entries = KVM_E500_TLB0_SIZE + KVM_E500_TLB1_SIZE; 904 905 if (e500_mmu_host_init(vcpu_e500)) 906 goto err; 907 908 vcpu_e500->gtlb_params[0].entries = KVM_E500_TLB0_SIZE; 909 vcpu_e500->gtlb_params[1].entries = KVM_E500_TLB1_SIZE; 910 911 vcpu_e500->gtlb_params[0].ways = KVM_E500_TLB0_WAY_NUM; 912 vcpu_e500->gtlb_params[0].sets = 913 KVM_E500_TLB0_SIZE / KVM_E500_TLB0_WAY_NUM; 914 915 vcpu_e500->gtlb_params[1].ways = KVM_E500_TLB1_SIZE; 916 vcpu_e500->gtlb_params[1].sets = 1; 917 918 vcpu_e500->gtlb_arch = kmalloc(entries * entry_size, GFP_KERNEL); 919 if (!vcpu_e500->gtlb_arch) 920 return -ENOMEM; 921 922 vcpu_e500->gtlb_offset[0] = 0; 923 vcpu_e500->gtlb_offset[1] = KVM_E500_TLB0_SIZE; 924 925 vcpu_e500->gtlb_priv[0] = kzalloc(sizeof(struct tlbe_ref) * 926 vcpu_e500->gtlb_params[0].entries, 927 GFP_KERNEL); 928 if (!vcpu_e500->gtlb_priv[0]) 929 goto err; 930 931 vcpu_e500->gtlb_priv[1] = kzalloc(sizeof(struct tlbe_ref) * 932 vcpu_e500->gtlb_params[1].entries, 933 GFP_KERNEL); 934 if (!vcpu_e500->gtlb_priv[1]) 935 goto err; 936 937 vcpu_e500->g2h_tlb1_map = kzalloc(sizeof(u64) * 938 vcpu_e500->gtlb_params[1].entries, 939 GFP_KERNEL); 940 if (!vcpu_e500->g2h_tlb1_map) 941 goto err; 942 943 vcpu_mmu_init(vcpu, vcpu_e500->gtlb_params); 944 945 kvmppc_recalc_tlb1map_range(vcpu_e500); 946 return 0; 947 948 err: 949 free_gtlb(vcpu_e500); 950 return -1; 951 } 952 953 void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500) 954 { 955 free_gtlb(vcpu_e500); 956 e500_mmu_host_uninit(vcpu_e500); 957 } 958