1 /* 2 * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved. 3 * 4 * Author: Yu Liu, yu.liu@freescale.com 5 * Scott Wood, scottwood@freescale.com 6 * Ashish Kalra, ashish.kalra@freescale.com 7 * Varun Sethi, varun.sethi@freescale.com 8 * Alexander Graf, agraf@suse.de 9 * 10 * Description: 11 * This file is based on arch/powerpc/kvm/44x_tlb.c, 12 * by Hollis Blanchard <hollisb@us.ibm.com>. 13 * 14 * This program is free software; you can redistribute it and/or modify 15 * it under the terms of the GNU General Public License, version 2, as 16 * published by the Free Software Foundation. 17 */ 18 19 #include <linux/kernel.h> 20 #include <linux/types.h> 21 #include <linux/slab.h> 22 #include <linux/string.h> 23 #include <linux/kvm.h> 24 #include <linux/kvm_host.h> 25 #include <linux/highmem.h> 26 #include <linux/log2.h> 27 #include <linux/uaccess.h> 28 #include <linux/sched.h> 29 #include <linux/rwsem.h> 30 #include <linux/vmalloc.h> 31 #include <linux/hugetlb.h> 32 #include <asm/kvm_ppc.h> 33 34 #include "e500.h" 35 #include "trace.h" 36 #include "timing.h" 37 #include "e500_mmu_host.h" 38 39 static inline unsigned int gtlb0_get_next_victim( 40 struct kvmppc_vcpu_e500 *vcpu_e500) 41 { 42 unsigned int victim; 43 44 victim = vcpu_e500->gtlb_nv[0]++; 45 if (unlikely(vcpu_e500->gtlb_nv[0] >= vcpu_e500->gtlb_params[0].ways)) 46 vcpu_e500->gtlb_nv[0] = 0; 47 48 return victim; 49 } 50 51 static int tlb0_set_base(gva_t addr, int sets, int ways) 52 { 53 int set_base; 54 55 set_base = (addr >> PAGE_SHIFT) & (sets - 1); 56 set_base *= ways; 57 58 return set_base; 59 } 60 61 static int gtlb0_set_base(struct kvmppc_vcpu_e500 *vcpu_e500, gva_t addr) 62 { 63 return tlb0_set_base(addr, vcpu_e500->gtlb_params[0].sets, 64 vcpu_e500->gtlb_params[0].ways); 65 } 66 67 static unsigned int get_tlb_esel(struct kvm_vcpu *vcpu, int tlbsel) 68 { 69 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 70 int esel = get_tlb_esel_bit(vcpu); 71 72 if (tlbsel == 0) { 73 esel &= vcpu_e500->gtlb_params[0].ways - 1; 74 esel += gtlb0_set_base(vcpu_e500, vcpu->arch.shared->mas2); 75 } else { 76 esel &= vcpu_e500->gtlb_params[tlbsel].entries - 1; 77 } 78 79 return esel; 80 } 81 82 /* Search the guest TLB for a matching entry. */ 83 static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500, 84 gva_t eaddr, int tlbsel, unsigned int pid, int as) 85 { 86 int size = vcpu_e500->gtlb_params[tlbsel].entries; 87 unsigned int set_base, offset; 88 int i; 89 90 if (tlbsel == 0) { 91 set_base = gtlb0_set_base(vcpu_e500, eaddr); 92 size = vcpu_e500->gtlb_params[0].ways; 93 } else { 94 if (eaddr < vcpu_e500->tlb1_min_eaddr || 95 eaddr > vcpu_e500->tlb1_max_eaddr) 96 return -1; 97 set_base = 0; 98 } 99 100 offset = vcpu_e500->gtlb_offset[tlbsel]; 101 102 for (i = 0; i < size; i++) { 103 struct kvm_book3e_206_tlb_entry *tlbe = 104 &vcpu_e500->gtlb_arch[offset + set_base + i]; 105 unsigned int tid; 106 107 if (eaddr < get_tlb_eaddr(tlbe)) 108 continue; 109 110 if (eaddr > get_tlb_end(tlbe)) 111 continue; 112 113 tid = get_tlb_tid(tlbe); 114 if (tid && (tid != pid)) 115 continue; 116 117 if (!get_tlb_v(tlbe)) 118 continue; 119 120 if (get_tlb_ts(tlbe) != as && as != -1) 121 continue; 122 123 return set_base + i; 124 } 125 126 return -1; 127 } 128 129 static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu, 130 unsigned int eaddr, int as) 131 { 132 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 133 unsigned int victim, tsized; 134 int tlbsel; 135 136 /* since we only have two TLBs, only lower bit is used. */ 137 tlbsel = (vcpu->arch.shared->mas4 >> 28) & 0x1; 138 victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0; 139 tsized = (vcpu->arch.shared->mas4 >> 7) & 0x1f; 140 141 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim) 142 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 143 vcpu->arch.shared->mas1 = MAS1_VALID | (as ? MAS1_TS : 0) 144 | MAS1_TID(get_tlbmiss_tid(vcpu)) 145 | MAS1_TSIZE(tsized); 146 vcpu->arch.shared->mas2 = (eaddr & MAS2_EPN) 147 | (vcpu->arch.shared->mas4 & MAS2_ATTRIB_MASK); 148 vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3; 149 vcpu->arch.shared->mas6 = (vcpu->arch.shared->mas6 & MAS6_SPID1) 150 | (get_cur_pid(vcpu) << 16) 151 | (as ? MAS6_SAS : 0); 152 } 153 154 static void kvmppc_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500) 155 { 156 int size = vcpu_e500->gtlb_params[1].entries; 157 unsigned int offset; 158 gva_t eaddr; 159 int i; 160 161 vcpu_e500->tlb1_min_eaddr = ~0UL; 162 vcpu_e500->tlb1_max_eaddr = 0; 163 offset = vcpu_e500->gtlb_offset[1]; 164 165 for (i = 0; i < size; i++) { 166 struct kvm_book3e_206_tlb_entry *tlbe = 167 &vcpu_e500->gtlb_arch[offset + i]; 168 169 if (!get_tlb_v(tlbe)) 170 continue; 171 172 eaddr = get_tlb_eaddr(tlbe); 173 vcpu_e500->tlb1_min_eaddr = 174 min(vcpu_e500->tlb1_min_eaddr, eaddr); 175 176 eaddr = get_tlb_end(tlbe); 177 vcpu_e500->tlb1_max_eaddr = 178 max(vcpu_e500->tlb1_max_eaddr, eaddr); 179 } 180 } 181 182 static int kvmppc_need_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500, 183 struct kvm_book3e_206_tlb_entry *gtlbe) 184 { 185 unsigned long start, end, size; 186 187 size = get_tlb_bytes(gtlbe); 188 start = get_tlb_eaddr(gtlbe) & ~(size - 1); 189 end = start + size - 1; 190 191 return vcpu_e500->tlb1_min_eaddr == start || 192 vcpu_e500->tlb1_max_eaddr == end; 193 } 194 195 /* This function is supposed to be called for a adding a new valid tlb entry */ 196 static void kvmppc_set_tlb1map_range(struct kvm_vcpu *vcpu, 197 struct kvm_book3e_206_tlb_entry *gtlbe) 198 { 199 unsigned long start, end, size; 200 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 201 202 if (!get_tlb_v(gtlbe)) 203 return; 204 205 size = get_tlb_bytes(gtlbe); 206 start = get_tlb_eaddr(gtlbe) & ~(size - 1); 207 end = start + size - 1; 208 209 vcpu_e500->tlb1_min_eaddr = min(vcpu_e500->tlb1_min_eaddr, start); 210 vcpu_e500->tlb1_max_eaddr = max(vcpu_e500->tlb1_max_eaddr, end); 211 } 212 213 static inline int kvmppc_e500_gtlbe_invalidate( 214 struct kvmppc_vcpu_e500 *vcpu_e500, 215 int tlbsel, int esel) 216 { 217 struct kvm_book3e_206_tlb_entry *gtlbe = 218 get_entry(vcpu_e500, tlbsel, esel); 219 220 if (unlikely(get_tlb_iprot(gtlbe))) 221 return -1; 222 223 if (tlbsel == 1 && kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe)) 224 kvmppc_recalc_tlb1map_range(vcpu_e500); 225 226 gtlbe->mas1 = 0; 227 228 return 0; 229 } 230 231 int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value) 232 { 233 int esel; 234 235 if (value & MMUCSR0_TLB0FI) 236 for (esel = 0; esel < vcpu_e500->gtlb_params[0].entries; esel++) 237 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel); 238 if (value & MMUCSR0_TLB1FI) 239 for (esel = 0; esel < vcpu_e500->gtlb_params[1].entries; esel++) 240 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel); 241 242 /* Invalidate all vcpu id mappings */ 243 kvmppc_e500_tlbil_all(vcpu_e500); 244 245 return EMULATE_DONE; 246 } 247 248 int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, gva_t ea) 249 { 250 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 251 unsigned int ia; 252 int esel, tlbsel; 253 254 ia = (ea >> 2) & 0x1; 255 256 /* since we only have two TLBs, only lower bit is used. */ 257 tlbsel = (ea >> 3) & 0x1; 258 259 if (ia) { 260 /* invalidate all entries */ 261 for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries; 262 esel++) 263 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 264 } else { 265 ea &= 0xfffff000; 266 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, 267 get_cur_pid(vcpu), -1); 268 if (esel >= 0) 269 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 270 } 271 272 /* Invalidate all vcpu id mappings */ 273 kvmppc_e500_tlbil_all(vcpu_e500); 274 275 return EMULATE_DONE; 276 } 277 278 static void tlbilx_all(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel, 279 int pid, int type) 280 { 281 struct kvm_book3e_206_tlb_entry *tlbe; 282 int tid, esel; 283 284 /* invalidate all entries */ 285 for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries; esel++) { 286 tlbe = get_entry(vcpu_e500, tlbsel, esel); 287 tid = get_tlb_tid(tlbe); 288 if (type == 0 || tid == pid) { 289 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel); 290 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 291 } 292 } 293 } 294 295 static void tlbilx_one(struct kvmppc_vcpu_e500 *vcpu_e500, int pid, 296 gva_t ea) 297 { 298 int tlbsel, esel; 299 300 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 301 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, -1); 302 if (esel >= 0) { 303 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel); 304 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 305 break; 306 } 307 } 308 } 309 310 int kvmppc_e500_emul_tlbilx(struct kvm_vcpu *vcpu, int type, gva_t ea) 311 { 312 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 313 int pid = get_cur_spid(vcpu); 314 315 if (type == 0 || type == 1) { 316 tlbilx_all(vcpu_e500, 0, pid, type); 317 tlbilx_all(vcpu_e500, 1, pid, type); 318 } else if (type == 3) { 319 tlbilx_one(vcpu_e500, pid, ea); 320 } 321 322 return EMULATE_DONE; 323 } 324 325 int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu) 326 { 327 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 328 int tlbsel, esel; 329 struct kvm_book3e_206_tlb_entry *gtlbe; 330 331 tlbsel = get_tlb_tlbsel(vcpu); 332 esel = get_tlb_esel(vcpu, tlbsel); 333 334 gtlbe = get_entry(vcpu_e500, tlbsel, esel); 335 vcpu->arch.shared->mas0 &= ~MAS0_NV(~0); 336 vcpu->arch.shared->mas0 |= MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 337 vcpu->arch.shared->mas1 = gtlbe->mas1; 338 vcpu->arch.shared->mas2 = gtlbe->mas2; 339 vcpu->arch.shared->mas7_3 = gtlbe->mas7_3; 340 341 return EMULATE_DONE; 342 } 343 344 int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, gva_t ea) 345 { 346 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 347 int as = !!get_cur_sas(vcpu); 348 unsigned int pid = get_cur_spid(vcpu); 349 int esel, tlbsel; 350 struct kvm_book3e_206_tlb_entry *gtlbe = NULL; 351 352 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 353 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as); 354 if (esel >= 0) { 355 gtlbe = get_entry(vcpu_e500, tlbsel, esel); 356 break; 357 } 358 } 359 360 if (gtlbe) { 361 esel &= vcpu_e500->gtlb_params[tlbsel].ways - 1; 362 363 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel) 364 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 365 vcpu->arch.shared->mas1 = gtlbe->mas1; 366 vcpu->arch.shared->mas2 = gtlbe->mas2; 367 vcpu->arch.shared->mas7_3 = gtlbe->mas7_3; 368 } else { 369 int victim; 370 371 /* since we only have two TLBs, only lower bit is used. */ 372 tlbsel = vcpu->arch.shared->mas4 >> 28 & 0x1; 373 victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0; 374 375 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) 376 | MAS0_ESEL(victim) 377 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); 378 vcpu->arch.shared->mas1 = 379 (vcpu->arch.shared->mas6 & MAS6_SPID0) 380 | (vcpu->arch.shared->mas6 & (MAS6_SAS ? MAS1_TS : 0)) 381 | (vcpu->arch.shared->mas4 & MAS4_TSIZED(~0)); 382 vcpu->arch.shared->mas2 &= MAS2_EPN; 383 vcpu->arch.shared->mas2 |= vcpu->arch.shared->mas4 & 384 MAS2_ATTRIB_MASK; 385 vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | 386 MAS3_U2 | MAS3_U3; 387 } 388 389 kvmppc_set_exit_type(vcpu, EMULATED_TLBSX_EXITS); 390 return EMULATE_DONE; 391 } 392 393 int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu) 394 { 395 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 396 struct kvm_book3e_206_tlb_entry *gtlbe; 397 int tlbsel, esel; 398 int recal = 0; 399 400 tlbsel = get_tlb_tlbsel(vcpu); 401 esel = get_tlb_esel(vcpu, tlbsel); 402 403 gtlbe = get_entry(vcpu_e500, tlbsel, esel); 404 405 if (get_tlb_v(gtlbe)) { 406 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel); 407 if ((tlbsel == 1) && 408 kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe)) 409 recal = 1; 410 } 411 412 gtlbe->mas1 = vcpu->arch.shared->mas1; 413 gtlbe->mas2 = vcpu->arch.shared->mas2; 414 if (!(vcpu->arch.shared->msr & MSR_CM)) 415 gtlbe->mas2 &= 0xffffffffUL; 416 gtlbe->mas7_3 = vcpu->arch.shared->mas7_3; 417 418 trace_kvm_booke206_gtlb_write(vcpu->arch.shared->mas0, gtlbe->mas1, 419 gtlbe->mas2, gtlbe->mas7_3); 420 421 if (tlbsel == 1) { 422 /* 423 * If a valid tlb1 entry is overwritten then recalculate the 424 * min/max TLB1 map address range otherwise no need to look 425 * in tlb1 array. 426 */ 427 if (recal) 428 kvmppc_recalc_tlb1map_range(vcpu_e500); 429 else 430 kvmppc_set_tlb1map_range(vcpu, gtlbe); 431 } 432 433 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */ 434 if (tlbe_is_host_safe(vcpu, gtlbe)) { 435 u64 eaddr = get_tlb_eaddr(gtlbe); 436 u64 raddr = get_tlb_raddr(gtlbe); 437 438 if (tlbsel == 0) { 439 gtlbe->mas1 &= ~MAS1_TSIZE(~0); 440 gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K); 441 } 442 443 /* Premap the faulting page */ 444 kvmppc_mmu_map(vcpu, eaddr, raddr, index_of(tlbsel, esel)); 445 } 446 447 kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS); 448 return EMULATE_DONE; 449 } 450 451 static int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu, 452 gva_t eaddr, unsigned int pid, int as) 453 { 454 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 455 int esel, tlbsel; 456 457 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 458 esel = kvmppc_e500_tlb_index(vcpu_e500, eaddr, tlbsel, pid, as); 459 if (esel >= 0) 460 return index_of(tlbsel, esel); 461 } 462 463 return -1; 464 } 465 466 /* 'linear_address' is actually an encoding of AS|PID|EADDR . */ 467 int kvmppc_core_vcpu_translate(struct kvm_vcpu *vcpu, 468 struct kvm_translation *tr) 469 { 470 int index; 471 gva_t eaddr; 472 u8 pid; 473 u8 as; 474 475 eaddr = tr->linear_address; 476 pid = (tr->linear_address >> 32) & 0xff; 477 as = (tr->linear_address >> 40) & 0x1; 478 479 index = kvmppc_e500_tlb_search(vcpu, eaddr, pid, as); 480 if (index < 0) { 481 tr->valid = 0; 482 return 0; 483 } 484 485 tr->physical_address = kvmppc_mmu_xlate(vcpu, index, eaddr); 486 /* XXX what does "writeable" and "usermode" even mean? */ 487 tr->valid = 1; 488 489 return 0; 490 } 491 492 493 int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) 494 { 495 unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS); 496 497 return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as); 498 } 499 500 int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) 501 { 502 unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS); 503 504 return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as); 505 } 506 507 void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu) 508 { 509 unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS); 510 511 kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.pc, as); 512 } 513 514 void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu) 515 { 516 unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS); 517 518 kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.fault_dear, as); 519 } 520 521 gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index, 522 gva_t eaddr) 523 { 524 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 525 struct kvm_book3e_206_tlb_entry *gtlbe; 526 u64 pgmask; 527 528 gtlbe = get_entry(vcpu_e500, tlbsel_of(index), esel_of(index)); 529 pgmask = get_tlb_bytes(gtlbe) - 1; 530 531 return get_tlb_raddr(gtlbe) | (eaddr & pgmask); 532 } 533 534 void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) 535 { 536 } 537 538 /*****************************************/ 539 540 static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500) 541 { 542 int i; 543 544 kvmppc_core_flush_tlb(&vcpu_e500->vcpu); 545 kfree(vcpu_e500->g2h_tlb1_map); 546 kfree(vcpu_e500->gtlb_priv[0]); 547 kfree(vcpu_e500->gtlb_priv[1]); 548 549 if (vcpu_e500->shared_tlb_pages) { 550 vfree((void *)(round_down((uintptr_t)vcpu_e500->gtlb_arch, 551 PAGE_SIZE))); 552 553 for (i = 0; i < vcpu_e500->num_shared_tlb_pages; i++) { 554 set_page_dirty_lock(vcpu_e500->shared_tlb_pages[i]); 555 put_page(vcpu_e500->shared_tlb_pages[i]); 556 } 557 558 vcpu_e500->num_shared_tlb_pages = 0; 559 560 kfree(vcpu_e500->shared_tlb_pages); 561 vcpu_e500->shared_tlb_pages = NULL; 562 } else { 563 kfree(vcpu_e500->gtlb_arch); 564 } 565 566 vcpu_e500->gtlb_arch = NULL; 567 } 568 569 void kvmppc_get_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 570 { 571 sregs->u.e.mas0 = vcpu->arch.shared->mas0; 572 sregs->u.e.mas1 = vcpu->arch.shared->mas1; 573 sregs->u.e.mas2 = vcpu->arch.shared->mas2; 574 sregs->u.e.mas7_3 = vcpu->arch.shared->mas7_3; 575 sregs->u.e.mas4 = vcpu->arch.shared->mas4; 576 sregs->u.e.mas6 = vcpu->arch.shared->mas6; 577 578 sregs->u.e.mmucfg = vcpu->arch.mmucfg; 579 sregs->u.e.tlbcfg[0] = vcpu->arch.tlbcfg[0]; 580 sregs->u.e.tlbcfg[1] = vcpu->arch.tlbcfg[1]; 581 sregs->u.e.tlbcfg[2] = 0; 582 sregs->u.e.tlbcfg[3] = 0; 583 } 584 585 int kvmppc_set_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 586 { 587 if (sregs->u.e.features & KVM_SREGS_E_ARCH206_MMU) { 588 vcpu->arch.shared->mas0 = sregs->u.e.mas0; 589 vcpu->arch.shared->mas1 = sregs->u.e.mas1; 590 vcpu->arch.shared->mas2 = sregs->u.e.mas2; 591 vcpu->arch.shared->mas7_3 = sregs->u.e.mas7_3; 592 vcpu->arch.shared->mas4 = sregs->u.e.mas4; 593 vcpu->arch.shared->mas6 = sregs->u.e.mas6; 594 } 595 596 return 0; 597 } 598 599 int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu, 600 struct kvm_config_tlb *cfg) 601 { 602 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 603 struct kvm_book3e_206_tlb_params params; 604 char *virt; 605 struct page **pages; 606 struct tlbe_priv *privs[2] = {}; 607 u64 *g2h_bitmap = NULL; 608 size_t array_len; 609 u32 sets; 610 int num_pages, ret, i; 611 612 if (cfg->mmu_type != KVM_MMU_FSL_BOOKE_NOHV) 613 return -EINVAL; 614 615 if (copy_from_user(¶ms, (void __user *)(uintptr_t)cfg->params, 616 sizeof(params))) 617 return -EFAULT; 618 619 if (params.tlb_sizes[1] > 64) 620 return -EINVAL; 621 if (params.tlb_ways[1] != params.tlb_sizes[1]) 622 return -EINVAL; 623 if (params.tlb_sizes[2] != 0 || params.tlb_sizes[3] != 0) 624 return -EINVAL; 625 if (params.tlb_ways[2] != 0 || params.tlb_ways[3] != 0) 626 return -EINVAL; 627 628 if (!is_power_of_2(params.tlb_ways[0])) 629 return -EINVAL; 630 631 sets = params.tlb_sizes[0] >> ilog2(params.tlb_ways[0]); 632 if (!is_power_of_2(sets)) 633 return -EINVAL; 634 635 array_len = params.tlb_sizes[0] + params.tlb_sizes[1]; 636 array_len *= sizeof(struct kvm_book3e_206_tlb_entry); 637 638 if (cfg->array_len < array_len) 639 return -EINVAL; 640 641 num_pages = DIV_ROUND_UP(cfg->array + array_len - 1, PAGE_SIZE) - 642 cfg->array / PAGE_SIZE; 643 pages = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL); 644 if (!pages) 645 return -ENOMEM; 646 647 ret = get_user_pages_fast(cfg->array, num_pages, 1, pages); 648 if (ret < 0) 649 goto err_pages; 650 651 if (ret != num_pages) { 652 num_pages = ret; 653 ret = -EFAULT; 654 goto err_put_page; 655 } 656 657 virt = vmap(pages, num_pages, VM_MAP, PAGE_KERNEL); 658 if (!virt) { 659 ret = -ENOMEM; 660 goto err_put_page; 661 } 662 663 privs[0] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[0], 664 GFP_KERNEL); 665 privs[1] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[1], 666 GFP_KERNEL); 667 668 if (!privs[0] || !privs[1]) { 669 ret = -ENOMEM; 670 goto err_privs; 671 } 672 673 g2h_bitmap = kzalloc(sizeof(u64) * params.tlb_sizes[1], 674 GFP_KERNEL); 675 if (!g2h_bitmap) { 676 ret = -ENOMEM; 677 goto err_privs; 678 } 679 680 free_gtlb(vcpu_e500); 681 682 vcpu_e500->gtlb_priv[0] = privs[0]; 683 vcpu_e500->gtlb_priv[1] = privs[1]; 684 vcpu_e500->g2h_tlb1_map = g2h_bitmap; 685 686 vcpu_e500->gtlb_arch = (struct kvm_book3e_206_tlb_entry *) 687 (virt + (cfg->array & (PAGE_SIZE - 1))); 688 689 vcpu_e500->gtlb_params[0].entries = params.tlb_sizes[0]; 690 vcpu_e500->gtlb_params[1].entries = params.tlb_sizes[1]; 691 692 vcpu_e500->gtlb_offset[0] = 0; 693 vcpu_e500->gtlb_offset[1] = params.tlb_sizes[0]; 694 695 vcpu->arch.mmucfg = mfspr(SPRN_MMUCFG) & ~MMUCFG_LPIDSIZE; 696 697 vcpu->arch.tlbcfg[0] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 698 if (params.tlb_sizes[0] <= 2048) 699 vcpu->arch.tlbcfg[0] |= params.tlb_sizes[0]; 700 vcpu->arch.tlbcfg[0] |= params.tlb_ways[0] << TLBnCFG_ASSOC_SHIFT; 701 702 vcpu->arch.tlbcfg[1] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 703 vcpu->arch.tlbcfg[1] |= params.tlb_sizes[1]; 704 vcpu->arch.tlbcfg[1] |= params.tlb_ways[1] << TLBnCFG_ASSOC_SHIFT; 705 706 vcpu_e500->shared_tlb_pages = pages; 707 vcpu_e500->num_shared_tlb_pages = num_pages; 708 709 vcpu_e500->gtlb_params[0].ways = params.tlb_ways[0]; 710 vcpu_e500->gtlb_params[0].sets = sets; 711 712 vcpu_e500->gtlb_params[1].ways = params.tlb_sizes[1]; 713 vcpu_e500->gtlb_params[1].sets = 1; 714 715 kvmppc_recalc_tlb1map_range(vcpu_e500); 716 return 0; 717 718 err_privs: 719 kfree(privs[0]); 720 kfree(privs[1]); 721 722 err_put_page: 723 for (i = 0; i < num_pages; i++) 724 put_page(pages[i]); 725 726 err_pages: 727 kfree(pages); 728 return ret; 729 } 730 731 int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu, 732 struct kvm_dirty_tlb *dirty) 733 { 734 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 735 kvmppc_recalc_tlb1map_range(vcpu_e500); 736 kvmppc_core_flush_tlb(vcpu); 737 return 0; 738 } 739 740 int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500) 741 { 742 struct kvm_vcpu *vcpu = &vcpu_e500->vcpu; 743 int entry_size = sizeof(struct kvm_book3e_206_tlb_entry); 744 int entries = KVM_E500_TLB0_SIZE + KVM_E500_TLB1_SIZE; 745 746 if (e500_mmu_host_init(vcpu_e500)) 747 goto err; 748 749 vcpu_e500->gtlb_params[0].entries = KVM_E500_TLB0_SIZE; 750 vcpu_e500->gtlb_params[1].entries = KVM_E500_TLB1_SIZE; 751 752 vcpu_e500->gtlb_params[0].ways = KVM_E500_TLB0_WAY_NUM; 753 vcpu_e500->gtlb_params[0].sets = 754 KVM_E500_TLB0_SIZE / KVM_E500_TLB0_WAY_NUM; 755 756 vcpu_e500->gtlb_params[1].ways = KVM_E500_TLB1_SIZE; 757 vcpu_e500->gtlb_params[1].sets = 1; 758 759 vcpu_e500->gtlb_arch = kmalloc(entries * entry_size, GFP_KERNEL); 760 if (!vcpu_e500->gtlb_arch) 761 return -ENOMEM; 762 763 vcpu_e500->gtlb_offset[0] = 0; 764 vcpu_e500->gtlb_offset[1] = KVM_E500_TLB0_SIZE; 765 766 vcpu_e500->gtlb_priv[0] = kzalloc(sizeof(struct tlbe_ref) * 767 vcpu_e500->gtlb_params[0].entries, 768 GFP_KERNEL); 769 if (!vcpu_e500->gtlb_priv[0]) 770 goto err; 771 772 vcpu_e500->gtlb_priv[1] = kzalloc(sizeof(struct tlbe_ref) * 773 vcpu_e500->gtlb_params[1].entries, 774 GFP_KERNEL); 775 if (!vcpu_e500->gtlb_priv[1]) 776 goto err; 777 778 vcpu_e500->g2h_tlb1_map = kzalloc(sizeof(u64) * 779 vcpu_e500->gtlb_params[1].entries, 780 GFP_KERNEL); 781 if (!vcpu_e500->g2h_tlb1_map) 782 goto err; 783 784 /* Init TLB configuration register */ 785 vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) & 786 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 787 vcpu->arch.tlbcfg[0] |= vcpu_e500->gtlb_params[0].entries; 788 vcpu->arch.tlbcfg[0] |= 789 vcpu_e500->gtlb_params[0].ways << TLBnCFG_ASSOC_SHIFT; 790 791 vcpu->arch.tlbcfg[1] = mfspr(SPRN_TLB1CFG) & 792 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC); 793 vcpu->arch.tlbcfg[1] |= vcpu_e500->gtlb_params[1].entries; 794 vcpu->arch.tlbcfg[1] |= 795 vcpu_e500->gtlb_params[1].ways << TLBnCFG_ASSOC_SHIFT; 796 797 kvmppc_recalc_tlb1map_range(vcpu_e500); 798 return 0; 799 800 err: 801 free_gtlb(vcpu_e500); 802 return -1; 803 } 804 805 void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500) 806 { 807 free_gtlb(vcpu_e500); 808 e500_mmu_host_uninit(vcpu_e500); 809 } 810