1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TLB flush routines for radix kernels. 4 * 5 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation. 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/hugetlb.h> 10 #include <linux/memblock.h> 11 #include <linux/mmu_context.h> 12 #include <linux/sched/mm.h> 13 14 #include <asm/ppc-opcode.h> 15 #include <asm/tlb.h> 16 #include <asm/tlbflush.h> 17 #include <asm/trace.h> 18 #include <asm/cputhreads.h> 19 #include <asm/plpar_wrappers.h> 20 21 #define RIC_FLUSH_TLB 0 22 #define RIC_FLUSH_PWC 1 23 #define RIC_FLUSH_ALL 2 24 25 /* 26 * tlbiel instruction for radix, set invalidation 27 * i.e., r=1 and is=01 or is=10 or is=11 28 */ 29 static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, 30 unsigned int pid, 31 unsigned int ric, unsigned int prs) 32 { 33 unsigned long rb; 34 unsigned long rs; 35 36 rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); 37 rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); 38 39 asm volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 40 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 41 : "memory"); 42 } 43 44 static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) 45 { 46 unsigned int set; 47 48 asm volatile("ptesync": : :"memory"); 49 50 /* 51 * Flush the first set of the TLB, and the entire Page Walk Cache 52 * and partition table entries. Then flush the remaining sets of the 53 * TLB. 54 */ 55 56 if (early_cpu_has_feature(CPU_FTR_HVMODE)) { 57 /* MSR[HV] should flush partition scope translations first. */ 58 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 59 60 if (!early_cpu_has_feature(CPU_FTR_ARCH_31)) { 61 for (set = 1; set < num_sets; set++) 62 tlbiel_radix_set_isa300(set, is, 0, 63 RIC_FLUSH_TLB, 0); 64 } 65 } 66 67 /* Flush process scoped entries. */ 68 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 69 70 if (!early_cpu_has_feature(CPU_FTR_ARCH_31)) { 71 for (set = 1; set < num_sets; set++) 72 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 73 } 74 75 ppc_after_tlbiel_barrier(); 76 } 77 78 void radix__tlbiel_all(unsigned int action) 79 { 80 unsigned int is; 81 82 switch (action) { 83 case TLB_INVAL_SCOPE_GLOBAL: 84 is = 3; 85 break; 86 case TLB_INVAL_SCOPE_LPID: 87 is = 2; 88 break; 89 default: 90 BUG(); 91 } 92 93 if (early_cpu_has_feature(CPU_FTR_ARCH_300)) 94 tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is); 95 else 96 WARN(1, "%s called on pre-POWER9 CPU\n", __func__); 97 98 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); 99 } 100 101 static __always_inline void __tlbiel_pid(unsigned long pid, int set, 102 unsigned long ric) 103 { 104 unsigned long rb,rs,prs,r; 105 106 rb = PPC_BIT(53); /* IS = 1 */ 107 rb |= set << PPC_BITLSHIFT(51); 108 rs = ((unsigned long)pid) << PPC_BITLSHIFT(31); 109 prs = 1; /* process scoped */ 110 r = 1; /* radix format */ 111 112 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 113 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 114 trace_tlbie(0, 1, rb, rs, ric, prs, r); 115 } 116 117 static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric) 118 { 119 unsigned long rb,rs,prs,r; 120 121 rb = PPC_BIT(53); /* IS = 1 */ 122 rs = pid << PPC_BITLSHIFT(31); 123 prs = 1; /* process scoped */ 124 r = 1; /* radix format */ 125 126 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 127 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 128 trace_tlbie(0, 0, rb, rs, ric, prs, r); 129 } 130 131 static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric) 132 { 133 unsigned long rb,rs,prs,r; 134 135 rb = PPC_BIT(52); /* IS = 2 */ 136 rs = lpid; 137 prs = 0; /* partition scoped */ 138 r = 1; /* radix format */ 139 140 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 141 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 142 trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 143 } 144 145 static __always_inline void __tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 146 { 147 unsigned long rb,rs,prs,r; 148 149 rb = PPC_BIT(52); /* IS = 2 */ 150 rs = lpid; 151 prs = 1; /* process scoped */ 152 r = 1; /* radix format */ 153 154 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 155 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 156 trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 157 } 158 159 static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid, 160 unsigned long ap, unsigned long ric) 161 { 162 unsigned long rb,rs,prs,r; 163 164 rb = va & ~(PPC_BITMASK(52, 63)); 165 rb |= ap << PPC_BITLSHIFT(58); 166 rs = pid << PPC_BITLSHIFT(31); 167 prs = 1; /* process scoped */ 168 r = 1; /* radix format */ 169 170 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 171 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 172 trace_tlbie(0, 1, rb, rs, ric, prs, r); 173 } 174 175 static __always_inline void __tlbie_va(unsigned long va, unsigned long pid, 176 unsigned long ap, unsigned long ric) 177 { 178 unsigned long rb,rs,prs,r; 179 180 rb = va & ~(PPC_BITMASK(52, 63)); 181 rb |= ap << PPC_BITLSHIFT(58); 182 rs = pid << PPC_BITLSHIFT(31); 183 prs = 1; /* process scoped */ 184 r = 1; /* radix format */ 185 186 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 187 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 188 trace_tlbie(0, 0, rb, rs, ric, prs, r); 189 } 190 191 static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, 192 unsigned long ap, unsigned long ric) 193 { 194 unsigned long rb,rs,prs,r; 195 196 rb = va & ~(PPC_BITMASK(52, 63)); 197 rb |= ap << PPC_BITLSHIFT(58); 198 rs = lpid; 199 prs = 0; /* partition scoped */ 200 r = 1; /* radix format */ 201 202 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 203 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 204 trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 205 } 206 207 208 static inline void fixup_tlbie_va(unsigned long va, unsigned long pid, 209 unsigned long ap) 210 { 211 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 212 asm volatile("ptesync": : :"memory"); 213 __tlbie_va(va, 0, ap, RIC_FLUSH_TLB); 214 } 215 216 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 217 asm volatile("ptesync": : :"memory"); 218 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 219 } 220 } 221 222 static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid, 223 unsigned long ap) 224 { 225 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 226 asm volatile("ptesync": : :"memory"); 227 __tlbie_pid(0, RIC_FLUSH_TLB); 228 } 229 230 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 231 asm volatile("ptesync": : :"memory"); 232 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 233 } 234 } 235 236 static inline void fixup_tlbie_pid(unsigned long pid) 237 { 238 /* 239 * We can use any address for the invalidation, pick one which is 240 * probably unused as an optimisation. 241 */ 242 unsigned long va = ((1UL << 52) - 1); 243 244 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 245 asm volatile("ptesync": : :"memory"); 246 __tlbie_pid(0, RIC_FLUSH_TLB); 247 } 248 249 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 250 asm volatile("ptesync": : :"memory"); 251 __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 252 } 253 } 254 255 256 static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid, 257 unsigned long ap) 258 { 259 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 260 asm volatile("ptesync": : :"memory"); 261 __tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB); 262 } 263 264 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 265 asm volatile("ptesync": : :"memory"); 266 __tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB); 267 } 268 } 269 270 static inline void fixup_tlbie_lpid(unsigned long lpid) 271 { 272 /* 273 * We can use any address for the invalidation, pick one which is 274 * probably unused as an optimisation. 275 */ 276 unsigned long va = ((1UL << 52) - 1); 277 278 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 279 asm volatile("ptesync": : :"memory"); 280 __tlbie_lpid(0, RIC_FLUSH_TLB); 281 } 282 283 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 284 asm volatile("ptesync": : :"memory"); 285 __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 286 } 287 } 288 289 /* 290 * We use 128 set in radix mode and 256 set in hpt mode. 291 */ 292 static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric) 293 { 294 int set; 295 296 asm volatile("ptesync": : :"memory"); 297 298 /* 299 * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL, 300 * also flush the entire Page Walk Cache. 301 */ 302 __tlbiel_pid(pid, 0, ric); 303 304 /* For PWC, only one flush is needed */ 305 if (ric == RIC_FLUSH_PWC) { 306 ppc_after_tlbiel_barrier(); 307 return; 308 } 309 310 if (!cpu_has_feature(CPU_FTR_ARCH_31)) { 311 /* For the remaining sets, just flush the TLB */ 312 for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++) 313 __tlbiel_pid(pid, set, RIC_FLUSH_TLB); 314 } 315 316 ppc_after_tlbiel_barrier(); 317 asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory"); 318 } 319 320 static inline void _tlbie_pid(unsigned long pid, unsigned long ric) 321 { 322 asm volatile("ptesync": : :"memory"); 323 324 /* 325 * Workaround the fact that the "ric" argument to __tlbie_pid 326 * must be a compile-time contraint to match the "i" constraint 327 * in the asm statement. 328 */ 329 switch (ric) { 330 case RIC_FLUSH_TLB: 331 __tlbie_pid(pid, RIC_FLUSH_TLB); 332 fixup_tlbie_pid(pid); 333 break; 334 case RIC_FLUSH_PWC: 335 __tlbie_pid(pid, RIC_FLUSH_PWC); 336 break; 337 case RIC_FLUSH_ALL: 338 default: 339 __tlbie_pid(pid, RIC_FLUSH_ALL); 340 fixup_tlbie_pid(pid); 341 } 342 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 343 } 344 345 struct tlbiel_pid { 346 unsigned long pid; 347 unsigned long ric; 348 }; 349 350 static void do_tlbiel_pid(void *info) 351 { 352 struct tlbiel_pid *t = info; 353 354 if (t->ric == RIC_FLUSH_TLB) 355 _tlbiel_pid(t->pid, RIC_FLUSH_TLB); 356 else if (t->ric == RIC_FLUSH_PWC) 357 _tlbiel_pid(t->pid, RIC_FLUSH_PWC); 358 else 359 _tlbiel_pid(t->pid, RIC_FLUSH_ALL); 360 } 361 362 static inline void _tlbiel_pid_multicast(struct mm_struct *mm, 363 unsigned long pid, unsigned long ric) 364 { 365 struct cpumask *cpus = mm_cpumask(mm); 366 struct tlbiel_pid t = { .pid = pid, .ric = ric }; 367 368 on_each_cpu_mask(cpus, do_tlbiel_pid, &t, 1); 369 /* 370 * Always want the CPU translations to be invalidated with tlbiel in 371 * these paths, so while coprocessors must use tlbie, we can not 372 * optimise away the tlbiel component. 373 */ 374 if (atomic_read(&mm->context.copros) > 0) 375 _tlbie_pid(pid, RIC_FLUSH_ALL); 376 } 377 378 static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) 379 { 380 asm volatile("ptesync": : :"memory"); 381 382 /* 383 * Workaround the fact that the "ric" argument to __tlbie_pid 384 * must be a compile-time contraint to match the "i" constraint 385 * in the asm statement. 386 */ 387 switch (ric) { 388 case RIC_FLUSH_TLB: 389 __tlbie_lpid(lpid, RIC_FLUSH_TLB); 390 fixup_tlbie_lpid(lpid); 391 break; 392 case RIC_FLUSH_PWC: 393 __tlbie_lpid(lpid, RIC_FLUSH_PWC); 394 break; 395 case RIC_FLUSH_ALL: 396 default: 397 __tlbie_lpid(lpid, RIC_FLUSH_ALL); 398 fixup_tlbie_lpid(lpid); 399 } 400 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 401 } 402 403 static __always_inline void _tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 404 { 405 /* 406 * Workaround the fact that the "ric" argument to __tlbie_pid 407 * must be a compile-time contraint to match the "i" constraint 408 * in the asm statement. 409 */ 410 switch (ric) { 411 case RIC_FLUSH_TLB: 412 __tlbie_lpid_guest(lpid, RIC_FLUSH_TLB); 413 break; 414 case RIC_FLUSH_PWC: 415 __tlbie_lpid_guest(lpid, RIC_FLUSH_PWC); 416 break; 417 case RIC_FLUSH_ALL: 418 default: 419 __tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 420 } 421 fixup_tlbie_lpid(lpid); 422 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 423 } 424 425 static inline void __tlbiel_va_range(unsigned long start, unsigned long end, 426 unsigned long pid, unsigned long page_size, 427 unsigned long psize) 428 { 429 unsigned long addr; 430 unsigned long ap = mmu_get_ap(psize); 431 432 for (addr = start; addr < end; addr += page_size) 433 __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB); 434 } 435 436 static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid, 437 unsigned long psize, unsigned long ric) 438 { 439 unsigned long ap = mmu_get_ap(psize); 440 441 asm volatile("ptesync": : :"memory"); 442 __tlbiel_va(va, pid, ap, ric); 443 ppc_after_tlbiel_barrier(); 444 } 445 446 static inline void _tlbiel_va_range(unsigned long start, unsigned long end, 447 unsigned long pid, unsigned long page_size, 448 unsigned long psize, bool also_pwc) 449 { 450 asm volatile("ptesync": : :"memory"); 451 if (also_pwc) 452 __tlbiel_pid(pid, 0, RIC_FLUSH_PWC); 453 __tlbiel_va_range(start, end, pid, page_size, psize); 454 ppc_after_tlbiel_barrier(); 455 } 456 457 static inline void __tlbie_va_range(unsigned long start, unsigned long end, 458 unsigned long pid, unsigned long page_size, 459 unsigned long psize) 460 { 461 unsigned long addr; 462 unsigned long ap = mmu_get_ap(psize); 463 464 for (addr = start; addr < end; addr += page_size) 465 __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB); 466 467 fixup_tlbie_va_range(addr - page_size, pid, ap); 468 } 469 470 static __always_inline void _tlbie_va(unsigned long va, unsigned long pid, 471 unsigned long psize, unsigned long ric) 472 { 473 unsigned long ap = mmu_get_ap(psize); 474 475 asm volatile("ptesync": : :"memory"); 476 __tlbie_va(va, pid, ap, ric); 477 fixup_tlbie_va(va, pid, ap); 478 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 479 } 480 481 struct tlbiel_va { 482 unsigned long pid; 483 unsigned long va; 484 unsigned long psize; 485 unsigned long ric; 486 }; 487 488 static void do_tlbiel_va(void *info) 489 { 490 struct tlbiel_va *t = info; 491 492 if (t->ric == RIC_FLUSH_TLB) 493 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB); 494 else if (t->ric == RIC_FLUSH_PWC) 495 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC); 496 else 497 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL); 498 } 499 500 static inline void _tlbiel_va_multicast(struct mm_struct *mm, 501 unsigned long va, unsigned long pid, 502 unsigned long psize, unsigned long ric) 503 { 504 struct cpumask *cpus = mm_cpumask(mm); 505 struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric }; 506 on_each_cpu_mask(cpus, do_tlbiel_va, &t, 1); 507 if (atomic_read(&mm->context.copros) > 0) 508 _tlbie_va(va, pid, psize, RIC_FLUSH_TLB); 509 } 510 511 struct tlbiel_va_range { 512 unsigned long pid; 513 unsigned long start; 514 unsigned long end; 515 unsigned long page_size; 516 unsigned long psize; 517 bool also_pwc; 518 }; 519 520 static void do_tlbiel_va_range(void *info) 521 { 522 struct tlbiel_va_range *t = info; 523 524 _tlbiel_va_range(t->start, t->end, t->pid, t->page_size, 525 t->psize, t->also_pwc); 526 } 527 528 static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid, 529 unsigned long psize, unsigned long ric) 530 { 531 unsigned long ap = mmu_get_ap(psize); 532 533 asm volatile("ptesync": : :"memory"); 534 __tlbie_lpid_va(va, lpid, ap, ric); 535 fixup_tlbie_lpid_va(va, lpid, ap); 536 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 537 } 538 539 static inline void _tlbie_va_range(unsigned long start, unsigned long end, 540 unsigned long pid, unsigned long page_size, 541 unsigned long psize, bool also_pwc) 542 { 543 asm volatile("ptesync": : :"memory"); 544 if (also_pwc) 545 __tlbie_pid(pid, RIC_FLUSH_PWC); 546 __tlbie_va_range(start, end, pid, page_size, psize); 547 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 548 } 549 550 static inline void _tlbiel_va_range_multicast(struct mm_struct *mm, 551 unsigned long start, unsigned long end, 552 unsigned long pid, unsigned long page_size, 553 unsigned long psize, bool also_pwc) 554 { 555 struct cpumask *cpus = mm_cpumask(mm); 556 struct tlbiel_va_range t = { .start = start, .end = end, 557 .pid = pid, .page_size = page_size, 558 .psize = psize, .also_pwc = also_pwc }; 559 560 on_each_cpu_mask(cpus, do_tlbiel_va_range, &t, 1); 561 if (atomic_read(&mm->context.copros) > 0) 562 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); 563 } 564 565 /* 566 * Base TLB flushing operations: 567 * 568 * - flush_tlb_mm(mm) flushes the specified mm context TLB's 569 * - flush_tlb_page(vma, vmaddr) flushes one page 570 * - flush_tlb_range(vma, start, end) flushes a range of pages 571 * - flush_tlb_kernel_range(start, end) flushes kernel pages 572 * 573 * - local_* variants of page and mm only apply to the current 574 * processor 575 */ 576 void radix__local_flush_tlb_mm(struct mm_struct *mm) 577 { 578 unsigned long pid; 579 580 preempt_disable(); 581 pid = mm->context.id; 582 if (pid != MMU_NO_CONTEXT) 583 _tlbiel_pid(pid, RIC_FLUSH_TLB); 584 preempt_enable(); 585 } 586 EXPORT_SYMBOL(radix__local_flush_tlb_mm); 587 588 #ifndef CONFIG_SMP 589 void radix__local_flush_all_mm(struct mm_struct *mm) 590 { 591 unsigned long pid; 592 593 preempt_disable(); 594 pid = mm->context.id; 595 if (pid != MMU_NO_CONTEXT) 596 _tlbiel_pid(pid, RIC_FLUSH_ALL); 597 preempt_enable(); 598 } 599 EXPORT_SYMBOL(radix__local_flush_all_mm); 600 601 static void __flush_all_mm(struct mm_struct *mm, bool fullmm) 602 { 603 radix__local_flush_all_mm(mm); 604 } 605 #endif /* CONFIG_SMP */ 606 607 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 608 int psize) 609 { 610 unsigned long pid; 611 612 preempt_disable(); 613 pid = mm->context.id; 614 if (pid != MMU_NO_CONTEXT) 615 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 616 preempt_enable(); 617 } 618 619 void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 620 { 621 #ifdef CONFIG_HUGETLB_PAGE 622 /* need the return fix for nohash.c */ 623 if (is_vm_hugetlb_page(vma)) 624 return radix__local_flush_hugetlb_page(vma, vmaddr); 625 #endif 626 radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 627 } 628 EXPORT_SYMBOL(radix__local_flush_tlb_page); 629 630 static bool mm_is_singlethreaded(struct mm_struct *mm) 631 { 632 if (atomic_read(&mm->context.copros) > 0) 633 return false; 634 if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) 635 return true; 636 return false; 637 } 638 639 static bool mm_needs_flush_escalation(struct mm_struct *mm) 640 { 641 /* 642 * P9 nest MMU has issues with the page walk cache 643 * caching PTEs and not flushing them properly when 644 * RIC = 0 for a PID/LPID invalidate 645 */ 646 if (atomic_read(&mm->context.copros) > 0) 647 return true; 648 return false; 649 } 650 651 #ifdef CONFIG_SMP 652 static void do_exit_flush_lazy_tlb(void *arg) 653 { 654 struct mm_struct *mm = arg; 655 unsigned long pid = mm->context.id; 656 657 /* 658 * A kthread could have done a mmget_not_zero() after the flushing CPU 659 * checked mm_is_singlethreaded, and be in the process of 660 * kthread_use_mm when interrupted here. In that case, current->mm will 661 * be set to mm, because kthread_use_mm() setting ->mm and switching to 662 * the mm is done with interrupts off. 663 */ 664 if (current->mm == mm) 665 goto out_flush; 666 667 if (current->active_mm == mm) { 668 WARN_ON_ONCE(current->mm != NULL); 669 /* Is a kernel thread and is using mm as the lazy tlb */ 670 mmgrab(&init_mm); 671 current->active_mm = &init_mm; 672 switch_mm_irqs_off(mm, &init_mm, current); 673 mmdrop(mm); 674 } 675 676 atomic_dec(&mm->context.active_cpus); 677 cpumask_clear_cpu(smp_processor_id(), mm_cpumask(mm)); 678 679 out_flush: 680 _tlbiel_pid(pid, RIC_FLUSH_ALL); 681 } 682 683 static void exit_flush_lazy_tlbs(struct mm_struct *mm) 684 { 685 /* 686 * Would be nice if this was async so it could be run in 687 * parallel with our local flush, but generic code does not 688 * give a good API for it. Could extend the generic code or 689 * make a special powerpc IPI for flushing TLBs. 690 * For now it's not too performance critical. 691 */ 692 smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb, 693 (void *)mm, 1); 694 } 695 696 void radix__flush_tlb_mm(struct mm_struct *mm) 697 { 698 unsigned long pid; 699 700 pid = mm->context.id; 701 if (unlikely(pid == MMU_NO_CONTEXT)) 702 return; 703 704 preempt_disable(); 705 /* 706 * Order loads of mm_cpumask vs previous stores to clear ptes before 707 * the invalidate. See barrier in switch_mm_irqs_off 708 */ 709 smp_mb(); 710 if (!mm_is_thread_local(mm)) { 711 if (unlikely(mm_is_singlethreaded(mm))) { 712 exit_flush_lazy_tlbs(mm); 713 goto local; 714 } 715 716 if (!mmu_has_feature(MMU_FTR_GTSE)) { 717 unsigned long tgt = H_RPTI_TARGET_CMMU; 718 719 if (atomic_read(&mm->context.copros) > 0) 720 tgt |= H_RPTI_TARGET_NMMU; 721 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, 722 H_RPTI_PAGE_ALL, 0, -1UL); 723 } else if (cputlb_use_tlbie()) { 724 if (mm_needs_flush_escalation(mm)) 725 _tlbie_pid(pid, RIC_FLUSH_ALL); 726 else 727 _tlbie_pid(pid, RIC_FLUSH_TLB); 728 } else { 729 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); 730 } 731 } else { 732 local: 733 _tlbiel_pid(pid, RIC_FLUSH_TLB); 734 } 735 preempt_enable(); 736 } 737 EXPORT_SYMBOL(radix__flush_tlb_mm); 738 739 static void __flush_all_mm(struct mm_struct *mm, bool fullmm) 740 { 741 unsigned long pid; 742 743 pid = mm->context.id; 744 if (unlikely(pid == MMU_NO_CONTEXT)) 745 return; 746 747 preempt_disable(); 748 smp_mb(); /* see radix__flush_tlb_mm */ 749 if (!mm_is_thread_local(mm)) { 750 if (unlikely(mm_is_singlethreaded(mm))) { 751 if (!fullmm) { 752 exit_flush_lazy_tlbs(mm); 753 goto local; 754 } 755 } 756 if (!mmu_has_feature(MMU_FTR_GTSE)) { 757 unsigned long tgt = H_RPTI_TARGET_CMMU; 758 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | 759 H_RPTI_TYPE_PRT; 760 761 if (atomic_read(&mm->context.copros) > 0) 762 tgt |= H_RPTI_TARGET_NMMU; 763 pseries_rpt_invalidate(pid, tgt, type, 764 H_RPTI_PAGE_ALL, 0, -1UL); 765 } else if (cputlb_use_tlbie()) 766 _tlbie_pid(pid, RIC_FLUSH_ALL); 767 else 768 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); 769 } else { 770 local: 771 _tlbiel_pid(pid, RIC_FLUSH_ALL); 772 } 773 preempt_enable(); 774 } 775 776 void radix__flush_all_mm(struct mm_struct *mm) 777 { 778 __flush_all_mm(mm, false); 779 } 780 EXPORT_SYMBOL(radix__flush_all_mm); 781 782 void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 783 int psize) 784 { 785 unsigned long pid; 786 787 pid = mm->context.id; 788 if (unlikely(pid == MMU_NO_CONTEXT)) 789 return; 790 791 preempt_disable(); 792 smp_mb(); /* see radix__flush_tlb_mm */ 793 if (!mm_is_thread_local(mm)) { 794 if (unlikely(mm_is_singlethreaded(mm))) { 795 exit_flush_lazy_tlbs(mm); 796 goto local; 797 } 798 if (!mmu_has_feature(MMU_FTR_GTSE)) { 799 unsigned long tgt, pg_sizes, size; 800 801 tgt = H_RPTI_TARGET_CMMU; 802 pg_sizes = psize_to_rpti_pgsize(psize); 803 size = 1UL << mmu_psize_to_shift(psize); 804 805 if (atomic_read(&mm->context.copros) > 0) 806 tgt |= H_RPTI_TARGET_NMMU; 807 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, 808 pg_sizes, vmaddr, 809 vmaddr + size); 810 } else if (cputlb_use_tlbie()) 811 _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 812 else 813 _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); 814 } else { 815 local: 816 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 817 } 818 preempt_enable(); 819 } 820 821 void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 822 { 823 #ifdef CONFIG_HUGETLB_PAGE 824 if (is_vm_hugetlb_page(vma)) 825 return radix__flush_hugetlb_page(vma, vmaddr); 826 #endif 827 radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 828 } 829 EXPORT_SYMBOL(radix__flush_tlb_page); 830 831 #else /* CONFIG_SMP */ 832 static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { } 833 #endif /* CONFIG_SMP */ 834 835 static void do_tlbiel_kernel(void *info) 836 { 837 _tlbiel_pid(0, RIC_FLUSH_ALL); 838 } 839 840 static inline void _tlbiel_kernel_broadcast(void) 841 { 842 on_each_cpu(do_tlbiel_kernel, NULL, 1); 843 if (tlbie_capable) { 844 /* 845 * Coherent accelerators don't refcount kernel memory mappings, 846 * so have to always issue a tlbie for them. This is quite a 847 * slow path anyway. 848 */ 849 _tlbie_pid(0, RIC_FLUSH_ALL); 850 } 851 } 852 853 /* 854 * If kernel TLBIs ever become local rather than global, then 855 * drivers/misc/ocxl/link.c:ocxl_link_add_pe will need some work, as it 856 * assumes kernel TLBIs are global. 857 */ 858 void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) 859 { 860 if (!mmu_has_feature(MMU_FTR_GTSE)) { 861 unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU; 862 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | 863 H_RPTI_TYPE_PRT; 864 865 pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL, 866 start, end); 867 } else if (cputlb_use_tlbie()) 868 _tlbie_pid(0, RIC_FLUSH_ALL); 869 else 870 _tlbiel_kernel_broadcast(); 871 } 872 EXPORT_SYMBOL(radix__flush_tlb_kernel_range); 873 874 #define TLB_FLUSH_ALL -1UL 875 876 /* 877 * Number of pages above which we invalidate the entire PID rather than 878 * flush individual pages, for local and global flushes respectively. 879 * 880 * tlbie goes out to the interconnect and individual ops are more costly. 881 * It also does not iterate over sets like the local tlbiel variant when 882 * invalidating a full PID, so it has a far lower threshold to change from 883 * individual page flushes to full-pid flushes. 884 */ 885 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; 886 static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2; 887 888 static inline void __radix__flush_tlb_range(struct mm_struct *mm, 889 unsigned long start, unsigned long end) 890 891 { 892 unsigned long pid; 893 unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; 894 unsigned long page_size = 1UL << page_shift; 895 unsigned long nr_pages = (end - start) >> page_shift; 896 bool local, full; 897 898 pid = mm->context.id; 899 if (unlikely(pid == MMU_NO_CONTEXT)) 900 return; 901 902 preempt_disable(); 903 smp_mb(); /* see radix__flush_tlb_mm */ 904 if (!mm_is_thread_local(mm)) { 905 if (unlikely(mm_is_singlethreaded(mm))) { 906 if (end != TLB_FLUSH_ALL) { 907 exit_flush_lazy_tlbs(mm); 908 goto is_local; 909 } 910 } 911 local = false; 912 full = (end == TLB_FLUSH_ALL || 913 nr_pages > tlb_single_page_flush_ceiling); 914 } else { 915 is_local: 916 local = true; 917 full = (end == TLB_FLUSH_ALL || 918 nr_pages > tlb_local_single_page_flush_ceiling); 919 } 920 921 if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { 922 unsigned long tgt = H_RPTI_TARGET_CMMU; 923 unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize); 924 925 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 926 pg_sizes |= psize_to_rpti_pgsize(MMU_PAGE_2M); 927 if (atomic_read(&mm->context.copros) > 0) 928 tgt |= H_RPTI_TARGET_NMMU; 929 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes, 930 start, end); 931 } else if (full) { 932 if (local) { 933 _tlbiel_pid(pid, RIC_FLUSH_TLB); 934 } else { 935 if (cputlb_use_tlbie()) { 936 if (mm_needs_flush_escalation(mm)) 937 _tlbie_pid(pid, RIC_FLUSH_ALL); 938 else 939 _tlbie_pid(pid, RIC_FLUSH_TLB); 940 } else { 941 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); 942 } 943 } 944 } else { 945 bool hflush = false; 946 unsigned long hstart, hend; 947 948 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 949 hstart = (start + PMD_SIZE - 1) & PMD_MASK; 950 hend = end & PMD_MASK; 951 if (hstart < hend) 952 hflush = true; 953 } 954 955 if (local) { 956 asm volatile("ptesync": : :"memory"); 957 __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); 958 if (hflush) 959 __tlbiel_va_range(hstart, hend, pid, 960 PMD_SIZE, MMU_PAGE_2M); 961 ppc_after_tlbiel_barrier(); 962 } else if (cputlb_use_tlbie()) { 963 asm volatile("ptesync": : :"memory"); 964 __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize); 965 if (hflush) 966 __tlbie_va_range(hstart, hend, pid, 967 PMD_SIZE, MMU_PAGE_2M); 968 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 969 } else { 970 _tlbiel_va_range_multicast(mm, 971 start, end, pid, page_size, mmu_virtual_psize, false); 972 if (hflush) 973 _tlbiel_va_range_multicast(mm, 974 hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false); 975 } 976 } 977 preempt_enable(); 978 } 979 980 void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 981 unsigned long end) 982 983 { 984 #ifdef CONFIG_HUGETLB_PAGE 985 if (is_vm_hugetlb_page(vma)) 986 return radix__flush_hugetlb_tlb_range(vma, start, end); 987 #endif 988 989 __radix__flush_tlb_range(vma->vm_mm, start, end); 990 } 991 EXPORT_SYMBOL(radix__flush_tlb_range); 992 993 static int radix_get_mmu_psize(int page_size) 994 { 995 int psize; 996 997 if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift)) 998 psize = mmu_virtual_psize; 999 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift)) 1000 psize = MMU_PAGE_2M; 1001 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift)) 1002 psize = MMU_PAGE_1G; 1003 else 1004 return -1; 1005 return psize; 1006 } 1007 1008 /* 1009 * Flush partition scoped LPID address translation for all CPUs. 1010 */ 1011 void radix__flush_tlb_lpid_page(unsigned int lpid, 1012 unsigned long addr, 1013 unsigned long page_size) 1014 { 1015 int psize = radix_get_mmu_psize(page_size); 1016 1017 _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB); 1018 } 1019 EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page); 1020 1021 /* 1022 * Flush partition scoped PWC from LPID for all CPUs. 1023 */ 1024 void radix__flush_pwc_lpid(unsigned int lpid) 1025 { 1026 _tlbie_lpid(lpid, RIC_FLUSH_PWC); 1027 } 1028 EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid); 1029 1030 /* 1031 * Flush partition scoped translations from LPID (=LPIDR) 1032 */ 1033 void radix__flush_all_lpid(unsigned int lpid) 1034 { 1035 _tlbie_lpid(lpid, RIC_FLUSH_ALL); 1036 } 1037 EXPORT_SYMBOL_GPL(radix__flush_all_lpid); 1038 1039 /* 1040 * Flush process scoped translations from LPID (=LPIDR) 1041 */ 1042 void radix__flush_all_lpid_guest(unsigned int lpid) 1043 { 1044 _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 1045 } 1046 1047 static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 1048 unsigned long end, int psize); 1049 1050 void radix__tlb_flush(struct mmu_gather *tlb) 1051 { 1052 int psize = 0; 1053 struct mm_struct *mm = tlb->mm; 1054 int page_size = tlb->page_size; 1055 unsigned long start = tlb->start; 1056 unsigned long end = tlb->end; 1057 1058 /* 1059 * if page size is not something we understand, do a full mm flush 1060 * 1061 * A "fullmm" flush must always do a flush_all_mm (RIC=2) flush 1062 * that flushes the process table entry cache upon process teardown. 1063 * See the comment for radix in arch_exit_mmap(). 1064 */ 1065 if (tlb->fullmm || tlb->need_flush_all) { 1066 __flush_all_mm(mm, true); 1067 } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) { 1068 if (!tlb->freed_tables) 1069 radix__flush_tlb_mm(mm); 1070 else 1071 radix__flush_all_mm(mm); 1072 } else { 1073 if (!tlb->freed_tables) 1074 radix__flush_tlb_range_psize(mm, start, end, psize); 1075 else 1076 radix__flush_tlb_pwc_range_psize(mm, start, end, psize); 1077 } 1078 } 1079 1080 static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, 1081 unsigned long start, unsigned long end, 1082 int psize, bool also_pwc) 1083 { 1084 unsigned long pid; 1085 unsigned int page_shift = mmu_psize_defs[psize].shift; 1086 unsigned long page_size = 1UL << page_shift; 1087 unsigned long nr_pages = (end - start) >> page_shift; 1088 bool local, full; 1089 1090 pid = mm->context.id; 1091 if (unlikely(pid == MMU_NO_CONTEXT)) 1092 return; 1093 1094 preempt_disable(); 1095 smp_mb(); /* see radix__flush_tlb_mm */ 1096 if (!mm_is_thread_local(mm)) { 1097 if (unlikely(mm_is_singlethreaded(mm))) { 1098 if (end != TLB_FLUSH_ALL) { 1099 exit_flush_lazy_tlbs(mm); 1100 goto is_local; 1101 } 1102 } 1103 local = false; 1104 full = (end == TLB_FLUSH_ALL || 1105 nr_pages > tlb_single_page_flush_ceiling); 1106 } else { 1107 is_local: 1108 local = true; 1109 full = (end == TLB_FLUSH_ALL || 1110 nr_pages > tlb_local_single_page_flush_ceiling); 1111 } 1112 1113 if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { 1114 unsigned long tgt = H_RPTI_TARGET_CMMU; 1115 unsigned long type = H_RPTI_TYPE_TLB; 1116 unsigned long pg_sizes = psize_to_rpti_pgsize(psize); 1117 1118 if (also_pwc) 1119 type |= H_RPTI_TYPE_PWC; 1120 if (atomic_read(&mm->context.copros) > 0) 1121 tgt |= H_RPTI_TARGET_NMMU; 1122 pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end); 1123 } else if (full) { 1124 if (local) { 1125 _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 1126 } else { 1127 if (cputlb_use_tlbie()) { 1128 if (mm_needs_flush_escalation(mm)) 1129 also_pwc = true; 1130 1131 _tlbie_pid(pid, 1132 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 1133 } else { 1134 _tlbiel_pid_multicast(mm, pid, 1135 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 1136 } 1137 1138 } 1139 } else { 1140 if (local) 1141 _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc); 1142 else if (cputlb_use_tlbie()) 1143 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); 1144 else 1145 _tlbiel_va_range_multicast(mm, 1146 start, end, pid, page_size, psize, also_pwc); 1147 } 1148 preempt_enable(); 1149 } 1150 1151 void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, 1152 unsigned long end, int psize) 1153 { 1154 return __radix__flush_tlb_range_psize(mm, start, end, psize, false); 1155 } 1156 1157 static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 1158 unsigned long end, int psize) 1159 { 1160 __radix__flush_tlb_range_psize(mm, start, end, psize, true); 1161 } 1162 1163 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1164 void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) 1165 { 1166 unsigned long pid, end; 1167 1168 pid = mm->context.id; 1169 if (unlikely(pid == MMU_NO_CONTEXT)) 1170 return; 1171 1172 /* 4k page size, just blow the world */ 1173 if (PAGE_SIZE == 0x1000) { 1174 radix__flush_all_mm(mm); 1175 return; 1176 } 1177 1178 end = addr + HPAGE_PMD_SIZE; 1179 1180 /* Otherwise first do the PWC, then iterate the pages. */ 1181 preempt_disable(); 1182 smp_mb(); /* see radix__flush_tlb_mm */ 1183 if (!mm_is_thread_local(mm)) { 1184 if (unlikely(mm_is_singlethreaded(mm))) { 1185 exit_flush_lazy_tlbs(mm); 1186 goto local; 1187 } 1188 if (!mmu_has_feature(MMU_FTR_GTSE)) { 1189 unsigned long tgt, type, pg_sizes; 1190 1191 tgt = H_RPTI_TARGET_CMMU; 1192 type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | 1193 H_RPTI_TYPE_PRT; 1194 pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize); 1195 1196 if (atomic_read(&mm->context.copros) > 0) 1197 tgt |= H_RPTI_TARGET_NMMU; 1198 pseries_rpt_invalidate(pid, tgt, type, pg_sizes, 1199 addr, end); 1200 } else if (cputlb_use_tlbie()) 1201 _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 1202 else 1203 _tlbiel_va_range_multicast(mm, 1204 addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 1205 } else { 1206 local: 1207 _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 1208 } 1209 1210 preempt_enable(); 1211 } 1212 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1213 1214 void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, 1215 unsigned long start, unsigned long end) 1216 { 1217 radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); 1218 } 1219 EXPORT_SYMBOL(radix__flush_pmd_tlb_range); 1220 1221 void radix__flush_tlb_all(void) 1222 { 1223 unsigned long rb,prs,r,rs; 1224 unsigned long ric = RIC_FLUSH_ALL; 1225 1226 rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */ 1227 prs = 0; /* partition scoped */ 1228 r = 1; /* radix format */ 1229 rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */ 1230 1231 asm volatile("ptesync": : :"memory"); 1232 /* 1233 * now flush guest entries by passing PRS = 1 and LPID != 0 1234 */ 1235 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 1236 : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory"); 1237 /* 1238 * now flush host entires by passing PRS = 0 and LPID == 0 1239 */ 1240 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 1241 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory"); 1242 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 1243 } 1244 1245 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1246 extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) 1247 { 1248 unsigned long pid = mm->context.id; 1249 1250 if (unlikely(pid == MMU_NO_CONTEXT)) 1251 return; 1252 1253 if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 1254 return; 1255 1256 /* 1257 * If this context hasn't run on that CPU before and KVM is 1258 * around, there's a slim chance that the guest on another 1259 * CPU just brought in obsolete translation into the TLB of 1260 * this CPU due to a bad prefetch using the guest PID on 1261 * the way into the hypervisor. 1262 * 1263 * We work around this here. If KVM is possible, we check if 1264 * any sibling thread is in KVM. If it is, the window may exist 1265 * and thus we flush that PID from the core. 1266 * 1267 * A potential future improvement would be to mark which PIDs 1268 * have never been used on the system and avoid it if the PID 1269 * is new and the process has no other cpumask bit set. 1270 */ 1271 if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) { 1272 int cpu = smp_processor_id(); 1273 int sib = cpu_first_thread_sibling(cpu); 1274 bool flush = false; 1275 1276 for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) { 1277 if (sib == cpu) 1278 continue; 1279 if (!cpu_possible(sib)) 1280 continue; 1281 if (paca_ptrs[sib]->kvm_hstate.kvm_vcpu) 1282 flush = true; 1283 } 1284 if (flush) 1285 _tlbiel_pid(pid, RIC_FLUSH_ALL); 1286 } 1287 } 1288 EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround); 1289 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 1290