1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TLB flush routines for radix kernels. 4 * 5 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation. 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/hugetlb.h> 10 #include <linux/memblock.h> 11 #include <linux/mmu_context.h> 12 #include <linux/sched/mm.h> 13 14 #include <asm/ppc-opcode.h> 15 #include <asm/tlb.h> 16 #include <asm/tlbflush.h> 17 #include <asm/trace.h> 18 #include <asm/cputhreads.h> 19 #include <asm/plpar_wrappers.h> 20 21 #define RIC_FLUSH_TLB 0 22 #define RIC_FLUSH_PWC 1 23 #define RIC_FLUSH_ALL 2 24 25 /* 26 * tlbiel instruction for radix, set invalidation 27 * i.e., r=1 and is=01 or is=10 or is=11 28 */ 29 static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, 30 unsigned int pid, 31 unsigned int ric, unsigned int prs) 32 { 33 unsigned long rb; 34 unsigned long rs; 35 36 rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); 37 rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); 38 39 asm volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 40 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 41 : "memory"); 42 } 43 44 static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) 45 { 46 unsigned int set; 47 48 asm volatile("ptesync": : :"memory"); 49 50 /* 51 * Flush the first set of the TLB, and the entire Page Walk Cache 52 * and partition table entries. Then flush the remaining sets of the 53 * TLB. 54 */ 55 56 if (early_cpu_has_feature(CPU_FTR_HVMODE)) { 57 /* MSR[HV] should flush partition scope translations first. */ 58 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 59 for (set = 1; set < num_sets; set++) 60 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 61 } 62 63 /* Flush process scoped entries. */ 64 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 65 for (set = 1; set < num_sets; set++) 66 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 67 68 asm volatile("ptesync": : :"memory"); 69 } 70 71 void radix__tlbiel_all(unsigned int action) 72 { 73 unsigned int is; 74 75 switch (action) { 76 case TLB_INVAL_SCOPE_GLOBAL: 77 is = 3; 78 break; 79 case TLB_INVAL_SCOPE_LPID: 80 is = 2; 81 break; 82 default: 83 BUG(); 84 } 85 86 if (early_cpu_has_feature(CPU_FTR_ARCH_300)) 87 tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is); 88 else 89 WARN(1, "%s called on pre-POWER9 CPU\n", __func__); 90 91 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); 92 } 93 94 static __always_inline void __tlbiel_pid(unsigned long pid, int set, 95 unsigned long ric) 96 { 97 unsigned long rb,rs,prs,r; 98 99 rb = PPC_BIT(53); /* IS = 1 */ 100 rb |= set << PPC_BITLSHIFT(51); 101 rs = ((unsigned long)pid) << PPC_BITLSHIFT(31); 102 prs = 1; /* process scoped */ 103 r = 1; /* radix format */ 104 105 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 106 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 107 trace_tlbie(0, 1, rb, rs, ric, prs, r); 108 } 109 110 static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric) 111 { 112 unsigned long rb,rs,prs,r; 113 114 rb = PPC_BIT(53); /* IS = 1 */ 115 rs = pid << PPC_BITLSHIFT(31); 116 prs = 1; /* process scoped */ 117 r = 1; /* radix format */ 118 119 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 120 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 121 trace_tlbie(0, 0, rb, rs, ric, prs, r); 122 } 123 124 static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric) 125 { 126 unsigned long rb,rs,prs,r; 127 128 rb = PPC_BIT(52); /* IS = 2 */ 129 rs = lpid; 130 prs = 0; /* partition scoped */ 131 r = 1; /* radix format */ 132 133 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 134 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 135 trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 136 } 137 138 static __always_inline void __tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 139 { 140 unsigned long rb,rs,prs,r; 141 142 rb = PPC_BIT(52); /* IS = 2 */ 143 rs = lpid; 144 prs = 1; /* process scoped */ 145 r = 1; /* radix format */ 146 147 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 148 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 149 trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 150 } 151 152 static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid, 153 unsigned long ap, unsigned long ric) 154 { 155 unsigned long rb,rs,prs,r; 156 157 rb = va & ~(PPC_BITMASK(52, 63)); 158 rb |= ap << PPC_BITLSHIFT(58); 159 rs = pid << PPC_BITLSHIFT(31); 160 prs = 1; /* process scoped */ 161 r = 1; /* radix format */ 162 163 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 164 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 165 trace_tlbie(0, 1, rb, rs, ric, prs, r); 166 } 167 168 static __always_inline void __tlbie_va(unsigned long va, unsigned long pid, 169 unsigned long ap, unsigned long ric) 170 { 171 unsigned long rb,rs,prs,r; 172 173 rb = va & ~(PPC_BITMASK(52, 63)); 174 rb |= ap << PPC_BITLSHIFT(58); 175 rs = pid << PPC_BITLSHIFT(31); 176 prs = 1; /* process scoped */ 177 r = 1; /* radix format */ 178 179 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 180 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 181 trace_tlbie(0, 0, rb, rs, ric, prs, r); 182 } 183 184 static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, 185 unsigned long ap, unsigned long ric) 186 { 187 unsigned long rb,rs,prs,r; 188 189 rb = va & ~(PPC_BITMASK(52, 63)); 190 rb |= ap << PPC_BITLSHIFT(58); 191 rs = lpid; 192 prs = 0; /* partition scoped */ 193 r = 1; /* radix format */ 194 195 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 196 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 197 trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 198 } 199 200 201 static inline void fixup_tlbie_va(unsigned long va, unsigned long pid, 202 unsigned long ap) 203 { 204 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 205 asm volatile("ptesync": : :"memory"); 206 __tlbie_va(va, 0, ap, RIC_FLUSH_TLB); 207 } 208 209 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 210 asm volatile("ptesync": : :"memory"); 211 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 212 } 213 } 214 215 static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid, 216 unsigned long ap) 217 { 218 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 219 asm volatile("ptesync": : :"memory"); 220 __tlbie_pid(0, RIC_FLUSH_TLB); 221 } 222 223 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 224 asm volatile("ptesync": : :"memory"); 225 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 226 } 227 } 228 229 static inline void fixup_tlbie_pid(unsigned long pid) 230 { 231 /* 232 * We can use any address for the invalidation, pick one which is 233 * probably unused as an optimisation. 234 */ 235 unsigned long va = ((1UL << 52) - 1); 236 237 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 238 asm volatile("ptesync": : :"memory"); 239 __tlbie_pid(0, RIC_FLUSH_TLB); 240 } 241 242 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 243 asm volatile("ptesync": : :"memory"); 244 __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 245 } 246 } 247 248 249 static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid, 250 unsigned long ap) 251 { 252 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 253 asm volatile("ptesync": : :"memory"); 254 __tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB); 255 } 256 257 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 258 asm volatile("ptesync": : :"memory"); 259 __tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB); 260 } 261 } 262 263 static inline void fixup_tlbie_lpid(unsigned long lpid) 264 { 265 /* 266 * We can use any address for the invalidation, pick one which is 267 * probably unused as an optimisation. 268 */ 269 unsigned long va = ((1UL << 52) - 1); 270 271 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 272 asm volatile("ptesync": : :"memory"); 273 __tlbie_lpid(0, RIC_FLUSH_TLB); 274 } 275 276 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 277 asm volatile("ptesync": : :"memory"); 278 __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 279 } 280 } 281 282 /* 283 * We use 128 set in radix mode and 256 set in hpt mode. 284 */ 285 static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric) 286 { 287 int set; 288 289 asm volatile("ptesync": : :"memory"); 290 291 /* 292 * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL, 293 * also flush the entire Page Walk Cache. 294 */ 295 __tlbiel_pid(pid, 0, ric); 296 297 /* For PWC, only one flush is needed */ 298 if (ric == RIC_FLUSH_PWC) { 299 asm volatile("ptesync": : :"memory"); 300 return; 301 } 302 303 /* For the remaining sets, just flush the TLB */ 304 for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++) 305 __tlbiel_pid(pid, set, RIC_FLUSH_TLB); 306 307 asm volatile("ptesync": : :"memory"); 308 asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory"); 309 } 310 311 static inline void _tlbie_pid(unsigned long pid, unsigned long ric) 312 { 313 asm volatile("ptesync": : :"memory"); 314 315 /* 316 * Workaround the fact that the "ric" argument to __tlbie_pid 317 * must be a compile-time contraint to match the "i" constraint 318 * in the asm statement. 319 */ 320 switch (ric) { 321 case RIC_FLUSH_TLB: 322 __tlbie_pid(pid, RIC_FLUSH_TLB); 323 fixup_tlbie_pid(pid); 324 break; 325 case RIC_FLUSH_PWC: 326 __tlbie_pid(pid, RIC_FLUSH_PWC); 327 break; 328 case RIC_FLUSH_ALL: 329 default: 330 __tlbie_pid(pid, RIC_FLUSH_ALL); 331 fixup_tlbie_pid(pid); 332 } 333 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 334 } 335 336 struct tlbiel_pid { 337 unsigned long pid; 338 unsigned long ric; 339 }; 340 341 static void do_tlbiel_pid(void *info) 342 { 343 struct tlbiel_pid *t = info; 344 345 if (t->ric == RIC_FLUSH_TLB) 346 _tlbiel_pid(t->pid, RIC_FLUSH_TLB); 347 else if (t->ric == RIC_FLUSH_PWC) 348 _tlbiel_pid(t->pid, RIC_FLUSH_PWC); 349 else 350 _tlbiel_pid(t->pid, RIC_FLUSH_ALL); 351 } 352 353 static inline void _tlbiel_pid_multicast(struct mm_struct *mm, 354 unsigned long pid, unsigned long ric) 355 { 356 struct cpumask *cpus = mm_cpumask(mm); 357 struct tlbiel_pid t = { .pid = pid, .ric = ric }; 358 359 on_each_cpu_mask(cpus, do_tlbiel_pid, &t, 1); 360 /* 361 * Always want the CPU translations to be invalidated with tlbiel in 362 * these paths, so while coprocessors must use tlbie, we can not 363 * optimise away the tlbiel component. 364 */ 365 if (atomic_read(&mm->context.copros) > 0) 366 _tlbie_pid(pid, RIC_FLUSH_ALL); 367 } 368 369 static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) 370 { 371 asm volatile("ptesync": : :"memory"); 372 373 /* 374 * Workaround the fact that the "ric" argument to __tlbie_pid 375 * must be a compile-time contraint to match the "i" constraint 376 * in the asm statement. 377 */ 378 switch (ric) { 379 case RIC_FLUSH_TLB: 380 __tlbie_lpid(lpid, RIC_FLUSH_TLB); 381 fixup_tlbie_lpid(lpid); 382 break; 383 case RIC_FLUSH_PWC: 384 __tlbie_lpid(lpid, RIC_FLUSH_PWC); 385 break; 386 case RIC_FLUSH_ALL: 387 default: 388 __tlbie_lpid(lpid, RIC_FLUSH_ALL); 389 fixup_tlbie_lpid(lpid); 390 } 391 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 392 } 393 394 static __always_inline void _tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 395 { 396 /* 397 * Workaround the fact that the "ric" argument to __tlbie_pid 398 * must be a compile-time contraint to match the "i" constraint 399 * in the asm statement. 400 */ 401 switch (ric) { 402 case RIC_FLUSH_TLB: 403 __tlbie_lpid_guest(lpid, RIC_FLUSH_TLB); 404 break; 405 case RIC_FLUSH_PWC: 406 __tlbie_lpid_guest(lpid, RIC_FLUSH_PWC); 407 break; 408 case RIC_FLUSH_ALL: 409 default: 410 __tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 411 } 412 fixup_tlbie_lpid(lpid); 413 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 414 } 415 416 static inline void __tlbiel_va_range(unsigned long start, unsigned long end, 417 unsigned long pid, unsigned long page_size, 418 unsigned long psize) 419 { 420 unsigned long addr; 421 unsigned long ap = mmu_get_ap(psize); 422 423 for (addr = start; addr < end; addr += page_size) 424 __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB); 425 } 426 427 static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid, 428 unsigned long psize, unsigned long ric) 429 { 430 unsigned long ap = mmu_get_ap(psize); 431 432 asm volatile("ptesync": : :"memory"); 433 __tlbiel_va(va, pid, ap, ric); 434 asm volatile("ptesync": : :"memory"); 435 } 436 437 static inline void _tlbiel_va_range(unsigned long start, unsigned long end, 438 unsigned long pid, unsigned long page_size, 439 unsigned long psize, bool also_pwc) 440 { 441 asm volatile("ptesync": : :"memory"); 442 if (also_pwc) 443 __tlbiel_pid(pid, 0, RIC_FLUSH_PWC); 444 __tlbiel_va_range(start, end, pid, page_size, psize); 445 asm volatile("ptesync": : :"memory"); 446 } 447 448 static inline void __tlbie_va_range(unsigned long start, unsigned long end, 449 unsigned long pid, unsigned long page_size, 450 unsigned long psize) 451 { 452 unsigned long addr; 453 unsigned long ap = mmu_get_ap(psize); 454 455 for (addr = start; addr < end; addr += page_size) 456 __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB); 457 458 fixup_tlbie_va_range(addr - page_size, pid, ap); 459 } 460 461 static __always_inline void _tlbie_va(unsigned long va, unsigned long pid, 462 unsigned long psize, unsigned long ric) 463 { 464 unsigned long ap = mmu_get_ap(psize); 465 466 asm volatile("ptesync": : :"memory"); 467 __tlbie_va(va, pid, ap, ric); 468 fixup_tlbie_va(va, pid, ap); 469 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 470 } 471 472 struct tlbiel_va { 473 unsigned long pid; 474 unsigned long va; 475 unsigned long psize; 476 unsigned long ric; 477 }; 478 479 static void do_tlbiel_va(void *info) 480 { 481 struct tlbiel_va *t = info; 482 483 if (t->ric == RIC_FLUSH_TLB) 484 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB); 485 else if (t->ric == RIC_FLUSH_PWC) 486 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC); 487 else 488 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL); 489 } 490 491 static inline void _tlbiel_va_multicast(struct mm_struct *mm, 492 unsigned long va, unsigned long pid, 493 unsigned long psize, unsigned long ric) 494 { 495 struct cpumask *cpus = mm_cpumask(mm); 496 struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric }; 497 on_each_cpu_mask(cpus, do_tlbiel_va, &t, 1); 498 if (atomic_read(&mm->context.copros) > 0) 499 _tlbie_va(va, pid, psize, RIC_FLUSH_TLB); 500 } 501 502 struct tlbiel_va_range { 503 unsigned long pid; 504 unsigned long start; 505 unsigned long end; 506 unsigned long page_size; 507 unsigned long psize; 508 bool also_pwc; 509 }; 510 511 static void do_tlbiel_va_range(void *info) 512 { 513 struct tlbiel_va_range *t = info; 514 515 _tlbiel_va_range(t->start, t->end, t->pid, t->page_size, 516 t->psize, t->also_pwc); 517 } 518 519 static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid, 520 unsigned long psize, unsigned long ric) 521 { 522 unsigned long ap = mmu_get_ap(psize); 523 524 asm volatile("ptesync": : :"memory"); 525 __tlbie_lpid_va(va, lpid, ap, ric); 526 fixup_tlbie_lpid_va(va, lpid, ap); 527 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 528 } 529 530 static inline void _tlbie_va_range(unsigned long start, unsigned long end, 531 unsigned long pid, unsigned long page_size, 532 unsigned long psize, bool also_pwc) 533 { 534 asm volatile("ptesync": : :"memory"); 535 if (also_pwc) 536 __tlbie_pid(pid, RIC_FLUSH_PWC); 537 __tlbie_va_range(start, end, pid, page_size, psize); 538 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 539 } 540 541 static inline void _tlbiel_va_range_multicast(struct mm_struct *mm, 542 unsigned long start, unsigned long end, 543 unsigned long pid, unsigned long page_size, 544 unsigned long psize, bool also_pwc) 545 { 546 struct cpumask *cpus = mm_cpumask(mm); 547 struct tlbiel_va_range t = { .start = start, .end = end, 548 .pid = pid, .page_size = page_size, 549 .psize = psize, .also_pwc = also_pwc }; 550 551 on_each_cpu_mask(cpus, do_tlbiel_va_range, &t, 1); 552 if (atomic_read(&mm->context.copros) > 0) 553 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); 554 } 555 556 /* 557 * Base TLB flushing operations: 558 * 559 * - flush_tlb_mm(mm) flushes the specified mm context TLB's 560 * - flush_tlb_page(vma, vmaddr) flushes one page 561 * - flush_tlb_range(vma, start, end) flushes a range of pages 562 * - flush_tlb_kernel_range(start, end) flushes kernel pages 563 * 564 * - local_* variants of page and mm only apply to the current 565 * processor 566 */ 567 void radix__local_flush_tlb_mm(struct mm_struct *mm) 568 { 569 unsigned long pid; 570 571 preempt_disable(); 572 pid = mm->context.id; 573 if (pid != MMU_NO_CONTEXT) 574 _tlbiel_pid(pid, RIC_FLUSH_TLB); 575 preempt_enable(); 576 } 577 EXPORT_SYMBOL(radix__local_flush_tlb_mm); 578 579 #ifndef CONFIG_SMP 580 void radix__local_flush_all_mm(struct mm_struct *mm) 581 { 582 unsigned long pid; 583 584 preempt_disable(); 585 pid = mm->context.id; 586 if (pid != MMU_NO_CONTEXT) 587 _tlbiel_pid(pid, RIC_FLUSH_ALL); 588 preempt_enable(); 589 } 590 EXPORT_SYMBOL(radix__local_flush_all_mm); 591 592 static void __flush_all_mm(struct mm_struct *mm, bool fullmm) 593 { 594 radix__local_flush_all_mm(mm); 595 } 596 #endif /* CONFIG_SMP */ 597 598 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 599 int psize) 600 { 601 unsigned long pid; 602 603 preempt_disable(); 604 pid = mm->context.id; 605 if (pid != MMU_NO_CONTEXT) 606 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 607 preempt_enable(); 608 } 609 610 void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 611 { 612 #ifdef CONFIG_HUGETLB_PAGE 613 /* need the return fix for nohash.c */ 614 if (is_vm_hugetlb_page(vma)) 615 return radix__local_flush_hugetlb_page(vma, vmaddr); 616 #endif 617 radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 618 } 619 EXPORT_SYMBOL(radix__local_flush_tlb_page); 620 621 static bool mm_is_singlethreaded(struct mm_struct *mm) 622 { 623 if (atomic_read(&mm->context.copros) > 0) 624 return false; 625 if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) 626 return true; 627 return false; 628 } 629 630 static bool mm_needs_flush_escalation(struct mm_struct *mm) 631 { 632 /* 633 * P9 nest MMU has issues with the page walk cache 634 * caching PTEs and not flushing them properly when 635 * RIC = 0 for a PID/LPID invalidate 636 */ 637 if (atomic_read(&mm->context.copros) > 0) 638 return true; 639 return false; 640 } 641 642 #ifdef CONFIG_SMP 643 static void do_exit_flush_lazy_tlb(void *arg) 644 { 645 struct mm_struct *mm = arg; 646 unsigned long pid = mm->context.id; 647 648 if (current->mm == mm) 649 return; /* Local CPU */ 650 651 if (current->active_mm == mm) { 652 /* 653 * Must be a kernel thread because sender is single-threaded. 654 */ 655 BUG_ON(current->mm); 656 mmgrab(&init_mm); 657 switch_mm(mm, &init_mm, current); 658 current->active_mm = &init_mm; 659 mmdrop(mm); 660 } 661 _tlbiel_pid(pid, RIC_FLUSH_ALL); 662 } 663 664 static void exit_flush_lazy_tlbs(struct mm_struct *mm) 665 { 666 /* 667 * Would be nice if this was async so it could be run in 668 * parallel with our local flush, but generic code does not 669 * give a good API for it. Could extend the generic code or 670 * make a special powerpc IPI for flushing TLBs. 671 * For now it's not too performance critical. 672 */ 673 smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb, 674 (void *)mm, 1); 675 mm_reset_thread_local(mm); 676 } 677 678 void radix__flush_tlb_mm(struct mm_struct *mm) 679 { 680 unsigned long pid; 681 682 pid = mm->context.id; 683 if (unlikely(pid == MMU_NO_CONTEXT)) 684 return; 685 686 preempt_disable(); 687 /* 688 * Order loads of mm_cpumask vs previous stores to clear ptes before 689 * the invalidate. See barrier in switch_mm_irqs_off 690 */ 691 smp_mb(); 692 if (!mm_is_thread_local(mm)) { 693 if (unlikely(mm_is_singlethreaded(mm))) { 694 exit_flush_lazy_tlbs(mm); 695 goto local; 696 } 697 698 if (!mmu_has_feature(MMU_FTR_GTSE)) { 699 unsigned long tgt = H_RPTI_TARGET_CMMU; 700 701 if (atomic_read(&mm->context.copros) > 0) 702 tgt |= H_RPTI_TARGET_NMMU; 703 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, 704 H_RPTI_PAGE_ALL, 0, -1UL); 705 } else if (cputlb_use_tlbie()) { 706 if (mm_needs_flush_escalation(mm)) 707 _tlbie_pid(pid, RIC_FLUSH_ALL); 708 else 709 _tlbie_pid(pid, RIC_FLUSH_TLB); 710 } else { 711 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); 712 } 713 } else { 714 local: 715 _tlbiel_pid(pid, RIC_FLUSH_TLB); 716 } 717 preempt_enable(); 718 } 719 EXPORT_SYMBOL(radix__flush_tlb_mm); 720 721 static void __flush_all_mm(struct mm_struct *mm, bool fullmm) 722 { 723 unsigned long pid; 724 725 pid = mm->context.id; 726 if (unlikely(pid == MMU_NO_CONTEXT)) 727 return; 728 729 preempt_disable(); 730 smp_mb(); /* see radix__flush_tlb_mm */ 731 if (!mm_is_thread_local(mm)) { 732 if (unlikely(mm_is_singlethreaded(mm))) { 733 if (!fullmm) { 734 exit_flush_lazy_tlbs(mm); 735 goto local; 736 } 737 } 738 if (!mmu_has_feature(MMU_FTR_GTSE)) { 739 unsigned long tgt = H_RPTI_TARGET_CMMU; 740 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | 741 H_RPTI_TYPE_PRT; 742 743 if (atomic_read(&mm->context.copros) > 0) 744 tgt |= H_RPTI_TARGET_NMMU; 745 pseries_rpt_invalidate(pid, tgt, type, 746 H_RPTI_PAGE_ALL, 0, -1UL); 747 } else if (cputlb_use_tlbie()) 748 _tlbie_pid(pid, RIC_FLUSH_ALL); 749 else 750 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); 751 } else { 752 local: 753 _tlbiel_pid(pid, RIC_FLUSH_ALL); 754 } 755 preempt_enable(); 756 } 757 758 void radix__flush_all_mm(struct mm_struct *mm) 759 { 760 __flush_all_mm(mm, false); 761 } 762 EXPORT_SYMBOL(radix__flush_all_mm); 763 764 void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 765 int psize) 766 { 767 unsigned long pid; 768 769 pid = mm->context.id; 770 if (unlikely(pid == MMU_NO_CONTEXT)) 771 return; 772 773 preempt_disable(); 774 smp_mb(); /* see radix__flush_tlb_mm */ 775 if (!mm_is_thread_local(mm)) { 776 if (unlikely(mm_is_singlethreaded(mm))) { 777 exit_flush_lazy_tlbs(mm); 778 goto local; 779 } 780 if (!mmu_has_feature(MMU_FTR_GTSE)) { 781 unsigned long tgt, pg_sizes, size; 782 783 tgt = H_RPTI_TARGET_CMMU; 784 pg_sizes = psize_to_rpti_pgsize(psize); 785 size = 1UL << mmu_psize_to_shift(psize); 786 787 if (atomic_read(&mm->context.copros) > 0) 788 tgt |= H_RPTI_TARGET_NMMU; 789 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, 790 pg_sizes, vmaddr, 791 vmaddr + size); 792 } else if (cputlb_use_tlbie()) 793 _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 794 else 795 _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); 796 } else { 797 local: 798 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 799 } 800 preempt_enable(); 801 } 802 803 void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 804 { 805 #ifdef CONFIG_HUGETLB_PAGE 806 if (is_vm_hugetlb_page(vma)) 807 return radix__flush_hugetlb_page(vma, vmaddr); 808 #endif 809 radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 810 } 811 EXPORT_SYMBOL(radix__flush_tlb_page); 812 813 #else /* CONFIG_SMP */ 814 static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { } 815 #endif /* CONFIG_SMP */ 816 817 static void do_tlbiel_kernel(void *info) 818 { 819 _tlbiel_pid(0, RIC_FLUSH_ALL); 820 } 821 822 static inline void _tlbiel_kernel_broadcast(void) 823 { 824 on_each_cpu(do_tlbiel_kernel, NULL, 1); 825 if (tlbie_capable) { 826 /* 827 * Coherent accelerators don't refcount kernel memory mappings, 828 * so have to always issue a tlbie for them. This is quite a 829 * slow path anyway. 830 */ 831 _tlbie_pid(0, RIC_FLUSH_ALL); 832 } 833 } 834 835 /* 836 * If kernel TLBIs ever become local rather than global, then 837 * drivers/misc/ocxl/link.c:ocxl_link_add_pe will need some work, as it 838 * assumes kernel TLBIs are global. 839 */ 840 void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) 841 { 842 if (!mmu_has_feature(MMU_FTR_GTSE)) { 843 unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU; 844 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | 845 H_RPTI_TYPE_PRT; 846 847 pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL, 848 start, end); 849 } else if (cputlb_use_tlbie()) 850 _tlbie_pid(0, RIC_FLUSH_ALL); 851 else 852 _tlbiel_kernel_broadcast(); 853 } 854 EXPORT_SYMBOL(radix__flush_tlb_kernel_range); 855 856 #define TLB_FLUSH_ALL -1UL 857 858 /* 859 * Number of pages above which we invalidate the entire PID rather than 860 * flush individual pages, for local and global flushes respectively. 861 * 862 * tlbie goes out to the interconnect and individual ops are more costly. 863 * It also does not iterate over sets like the local tlbiel variant when 864 * invalidating a full PID, so it has a far lower threshold to change from 865 * individual page flushes to full-pid flushes. 866 */ 867 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; 868 static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2; 869 870 static inline void __radix__flush_tlb_range(struct mm_struct *mm, 871 unsigned long start, unsigned long end) 872 873 { 874 unsigned long pid; 875 unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; 876 unsigned long page_size = 1UL << page_shift; 877 unsigned long nr_pages = (end - start) >> page_shift; 878 bool local, full; 879 880 pid = mm->context.id; 881 if (unlikely(pid == MMU_NO_CONTEXT)) 882 return; 883 884 preempt_disable(); 885 smp_mb(); /* see radix__flush_tlb_mm */ 886 if (!mm_is_thread_local(mm)) { 887 if (unlikely(mm_is_singlethreaded(mm))) { 888 if (end != TLB_FLUSH_ALL) { 889 exit_flush_lazy_tlbs(mm); 890 goto is_local; 891 } 892 } 893 local = false; 894 full = (end == TLB_FLUSH_ALL || 895 nr_pages > tlb_single_page_flush_ceiling); 896 } else { 897 is_local: 898 local = true; 899 full = (end == TLB_FLUSH_ALL || 900 nr_pages > tlb_local_single_page_flush_ceiling); 901 } 902 903 if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { 904 unsigned long tgt = H_RPTI_TARGET_CMMU; 905 unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize); 906 907 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 908 pg_sizes |= psize_to_rpti_pgsize(MMU_PAGE_2M); 909 if (atomic_read(&mm->context.copros) > 0) 910 tgt |= H_RPTI_TARGET_NMMU; 911 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes, 912 start, end); 913 } else if (full) { 914 if (local) { 915 _tlbiel_pid(pid, RIC_FLUSH_TLB); 916 } else { 917 if (cputlb_use_tlbie()) { 918 if (mm_needs_flush_escalation(mm)) 919 _tlbie_pid(pid, RIC_FLUSH_ALL); 920 else 921 _tlbie_pid(pid, RIC_FLUSH_TLB); 922 } else { 923 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); 924 } 925 } 926 } else { 927 bool hflush = false; 928 unsigned long hstart, hend; 929 930 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 931 hstart = (start + PMD_SIZE - 1) & PMD_MASK; 932 hend = end & PMD_MASK; 933 if (hstart < hend) 934 hflush = true; 935 } 936 937 if (local) { 938 asm volatile("ptesync": : :"memory"); 939 __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); 940 if (hflush) 941 __tlbiel_va_range(hstart, hend, pid, 942 PMD_SIZE, MMU_PAGE_2M); 943 asm volatile("ptesync": : :"memory"); 944 } else if (cputlb_use_tlbie()) { 945 asm volatile("ptesync": : :"memory"); 946 __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize); 947 if (hflush) 948 __tlbie_va_range(hstart, hend, pid, 949 PMD_SIZE, MMU_PAGE_2M); 950 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 951 } else { 952 _tlbiel_va_range_multicast(mm, 953 start, end, pid, page_size, mmu_virtual_psize, false); 954 if (hflush) 955 _tlbiel_va_range_multicast(mm, 956 hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false); 957 } 958 } 959 preempt_enable(); 960 } 961 962 void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 963 unsigned long end) 964 965 { 966 #ifdef CONFIG_HUGETLB_PAGE 967 if (is_vm_hugetlb_page(vma)) 968 return radix__flush_hugetlb_tlb_range(vma, start, end); 969 #endif 970 971 __radix__flush_tlb_range(vma->vm_mm, start, end); 972 } 973 EXPORT_SYMBOL(radix__flush_tlb_range); 974 975 static int radix_get_mmu_psize(int page_size) 976 { 977 int psize; 978 979 if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift)) 980 psize = mmu_virtual_psize; 981 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift)) 982 psize = MMU_PAGE_2M; 983 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift)) 984 psize = MMU_PAGE_1G; 985 else 986 return -1; 987 return psize; 988 } 989 990 /* 991 * Flush partition scoped LPID address translation for all CPUs. 992 */ 993 void radix__flush_tlb_lpid_page(unsigned int lpid, 994 unsigned long addr, 995 unsigned long page_size) 996 { 997 int psize = radix_get_mmu_psize(page_size); 998 999 _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB); 1000 } 1001 EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page); 1002 1003 /* 1004 * Flush partition scoped PWC from LPID for all CPUs. 1005 */ 1006 void radix__flush_pwc_lpid(unsigned int lpid) 1007 { 1008 _tlbie_lpid(lpid, RIC_FLUSH_PWC); 1009 } 1010 EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid); 1011 1012 /* 1013 * Flush partition scoped translations from LPID (=LPIDR) 1014 */ 1015 void radix__flush_all_lpid(unsigned int lpid) 1016 { 1017 _tlbie_lpid(lpid, RIC_FLUSH_ALL); 1018 } 1019 EXPORT_SYMBOL_GPL(radix__flush_all_lpid); 1020 1021 /* 1022 * Flush process scoped translations from LPID (=LPIDR) 1023 */ 1024 void radix__flush_all_lpid_guest(unsigned int lpid) 1025 { 1026 _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 1027 } 1028 1029 static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 1030 unsigned long end, int psize); 1031 1032 void radix__tlb_flush(struct mmu_gather *tlb) 1033 { 1034 int psize = 0; 1035 struct mm_struct *mm = tlb->mm; 1036 int page_size = tlb->page_size; 1037 unsigned long start = tlb->start; 1038 unsigned long end = tlb->end; 1039 1040 /* 1041 * if page size is not something we understand, do a full mm flush 1042 * 1043 * A "fullmm" flush must always do a flush_all_mm (RIC=2) flush 1044 * that flushes the process table entry cache upon process teardown. 1045 * See the comment for radix in arch_exit_mmap(). 1046 */ 1047 if (tlb->fullmm || tlb->need_flush_all) { 1048 __flush_all_mm(mm, true); 1049 } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) { 1050 if (!tlb->freed_tables) 1051 radix__flush_tlb_mm(mm); 1052 else 1053 radix__flush_all_mm(mm); 1054 } else { 1055 if (!tlb->freed_tables) 1056 radix__flush_tlb_range_psize(mm, start, end, psize); 1057 else 1058 radix__flush_tlb_pwc_range_psize(mm, start, end, psize); 1059 } 1060 } 1061 1062 static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, 1063 unsigned long start, unsigned long end, 1064 int psize, bool also_pwc) 1065 { 1066 unsigned long pid; 1067 unsigned int page_shift = mmu_psize_defs[psize].shift; 1068 unsigned long page_size = 1UL << page_shift; 1069 unsigned long nr_pages = (end - start) >> page_shift; 1070 bool local, full; 1071 1072 pid = mm->context.id; 1073 if (unlikely(pid == MMU_NO_CONTEXT)) 1074 return; 1075 1076 preempt_disable(); 1077 smp_mb(); /* see radix__flush_tlb_mm */ 1078 if (!mm_is_thread_local(mm)) { 1079 if (unlikely(mm_is_singlethreaded(mm))) { 1080 if (end != TLB_FLUSH_ALL) { 1081 exit_flush_lazy_tlbs(mm); 1082 goto is_local; 1083 } 1084 } 1085 local = false; 1086 full = (end == TLB_FLUSH_ALL || 1087 nr_pages > tlb_single_page_flush_ceiling); 1088 } else { 1089 is_local: 1090 local = true; 1091 full = (end == TLB_FLUSH_ALL || 1092 nr_pages > tlb_local_single_page_flush_ceiling); 1093 } 1094 1095 if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { 1096 unsigned long tgt = H_RPTI_TARGET_CMMU; 1097 unsigned long type = H_RPTI_TYPE_TLB; 1098 unsigned long pg_sizes = psize_to_rpti_pgsize(psize); 1099 1100 if (also_pwc) 1101 type |= H_RPTI_TYPE_PWC; 1102 if (atomic_read(&mm->context.copros) > 0) 1103 tgt |= H_RPTI_TARGET_NMMU; 1104 pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end); 1105 } else if (full) { 1106 if (local) { 1107 _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 1108 } else { 1109 if (cputlb_use_tlbie()) { 1110 if (mm_needs_flush_escalation(mm)) 1111 also_pwc = true; 1112 1113 _tlbie_pid(pid, 1114 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 1115 } else { 1116 _tlbiel_pid_multicast(mm, pid, 1117 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 1118 } 1119 1120 } 1121 } else { 1122 if (local) 1123 _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc); 1124 else if (cputlb_use_tlbie()) 1125 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); 1126 else 1127 _tlbiel_va_range_multicast(mm, 1128 start, end, pid, page_size, psize, also_pwc); 1129 } 1130 preempt_enable(); 1131 } 1132 1133 void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, 1134 unsigned long end, int psize) 1135 { 1136 return __radix__flush_tlb_range_psize(mm, start, end, psize, false); 1137 } 1138 1139 static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 1140 unsigned long end, int psize) 1141 { 1142 __radix__flush_tlb_range_psize(mm, start, end, psize, true); 1143 } 1144 1145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1146 void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) 1147 { 1148 unsigned long pid, end; 1149 1150 pid = mm->context.id; 1151 if (unlikely(pid == MMU_NO_CONTEXT)) 1152 return; 1153 1154 /* 4k page size, just blow the world */ 1155 if (PAGE_SIZE == 0x1000) { 1156 radix__flush_all_mm(mm); 1157 return; 1158 } 1159 1160 end = addr + HPAGE_PMD_SIZE; 1161 1162 /* Otherwise first do the PWC, then iterate the pages. */ 1163 preempt_disable(); 1164 smp_mb(); /* see radix__flush_tlb_mm */ 1165 if (!mm_is_thread_local(mm)) { 1166 if (unlikely(mm_is_singlethreaded(mm))) { 1167 exit_flush_lazy_tlbs(mm); 1168 goto local; 1169 } 1170 if (!mmu_has_feature(MMU_FTR_GTSE)) { 1171 unsigned long tgt, type, pg_sizes; 1172 1173 tgt = H_RPTI_TARGET_CMMU; 1174 type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | 1175 H_RPTI_TYPE_PRT; 1176 pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize); 1177 1178 if (atomic_read(&mm->context.copros) > 0) 1179 tgt |= H_RPTI_TARGET_NMMU; 1180 pseries_rpt_invalidate(pid, tgt, type, pg_sizes, 1181 addr, end); 1182 } else if (cputlb_use_tlbie()) 1183 _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 1184 else 1185 _tlbiel_va_range_multicast(mm, 1186 addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 1187 } else { 1188 local: 1189 _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 1190 } 1191 1192 preempt_enable(); 1193 } 1194 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1195 1196 void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, 1197 unsigned long start, unsigned long end) 1198 { 1199 radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); 1200 } 1201 EXPORT_SYMBOL(radix__flush_pmd_tlb_range); 1202 1203 void radix__flush_tlb_all(void) 1204 { 1205 unsigned long rb,prs,r,rs; 1206 unsigned long ric = RIC_FLUSH_ALL; 1207 1208 rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */ 1209 prs = 0; /* partition scoped */ 1210 r = 1; /* radix format */ 1211 rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */ 1212 1213 asm volatile("ptesync": : :"memory"); 1214 /* 1215 * now flush guest entries by passing PRS = 1 and LPID != 0 1216 */ 1217 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 1218 : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory"); 1219 /* 1220 * now flush host entires by passing PRS = 0 and LPID == 0 1221 */ 1222 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 1223 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory"); 1224 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 1225 } 1226 1227 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1228 extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) 1229 { 1230 unsigned long pid = mm->context.id; 1231 1232 if (unlikely(pid == MMU_NO_CONTEXT)) 1233 return; 1234 1235 if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 1236 return; 1237 1238 /* 1239 * If this context hasn't run on that CPU before and KVM is 1240 * around, there's a slim chance that the guest on another 1241 * CPU just brought in obsolete translation into the TLB of 1242 * this CPU due to a bad prefetch using the guest PID on 1243 * the way into the hypervisor. 1244 * 1245 * We work around this here. If KVM is possible, we check if 1246 * any sibling thread is in KVM. If it is, the window may exist 1247 * and thus we flush that PID from the core. 1248 * 1249 * A potential future improvement would be to mark which PIDs 1250 * have never been used on the system and avoid it if the PID 1251 * is new and the process has no other cpumask bit set. 1252 */ 1253 if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) { 1254 int cpu = smp_processor_id(); 1255 int sib = cpu_first_thread_sibling(cpu); 1256 bool flush = false; 1257 1258 for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) { 1259 if (sib == cpu) 1260 continue; 1261 if (!cpu_possible(sib)) 1262 continue; 1263 if (paca_ptrs[sib]->kvm_hstate.kvm_vcpu) 1264 flush = true; 1265 } 1266 if (flush) 1267 _tlbiel_pid(pid, RIC_FLUSH_ALL); 1268 } 1269 } 1270 EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround); 1271 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 1272