12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 247d99948SChristophe Leroy /* 347d99948SChristophe Leroy * TLB flush routines for radix kernels. 447d99948SChristophe Leroy * 547d99948SChristophe Leroy * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation. 647d99948SChristophe Leroy */ 747d99948SChristophe Leroy 847d99948SChristophe Leroy #include <linux/mm.h> 947d99948SChristophe Leroy #include <linux/hugetlb.h> 1047d99948SChristophe Leroy #include <linux/memblock.h> 1147d99948SChristophe Leroy #include <linux/mmu_context.h> 1247d99948SChristophe Leroy #include <linux/sched/mm.h> 1347d99948SChristophe Leroy 1447d99948SChristophe Leroy #include <asm/ppc-opcode.h> 1547d99948SChristophe Leroy #include <asm/tlb.h> 1647d99948SChristophe Leroy #include <asm/tlbflush.h> 1747d99948SChristophe Leroy #include <asm/trace.h> 1847d99948SChristophe Leroy #include <asm/cputhreads.h> 1947d99948SChristophe Leroy 2047d99948SChristophe Leroy #define RIC_FLUSH_TLB 0 2147d99948SChristophe Leroy #define RIC_FLUSH_PWC 1 2247d99948SChristophe Leroy #define RIC_FLUSH_ALL 2 2347d99948SChristophe Leroy 2447d99948SChristophe Leroy /* 2547d99948SChristophe Leroy * tlbiel instruction for radix, set invalidation 2647d99948SChristophe Leroy * i.e., r=1 and is=01 or is=10 or is=11 2747d99948SChristophe Leroy */ 286d3ca7e7SMasahiro Yamada static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, 2947d99948SChristophe Leroy unsigned int pid, 3047d99948SChristophe Leroy unsigned int ric, unsigned int prs) 3147d99948SChristophe Leroy { 3247d99948SChristophe Leroy unsigned long rb; 3347d99948SChristophe Leroy unsigned long rs; 3447d99948SChristophe Leroy 3547d99948SChristophe Leroy rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); 3647d99948SChristophe Leroy rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); 3747d99948SChristophe Leroy 3847d99948SChristophe Leroy asm volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 3947d99948SChristophe Leroy : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 4047d99948SChristophe Leroy : "memory"); 4147d99948SChristophe Leroy } 4247d99948SChristophe Leroy 4347d99948SChristophe Leroy static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) 4447d99948SChristophe Leroy { 4547d99948SChristophe Leroy unsigned int set; 4647d99948SChristophe Leroy 4747d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 4847d99948SChristophe Leroy 4947d99948SChristophe Leroy /* 5047d99948SChristophe Leroy * Flush the first set of the TLB, and the entire Page Walk Cache 5147d99948SChristophe Leroy * and partition table entries. Then flush the remaining sets of the 5247d99948SChristophe Leroy * TLB. 5347d99948SChristophe Leroy */ 547e71c428SNicholas Piggin 557e71c428SNicholas Piggin if (early_cpu_has_feature(CPU_FTR_HVMODE)) { 567e71c428SNicholas Piggin /* MSR[HV] should flush partition scope translations first. */ 5747d99948SChristophe Leroy tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 5847d99948SChristophe Leroy for (set = 1; set < num_sets; set++) 5947d99948SChristophe Leroy tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 607e71c428SNicholas Piggin } 6147d99948SChristophe Leroy 627e71c428SNicholas Piggin /* Flush process scoped entries. */ 6347d99948SChristophe Leroy tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 6447d99948SChristophe Leroy for (set = 1; set < num_sets; set++) 6547d99948SChristophe Leroy tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 6647d99948SChristophe Leroy 6747d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 6847d99948SChristophe Leroy } 6947d99948SChristophe Leroy 7047d99948SChristophe Leroy void radix__tlbiel_all(unsigned int action) 7147d99948SChristophe Leroy { 7247d99948SChristophe Leroy unsigned int is; 7347d99948SChristophe Leroy 7447d99948SChristophe Leroy switch (action) { 7547d99948SChristophe Leroy case TLB_INVAL_SCOPE_GLOBAL: 7647d99948SChristophe Leroy is = 3; 7747d99948SChristophe Leroy break; 7847d99948SChristophe Leroy case TLB_INVAL_SCOPE_LPID: 7947d99948SChristophe Leroy is = 2; 8047d99948SChristophe Leroy break; 8147d99948SChristophe Leroy default: 8247d99948SChristophe Leroy BUG(); 8347d99948SChristophe Leroy } 8447d99948SChristophe Leroy 8547d99948SChristophe Leroy if (early_cpu_has_feature(CPU_FTR_ARCH_300)) 8647d99948SChristophe Leroy tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is); 8747d99948SChristophe Leroy else 8847d99948SChristophe Leroy WARN(1, "%s called on pre-POWER9 CPU\n", __func__); 8947d99948SChristophe Leroy 90fe7946ceSNicholas Piggin asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); 9147d99948SChristophe Leroy } 9247d99948SChristophe Leroy 93efc344c5SMasahiro Yamada static __always_inline void __tlbiel_pid(unsigned long pid, int set, 9447d99948SChristophe Leroy unsigned long ric) 9547d99948SChristophe Leroy { 9647d99948SChristophe Leroy unsigned long rb,rs,prs,r; 9747d99948SChristophe Leroy 9847d99948SChristophe Leroy rb = PPC_BIT(53); /* IS = 1 */ 9947d99948SChristophe Leroy rb |= set << PPC_BITLSHIFT(51); 10047d99948SChristophe Leroy rs = ((unsigned long)pid) << PPC_BITLSHIFT(31); 10147d99948SChristophe Leroy prs = 1; /* process scoped */ 10247d99948SChristophe Leroy r = 1; /* radix format */ 10347d99948SChristophe Leroy 10447d99948SChristophe Leroy asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 10547d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 10647d99948SChristophe Leroy trace_tlbie(0, 1, rb, rs, ric, prs, r); 10747d99948SChristophe Leroy } 10847d99948SChristophe Leroy 109efc344c5SMasahiro Yamada static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric) 11047d99948SChristophe Leroy { 11147d99948SChristophe Leroy unsigned long rb,rs,prs,r; 11247d99948SChristophe Leroy 11347d99948SChristophe Leroy rb = PPC_BIT(53); /* IS = 1 */ 11447d99948SChristophe Leroy rs = pid << PPC_BITLSHIFT(31); 11547d99948SChristophe Leroy prs = 1; /* process scoped */ 11647d99948SChristophe Leroy r = 1; /* radix format */ 11747d99948SChristophe Leroy 11847d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 11947d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 12047d99948SChristophe Leroy trace_tlbie(0, 0, rb, rs, ric, prs, r); 12147d99948SChristophe Leroy } 12247d99948SChristophe Leroy 123efc344c5SMasahiro Yamada static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric) 12447d99948SChristophe Leroy { 12547d99948SChristophe Leroy unsigned long rb,rs,prs,r; 12647d99948SChristophe Leroy 12747d99948SChristophe Leroy rb = PPC_BIT(52); /* IS = 2 */ 12847d99948SChristophe Leroy rs = lpid; 12947d99948SChristophe Leroy prs = 0; /* partition scoped */ 13047d99948SChristophe Leroy r = 1; /* radix format */ 13147d99948SChristophe Leroy 13247d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 13347d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 13447d99948SChristophe Leroy trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 13547d99948SChristophe Leroy } 13647d99948SChristophe Leroy 13799161de3SNicholas Piggin static __always_inline void __tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 13847d99948SChristophe Leroy { 13947d99948SChristophe Leroy unsigned long rb,rs,prs,r; 14047d99948SChristophe Leroy 14147d99948SChristophe Leroy rb = PPC_BIT(52); /* IS = 2 */ 14299161de3SNicholas Piggin rs = lpid; 14347d99948SChristophe Leroy prs = 1; /* process scoped */ 14447d99948SChristophe Leroy r = 1; /* radix format */ 14547d99948SChristophe Leroy 14699161de3SNicholas Piggin asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 14747d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 14899161de3SNicholas Piggin trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 14947d99948SChristophe Leroy } 15047d99948SChristophe Leroy 1516d3ca7e7SMasahiro Yamada static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid, 15247d99948SChristophe Leroy unsigned long ap, unsigned long ric) 15347d99948SChristophe Leroy { 15447d99948SChristophe Leroy unsigned long rb,rs,prs,r; 15547d99948SChristophe Leroy 15647d99948SChristophe Leroy rb = va & ~(PPC_BITMASK(52, 63)); 15747d99948SChristophe Leroy rb |= ap << PPC_BITLSHIFT(58); 15847d99948SChristophe Leroy rs = pid << PPC_BITLSHIFT(31); 15947d99948SChristophe Leroy prs = 1; /* process scoped */ 16047d99948SChristophe Leroy r = 1; /* radix format */ 16147d99948SChristophe Leroy 16247d99948SChristophe Leroy asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 16347d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 16447d99948SChristophe Leroy trace_tlbie(0, 1, rb, rs, ric, prs, r); 16547d99948SChristophe Leroy } 16647d99948SChristophe Leroy 1676d3ca7e7SMasahiro Yamada static __always_inline void __tlbie_va(unsigned long va, unsigned long pid, 16847d99948SChristophe Leroy unsigned long ap, unsigned long ric) 16947d99948SChristophe Leroy { 17047d99948SChristophe Leroy unsigned long rb,rs,prs,r; 17147d99948SChristophe Leroy 17247d99948SChristophe Leroy rb = va & ~(PPC_BITMASK(52, 63)); 17347d99948SChristophe Leroy rb |= ap << PPC_BITLSHIFT(58); 17447d99948SChristophe Leroy rs = pid << PPC_BITLSHIFT(31); 17547d99948SChristophe Leroy prs = 1; /* process scoped */ 17647d99948SChristophe Leroy r = 1; /* radix format */ 17747d99948SChristophe Leroy 17847d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 17947d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 18047d99948SChristophe Leroy trace_tlbie(0, 0, rb, rs, ric, prs, r); 18147d99948SChristophe Leroy } 18247d99948SChristophe Leroy 1836d3ca7e7SMasahiro Yamada static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, 18447d99948SChristophe Leroy unsigned long ap, unsigned long ric) 18547d99948SChristophe Leroy { 18647d99948SChristophe Leroy unsigned long rb,rs,prs,r; 18747d99948SChristophe Leroy 18847d99948SChristophe Leroy rb = va & ~(PPC_BITMASK(52, 63)); 18947d99948SChristophe Leroy rb |= ap << PPC_BITLSHIFT(58); 19047d99948SChristophe Leroy rs = lpid; 19147d99948SChristophe Leroy prs = 0; /* partition scoped */ 19247d99948SChristophe Leroy r = 1; /* radix format */ 19347d99948SChristophe Leroy 19447d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 19547d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 19647d99948SChristophe Leroy trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 19747d99948SChristophe Leroy } 19847d99948SChristophe Leroy 199047e6575SAneesh Kumar K.V 200047e6575SAneesh Kumar K.V static inline void fixup_tlbie_va(unsigned long va, unsigned long pid, 201047e6575SAneesh Kumar K.V unsigned long ap) 20247d99948SChristophe Leroy { 203047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 204047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 205047e6575SAneesh Kumar K.V __tlbie_va(va, 0, ap, RIC_FLUSH_TLB); 206047e6575SAneesh Kumar K.V } 207047e6575SAneesh Kumar K.V 208047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 209047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 210047e6575SAneesh Kumar K.V __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 211047e6575SAneesh Kumar K.V } 212047e6575SAneesh Kumar K.V } 213047e6575SAneesh Kumar K.V 214047e6575SAneesh Kumar K.V static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid, 215047e6575SAneesh Kumar K.V unsigned long ap) 216047e6575SAneesh Kumar K.V { 217047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 218047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 219047e6575SAneesh Kumar K.V __tlbie_pid(0, RIC_FLUSH_TLB); 220047e6575SAneesh Kumar K.V } 221047e6575SAneesh Kumar K.V 222047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 223047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 224047e6575SAneesh Kumar K.V __tlbie_va(va, pid, ap, RIC_FLUSH_TLB); 225047e6575SAneesh Kumar K.V } 226047e6575SAneesh Kumar K.V } 227047e6575SAneesh Kumar K.V 228047e6575SAneesh Kumar K.V static inline void fixup_tlbie_pid(unsigned long pid) 229047e6575SAneesh Kumar K.V { 230047e6575SAneesh Kumar K.V /* 231047e6575SAneesh Kumar K.V * We can use any address for the invalidation, pick one which is 232047e6575SAneesh Kumar K.V * probably unused as an optimisation. 233047e6575SAneesh Kumar K.V */ 23447d99948SChristophe Leroy unsigned long va = ((1UL << 52) - 1); 23547d99948SChristophe Leroy 236047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 237047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 238047e6575SAneesh Kumar K.V __tlbie_pid(0, RIC_FLUSH_TLB); 239047e6575SAneesh Kumar K.V } 240047e6575SAneesh Kumar K.V 24109ce98caSAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 24247d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 24347d99948SChristophe Leroy __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 24447d99948SChristophe Leroy } 24547d99948SChristophe Leroy } 24647d99948SChristophe Leroy 247047e6575SAneesh Kumar K.V 248047e6575SAneesh Kumar K.V static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid, 249047e6575SAneesh Kumar K.V unsigned long ap) 250047e6575SAneesh Kumar K.V { 251047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 252047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 253047e6575SAneesh Kumar K.V __tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB); 254047e6575SAneesh Kumar K.V } 255047e6575SAneesh Kumar K.V 256047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 257047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 258047e6575SAneesh Kumar K.V __tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB); 259047e6575SAneesh Kumar K.V } 260047e6575SAneesh Kumar K.V } 261047e6575SAneesh Kumar K.V 26247d99948SChristophe Leroy static inline void fixup_tlbie_lpid(unsigned long lpid) 26347d99948SChristophe Leroy { 264047e6575SAneesh Kumar K.V /* 265047e6575SAneesh Kumar K.V * We can use any address for the invalidation, pick one which is 266047e6575SAneesh Kumar K.V * probably unused as an optimisation. 267047e6575SAneesh Kumar K.V */ 26847d99948SChristophe Leroy unsigned long va = ((1UL << 52) - 1); 26947d99948SChristophe Leroy 270047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { 271047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory"); 272047e6575SAneesh Kumar K.V __tlbie_lpid(0, RIC_FLUSH_TLB); 273047e6575SAneesh Kumar K.V } 274047e6575SAneesh Kumar K.V 27509ce98caSAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { 27647d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 27747d99948SChristophe Leroy __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 27847d99948SChristophe Leroy } 27947d99948SChristophe Leroy } 28047d99948SChristophe Leroy 28147d99948SChristophe Leroy /* 28247d99948SChristophe Leroy * We use 128 set in radix mode and 256 set in hpt mode. 28347d99948SChristophe Leroy */ 2846d3ca7e7SMasahiro Yamada static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric) 28547d99948SChristophe Leroy { 28647d99948SChristophe Leroy int set; 28747d99948SChristophe Leroy 28847d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 28947d99948SChristophe Leroy 29047d99948SChristophe Leroy /* 29147d99948SChristophe Leroy * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL, 29247d99948SChristophe Leroy * also flush the entire Page Walk Cache. 29347d99948SChristophe Leroy */ 29447d99948SChristophe Leroy __tlbiel_pid(pid, 0, ric); 29547d99948SChristophe Leroy 29647d99948SChristophe Leroy /* For PWC, only one flush is needed */ 29747d99948SChristophe Leroy if (ric == RIC_FLUSH_PWC) { 29847d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 29947d99948SChristophe Leroy return; 30047d99948SChristophe Leroy } 30147d99948SChristophe Leroy 30247d99948SChristophe Leroy /* For the remaining sets, just flush the TLB */ 30347d99948SChristophe Leroy for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++) 30447d99948SChristophe Leroy __tlbiel_pid(pid, set, RIC_FLUSH_TLB); 30547d99948SChristophe Leroy 30647d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 3076c46fcceSNicholas Piggin asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory"); 30847d99948SChristophe Leroy } 30947d99948SChristophe Leroy 31047d99948SChristophe Leroy static inline void _tlbie_pid(unsigned long pid, unsigned long ric) 31147d99948SChristophe Leroy { 31247d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 31347d99948SChristophe Leroy 31447d99948SChristophe Leroy /* 31547d99948SChristophe Leroy * Workaround the fact that the "ric" argument to __tlbie_pid 31647d99948SChristophe Leroy * must be a compile-time contraint to match the "i" constraint 31747d99948SChristophe Leroy * in the asm statement. 31847d99948SChristophe Leroy */ 31947d99948SChristophe Leroy switch (ric) { 32047d99948SChristophe Leroy case RIC_FLUSH_TLB: 32147d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_TLB); 322047e6575SAneesh Kumar K.V fixup_tlbie_pid(pid); 32347d99948SChristophe Leroy break; 32447d99948SChristophe Leroy case RIC_FLUSH_PWC: 32547d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_PWC); 32647d99948SChristophe Leroy break; 32747d99948SChristophe Leroy case RIC_FLUSH_ALL: 32847d99948SChristophe Leroy default: 32947d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_ALL); 330047e6575SAneesh Kumar K.V fixup_tlbie_pid(pid); 33147d99948SChristophe Leroy } 33247d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 33347d99948SChristophe Leroy } 33447d99948SChristophe Leroy 3352275d7b5SNicholas Piggin struct tlbiel_pid { 3362275d7b5SNicholas Piggin unsigned long pid; 3372275d7b5SNicholas Piggin unsigned long ric; 3382275d7b5SNicholas Piggin }; 3392275d7b5SNicholas Piggin 3402275d7b5SNicholas Piggin static void do_tlbiel_pid(void *info) 3412275d7b5SNicholas Piggin { 3422275d7b5SNicholas Piggin struct tlbiel_pid *t = info; 3432275d7b5SNicholas Piggin 3442275d7b5SNicholas Piggin if (t->ric == RIC_FLUSH_TLB) 3452275d7b5SNicholas Piggin _tlbiel_pid(t->pid, RIC_FLUSH_TLB); 3462275d7b5SNicholas Piggin else if (t->ric == RIC_FLUSH_PWC) 3472275d7b5SNicholas Piggin _tlbiel_pid(t->pid, RIC_FLUSH_PWC); 3482275d7b5SNicholas Piggin else 3492275d7b5SNicholas Piggin _tlbiel_pid(t->pid, RIC_FLUSH_ALL); 3502275d7b5SNicholas Piggin } 3512275d7b5SNicholas Piggin 3522275d7b5SNicholas Piggin static inline void _tlbiel_pid_multicast(struct mm_struct *mm, 3532275d7b5SNicholas Piggin unsigned long pid, unsigned long ric) 3542275d7b5SNicholas Piggin { 3552275d7b5SNicholas Piggin struct cpumask *cpus = mm_cpumask(mm); 3562275d7b5SNicholas Piggin struct tlbiel_pid t = { .pid = pid, .ric = ric }; 3572275d7b5SNicholas Piggin 3582275d7b5SNicholas Piggin on_each_cpu_mask(cpus, do_tlbiel_pid, &t, 1); 3592275d7b5SNicholas Piggin /* 3602275d7b5SNicholas Piggin * Always want the CPU translations to be invalidated with tlbiel in 3612275d7b5SNicholas Piggin * these paths, so while coprocessors must use tlbie, we can not 3622275d7b5SNicholas Piggin * optimise away the tlbiel component. 3632275d7b5SNicholas Piggin */ 3642275d7b5SNicholas Piggin if (atomic_read(&mm->context.copros) > 0) 3652275d7b5SNicholas Piggin _tlbie_pid(pid, RIC_FLUSH_ALL); 3662275d7b5SNicholas Piggin } 3672275d7b5SNicholas Piggin 36847d99948SChristophe Leroy static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) 36947d99948SChristophe Leroy { 37047d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 37147d99948SChristophe Leroy 37247d99948SChristophe Leroy /* 37347d99948SChristophe Leroy * Workaround the fact that the "ric" argument to __tlbie_pid 37447d99948SChristophe Leroy * must be a compile-time contraint to match the "i" constraint 37547d99948SChristophe Leroy * in the asm statement. 37647d99948SChristophe Leroy */ 37747d99948SChristophe Leroy switch (ric) { 37847d99948SChristophe Leroy case RIC_FLUSH_TLB: 37947d99948SChristophe Leroy __tlbie_lpid(lpid, RIC_FLUSH_TLB); 380047e6575SAneesh Kumar K.V fixup_tlbie_lpid(lpid); 38147d99948SChristophe Leroy break; 38247d99948SChristophe Leroy case RIC_FLUSH_PWC: 38347d99948SChristophe Leroy __tlbie_lpid(lpid, RIC_FLUSH_PWC); 38447d99948SChristophe Leroy break; 38547d99948SChristophe Leroy case RIC_FLUSH_ALL: 38647d99948SChristophe Leroy default: 38747d99948SChristophe Leroy __tlbie_lpid(lpid, RIC_FLUSH_ALL); 38847d99948SChristophe Leroy fixup_tlbie_lpid(lpid); 389047e6575SAneesh Kumar K.V } 39047d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 39147d99948SChristophe Leroy } 39247d99948SChristophe Leroy 39399161de3SNicholas Piggin static __always_inline void _tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 39447d99948SChristophe Leroy { 39547d99948SChristophe Leroy /* 39699161de3SNicholas Piggin * Workaround the fact that the "ric" argument to __tlbie_pid 39799161de3SNicholas Piggin * must be a compile-time contraint to match the "i" constraint 39899161de3SNicholas Piggin * in the asm statement. 39947d99948SChristophe Leroy */ 40099161de3SNicholas Piggin switch (ric) { 40199161de3SNicholas Piggin case RIC_FLUSH_TLB: 40299161de3SNicholas Piggin __tlbie_lpid_guest(lpid, RIC_FLUSH_TLB); 40399161de3SNicholas Piggin break; 40499161de3SNicholas Piggin case RIC_FLUSH_PWC: 40599161de3SNicholas Piggin __tlbie_lpid_guest(lpid, RIC_FLUSH_PWC); 40699161de3SNicholas Piggin break; 40799161de3SNicholas Piggin case RIC_FLUSH_ALL: 40899161de3SNicholas Piggin default: 40999161de3SNicholas Piggin __tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 41047d99948SChristophe Leroy } 41199161de3SNicholas Piggin fixup_tlbie_lpid(lpid); 41299161de3SNicholas Piggin asm volatile("eieio; tlbsync; ptesync": : :"memory"); 41347d99948SChristophe Leroy } 41447d99948SChristophe Leroy 41547d99948SChristophe Leroy static inline void __tlbiel_va_range(unsigned long start, unsigned long end, 41647d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 41747d99948SChristophe Leroy unsigned long psize) 41847d99948SChristophe Leroy { 41947d99948SChristophe Leroy unsigned long addr; 42047d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 42147d99948SChristophe Leroy 42247d99948SChristophe Leroy for (addr = start; addr < end; addr += page_size) 42347d99948SChristophe Leroy __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB); 42447d99948SChristophe Leroy } 42547d99948SChristophe Leroy 4266d3ca7e7SMasahiro Yamada static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid, 42747d99948SChristophe Leroy unsigned long psize, unsigned long ric) 42847d99948SChristophe Leroy { 42947d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 43047d99948SChristophe Leroy 43147d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 43247d99948SChristophe Leroy __tlbiel_va(va, pid, ap, ric); 43347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 43447d99948SChristophe Leroy } 43547d99948SChristophe Leroy 43647d99948SChristophe Leroy static inline void _tlbiel_va_range(unsigned long start, unsigned long end, 43747d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 43847d99948SChristophe Leroy unsigned long psize, bool also_pwc) 43947d99948SChristophe Leroy { 44047d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 44147d99948SChristophe Leroy if (also_pwc) 44247d99948SChristophe Leroy __tlbiel_pid(pid, 0, RIC_FLUSH_PWC); 44347d99948SChristophe Leroy __tlbiel_va_range(start, end, pid, page_size, psize); 44447d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 44547d99948SChristophe Leroy } 44647d99948SChristophe Leroy 44747d99948SChristophe Leroy static inline void __tlbie_va_range(unsigned long start, unsigned long end, 44847d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 44947d99948SChristophe Leroy unsigned long psize) 45047d99948SChristophe Leroy { 45147d99948SChristophe Leroy unsigned long addr; 45247d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 45347d99948SChristophe Leroy 45447d99948SChristophe Leroy for (addr = start; addr < end; addr += page_size) 45547d99948SChristophe Leroy __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB); 456047e6575SAneesh Kumar K.V 457047e6575SAneesh Kumar K.V fixup_tlbie_va_range(addr - page_size, pid, ap); 45847d99948SChristophe Leroy } 45947d99948SChristophe Leroy 4606d3ca7e7SMasahiro Yamada static __always_inline void _tlbie_va(unsigned long va, unsigned long pid, 46147d99948SChristophe Leroy unsigned long psize, unsigned long ric) 46247d99948SChristophe Leroy { 46347d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 46447d99948SChristophe Leroy 46547d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 46647d99948SChristophe Leroy __tlbie_va(va, pid, ap, ric); 467047e6575SAneesh Kumar K.V fixup_tlbie_va(va, pid, ap); 46847d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 46947d99948SChristophe Leroy } 47047d99948SChristophe Leroy 4712275d7b5SNicholas Piggin struct tlbiel_va { 4722275d7b5SNicholas Piggin unsigned long pid; 4732275d7b5SNicholas Piggin unsigned long va; 4742275d7b5SNicholas Piggin unsigned long psize; 4752275d7b5SNicholas Piggin unsigned long ric; 4762275d7b5SNicholas Piggin }; 4772275d7b5SNicholas Piggin 4782275d7b5SNicholas Piggin static void do_tlbiel_va(void *info) 4792275d7b5SNicholas Piggin { 4802275d7b5SNicholas Piggin struct tlbiel_va *t = info; 4812275d7b5SNicholas Piggin 4822275d7b5SNicholas Piggin if (t->ric == RIC_FLUSH_TLB) 4832275d7b5SNicholas Piggin _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB); 4842275d7b5SNicholas Piggin else if (t->ric == RIC_FLUSH_PWC) 4852275d7b5SNicholas Piggin _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC); 4862275d7b5SNicholas Piggin else 4872275d7b5SNicholas Piggin _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL); 4882275d7b5SNicholas Piggin } 4892275d7b5SNicholas Piggin 4902275d7b5SNicholas Piggin static inline void _tlbiel_va_multicast(struct mm_struct *mm, 4912275d7b5SNicholas Piggin unsigned long va, unsigned long pid, 4922275d7b5SNicholas Piggin unsigned long psize, unsigned long ric) 4932275d7b5SNicholas Piggin { 4942275d7b5SNicholas Piggin struct cpumask *cpus = mm_cpumask(mm); 4952275d7b5SNicholas Piggin struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric }; 4962275d7b5SNicholas Piggin on_each_cpu_mask(cpus, do_tlbiel_va, &t, 1); 4972275d7b5SNicholas Piggin if (atomic_read(&mm->context.copros) > 0) 4982275d7b5SNicholas Piggin _tlbie_va(va, pid, psize, RIC_FLUSH_TLB); 4992275d7b5SNicholas Piggin } 5002275d7b5SNicholas Piggin 5012275d7b5SNicholas Piggin struct tlbiel_va_range { 5022275d7b5SNicholas Piggin unsigned long pid; 5032275d7b5SNicholas Piggin unsigned long start; 5042275d7b5SNicholas Piggin unsigned long end; 5052275d7b5SNicholas Piggin unsigned long page_size; 5062275d7b5SNicholas Piggin unsigned long psize; 5072275d7b5SNicholas Piggin bool also_pwc; 5082275d7b5SNicholas Piggin }; 5092275d7b5SNicholas Piggin 5102275d7b5SNicholas Piggin static void do_tlbiel_va_range(void *info) 5112275d7b5SNicholas Piggin { 5122275d7b5SNicholas Piggin struct tlbiel_va_range *t = info; 5132275d7b5SNicholas Piggin 5142275d7b5SNicholas Piggin _tlbiel_va_range(t->start, t->end, t->pid, t->page_size, 5152275d7b5SNicholas Piggin t->psize, t->also_pwc); 5162275d7b5SNicholas Piggin } 5172275d7b5SNicholas Piggin 5186d3ca7e7SMasahiro Yamada static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid, 51947d99948SChristophe Leroy unsigned long psize, unsigned long ric) 52047d99948SChristophe Leroy { 52147d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 52247d99948SChristophe Leroy 52347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 52447d99948SChristophe Leroy __tlbie_lpid_va(va, lpid, ap, ric); 525047e6575SAneesh Kumar K.V fixup_tlbie_lpid_va(va, lpid, ap); 52647d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 52747d99948SChristophe Leroy } 52847d99948SChristophe Leroy 52947d99948SChristophe Leroy static inline void _tlbie_va_range(unsigned long start, unsigned long end, 53047d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 53147d99948SChristophe Leroy unsigned long psize, bool also_pwc) 53247d99948SChristophe Leroy { 53347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 53447d99948SChristophe Leroy if (also_pwc) 53547d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_PWC); 53647d99948SChristophe Leroy __tlbie_va_range(start, end, pid, page_size, psize); 53747d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 53847d99948SChristophe Leroy } 53947d99948SChristophe Leroy 5402275d7b5SNicholas Piggin static inline void _tlbiel_va_range_multicast(struct mm_struct *mm, 5412275d7b5SNicholas Piggin unsigned long start, unsigned long end, 5422275d7b5SNicholas Piggin unsigned long pid, unsigned long page_size, 5432275d7b5SNicholas Piggin unsigned long psize, bool also_pwc) 5442275d7b5SNicholas Piggin { 5452275d7b5SNicholas Piggin struct cpumask *cpus = mm_cpumask(mm); 5462275d7b5SNicholas Piggin struct tlbiel_va_range t = { .start = start, .end = end, 5472275d7b5SNicholas Piggin .pid = pid, .page_size = page_size, 5482275d7b5SNicholas Piggin .psize = psize, .also_pwc = also_pwc }; 5492275d7b5SNicholas Piggin 5502275d7b5SNicholas Piggin on_each_cpu_mask(cpus, do_tlbiel_va_range, &t, 1); 5512275d7b5SNicholas Piggin if (atomic_read(&mm->context.copros) > 0) 5522275d7b5SNicholas Piggin _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); 5532275d7b5SNicholas Piggin } 5542275d7b5SNicholas Piggin 55547d99948SChristophe Leroy /* 55647d99948SChristophe Leroy * Base TLB flushing operations: 55747d99948SChristophe Leroy * 55847d99948SChristophe Leroy * - flush_tlb_mm(mm) flushes the specified mm context TLB's 55947d99948SChristophe Leroy * - flush_tlb_page(vma, vmaddr) flushes one page 56047d99948SChristophe Leroy * - flush_tlb_range(vma, start, end) flushes a range of pages 56147d99948SChristophe Leroy * - flush_tlb_kernel_range(start, end) flushes kernel pages 56247d99948SChristophe Leroy * 56347d99948SChristophe Leroy * - local_* variants of page and mm only apply to the current 56447d99948SChristophe Leroy * processor 56547d99948SChristophe Leroy */ 56647d99948SChristophe Leroy void radix__local_flush_tlb_mm(struct mm_struct *mm) 56747d99948SChristophe Leroy { 56847d99948SChristophe Leroy unsigned long pid; 56947d99948SChristophe Leroy 57047d99948SChristophe Leroy preempt_disable(); 57147d99948SChristophe Leroy pid = mm->context.id; 57247d99948SChristophe Leroy if (pid != MMU_NO_CONTEXT) 57347d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_TLB); 57447d99948SChristophe Leroy preempt_enable(); 57547d99948SChristophe Leroy } 57647d99948SChristophe Leroy EXPORT_SYMBOL(radix__local_flush_tlb_mm); 57747d99948SChristophe Leroy 57847d99948SChristophe Leroy #ifndef CONFIG_SMP 57947d99948SChristophe Leroy void radix__local_flush_all_mm(struct mm_struct *mm) 58047d99948SChristophe Leroy { 58147d99948SChristophe Leroy unsigned long pid; 58247d99948SChristophe Leroy 58347d99948SChristophe Leroy preempt_disable(); 58447d99948SChristophe Leroy pid = mm->context.id; 58547d99948SChristophe Leroy if (pid != MMU_NO_CONTEXT) 58647d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 58747d99948SChristophe Leroy preempt_enable(); 58847d99948SChristophe Leroy } 58947d99948SChristophe Leroy EXPORT_SYMBOL(radix__local_flush_all_mm); 59047d99948SChristophe Leroy #endif /* CONFIG_SMP */ 59147d99948SChristophe Leroy 59247d99948SChristophe Leroy void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 59347d99948SChristophe Leroy int psize) 59447d99948SChristophe Leroy { 59547d99948SChristophe Leroy unsigned long pid; 59647d99948SChristophe Leroy 59747d99948SChristophe Leroy preempt_disable(); 59847d99948SChristophe Leroy pid = mm->context.id; 59947d99948SChristophe Leroy if (pid != MMU_NO_CONTEXT) 60047d99948SChristophe Leroy _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 60147d99948SChristophe Leroy preempt_enable(); 60247d99948SChristophe Leroy } 60347d99948SChristophe Leroy 60447d99948SChristophe Leroy void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 60547d99948SChristophe Leroy { 60647d99948SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 60747d99948SChristophe Leroy /* need the return fix for nohash.c */ 60847d99948SChristophe Leroy if (is_vm_hugetlb_page(vma)) 60947d99948SChristophe Leroy return radix__local_flush_hugetlb_page(vma, vmaddr); 61047d99948SChristophe Leroy #endif 61147d99948SChristophe Leroy radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 61247d99948SChristophe Leroy } 61347d99948SChristophe Leroy EXPORT_SYMBOL(radix__local_flush_tlb_page); 61447d99948SChristophe Leroy 61547d99948SChristophe Leroy static bool mm_is_singlethreaded(struct mm_struct *mm) 61647d99948SChristophe Leroy { 61747d99948SChristophe Leroy if (atomic_read(&mm->context.copros) > 0) 61847d99948SChristophe Leroy return false; 61947d99948SChristophe Leroy if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) 62047d99948SChristophe Leroy return true; 62147d99948SChristophe Leroy return false; 62247d99948SChristophe Leroy } 62347d99948SChristophe Leroy 62447d99948SChristophe Leroy static bool mm_needs_flush_escalation(struct mm_struct *mm) 62547d99948SChristophe Leroy { 62647d99948SChristophe Leroy /* 62747d99948SChristophe Leroy * P9 nest MMU has issues with the page walk cache 62847d99948SChristophe Leroy * caching PTEs and not flushing them properly when 62947d99948SChristophe Leroy * RIC = 0 for a PID/LPID invalidate 63047d99948SChristophe Leroy */ 63147d99948SChristophe Leroy if (atomic_read(&mm->context.copros) > 0) 63247d99948SChristophe Leroy return true; 63347d99948SChristophe Leroy return false; 63447d99948SChristophe Leroy } 63547d99948SChristophe Leroy 63647d99948SChristophe Leroy #ifdef CONFIG_SMP 63747d99948SChristophe Leroy static void do_exit_flush_lazy_tlb(void *arg) 63847d99948SChristophe Leroy { 63947d99948SChristophe Leroy struct mm_struct *mm = arg; 64047d99948SChristophe Leroy unsigned long pid = mm->context.id; 64147d99948SChristophe Leroy 64247d99948SChristophe Leroy if (current->mm == mm) 64347d99948SChristophe Leroy return; /* Local CPU */ 64447d99948SChristophe Leroy 64547d99948SChristophe Leroy if (current->active_mm == mm) { 64647d99948SChristophe Leroy /* 64747d99948SChristophe Leroy * Must be a kernel thread because sender is single-threaded. 64847d99948SChristophe Leroy */ 64947d99948SChristophe Leroy BUG_ON(current->mm); 65047d99948SChristophe Leroy mmgrab(&init_mm); 65147d99948SChristophe Leroy switch_mm(mm, &init_mm, current); 65247d99948SChristophe Leroy current->active_mm = &init_mm; 65347d99948SChristophe Leroy mmdrop(mm); 65447d99948SChristophe Leroy } 65547d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 65647d99948SChristophe Leroy } 65747d99948SChristophe Leroy 65847d99948SChristophe Leroy static void exit_flush_lazy_tlbs(struct mm_struct *mm) 65947d99948SChristophe Leroy { 66047d99948SChristophe Leroy /* 66147d99948SChristophe Leroy * Would be nice if this was async so it could be run in 66247d99948SChristophe Leroy * parallel with our local flush, but generic code does not 66347d99948SChristophe Leroy * give a good API for it. Could extend the generic code or 66447d99948SChristophe Leroy * make a special powerpc IPI for flushing TLBs. 66547d99948SChristophe Leroy * For now it's not too performance critical. 66647d99948SChristophe Leroy */ 66747d99948SChristophe Leroy smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb, 66847d99948SChristophe Leroy (void *)mm, 1); 66947d99948SChristophe Leroy mm_reset_thread_local(mm); 67047d99948SChristophe Leroy } 67147d99948SChristophe Leroy 67247d99948SChristophe Leroy void radix__flush_tlb_mm(struct mm_struct *mm) 67347d99948SChristophe Leroy { 67447d99948SChristophe Leroy unsigned long pid; 67547d99948SChristophe Leroy 67647d99948SChristophe Leroy pid = mm->context.id; 67747d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 67847d99948SChristophe Leroy return; 67947d99948SChristophe Leroy 68047d99948SChristophe Leroy preempt_disable(); 68147d99948SChristophe Leroy /* 68247d99948SChristophe Leroy * Order loads of mm_cpumask vs previous stores to clear ptes before 68347d99948SChristophe Leroy * the invalidate. See barrier in switch_mm_irqs_off 68447d99948SChristophe Leroy */ 68547d99948SChristophe Leroy smp_mb(); 68647d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 68747d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 68847d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 68947d99948SChristophe Leroy goto local; 69047d99948SChristophe Leroy } 69147d99948SChristophe Leroy 6922275d7b5SNicholas Piggin if (cputlb_use_tlbie()) { 69347d99948SChristophe Leroy if (mm_needs_flush_escalation(mm)) 69447d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_ALL); 69547d99948SChristophe Leroy else 69647d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_TLB); 69747d99948SChristophe Leroy } else { 6982275d7b5SNicholas Piggin _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); 6992275d7b5SNicholas Piggin } 7002275d7b5SNicholas Piggin } else { 70147d99948SChristophe Leroy local: 70247d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_TLB); 70347d99948SChristophe Leroy } 70447d99948SChristophe Leroy preempt_enable(); 70547d99948SChristophe Leroy } 70647d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_mm); 70747d99948SChristophe Leroy 70847d99948SChristophe Leroy static void __flush_all_mm(struct mm_struct *mm, bool fullmm) 70947d99948SChristophe Leroy { 71047d99948SChristophe Leroy unsigned long pid; 71147d99948SChristophe Leroy 71247d99948SChristophe Leroy pid = mm->context.id; 71347d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 71447d99948SChristophe Leroy return; 71547d99948SChristophe Leroy 71647d99948SChristophe Leroy preempt_disable(); 71747d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 71847d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 71947d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 72047d99948SChristophe Leroy if (!fullmm) { 72147d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 72247d99948SChristophe Leroy goto local; 72347d99948SChristophe Leroy } 72447d99948SChristophe Leroy } 7252275d7b5SNicholas Piggin if (cputlb_use_tlbie()) 72647d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_ALL); 7272275d7b5SNicholas Piggin else 7282275d7b5SNicholas Piggin _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); 72947d99948SChristophe Leroy } else { 73047d99948SChristophe Leroy local: 73147d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 73247d99948SChristophe Leroy } 73347d99948SChristophe Leroy preempt_enable(); 73447d99948SChristophe Leroy } 73547d99948SChristophe Leroy void radix__flush_all_mm(struct mm_struct *mm) 73647d99948SChristophe Leroy { 73747d99948SChristophe Leroy __flush_all_mm(mm, false); 73847d99948SChristophe Leroy } 73947d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_all_mm); 74047d99948SChristophe Leroy 74147d99948SChristophe Leroy void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr) 74247d99948SChristophe Leroy { 74347d99948SChristophe Leroy tlb->need_flush_all = 1; 74447d99948SChristophe Leroy } 74547d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_pwc); 74647d99948SChristophe Leroy 74747d99948SChristophe Leroy void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 74847d99948SChristophe Leroy int psize) 74947d99948SChristophe Leroy { 75047d99948SChristophe Leroy unsigned long pid; 75147d99948SChristophe Leroy 75247d99948SChristophe Leroy pid = mm->context.id; 75347d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 75447d99948SChristophe Leroy return; 75547d99948SChristophe Leroy 75647d99948SChristophe Leroy preempt_disable(); 75747d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 75847d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 75947d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 76047d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 76147d99948SChristophe Leroy goto local; 76247d99948SChristophe Leroy } 7632275d7b5SNicholas Piggin if (cputlb_use_tlbie()) 76447d99948SChristophe Leroy _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 7652275d7b5SNicholas Piggin else 7662275d7b5SNicholas Piggin _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); 76747d99948SChristophe Leroy } else { 76847d99948SChristophe Leroy local: 76947d99948SChristophe Leroy _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 77047d99948SChristophe Leroy } 77147d99948SChristophe Leroy preempt_enable(); 77247d99948SChristophe Leroy } 77347d99948SChristophe Leroy 77447d99948SChristophe Leroy void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 77547d99948SChristophe Leroy { 77647d99948SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 77747d99948SChristophe Leroy if (is_vm_hugetlb_page(vma)) 77847d99948SChristophe Leroy return radix__flush_hugetlb_page(vma, vmaddr); 77947d99948SChristophe Leroy #endif 78047d99948SChristophe Leroy radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 78147d99948SChristophe Leroy } 78247d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_page); 78347d99948SChristophe Leroy 78447d99948SChristophe Leroy #else /* CONFIG_SMP */ 78547d99948SChristophe Leroy #define radix__flush_all_mm radix__local_flush_all_mm 78647d99948SChristophe Leroy #endif /* CONFIG_SMP */ 78747d99948SChristophe Leroy 7882275d7b5SNicholas Piggin static void do_tlbiel_kernel(void *info) 7892275d7b5SNicholas Piggin { 7902275d7b5SNicholas Piggin _tlbiel_pid(0, RIC_FLUSH_ALL); 7912275d7b5SNicholas Piggin } 7922275d7b5SNicholas Piggin 7932275d7b5SNicholas Piggin static inline void _tlbiel_kernel_broadcast(void) 7942275d7b5SNicholas Piggin { 7952275d7b5SNicholas Piggin on_each_cpu(do_tlbiel_kernel, NULL, 1); 7962275d7b5SNicholas Piggin if (tlbie_capable) { 7972275d7b5SNicholas Piggin /* 7982275d7b5SNicholas Piggin * Coherent accelerators don't refcount kernel memory mappings, 7992275d7b5SNicholas Piggin * so have to always issue a tlbie for them. This is quite a 8002275d7b5SNicholas Piggin * slow path anyway. 8012275d7b5SNicholas Piggin */ 8022275d7b5SNicholas Piggin _tlbie_pid(0, RIC_FLUSH_ALL); 8032275d7b5SNicholas Piggin } 8042275d7b5SNicholas Piggin } 8052275d7b5SNicholas Piggin 80660e8523eSAlastair D'Silva /* 80760e8523eSAlastair D'Silva * If kernel TLBIs ever become local rather than global, then 80860e8523eSAlastair D'Silva * drivers/misc/ocxl/link.c:ocxl_link_add_pe will need some work, as it 80960e8523eSAlastair D'Silva * assumes kernel TLBIs are global. 81060e8523eSAlastair D'Silva */ 81147d99948SChristophe Leroy void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) 81247d99948SChristophe Leroy { 8132275d7b5SNicholas Piggin if (cputlb_use_tlbie()) 81447d99948SChristophe Leroy _tlbie_pid(0, RIC_FLUSH_ALL); 8152275d7b5SNicholas Piggin else 8162275d7b5SNicholas Piggin _tlbiel_kernel_broadcast(); 81747d99948SChristophe Leroy } 81847d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_kernel_range); 81947d99948SChristophe Leroy 82047d99948SChristophe Leroy #define TLB_FLUSH_ALL -1UL 82147d99948SChristophe Leroy 82247d99948SChristophe Leroy /* 82347d99948SChristophe Leroy * Number of pages above which we invalidate the entire PID rather than 82447d99948SChristophe Leroy * flush individual pages, for local and global flushes respectively. 82547d99948SChristophe Leroy * 82647d99948SChristophe Leroy * tlbie goes out to the interconnect and individual ops are more costly. 82747d99948SChristophe Leroy * It also does not iterate over sets like the local tlbiel variant when 82847d99948SChristophe Leroy * invalidating a full PID, so it has a far lower threshold to change from 82947d99948SChristophe Leroy * individual page flushes to full-pid flushes. 83047d99948SChristophe Leroy */ 83147d99948SChristophe Leroy static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; 83247d99948SChristophe Leroy static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2; 83347d99948SChristophe Leroy 83447d99948SChristophe Leroy static inline void __radix__flush_tlb_range(struct mm_struct *mm, 835a42d6ba8SAneesh Kumar K.V unsigned long start, unsigned long end) 83647d99948SChristophe Leroy 83747d99948SChristophe Leroy { 83847d99948SChristophe Leroy unsigned long pid; 83947d99948SChristophe Leroy unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; 84047d99948SChristophe Leroy unsigned long page_size = 1UL << page_shift; 84147d99948SChristophe Leroy unsigned long nr_pages = (end - start) >> page_shift; 84247d99948SChristophe Leroy bool local, full; 84347d99948SChristophe Leroy 84447d99948SChristophe Leroy pid = mm->context.id; 84547d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 84647d99948SChristophe Leroy return; 84747d99948SChristophe Leroy 84847d99948SChristophe Leroy preempt_disable(); 84947d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 85047d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 85147d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 85247d99948SChristophe Leroy if (end != TLB_FLUSH_ALL) { 85347d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 85447d99948SChristophe Leroy goto is_local; 85547d99948SChristophe Leroy } 85647d99948SChristophe Leroy } 85747d99948SChristophe Leroy local = false; 85847d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 85947d99948SChristophe Leroy nr_pages > tlb_single_page_flush_ceiling); 86047d99948SChristophe Leroy } else { 86147d99948SChristophe Leroy is_local: 86247d99948SChristophe Leroy local = true; 86347d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 86447d99948SChristophe Leroy nr_pages > tlb_local_single_page_flush_ceiling); 86547d99948SChristophe Leroy } 86647d99948SChristophe Leroy 86747d99948SChristophe Leroy if (full) { 86847d99948SChristophe Leroy if (local) { 86947d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_TLB); 87047d99948SChristophe Leroy } else { 8712275d7b5SNicholas Piggin if (cputlb_use_tlbie()) { 87247d99948SChristophe Leroy if (mm_needs_flush_escalation(mm)) 87347d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_ALL); 87447d99948SChristophe Leroy else 87547d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_TLB); 8762275d7b5SNicholas Piggin } else { 8772275d7b5SNicholas Piggin _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); 8782275d7b5SNicholas Piggin } 87947d99948SChristophe Leroy } 88047d99948SChristophe Leroy } else { 881a42d6ba8SAneesh Kumar K.V bool hflush = false; 88247d99948SChristophe Leroy unsigned long hstart, hend; 88347d99948SChristophe Leroy 884a42d6ba8SAneesh Kumar K.V if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 88547d99948SChristophe Leroy hstart = (start + PMD_SIZE - 1) & PMD_MASK; 88647d99948SChristophe Leroy hend = end & PMD_MASK; 88747d99948SChristophe Leroy if (hstart == hend) 88847d99948SChristophe Leroy hflush = false; 889a42d6ba8SAneesh Kumar K.V else 890a42d6ba8SAneesh Kumar K.V hflush = true; 89147d99948SChristophe Leroy } 89247d99948SChristophe Leroy 89347d99948SChristophe Leroy if (local) { 8942275d7b5SNicholas Piggin asm volatile("ptesync": : :"memory"); 89547d99948SChristophe Leroy __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); 89647d99948SChristophe Leroy if (hflush) 89747d99948SChristophe Leroy __tlbiel_va_range(hstart, hend, pid, 89847d99948SChristophe Leroy PMD_SIZE, MMU_PAGE_2M); 89947d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 9002275d7b5SNicholas Piggin } else if (cputlb_use_tlbie()) { 9012275d7b5SNicholas Piggin asm volatile("ptesync": : :"memory"); 90247d99948SChristophe Leroy __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize); 90347d99948SChristophe Leroy if (hflush) 90447d99948SChristophe Leroy __tlbie_va_range(hstart, hend, pid, 90547d99948SChristophe Leroy PMD_SIZE, MMU_PAGE_2M); 90647d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 9072275d7b5SNicholas Piggin } else { 9082275d7b5SNicholas Piggin _tlbiel_va_range_multicast(mm, 9092275d7b5SNicholas Piggin start, end, pid, page_size, mmu_virtual_psize, false); 9102275d7b5SNicholas Piggin if (hflush) 9112275d7b5SNicholas Piggin _tlbiel_va_range_multicast(mm, 9122275d7b5SNicholas Piggin hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false); 91347d99948SChristophe Leroy } 91447d99948SChristophe Leroy } 91547d99948SChristophe Leroy preempt_enable(); 91647d99948SChristophe Leroy } 91747d99948SChristophe Leroy 91847d99948SChristophe Leroy void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 91947d99948SChristophe Leroy unsigned long end) 92047d99948SChristophe Leroy 92147d99948SChristophe Leroy { 92247d99948SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 92347d99948SChristophe Leroy if (is_vm_hugetlb_page(vma)) 92447d99948SChristophe Leroy return radix__flush_hugetlb_tlb_range(vma, start, end); 92547d99948SChristophe Leroy #endif 92647d99948SChristophe Leroy 927a42d6ba8SAneesh Kumar K.V __radix__flush_tlb_range(vma->vm_mm, start, end); 92847d99948SChristophe Leroy } 92947d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_range); 93047d99948SChristophe Leroy 93147d99948SChristophe Leroy static int radix_get_mmu_psize(int page_size) 93247d99948SChristophe Leroy { 93347d99948SChristophe Leroy int psize; 93447d99948SChristophe Leroy 93547d99948SChristophe Leroy if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift)) 93647d99948SChristophe Leroy psize = mmu_virtual_psize; 93747d99948SChristophe Leroy else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift)) 93847d99948SChristophe Leroy psize = MMU_PAGE_2M; 93947d99948SChristophe Leroy else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift)) 94047d99948SChristophe Leroy psize = MMU_PAGE_1G; 94147d99948SChristophe Leroy else 94247d99948SChristophe Leroy return -1; 94347d99948SChristophe Leroy return psize; 94447d99948SChristophe Leroy } 94547d99948SChristophe Leroy 94647d99948SChristophe Leroy /* 94747d99948SChristophe Leroy * Flush partition scoped LPID address translation for all CPUs. 94847d99948SChristophe Leroy */ 94947d99948SChristophe Leroy void radix__flush_tlb_lpid_page(unsigned int lpid, 95047d99948SChristophe Leroy unsigned long addr, 95147d99948SChristophe Leroy unsigned long page_size) 95247d99948SChristophe Leroy { 95347d99948SChristophe Leroy int psize = radix_get_mmu_psize(page_size); 95447d99948SChristophe Leroy 95547d99948SChristophe Leroy _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB); 95647d99948SChristophe Leroy } 95747d99948SChristophe Leroy EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page); 95847d99948SChristophe Leroy 95947d99948SChristophe Leroy /* 96047d99948SChristophe Leroy * Flush partition scoped PWC from LPID for all CPUs. 96147d99948SChristophe Leroy */ 96247d99948SChristophe Leroy void radix__flush_pwc_lpid(unsigned int lpid) 96347d99948SChristophe Leroy { 96447d99948SChristophe Leroy _tlbie_lpid(lpid, RIC_FLUSH_PWC); 96547d99948SChristophe Leroy } 96647d99948SChristophe Leroy EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid); 96747d99948SChristophe Leroy 96847d99948SChristophe Leroy /* 96947d99948SChristophe Leroy * Flush partition scoped translations from LPID (=LPIDR) 97047d99948SChristophe Leroy */ 97199161de3SNicholas Piggin void radix__flush_all_lpid(unsigned int lpid) 97247d99948SChristophe Leroy { 97347d99948SChristophe Leroy _tlbie_lpid(lpid, RIC_FLUSH_ALL); 97447d99948SChristophe Leroy } 97599161de3SNicholas Piggin EXPORT_SYMBOL_GPL(radix__flush_all_lpid); 97647d99948SChristophe Leroy 97747d99948SChristophe Leroy /* 97899161de3SNicholas Piggin * Flush process scoped translations from LPID (=LPIDR) 97947d99948SChristophe Leroy */ 98099161de3SNicholas Piggin void radix__flush_all_lpid_guest(unsigned int lpid) 98147d99948SChristophe Leroy { 98299161de3SNicholas Piggin _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 98347d99948SChristophe Leroy } 98447d99948SChristophe Leroy 98547d99948SChristophe Leroy static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 98647d99948SChristophe Leroy unsigned long end, int psize); 98747d99948SChristophe Leroy 98847d99948SChristophe Leroy void radix__tlb_flush(struct mmu_gather *tlb) 98947d99948SChristophe Leroy { 99047d99948SChristophe Leroy int psize = 0; 99147d99948SChristophe Leroy struct mm_struct *mm = tlb->mm; 99247d99948SChristophe Leroy int page_size = tlb->page_size; 99347d99948SChristophe Leroy unsigned long start = tlb->start; 99447d99948SChristophe Leroy unsigned long end = tlb->end; 99547d99948SChristophe Leroy 99647d99948SChristophe Leroy /* 99747d99948SChristophe Leroy * if page size is not something we understand, do a full mm flush 99847d99948SChristophe Leroy * 99947d99948SChristophe Leroy * A "fullmm" flush must always do a flush_all_mm (RIC=2) flush 100047d99948SChristophe Leroy * that flushes the process table entry cache upon process teardown. 100147d99948SChristophe Leroy * See the comment for radix in arch_exit_mmap(). 100247d99948SChristophe Leroy */ 100347d99948SChristophe Leroy if (tlb->fullmm) { 100447d99948SChristophe Leroy __flush_all_mm(mm, true); 100547d99948SChristophe Leroy } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) { 100647d99948SChristophe Leroy if (!tlb->need_flush_all) 100747d99948SChristophe Leroy radix__flush_tlb_mm(mm); 100847d99948SChristophe Leroy else 100947d99948SChristophe Leroy radix__flush_all_mm(mm); 101047d99948SChristophe Leroy } else { 101147d99948SChristophe Leroy if (!tlb->need_flush_all) 101247d99948SChristophe Leroy radix__flush_tlb_range_psize(mm, start, end, psize); 101347d99948SChristophe Leroy else 101447d99948SChristophe Leroy radix__flush_tlb_pwc_range_psize(mm, start, end, psize); 101547d99948SChristophe Leroy } 101647d99948SChristophe Leroy tlb->need_flush_all = 0; 101747d99948SChristophe Leroy } 101847d99948SChristophe Leroy 1019e12d6d7dSMasahiro Yamada static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, 102047d99948SChristophe Leroy unsigned long start, unsigned long end, 102147d99948SChristophe Leroy int psize, bool also_pwc) 102247d99948SChristophe Leroy { 102347d99948SChristophe Leroy unsigned long pid; 102447d99948SChristophe Leroy unsigned int page_shift = mmu_psize_defs[psize].shift; 102547d99948SChristophe Leroy unsigned long page_size = 1UL << page_shift; 102647d99948SChristophe Leroy unsigned long nr_pages = (end - start) >> page_shift; 102747d99948SChristophe Leroy bool local, full; 102847d99948SChristophe Leroy 102947d99948SChristophe Leroy pid = mm->context.id; 103047d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 103147d99948SChristophe Leroy return; 103247d99948SChristophe Leroy 103347d99948SChristophe Leroy preempt_disable(); 103447d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 103547d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 103647d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 103747d99948SChristophe Leroy if (end != TLB_FLUSH_ALL) { 103847d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 103947d99948SChristophe Leroy goto is_local; 104047d99948SChristophe Leroy } 104147d99948SChristophe Leroy } 104247d99948SChristophe Leroy local = false; 104347d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 104447d99948SChristophe Leroy nr_pages > tlb_single_page_flush_ceiling); 104547d99948SChristophe Leroy } else { 104647d99948SChristophe Leroy is_local: 104747d99948SChristophe Leroy local = true; 104847d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 104947d99948SChristophe Leroy nr_pages > tlb_local_single_page_flush_ceiling); 105047d99948SChristophe Leroy } 105147d99948SChristophe Leroy 105247d99948SChristophe Leroy if (full) { 105347d99948SChristophe Leroy if (local) { 105447d99948SChristophe Leroy _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 105547d99948SChristophe Leroy } else { 10562275d7b5SNicholas Piggin if (cputlb_use_tlbie()) { 105747d99948SChristophe Leroy if (mm_needs_flush_escalation(mm)) 105847d99948SChristophe Leroy also_pwc = true; 105947d99948SChristophe Leroy 10602275d7b5SNicholas Piggin _tlbie_pid(pid, 10612275d7b5SNicholas Piggin also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 10622275d7b5SNicholas Piggin } else { 10632275d7b5SNicholas Piggin _tlbiel_pid_multicast(mm, pid, 10642275d7b5SNicholas Piggin also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 10652275d7b5SNicholas Piggin } 10662275d7b5SNicholas Piggin 106747d99948SChristophe Leroy } 106847d99948SChristophe Leroy } else { 106947d99948SChristophe Leroy if (local) 107047d99948SChristophe Leroy _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc); 10712275d7b5SNicholas Piggin else if (cputlb_use_tlbie()) 107247d99948SChristophe Leroy _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); 10732275d7b5SNicholas Piggin else 10742275d7b5SNicholas Piggin _tlbiel_va_range_multicast(mm, 10752275d7b5SNicholas Piggin start, end, pid, page_size, psize, also_pwc); 107647d99948SChristophe Leroy } 107747d99948SChristophe Leroy preempt_enable(); 107847d99948SChristophe Leroy } 107947d99948SChristophe Leroy 108047d99948SChristophe Leroy void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, 108147d99948SChristophe Leroy unsigned long end, int psize) 108247d99948SChristophe Leroy { 108347d99948SChristophe Leroy return __radix__flush_tlb_range_psize(mm, start, end, psize, false); 108447d99948SChristophe Leroy } 108547d99948SChristophe Leroy 108647d99948SChristophe Leroy static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 108747d99948SChristophe Leroy unsigned long end, int psize) 108847d99948SChristophe Leroy { 108947d99948SChristophe Leroy __radix__flush_tlb_range_psize(mm, start, end, psize, true); 109047d99948SChristophe Leroy } 109147d99948SChristophe Leroy 109247d99948SChristophe Leroy #ifdef CONFIG_TRANSPARENT_HUGEPAGE 109347d99948SChristophe Leroy void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) 109447d99948SChristophe Leroy { 109547d99948SChristophe Leroy unsigned long pid, end; 109647d99948SChristophe Leroy 109747d99948SChristophe Leroy pid = mm->context.id; 109847d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 109947d99948SChristophe Leroy return; 110047d99948SChristophe Leroy 110147d99948SChristophe Leroy /* 4k page size, just blow the world */ 110247d99948SChristophe Leroy if (PAGE_SIZE == 0x1000) { 110347d99948SChristophe Leroy radix__flush_all_mm(mm); 110447d99948SChristophe Leroy return; 110547d99948SChristophe Leroy } 110647d99948SChristophe Leroy 110747d99948SChristophe Leroy end = addr + HPAGE_PMD_SIZE; 110847d99948SChristophe Leroy 110947d99948SChristophe Leroy /* Otherwise first do the PWC, then iterate the pages. */ 111047d99948SChristophe Leroy preempt_disable(); 111147d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 111247d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 111347d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 111447d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 111547d99948SChristophe Leroy goto local; 111647d99948SChristophe Leroy } 11172275d7b5SNicholas Piggin if (cputlb_use_tlbie()) 111847d99948SChristophe Leroy _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 11192275d7b5SNicholas Piggin else 11202275d7b5SNicholas Piggin _tlbiel_va_range_multicast(mm, 11212275d7b5SNicholas Piggin addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 112247d99948SChristophe Leroy } else { 112347d99948SChristophe Leroy local: 112447d99948SChristophe Leroy _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 112547d99948SChristophe Leroy } 112647d99948SChristophe Leroy 112747d99948SChristophe Leroy preempt_enable(); 112847d99948SChristophe Leroy } 112947d99948SChristophe Leroy #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 113047d99948SChristophe Leroy 113147d99948SChristophe Leroy void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, 113247d99948SChristophe Leroy unsigned long start, unsigned long end) 113347d99948SChristophe Leroy { 113447d99948SChristophe Leroy radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); 113547d99948SChristophe Leroy } 113647d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_pmd_tlb_range); 113747d99948SChristophe Leroy 113847d99948SChristophe Leroy void radix__flush_tlb_all(void) 113947d99948SChristophe Leroy { 114047d99948SChristophe Leroy unsigned long rb,prs,r,rs; 114147d99948SChristophe Leroy unsigned long ric = RIC_FLUSH_ALL; 114247d99948SChristophe Leroy 114347d99948SChristophe Leroy rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */ 114447d99948SChristophe Leroy prs = 0; /* partition scoped */ 114547d99948SChristophe Leroy r = 1; /* radix format */ 114647d99948SChristophe Leroy rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */ 114747d99948SChristophe Leroy 114847d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 114947d99948SChristophe Leroy /* 115047d99948SChristophe Leroy * now flush guest entries by passing PRS = 1 and LPID != 0 115147d99948SChristophe Leroy */ 115247d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 115347d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory"); 115447d99948SChristophe Leroy /* 115547d99948SChristophe Leroy * now flush host entires by passing PRS = 0 and LPID == 0 115647d99948SChristophe Leroy */ 115747d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 115847d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory"); 115947d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 116047d99948SChristophe Leroy } 116147d99948SChristophe Leroy 116247d99948SChristophe Leroy #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 116347d99948SChristophe Leroy extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) 116447d99948SChristophe Leroy { 116547d99948SChristophe Leroy unsigned long pid = mm->context.id; 116647d99948SChristophe Leroy 116747d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 116847d99948SChristophe Leroy return; 116947d99948SChristophe Leroy 117047d99948SChristophe Leroy /* 117147d99948SChristophe Leroy * If this context hasn't run on that CPU before and KVM is 117247d99948SChristophe Leroy * around, there's a slim chance that the guest on another 117347d99948SChristophe Leroy * CPU just brought in obsolete translation into the TLB of 117447d99948SChristophe Leroy * this CPU due to a bad prefetch using the guest PID on 117547d99948SChristophe Leroy * the way into the hypervisor. 117647d99948SChristophe Leroy * 117747d99948SChristophe Leroy * We work around this here. If KVM is possible, we check if 117847d99948SChristophe Leroy * any sibling thread is in KVM. If it is, the window may exist 117947d99948SChristophe Leroy * and thus we flush that PID from the core. 118047d99948SChristophe Leroy * 118147d99948SChristophe Leroy * A potential future improvement would be to mark which PIDs 118247d99948SChristophe Leroy * have never been used on the system and avoid it if the PID 118347d99948SChristophe Leroy * is new and the process has no other cpumask bit set. 118447d99948SChristophe Leroy */ 118547d99948SChristophe Leroy if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) { 118647d99948SChristophe Leroy int cpu = smp_processor_id(); 118747d99948SChristophe Leroy int sib = cpu_first_thread_sibling(cpu); 118847d99948SChristophe Leroy bool flush = false; 118947d99948SChristophe Leroy 119047d99948SChristophe Leroy for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) { 119147d99948SChristophe Leroy if (sib == cpu) 119247d99948SChristophe Leroy continue; 119347d99948SChristophe Leroy if (!cpu_possible(sib)) 119447d99948SChristophe Leroy continue; 119547d99948SChristophe Leroy if (paca_ptrs[sib]->kvm_hstate.kvm_vcpu) 119647d99948SChristophe Leroy flush = true; 119747d99948SChristophe Leroy } 119847d99948SChristophe Leroy if (flush) 119947d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 120047d99948SChristophe Leroy } 120147d99948SChristophe Leroy } 120247d99948SChristophe Leroy EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround); 120347d99948SChristophe Leroy #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 1204