12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 247d99948SChristophe Leroy /* 347d99948SChristophe Leroy * TLB flush routines for radix kernels. 447d99948SChristophe Leroy * 547d99948SChristophe Leroy * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation. 647d99948SChristophe Leroy */ 747d99948SChristophe Leroy 847d99948SChristophe Leroy #include <linux/mm.h> 947d99948SChristophe Leroy #include <linux/hugetlb.h> 1047d99948SChristophe Leroy #include <linux/memblock.h> 1147d99948SChristophe Leroy #include <linux/mmu_context.h> 1247d99948SChristophe Leroy #include <linux/sched/mm.h> 1347d99948SChristophe Leroy 1447d99948SChristophe Leroy #include <asm/ppc-opcode.h> 1547d99948SChristophe Leroy #include <asm/tlb.h> 1647d99948SChristophe Leroy #include <asm/tlbflush.h> 1747d99948SChristophe Leroy #include <asm/trace.h> 1847d99948SChristophe Leroy #include <asm/cputhreads.h> 1947d99948SChristophe Leroy 2047d99948SChristophe Leroy #define RIC_FLUSH_TLB 0 2147d99948SChristophe Leroy #define RIC_FLUSH_PWC 1 2247d99948SChristophe Leroy #define RIC_FLUSH_ALL 2 2347d99948SChristophe Leroy 2447d99948SChristophe Leroy /* 2547d99948SChristophe Leroy * tlbiel instruction for radix, set invalidation 2647d99948SChristophe Leroy * i.e., r=1 and is=01 or is=10 or is=11 2747d99948SChristophe Leroy */ 286d3ca7e7SMasahiro Yamada static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, 2947d99948SChristophe Leroy unsigned int pid, 3047d99948SChristophe Leroy unsigned int ric, unsigned int prs) 3147d99948SChristophe Leroy { 3247d99948SChristophe Leroy unsigned long rb; 3347d99948SChristophe Leroy unsigned long rs; 3447d99948SChristophe Leroy 3547d99948SChristophe Leroy rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); 3647d99948SChristophe Leroy rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); 3747d99948SChristophe Leroy 3847d99948SChristophe Leroy asm volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 3947d99948SChristophe Leroy : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 4047d99948SChristophe Leroy : "memory"); 4147d99948SChristophe Leroy } 4247d99948SChristophe Leroy 4347d99948SChristophe Leroy static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) 4447d99948SChristophe Leroy { 4547d99948SChristophe Leroy unsigned int set; 4647d99948SChristophe Leroy 4747d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 4847d99948SChristophe Leroy 4947d99948SChristophe Leroy /* 5047d99948SChristophe Leroy * Flush the first set of the TLB, and the entire Page Walk Cache 5147d99948SChristophe Leroy * and partition table entries. Then flush the remaining sets of the 5247d99948SChristophe Leroy * TLB. 5347d99948SChristophe Leroy */ 5447d99948SChristophe Leroy tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 5547d99948SChristophe Leroy for (set = 1; set < num_sets; set++) 5647d99948SChristophe Leroy tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 5747d99948SChristophe Leroy 5847d99948SChristophe Leroy /* Do the same for process scoped entries. */ 5947d99948SChristophe Leroy tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 6047d99948SChristophe Leroy for (set = 1; set < num_sets; set++) 6147d99948SChristophe Leroy tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 6247d99948SChristophe Leroy 6347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 6447d99948SChristophe Leroy } 6547d99948SChristophe Leroy 6647d99948SChristophe Leroy void radix__tlbiel_all(unsigned int action) 6747d99948SChristophe Leroy { 6847d99948SChristophe Leroy unsigned int is; 6947d99948SChristophe Leroy 7047d99948SChristophe Leroy switch (action) { 7147d99948SChristophe Leroy case TLB_INVAL_SCOPE_GLOBAL: 7247d99948SChristophe Leroy is = 3; 7347d99948SChristophe Leroy break; 7447d99948SChristophe Leroy case TLB_INVAL_SCOPE_LPID: 7547d99948SChristophe Leroy is = 2; 7647d99948SChristophe Leroy break; 7747d99948SChristophe Leroy default: 7847d99948SChristophe Leroy BUG(); 7947d99948SChristophe Leroy } 8047d99948SChristophe Leroy 8147d99948SChristophe Leroy if (early_cpu_has_feature(CPU_FTR_ARCH_300)) 8247d99948SChristophe Leroy tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is); 8347d99948SChristophe Leroy else 8447d99948SChristophe Leroy WARN(1, "%s called on pre-POWER9 CPU\n", __func__); 8547d99948SChristophe Leroy 86fe7946ceSNicholas Piggin asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); 8747d99948SChristophe Leroy } 8847d99948SChristophe Leroy 89efc344c5SMasahiro Yamada static __always_inline void __tlbiel_pid(unsigned long pid, int set, 9047d99948SChristophe Leroy unsigned long ric) 9147d99948SChristophe Leroy { 9247d99948SChristophe Leroy unsigned long rb,rs,prs,r; 9347d99948SChristophe Leroy 9447d99948SChristophe Leroy rb = PPC_BIT(53); /* IS = 1 */ 9547d99948SChristophe Leroy rb |= set << PPC_BITLSHIFT(51); 9647d99948SChristophe Leroy rs = ((unsigned long)pid) << PPC_BITLSHIFT(31); 9747d99948SChristophe Leroy prs = 1; /* process scoped */ 9847d99948SChristophe Leroy r = 1; /* radix format */ 9947d99948SChristophe Leroy 10047d99948SChristophe Leroy asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 10147d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 10247d99948SChristophe Leroy trace_tlbie(0, 1, rb, rs, ric, prs, r); 10347d99948SChristophe Leroy } 10447d99948SChristophe Leroy 105efc344c5SMasahiro Yamada static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric) 10647d99948SChristophe Leroy { 10747d99948SChristophe Leroy unsigned long rb,rs,prs,r; 10847d99948SChristophe Leroy 10947d99948SChristophe Leroy rb = PPC_BIT(53); /* IS = 1 */ 11047d99948SChristophe Leroy rs = pid << PPC_BITLSHIFT(31); 11147d99948SChristophe Leroy prs = 1; /* process scoped */ 11247d99948SChristophe Leroy r = 1; /* radix format */ 11347d99948SChristophe Leroy 11447d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 11547d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 11647d99948SChristophe Leroy trace_tlbie(0, 0, rb, rs, ric, prs, r); 11747d99948SChristophe Leroy } 11847d99948SChristophe Leroy 119efc344c5SMasahiro Yamada static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric) 12047d99948SChristophe Leroy { 12147d99948SChristophe Leroy unsigned long rb,rs,prs,r; 12247d99948SChristophe Leroy 12347d99948SChristophe Leroy rb = PPC_BIT(52); /* IS = 2 */ 12447d99948SChristophe Leroy rs = lpid; 12547d99948SChristophe Leroy prs = 0; /* partition scoped */ 12647d99948SChristophe Leroy r = 1; /* radix format */ 12747d99948SChristophe Leroy 12847d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 12947d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 13047d99948SChristophe Leroy trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 13147d99948SChristophe Leroy } 13247d99948SChristophe Leroy 13399161de3SNicholas Piggin static __always_inline void __tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 13447d99948SChristophe Leroy { 13547d99948SChristophe Leroy unsigned long rb,rs,prs,r; 13647d99948SChristophe Leroy 13747d99948SChristophe Leroy rb = PPC_BIT(52); /* IS = 2 */ 13899161de3SNicholas Piggin rs = lpid; 13947d99948SChristophe Leroy prs = 1; /* process scoped */ 14047d99948SChristophe Leroy r = 1; /* radix format */ 14147d99948SChristophe Leroy 14299161de3SNicholas Piggin asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 14347d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 14499161de3SNicholas Piggin trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 14547d99948SChristophe Leroy } 14647d99948SChristophe Leroy 1476d3ca7e7SMasahiro Yamada static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid, 14847d99948SChristophe Leroy unsigned long ap, unsigned long ric) 14947d99948SChristophe Leroy { 15047d99948SChristophe Leroy unsigned long rb,rs,prs,r; 15147d99948SChristophe Leroy 15247d99948SChristophe Leroy rb = va & ~(PPC_BITMASK(52, 63)); 15347d99948SChristophe Leroy rb |= ap << PPC_BITLSHIFT(58); 15447d99948SChristophe Leroy rs = pid << PPC_BITLSHIFT(31); 15547d99948SChristophe Leroy prs = 1; /* process scoped */ 15647d99948SChristophe Leroy r = 1; /* radix format */ 15747d99948SChristophe Leroy 15847d99948SChristophe Leroy asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 15947d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 16047d99948SChristophe Leroy trace_tlbie(0, 1, rb, rs, ric, prs, r); 16147d99948SChristophe Leroy } 16247d99948SChristophe Leroy 1636d3ca7e7SMasahiro Yamada static __always_inline void __tlbie_va(unsigned long va, unsigned long pid, 16447d99948SChristophe Leroy unsigned long ap, unsigned long ric) 16547d99948SChristophe Leroy { 16647d99948SChristophe Leroy unsigned long rb,rs,prs,r; 16747d99948SChristophe Leroy 16847d99948SChristophe Leroy rb = va & ~(PPC_BITMASK(52, 63)); 16947d99948SChristophe Leroy rb |= ap << PPC_BITLSHIFT(58); 17047d99948SChristophe Leroy rs = pid << PPC_BITLSHIFT(31); 17147d99948SChristophe Leroy prs = 1; /* process scoped */ 17247d99948SChristophe Leroy r = 1; /* radix format */ 17347d99948SChristophe Leroy 17447d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 17547d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 17647d99948SChristophe Leroy trace_tlbie(0, 0, rb, rs, ric, prs, r); 17747d99948SChristophe Leroy } 17847d99948SChristophe Leroy 1796d3ca7e7SMasahiro Yamada static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, 18047d99948SChristophe Leroy unsigned long ap, unsigned long ric) 18147d99948SChristophe Leroy { 18247d99948SChristophe Leroy unsigned long rb,rs,prs,r; 18347d99948SChristophe Leroy 18447d99948SChristophe Leroy rb = va & ~(PPC_BITMASK(52, 63)); 18547d99948SChristophe Leroy rb |= ap << PPC_BITLSHIFT(58); 18647d99948SChristophe Leroy rs = lpid; 18747d99948SChristophe Leroy prs = 0; /* partition scoped */ 18847d99948SChristophe Leroy r = 1; /* radix format */ 18947d99948SChristophe Leroy 19047d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 19147d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); 19247d99948SChristophe Leroy trace_tlbie(lpid, 0, rb, rs, ric, prs, r); 19347d99948SChristophe Leroy } 19447d99948SChristophe Leroy 19547d99948SChristophe Leroy static inline void fixup_tlbie(void) 19647d99948SChristophe Leroy { 19747d99948SChristophe Leroy unsigned long pid = 0; 19847d99948SChristophe Leroy unsigned long va = ((1UL << 52) - 1); 19947d99948SChristophe Leroy 20047d99948SChristophe Leroy if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) { 20147d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 20247d99948SChristophe Leroy __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 20347d99948SChristophe Leroy } 20447d99948SChristophe Leroy } 20547d99948SChristophe Leroy 20647d99948SChristophe Leroy static inline void fixup_tlbie_lpid(unsigned long lpid) 20747d99948SChristophe Leroy { 20847d99948SChristophe Leroy unsigned long va = ((1UL << 52) - 1); 20947d99948SChristophe Leroy 21047d99948SChristophe Leroy if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) { 21147d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 21247d99948SChristophe Leroy __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); 21347d99948SChristophe Leroy } 21447d99948SChristophe Leroy } 21547d99948SChristophe Leroy 21647d99948SChristophe Leroy /* 21747d99948SChristophe Leroy * We use 128 set in radix mode and 256 set in hpt mode. 21847d99948SChristophe Leroy */ 2196d3ca7e7SMasahiro Yamada static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric) 22047d99948SChristophe Leroy { 22147d99948SChristophe Leroy int set; 22247d99948SChristophe Leroy 22347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 22447d99948SChristophe Leroy 22547d99948SChristophe Leroy /* 22647d99948SChristophe Leroy * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL, 22747d99948SChristophe Leroy * also flush the entire Page Walk Cache. 22847d99948SChristophe Leroy */ 22947d99948SChristophe Leroy __tlbiel_pid(pid, 0, ric); 23047d99948SChristophe Leroy 23147d99948SChristophe Leroy /* For PWC, only one flush is needed */ 23247d99948SChristophe Leroy if (ric == RIC_FLUSH_PWC) { 23347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 23447d99948SChristophe Leroy return; 23547d99948SChristophe Leroy } 23647d99948SChristophe Leroy 23747d99948SChristophe Leroy /* For the remaining sets, just flush the TLB */ 23847d99948SChristophe Leroy for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++) 23947d99948SChristophe Leroy __tlbiel_pid(pid, set, RIC_FLUSH_TLB); 24047d99948SChristophe Leroy 24147d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 2426c46fcceSNicholas Piggin asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory"); 24347d99948SChristophe Leroy } 24447d99948SChristophe Leroy 24547d99948SChristophe Leroy static inline void _tlbie_pid(unsigned long pid, unsigned long ric) 24647d99948SChristophe Leroy { 24747d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 24847d99948SChristophe Leroy 24947d99948SChristophe Leroy /* 25047d99948SChristophe Leroy * Workaround the fact that the "ric" argument to __tlbie_pid 25147d99948SChristophe Leroy * must be a compile-time contraint to match the "i" constraint 25247d99948SChristophe Leroy * in the asm statement. 25347d99948SChristophe Leroy */ 25447d99948SChristophe Leroy switch (ric) { 25547d99948SChristophe Leroy case RIC_FLUSH_TLB: 25647d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_TLB); 25747d99948SChristophe Leroy break; 25847d99948SChristophe Leroy case RIC_FLUSH_PWC: 25947d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_PWC); 26047d99948SChristophe Leroy break; 26147d99948SChristophe Leroy case RIC_FLUSH_ALL: 26247d99948SChristophe Leroy default: 26347d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_ALL); 26447d99948SChristophe Leroy } 26547d99948SChristophe Leroy fixup_tlbie(); 26647d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 26747d99948SChristophe Leroy } 26847d99948SChristophe Leroy 26947d99948SChristophe Leroy static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) 27047d99948SChristophe Leroy { 27147d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 27247d99948SChristophe Leroy 27347d99948SChristophe Leroy /* 27447d99948SChristophe Leroy * Workaround the fact that the "ric" argument to __tlbie_pid 27547d99948SChristophe Leroy * must be a compile-time contraint to match the "i" constraint 27647d99948SChristophe Leroy * in the asm statement. 27747d99948SChristophe Leroy */ 27847d99948SChristophe Leroy switch (ric) { 27947d99948SChristophe Leroy case RIC_FLUSH_TLB: 28047d99948SChristophe Leroy __tlbie_lpid(lpid, RIC_FLUSH_TLB); 28147d99948SChristophe Leroy break; 28247d99948SChristophe Leroy case RIC_FLUSH_PWC: 28347d99948SChristophe Leroy __tlbie_lpid(lpid, RIC_FLUSH_PWC); 28447d99948SChristophe Leroy break; 28547d99948SChristophe Leroy case RIC_FLUSH_ALL: 28647d99948SChristophe Leroy default: 28747d99948SChristophe Leroy __tlbie_lpid(lpid, RIC_FLUSH_ALL); 28847d99948SChristophe Leroy } 28947d99948SChristophe Leroy fixup_tlbie_lpid(lpid); 29047d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 29147d99948SChristophe Leroy } 29247d99948SChristophe Leroy 29399161de3SNicholas Piggin static __always_inline void _tlbie_lpid_guest(unsigned long lpid, unsigned long ric) 29447d99948SChristophe Leroy { 29547d99948SChristophe Leroy /* 29699161de3SNicholas Piggin * Workaround the fact that the "ric" argument to __tlbie_pid 29799161de3SNicholas Piggin * must be a compile-time contraint to match the "i" constraint 29899161de3SNicholas Piggin * in the asm statement. 29947d99948SChristophe Leroy */ 30099161de3SNicholas Piggin switch (ric) { 30199161de3SNicholas Piggin case RIC_FLUSH_TLB: 30299161de3SNicholas Piggin __tlbie_lpid_guest(lpid, RIC_FLUSH_TLB); 30399161de3SNicholas Piggin break; 30499161de3SNicholas Piggin case RIC_FLUSH_PWC: 30599161de3SNicholas Piggin __tlbie_lpid_guest(lpid, RIC_FLUSH_PWC); 30699161de3SNicholas Piggin break; 30799161de3SNicholas Piggin case RIC_FLUSH_ALL: 30899161de3SNicholas Piggin default: 30999161de3SNicholas Piggin __tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 31047d99948SChristophe Leroy } 31199161de3SNicholas Piggin fixup_tlbie_lpid(lpid); 31299161de3SNicholas Piggin asm volatile("eieio; tlbsync; ptesync": : :"memory"); 31347d99948SChristophe Leroy } 31447d99948SChristophe Leroy 31547d99948SChristophe Leroy static inline void __tlbiel_va_range(unsigned long start, unsigned long end, 31647d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 31747d99948SChristophe Leroy unsigned long psize) 31847d99948SChristophe Leroy { 31947d99948SChristophe Leroy unsigned long addr; 32047d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 32147d99948SChristophe Leroy 32247d99948SChristophe Leroy for (addr = start; addr < end; addr += page_size) 32347d99948SChristophe Leroy __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB); 32447d99948SChristophe Leroy } 32547d99948SChristophe Leroy 3266d3ca7e7SMasahiro Yamada static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid, 32747d99948SChristophe Leroy unsigned long psize, unsigned long ric) 32847d99948SChristophe Leroy { 32947d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 33047d99948SChristophe Leroy 33147d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 33247d99948SChristophe Leroy __tlbiel_va(va, pid, ap, ric); 33347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 33447d99948SChristophe Leroy } 33547d99948SChristophe Leroy 33647d99948SChristophe Leroy static inline void _tlbiel_va_range(unsigned long start, unsigned long end, 33747d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 33847d99948SChristophe Leroy unsigned long psize, bool also_pwc) 33947d99948SChristophe Leroy { 34047d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 34147d99948SChristophe Leroy if (also_pwc) 34247d99948SChristophe Leroy __tlbiel_pid(pid, 0, RIC_FLUSH_PWC); 34347d99948SChristophe Leroy __tlbiel_va_range(start, end, pid, page_size, psize); 34447d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 34547d99948SChristophe Leroy } 34647d99948SChristophe Leroy 34747d99948SChristophe Leroy static inline void __tlbie_va_range(unsigned long start, unsigned long end, 34847d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 34947d99948SChristophe Leroy unsigned long psize) 35047d99948SChristophe Leroy { 35147d99948SChristophe Leroy unsigned long addr; 35247d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 35347d99948SChristophe Leroy 35447d99948SChristophe Leroy for (addr = start; addr < end; addr += page_size) 35547d99948SChristophe Leroy __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB); 35647d99948SChristophe Leroy } 35747d99948SChristophe Leroy 3586d3ca7e7SMasahiro Yamada static __always_inline void _tlbie_va(unsigned long va, unsigned long pid, 35947d99948SChristophe Leroy unsigned long psize, unsigned long ric) 36047d99948SChristophe Leroy { 36147d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 36247d99948SChristophe Leroy 36347d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 36447d99948SChristophe Leroy __tlbie_va(va, pid, ap, ric); 36547d99948SChristophe Leroy fixup_tlbie(); 36647d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 36747d99948SChristophe Leroy } 36847d99948SChristophe Leroy 3696d3ca7e7SMasahiro Yamada static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid, 37047d99948SChristophe Leroy unsigned long psize, unsigned long ric) 37147d99948SChristophe Leroy { 37247d99948SChristophe Leroy unsigned long ap = mmu_get_ap(psize); 37347d99948SChristophe Leroy 37447d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 37547d99948SChristophe Leroy __tlbie_lpid_va(va, lpid, ap, ric); 37647d99948SChristophe Leroy fixup_tlbie_lpid(lpid); 37747d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 37847d99948SChristophe Leroy } 37947d99948SChristophe Leroy 38047d99948SChristophe Leroy static inline void _tlbie_va_range(unsigned long start, unsigned long end, 38147d99948SChristophe Leroy unsigned long pid, unsigned long page_size, 38247d99948SChristophe Leroy unsigned long psize, bool also_pwc) 38347d99948SChristophe Leroy { 38447d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 38547d99948SChristophe Leroy if (also_pwc) 38647d99948SChristophe Leroy __tlbie_pid(pid, RIC_FLUSH_PWC); 38747d99948SChristophe Leroy __tlbie_va_range(start, end, pid, page_size, psize); 38847d99948SChristophe Leroy fixup_tlbie(); 38947d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 39047d99948SChristophe Leroy } 39147d99948SChristophe Leroy 39247d99948SChristophe Leroy /* 39347d99948SChristophe Leroy * Base TLB flushing operations: 39447d99948SChristophe Leroy * 39547d99948SChristophe Leroy * - flush_tlb_mm(mm) flushes the specified mm context TLB's 39647d99948SChristophe Leroy * - flush_tlb_page(vma, vmaddr) flushes one page 39747d99948SChristophe Leroy * - flush_tlb_range(vma, start, end) flushes a range of pages 39847d99948SChristophe Leroy * - flush_tlb_kernel_range(start, end) flushes kernel pages 39947d99948SChristophe Leroy * 40047d99948SChristophe Leroy * - local_* variants of page and mm only apply to the current 40147d99948SChristophe Leroy * processor 40247d99948SChristophe Leroy */ 40347d99948SChristophe Leroy void radix__local_flush_tlb_mm(struct mm_struct *mm) 40447d99948SChristophe Leroy { 40547d99948SChristophe Leroy unsigned long pid; 40647d99948SChristophe Leroy 40747d99948SChristophe Leroy preempt_disable(); 40847d99948SChristophe Leroy pid = mm->context.id; 40947d99948SChristophe Leroy if (pid != MMU_NO_CONTEXT) 41047d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_TLB); 41147d99948SChristophe Leroy preempt_enable(); 41247d99948SChristophe Leroy } 41347d99948SChristophe Leroy EXPORT_SYMBOL(radix__local_flush_tlb_mm); 41447d99948SChristophe Leroy 41547d99948SChristophe Leroy #ifndef CONFIG_SMP 41647d99948SChristophe Leroy void radix__local_flush_all_mm(struct mm_struct *mm) 41747d99948SChristophe Leroy { 41847d99948SChristophe Leroy unsigned long pid; 41947d99948SChristophe Leroy 42047d99948SChristophe Leroy preempt_disable(); 42147d99948SChristophe Leroy pid = mm->context.id; 42247d99948SChristophe Leroy if (pid != MMU_NO_CONTEXT) 42347d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 42447d99948SChristophe Leroy preempt_enable(); 42547d99948SChristophe Leroy } 42647d99948SChristophe Leroy EXPORT_SYMBOL(radix__local_flush_all_mm); 42747d99948SChristophe Leroy #endif /* CONFIG_SMP */ 42847d99948SChristophe Leroy 42947d99948SChristophe Leroy void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 43047d99948SChristophe Leroy int psize) 43147d99948SChristophe Leroy { 43247d99948SChristophe Leroy unsigned long pid; 43347d99948SChristophe Leroy 43447d99948SChristophe Leroy preempt_disable(); 43547d99948SChristophe Leroy pid = mm->context.id; 43647d99948SChristophe Leroy if (pid != MMU_NO_CONTEXT) 43747d99948SChristophe Leroy _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 43847d99948SChristophe Leroy preempt_enable(); 43947d99948SChristophe Leroy } 44047d99948SChristophe Leroy 44147d99948SChristophe Leroy void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 44247d99948SChristophe Leroy { 44347d99948SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 44447d99948SChristophe Leroy /* need the return fix for nohash.c */ 44547d99948SChristophe Leroy if (is_vm_hugetlb_page(vma)) 44647d99948SChristophe Leroy return radix__local_flush_hugetlb_page(vma, vmaddr); 44747d99948SChristophe Leroy #endif 44847d99948SChristophe Leroy radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 44947d99948SChristophe Leroy } 45047d99948SChristophe Leroy EXPORT_SYMBOL(radix__local_flush_tlb_page); 45147d99948SChristophe Leroy 45247d99948SChristophe Leroy static bool mm_is_singlethreaded(struct mm_struct *mm) 45347d99948SChristophe Leroy { 45447d99948SChristophe Leroy if (atomic_read(&mm->context.copros) > 0) 45547d99948SChristophe Leroy return false; 45647d99948SChristophe Leroy if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) 45747d99948SChristophe Leroy return true; 45847d99948SChristophe Leroy return false; 45947d99948SChristophe Leroy } 46047d99948SChristophe Leroy 46147d99948SChristophe Leroy static bool mm_needs_flush_escalation(struct mm_struct *mm) 46247d99948SChristophe Leroy { 46347d99948SChristophe Leroy /* 46447d99948SChristophe Leroy * P9 nest MMU has issues with the page walk cache 46547d99948SChristophe Leroy * caching PTEs and not flushing them properly when 46647d99948SChristophe Leroy * RIC = 0 for a PID/LPID invalidate 46747d99948SChristophe Leroy */ 46847d99948SChristophe Leroy if (atomic_read(&mm->context.copros) > 0) 46947d99948SChristophe Leroy return true; 47047d99948SChristophe Leroy return false; 47147d99948SChristophe Leroy } 47247d99948SChristophe Leroy 47347d99948SChristophe Leroy #ifdef CONFIG_SMP 47447d99948SChristophe Leroy static void do_exit_flush_lazy_tlb(void *arg) 47547d99948SChristophe Leroy { 47647d99948SChristophe Leroy struct mm_struct *mm = arg; 47747d99948SChristophe Leroy unsigned long pid = mm->context.id; 47847d99948SChristophe Leroy 47947d99948SChristophe Leroy if (current->mm == mm) 48047d99948SChristophe Leroy return; /* Local CPU */ 48147d99948SChristophe Leroy 48247d99948SChristophe Leroy if (current->active_mm == mm) { 48347d99948SChristophe Leroy /* 48447d99948SChristophe Leroy * Must be a kernel thread because sender is single-threaded. 48547d99948SChristophe Leroy */ 48647d99948SChristophe Leroy BUG_ON(current->mm); 48747d99948SChristophe Leroy mmgrab(&init_mm); 48847d99948SChristophe Leroy switch_mm(mm, &init_mm, current); 48947d99948SChristophe Leroy current->active_mm = &init_mm; 49047d99948SChristophe Leroy mmdrop(mm); 49147d99948SChristophe Leroy } 49247d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 49347d99948SChristophe Leroy } 49447d99948SChristophe Leroy 49547d99948SChristophe Leroy static void exit_flush_lazy_tlbs(struct mm_struct *mm) 49647d99948SChristophe Leroy { 49747d99948SChristophe Leroy /* 49847d99948SChristophe Leroy * Would be nice if this was async so it could be run in 49947d99948SChristophe Leroy * parallel with our local flush, but generic code does not 50047d99948SChristophe Leroy * give a good API for it. Could extend the generic code or 50147d99948SChristophe Leroy * make a special powerpc IPI for flushing TLBs. 50247d99948SChristophe Leroy * For now it's not too performance critical. 50347d99948SChristophe Leroy */ 50447d99948SChristophe Leroy smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb, 50547d99948SChristophe Leroy (void *)mm, 1); 50647d99948SChristophe Leroy mm_reset_thread_local(mm); 50747d99948SChristophe Leroy } 50847d99948SChristophe Leroy 50947d99948SChristophe Leroy void radix__flush_tlb_mm(struct mm_struct *mm) 51047d99948SChristophe Leroy { 51147d99948SChristophe Leroy unsigned long pid; 51247d99948SChristophe Leroy 51347d99948SChristophe Leroy pid = mm->context.id; 51447d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 51547d99948SChristophe Leroy return; 51647d99948SChristophe Leroy 51747d99948SChristophe Leroy preempt_disable(); 51847d99948SChristophe Leroy /* 51947d99948SChristophe Leroy * Order loads of mm_cpumask vs previous stores to clear ptes before 52047d99948SChristophe Leroy * the invalidate. See barrier in switch_mm_irqs_off 52147d99948SChristophe Leroy */ 52247d99948SChristophe Leroy smp_mb(); 52347d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 52447d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 52547d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 52647d99948SChristophe Leroy goto local; 52747d99948SChristophe Leroy } 52847d99948SChristophe Leroy 52947d99948SChristophe Leroy if (mm_needs_flush_escalation(mm)) 53047d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_ALL); 53147d99948SChristophe Leroy else 53247d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_TLB); 53347d99948SChristophe Leroy } else { 53447d99948SChristophe Leroy local: 53547d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_TLB); 53647d99948SChristophe Leroy } 53747d99948SChristophe Leroy preempt_enable(); 53847d99948SChristophe Leroy } 53947d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_mm); 54047d99948SChristophe Leroy 54147d99948SChristophe Leroy static void __flush_all_mm(struct mm_struct *mm, bool fullmm) 54247d99948SChristophe Leroy { 54347d99948SChristophe Leroy unsigned long pid; 54447d99948SChristophe Leroy 54547d99948SChristophe Leroy pid = mm->context.id; 54647d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 54747d99948SChristophe Leroy return; 54847d99948SChristophe Leroy 54947d99948SChristophe Leroy preempt_disable(); 55047d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 55147d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 55247d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 55347d99948SChristophe Leroy if (!fullmm) { 55447d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 55547d99948SChristophe Leroy goto local; 55647d99948SChristophe Leroy } 55747d99948SChristophe Leroy } 55847d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_ALL); 55947d99948SChristophe Leroy } else { 56047d99948SChristophe Leroy local: 56147d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 56247d99948SChristophe Leroy } 56347d99948SChristophe Leroy preempt_enable(); 56447d99948SChristophe Leroy } 56547d99948SChristophe Leroy void radix__flush_all_mm(struct mm_struct *mm) 56647d99948SChristophe Leroy { 56747d99948SChristophe Leroy __flush_all_mm(mm, false); 56847d99948SChristophe Leroy } 56947d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_all_mm); 57047d99948SChristophe Leroy 57147d99948SChristophe Leroy void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr) 57247d99948SChristophe Leroy { 57347d99948SChristophe Leroy tlb->need_flush_all = 1; 57447d99948SChristophe Leroy } 57547d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_pwc); 57647d99948SChristophe Leroy 57747d99948SChristophe Leroy void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, 57847d99948SChristophe Leroy int psize) 57947d99948SChristophe Leroy { 58047d99948SChristophe Leroy unsigned long pid; 58147d99948SChristophe Leroy 58247d99948SChristophe Leroy pid = mm->context.id; 58347d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 58447d99948SChristophe Leroy return; 58547d99948SChristophe Leroy 58647d99948SChristophe Leroy preempt_disable(); 58747d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 58847d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 58947d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 59047d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 59147d99948SChristophe Leroy goto local; 59247d99948SChristophe Leroy } 59347d99948SChristophe Leroy _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 59447d99948SChristophe Leroy } else { 59547d99948SChristophe Leroy local: 59647d99948SChristophe Leroy _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); 59747d99948SChristophe Leroy } 59847d99948SChristophe Leroy preempt_enable(); 59947d99948SChristophe Leroy } 60047d99948SChristophe Leroy 60147d99948SChristophe Leroy void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 60247d99948SChristophe Leroy { 60347d99948SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 60447d99948SChristophe Leroy if (is_vm_hugetlb_page(vma)) 60547d99948SChristophe Leroy return radix__flush_hugetlb_page(vma, vmaddr); 60647d99948SChristophe Leroy #endif 60747d99948SChristophe Leroy radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); 60847d99948SChristophe Leroy } 60947d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_page); 61047d99948SChristophe Leroy 61147d99948SChristophe Leroy #else /* CONFIG_SMP */ 61247d99948SChristophe Leroy #define radix__flush_all_mm radix__local_flush_all_mm 61347d99948SChristophe Leroy #endif /* CONFIG_SMP */ 61447d99948SChristophe Leroy 61560e8523eSAlastair D'Silva /* 61660e8523eSAlastair D'Silva * If kernel TLBIs ever become local rather than global, then 61760e8523eSAlastair D'Silva * drivers/misc/ocxl/link.c:ocxl_link_add_pe will need some work, as it 61860e8523eSAlastair D'Silva * assumes kernel TLBIs are global. 61960e8523eSAlastair D'Silva */ 62047d99948SChristophe Leroy void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) 62147d99948SChristophe Leroy { 62247d99948SChristophe Leroy _tlbie_pid(0, RIC_FLUSH_ALL); 62347d99948SChristophe Leroy } 62447d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_kernel_range); 62547d99948SChristophe Leroy 62647d99948SChristophe Leroy #define TLB_FLUSH_ALL -1UL 62747d99948SChristophe Leroy 62847d99948SChristophe Leroy /* 62947d99948SChristophe Leroy * Number of pages above which we invalidate the entire PID rather than 63047d99948SChristophe Leroy * flush individual pages, for local and global flushes respectively. 63147d99948SChristophe Leroy * 63247d99948SChristophe Leroy * tlbie goes out to the interconnect and individual ops are more costly. 63347d99948SChristophe Leroy * It also does not iterate over sets like the local tlbiel variant when 63447d99948SChristophe Leroy * invalidating a full PID, so it has a far lower threshold to change from 63547d99948SChristophe Leroy * individual page flushes to full-pid flushes. 63647d99948SChristophe Leroy */ 63747d99948SChristophe Leroy static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; 63847d99948SChristophe Leroy static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2; 63947d99948SChristophe Leroy 64047d99948SChristophe Leroy static inline void __radix__flush_tlb_range(struct mm_struct *mm, 64147d99948SChristophe Leroy unsigned long start, unsigned long end, 64247d99948SChristophe Leroy bool flush_all_sizes) 64347d99948SChristophe Leroy 64447d99948SChristophe Leroy { 64547d99948SChristophe Leroy unsigned long pid; 64647d99948SChristophe Leroy unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; 64747d99948SChristophe Leroy unsigned long page_size = 1UL << page_shift; 64847d99948SChristophe Leroy unsigned long nr_pages = (end - start) >> page_shift; 64947d99948SChristophe Leroy bool local, full; 65047d99948SChristophe Leroy 65147d99948SChristophe Leroy pid = mm->context.id; 65247d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 65347d99948SChristophe Leroy return; 65447d99948SChristophe Leroy 65547d99948SChristophe Leroy preempt_disable(); 65647d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 65747d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 65847d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 65947d99948SChristophe Leroy if (end != TLB_FLUSH_ALL) { 66047d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 66147d99948SChristophe Leroy goto is_local; 66247d99948SChristophe Leroy } 66347d99948SChristophe Leroy } 66447d99948SChristophe Leroy local = false; 66547d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 66647d99948SChristophe Leroy nr_pages > tlb_single_page_flush_ceiling); 66747d99948SChristophe Leroy } else { 66847d99948SChristophe Leroy is_local: 66947d99948SChristophe Leroy local = true; 67047d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 67147d99948SChristophe Leroy nr_pages > tlb_local_single_page_flush_ceiling); 67247d99948SChristophe Leroy } 67347d99948SChristophe Leroy 67447d99948SChristophe Leroy if (full) { 67547d99948SChristophe Leroy if (local) { 67647d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_TLB); 67747d99948SChristophe Leroy } else { 67847d99948SChristophe Leroy if (mm_needs_flush_escalation(mm)) 67947d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_ALL); 68047d99948SChristophe Leroy else 68147d99948SChristophe Leroy _tlbie_pid(pid, RIC_FLUSH_TLB); 68247d99948SChristophe Leroy } 68347d99948SChristophe Leroy } else { 68447d99948SChristophe Leroy bool hflush = flush_all_sizes; 68547d99948SChristophe Leroy bool gflush = flush_all_sizes; 68647d99948SChristophe Leroy unsigned long hstart, hend; 68747d99948SChristophe Leroy unsigned long gstart, gend; 68847d99948SChristophe Leroy 68947d99948SChristophe Leroy if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 69047d99948SChristophe Leroy hflush = true; 69147d99948SChristophe Leroy 69247d99948SChristophe Leroy if (hflush) { 69347d99948SChristophe Leroy hstart = (start + PMD_SIZE - 1) & PMD_MASK; 69447d99948SChristophe Leroy hend = end & PMD_MASK; 69547d99948SChristophe Leroy if (hstart == hend) 69647d99948SChristophe Leroy hflush = false; 69747d99948SChristophe Leroy } 69847d99948SChristophe Leroy 69947d99948SChristophe Leroy if (gflush) { 70047d99948SChristophe Leroy gstart = (start + PUD_SIZE - 1) & PUD_MASK; 70147d99948SChristophe Leroy gend = end & PUD_MASK; 70247d99948SChristophe Leroy if (gstart == gend) 70347d99948SChristophe Leroy gflush = false; 70447d99948SChristophe Leroy } 70547d99948SChristophe Leroy 70647d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 70747d99948SChristophe Leroy if (local) { 70847d99948SChristophe Leroy __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); 70947d99948SChristophe Leroy if (hflush) 71047d99948SChristophe Leroy __tlbiel_va_range(hstart, hend, pid, 71147d99948SChristophe Leroy PMD_SIZE, MMU_PAGE_2M); 71247d99948SChristophe Leroy if (gflush) 71347d99948SChristophe Leroy __tlbiel_va_range(gstart, gend, pid, 71447d99948SChristophe Leroy PUD_SIZE, MMU_PAGE_1G); 71547d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 71647d99948SChristophe Leroy } else { 71747d99948SChristophe Leroy __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize); 71847d99948SChristophe Leroy if (hflush) 71947d99948SChristophe Leroy __tlbie_va_range(hstart, hend, pid, 72047d99948SChristophe Leroy PMD_SIZE, MMU_PAGE_2M); 72147d99948SChristophe Leroy if (gflush) 72247d99948SChristophe Leroy __tlbie_va_range(gstart, gend, pid, 72347d99948SChristophe Leroy PUD_SIZE, MMU_PAGE_1G); 72447d99948SChristophe Leroy fixup_tlbie(); 72547d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 72647d99948SChristophe Leroy } 72747d99948SChristophe Leroy } 72847d99948SChristophe Leroy preempt_enable(); 72947d99948SChristophe Leroy } 73047d99948SChristophe Leroy 73147d99948SChristophe Leroy void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 73247d99948SChristophe Leroy unsigned long end) 73347d99948SChristophe Leroy 73447d99948SChristophe Leroy { 73547d99948SChristophe Leroy #ifdef CONFIG_HUGETLB_PAGE 73647d99948SChristophe Leroy if (is_vm_hugetlb_page(vma)) 73747d99948SChristophe Leroy return radix__flush_hugetlb_tlb_range(vma, start, end); 73847d99948SChristophe Leroy #endif 73947d99948SChristophe Leroy 74047d99948SChristophe Leroy __radix__flush_tlb_range(vma->vm_mm, start, end, false); 74147d99948SChristophe Leroy } 74247d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_tlb_range); 74347d99948SChristophe Leroy 74447d99948SChristophe Leroy static int radix_get_mmu_psize(int page_size) 74547d99948SChristophe Leroy { 74647d99948SChristophe Leroy int psize; 74747d99948SChristophe Leroy 74847d99948SChristophe Leroy if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift)) 74947d99948SChristophe Leroy psize = mmu_virtual_psize; 75047d99948SChristophe Leroy else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift)) 75147d99948SChristophe Leroy psize = MMU_PAGE_2M; 75247d99948SChristophe Leroy else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift)) 75347d99948SChristophe Leroy psize = MMU_PAGE_1G; 75447d99948SChristophe Leroy else 75547d99948SChristophe Leroy return -1; 75647d99948SChristophe Leroy return psize; 75747d99948SChristophe Leroy } 75847d99948SChristophe Leroy 75947d99948SChristophe Leroy /* 76047d99948SChristophe Leroy * Flush partition scoped LPID address translation for all CPUs. 76147d99948SChristophe Leroy */ 76247d99948SChristophe Leroy void radix__flush_tlb_lpid_page(unsigned int lpid, 76347d99948SChristophe Leroy unsigned long addr, 76447d99948SChristophe Leroy unsigned long page_size) 76547d99948SChristophe Leroy { 76647d99948SChristophe Leroy int psize = radix_get_mmu_psize(page_size); 76747d99948SChristophe Leroy 76847d99948SChristophe Leroy _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB); 76947d99948SChristophe Leroy } 77047d99948SChristophe Leroy EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page); 77147d99948SChristophe Leroy 77247d99948SChristophe Leroy /* 77347d99948SChristophe Leroy * Flush partition scoped PWC from LPID for all CPUs. 77447d99948SChristophe Leroy */ 77547d99948SChristophe Leroy void radix__flush_pwc_lpid(unsigned int lpid) 77647d99948SChristophe Leroy { 77747d99948SChristophe Leroy _tlbie_lpid(lpid, RIC_FLUSH_PWC); 77847d99948SChristophe Leroy } 77947d99948SChristophe Leroy EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid); 78047d99948SChristophe Leroy 78147d99948SChristophe Leroy /* 78247d99948SChristophe Leroy * Flush partition scoped translations from LPID (=LPIDR) 78347d99948SChristophe Leroy */ 78499161de3SNicholas Piggin void radix__flush_all_lpid(unsigned int lpid) 78547d99948SChristophe Leroy { 78647d99948SChristophe Leroy _tlbie_lpid(lpid, RIC_FLUSH_ALL); 78747d99948SChristophe Leroy } 78899161de3SNicholas Piggin EXPORT_SYMBOL_GPL(radix__flush_all_lpid); 78947d99948SChristophe Leroy 79047d99948SChristophe Leroy /* 79199161de3SNicholas Piggin * Flush process scoped translations from LPID (=LPIDR) 79247d99948SChristophe Leroy */ 79399161de3SNicholas Piggin void radix__flush_all_lpid_guest(unsigned int lpid) 79447d99948SChristophe Leroy { 79599161de3SNicholas Piggin _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); 79647d99948SChristophe Leroy } 79747d99948SChristophe Leroy 79847d99948SChristophe Leroy static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 79947d99948SChristophe Leroy unsigned long end, int psize); 80047d99948SChristophe Leroy 80147d99948SChristophe Leroy void radix__tlb_flush(struct mmu_gather *tlb) 80247d99948SChristophe Leroy { 80347d99948SChristophe Leroy int psize = 0; 80447d99948SChristophe Leroy struct mm_struct *mm = tlb->mm; 80547d99948SChristophe Leroy int page_size = tlb->page_size; 80647d99948SChristophe Leroy unsigned long start = tlb->start; 80747d99948SChristophe Leroy unsigned long end = tlb->end; 80847d99948SChristophe Leroy 80947d99948SChristophe Leroy /* 81047d99948SChristophe Leroy * if page size is not something we understand, do a full mm flush 81147d99948SChristophe Leroy * 81247d99948SChristophe Leroy * A "fullmm" flush must always do a flush_all_mm (RIC=2) flush 81347d99948SChristophe Leroy * that flushes the process table entry cache upon process teardown. 81447d99948SChristophe Leroy * See the comment for radix in arch_exit_mmap(). 81547d99948SChristophe Leroy */ 81647d99948SChristophe Leroy if (tlb->fullmm) { 81747d99948SChristophe Leroy __flush_all_mm(mm, true); 81847d99948SChristophe Leroy #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) 81947d99948SChristophe Leroy } else if (mm_tlb_flush_nested(mm)) { 82047d99948SChristophe Leroy /* 82147d99948SChristophe Leroy * If there is a concurrent invalidation that is clearing ptes, 82247d99948SChristophe Leroy * then it's possible this invalidation will miss one of those 82347d99948SChristophe Leroy * cleared ptes and miss flushing the TLB. If this invalidate 82447d99948SChristophe Leroy * returns before the other one flushes TLBs, that can result 82547d99948SChristophe Leroy * in it returning while there are still valid TLBs inside the 82647d99948SChristophe Leroy * range to be invalidated. 82747d99948SChristophe Leroy * 82847d99948SChristophe Leroy * See mm/memory.c:tlb_finish_mmu() for more details. 82947d99948SChristophe Leroy * 83047d99948SChristophe Leroy * The solution to this is ensure the entire range is always 83147d99948SChristophe Leroy * flushed here. The problem for powerpc is that the flushes 83247d99948SChristophe Leroy * are page size specific, so this "forced flush" would not 83347d99948SChristophe Leroy * do the right thing if there are a mix of page sizes in 83447d99948SChristophe Leroy * the range to be invalidated. So use __flush_tlb_range 83547d99948SChristophe Leroy * which invalidates all possible page sizes in the range. 83647d99948SChristophe Leroy * 83747d99948SChristophe Leroy * PWC flush probably is not be required because the core code 83847d99948SChristophe Leroy * shouldn't free page tables in this path, but accounting 83947d99948SChristophe Leroy * for the possibility makes us a bit more robust. 84047d99948SChristophe Leroy * 84147d99948SChristophe Leroy * need_flush_all is an uncommon case because page table 84247d99948SChristophe Leroy * teardown should be done with exclusive locks held (but 84347d99948SChristophe Leroy * after locks are dropped another invalidate could come 84447d99948SChristophe Leroy * in), it could be optimized further if necessary. 84547d99948SChristophe Leroy */ 84647d99948SChristophe Leroy if (!tlb->need_flush_all) 84747d99948SChristophe Leroy __radix__flush_tlb_range(mm, start, end, true); 84847d99948SChristophe Leroy else 84947d99948SChristophe Leroy radix__flush_all_mm(mm); 85047d99948SChristophe Leroy #endif 85147d99948SChristophe Leroy } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) { 85247d99948SChristophe Leroy if (!tlb->need_flush_all) 85347d99948SChristophe Leroy radix__flush_tlb_mm(mm); 85447d99948SChristophe Leroy else 85547d99948SChristophe Leroy radix__flush_all_mm(mm); 85647d99948SChristophe Leroy } else { 85747d99948SChristophe Leroy if (!tlb->need_flush_all) 85847d99948SChristophe Leroy radix__flush_tlb_range_psize(mm, start, end, psize); 85947d99948SChristophe Leroy else 86047d99948SChristophe Leroy radix__flush_tlb_pwc_range_psize(mm, start, end, psize); 86147d99948SChristophe Leroy } 86247d99948SChristophe Leroy tlb->need_flush_all = 0; 86347d99948SChristophe Leroy } 86447d99948SChristophe Leroy 865e12d6d7dSMasahiro Yamada static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, 86647d99948SChristophe Leroy unsigned long start, unsigned long end, 86747d99948SChristophe Leroy int psize, bool also_pwc) 86847d99948SChristophe Leroy { 86947d99948SChristophe Leroy unsigned long pid; 87047d99948SChristophe Leroy unsigned int page_shift = mmu_psize_defs[psize].shift; 87147d99948SChristophe Leroy unsigned long page_size = 1UL << page_shift; 87247d99948SChristophe Leroy unsigned long nr_pages = (end - start) >> page_shift; 87347d99948SChristophe Leroy bool local, full; 87447d99948SChristophe Leroy 87547d99948SChristophe Leroy pid = mm->context.id; 87647d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 87747d99948SChristophe Leroy return; 87847d99948SChristophe Leroy 87947d99948SChristophe Leroy preempt_disable(); 88047d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 88147d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 88247d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 88347d99948SChristophe Leroy if (end != TLB_FLUSH_ALL) { 88447d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 88547d99948SChristophe Leroy goto is_local; 88647d99948SChristophe Leroy } 88747d99948SChristophe Leroy } 88847d99948SChristophe Leroy local = false; 88947d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 89047d99948SChristophe Leroy nr_pages > tlb_single_page_flush_ceiling); 89147d99948SChristophe Leroy } else { 89247d99948SChristophe Leroy is_local: 89347d99948SChristophe Leroy local = true; 89447d99948SChristophe Leroy full = (end == TLB_FLUSH_ALL || 89547d99948SChristophe Leroy nr_pages > tlb_local_single_page_flush_ceiling); 89647d99948SChristophe Leroy } 89747d99948SChristophe Leroy 89847d99948SChristophe Leroy if (full) { 89947d99948SChristophe Leroy if (local) { 90047d99948SChristophe Leroy _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 90147d99948SChristophe Leroy } else { 90247d99948SChristophe Leroy if (mm_needs_flush_escalation(mm)) 90347d99948SChristophe Leroy also_pwc = true; 90447d99948SChristophe Leroy 90547d99948SChristophe Leroy _tlbie_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); 90647d99948SChristophe Leroy } 90747d99948SChristophe Leroy } else { 90847d99948SChristophe Leroy if (local) 90947d99948SChristophe Leroy _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc); 91047d99948SChristophe Leroy else 91147d99948SChristophe Leroy _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); 91247d99948SChristophe Leroy } 91347d99948SChristophe Leroy preempt_enable(); 91447d99948SChristophe Leroy } 91547d99948SChristophe Leroy 91647d99948SChristophe Leroy void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, 91747d99948SChristophe Leroy unsigned long end, int psize) 91847d99948SChristophe Leroy { 91947d99948SChristophe Leroy return __radix__flush_tlb_range_psize(mm, start, end, psize, false); 92047d99948SChristophe Leroy } 92147d99948SChristophe Leroy 92247d99948SChristophe Leroy static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, 92347d99948SChristophe Leroy unsigned long end, int psize) 92447d99948SChristophe Leroy { 92547d99948SChristophe Leroy __radix__flush_tlb_range_psize(mm, start, end, psize, true); 92647d99948SChristophe Leroy } 92747d99948SChristophe Leroy 92847d99948SChristophe Leroy #ifdef CONFIG_TRANSPARENT_HUGEPAGE 92947d99948SChristophe Leroy void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) 93047d99948SChristophe Leroy { 93147d99948SChristophe Leroy unsigned long pid, end; 93247d99948SChristophe Leroy 93347d99948SChristophe Leroy pid = mm->context.id; 93447d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 93547d99948SChristophe Leroy return; 93647d99948SChristophe Leroy 93747d99948SChristophe Leroy /* 4k page size, just blow the world */ 93847d99948SChristophe Leroy if (PAGE_SIZE == 0x1000) { 93947d99948SChristophe Leroy radix__flush_all_mm(mm); 94047d99948SChristophe Leroy return; 94147d99948SChristophe Leroy } 94247d99948SChristophe Leroy 94347d99948SChristophe Leroy end = addr + HPAGE_PMD_SIZE; 94447d99948SChristophe Leroy 94547d99948SChristophe Leroy /* Otherwise first do the PWC, then iterate the pages. */ 94647d99948SChristophe Leroy preempt_disable(); 94747d99948SChristophe Leroy smp_mb(); /* see radix__flush_tlb_mm */ 94847d99948SChristophe Leroy if (!mm_is_thread_local(mm)) { 94947d99948SChristophe Leroy if (unlikely(mm_is_singlethreaded(mm))) { 95047d99948SChristophe Leroy exit_flush_lazy_tlbs(mm); 95147d99948SChristophe Leroy goto local; 95247d99948SChristophe Leroy } 95347d99948SChristophe Leroy _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 95447d99948SChristophe Leroy } else { 95547d99948SChristophe Leroy local: 95647d99948SChristophe Leroy _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); 95747d99948SChristophe Leroy } 95847d99948SChristophe Leroy 95947d99948SChristophe Leroy preempt_enable(); 96047d99948SChristophe Leroy } 96147d99948SChristophe Leroy #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 96247d99948SChristophe Leroy 96347d99948SChristophe Leroy void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, 96447d99948SChristophe Leroy unsigned long start, unsigned long end) 96547d99948SChristophe Leroy { 96647d99948SChristophe Leroy radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); 96747d99948SChristophe Leroy } 96847d99948SChristophe Leroy EXPORT_SYMBOL(radix__flush_pmd_tlb_range); 96947d99948SChristophe Leroy 97047d99948SChristophe Leroy void radix__flush_tlb_all(void) 97147d99948SChristophe Leroy { 97247d99948SChristophe Leroy unsigned long rb,prs,r,rs; 97347d99948SChristophe Leroy unsigned long ric = RIC_FLUSH_ALL; 97447d99948SChristophe Leroy 97547d99948SChristophe Leroy rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */ 97647d99948SChristophe Leroy prs = 0; /* partition scoped */ 97747d99948SChristophe Leroy r = 1; /* radix format */ 97847d99948SChristophe Leroy rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */ 97947d99948SChristophe Leroy 98047d99948SChristophe Leroy asm volatile("ptesync": : :"memory"); 98147d99948SChristophe Leroy /* 98247d99948SChristophe Leroy * now flush guest entries by passing PRS = 1 and LPID != 0 98347d99948SChristophe Leroy */ 98447d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 98547d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory"); 98647d99948SChristophe Leroy /* 98747d99948SChristophe Leroy * now flush host entires by passing PRS = 0 and LPID == 0 98847d99948SChristophe Leroy */ 98947d99948SChristophe Leroy asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) 99047d99948SChristophe Leroy : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory"); 99147d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory"); 99247d99948SChristophe Leroy } 99347d99948SChristophe Leroy 99447d99948SChristophe Leroy #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 99547d99948SChristophe Leroy extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) 99647d99948SChristophe Leroy { 99747d99948SChristophe Leroy unsigned long pid = mm->context.id; 99847d99948SChristophe Leroy 99947d99948SChristophe Leroy if (unlikely(pid == MMU_NO_CONTEXT)) 100047d99948SChristophe Leroy return; 100147d99948SChristophe Leroy 100247d99948SChristophe Leroy /* 100347d99948SChristophe Leroy * If this context hasn't run on that CPU before and KVM is 100447d99948SChristophe Leroy * around, there's a slim chance that the guest on another 100547d99948SChristophe Leroy * CPU just brought in obsolete translation into the TLB of 100647d99948SChristophe Leroy * this CPU due to a bad prefetch using the guest PID on 100747d99948SChristophe Leroy * the way into the hypervisor. 100847d99948SChristophe Leroy * 100947d99948SChristophe Leroy * We work around this here. If KVM is possible, we check if 101047d99948SChristophe Leroy * any sibling thread is in KVM. If it is, the window may exist 101147d99948SChristophe Leroy * and thus we flush that PID from the core. 101247d99948SChristophe Leroy * 101347d99948SChristophe Leroy * A potential future improvement would be to mark which PIDs 101447d99948SChristophe Leroy * have never been used on the system and avoid it if the PID 101547d99948SChristophe Leroy * is new and the process has no other cpumask bit set. 101647d99948SChristophe Leroy */ 101747d99948SChristophe Leroy if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) { 101847d99948SChristophe Leroy int cpu = smp_processor_id(); 101947d99948SChristophe Leroy int sib = cpu_first_thread_sibling(cpu); 102047d99948SChristophe Leroy bool flush = false; 102147d99948SChristophe Leroy 102247d99948SChristophe Leroy for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) { 102347d99948SChristophe Leroy if (sib == cpu) 102447d99948SChristophe Leroy continue; 102547d99948SChristophe Leroy if (!cpu_possible(sib)) 102647d99948SChristophe Leroy continue; 102747d99948SChristophe Leroy if (paca_ptrs[sib]->kvm_hstate.kvm_vcpu) 102847d99948SChristophe Leroy flush = true; 102947d99948SChristophe Leroy } 103047d99948SChristophe Leroy if (flush) 103147d99948SChristophe Leroy _tlbiel_pid(pid, RIC_FLUSH_ALL); 103247d99948SChristophe Leroy } 103347d99948SChristophe Leroy } 103447d99948SChristophe Leroy EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround); 103547d99948SChristophe Leroy #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ 1036