12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
247d99948SChristophe Leroy /*
347d99948SChristophe Leroy * native hashtable management.
447d99948SChristophe Leroy *
547d99948SChristophe Leroy * SMP scalability work:
647d99948SChristophe Leroy * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
747d99948SChristophe Leroy */
847d99948SChristophe Leroy
947d99948SChristophe Leroy #undef DEBUG_LOW
1047d99948SChristophe Leroy
1147d99948SChristophe Leroy #include <linux/spinlock.h>
1247d99948SChristophe Leroy #include <linux/bitops.h>
1347d99948SChristophe Leroy #include <linux/of.h>
1447d99948SChristophe Leroy #include <linux/processor.h>
1547d99948SChristophe Leroy #include <linux/threads.h>
1647d99948SChristophe Leroy #include <linux/smp.h>
1765fddcfcSMike Rapoport #include <linux/pgtable.h>
1847d99948SChristophe Leroy
1947d99948SChristophe Leroy #include <asm/machdep.h>
2047d99948SChristophe Leroy #include <asm/mmu.h>
2147d99948SChristophe Leroy #include <asm/mmu_context.h>
2247d99948SChristophe Leroy #include <asm/trace.h>
2347d99948SChristophe Leroy #include <asm/tlb.h>
2447d99948SChristophe Leroy #include <asm/cputable.h>
2547d99948SChristophe Leroy #include <asm/udbg.h>
2647d99948SChristophe Leroy #include <asm/kexec.h>
2747d99948SChristophe Leroy #include <asm/ppc-opcode.h>
2847d99948SChristophe Leroy #include <asm/feature-fixups.h>
2947d99948SChristophe Leroy
3047d99948SChristophe Leroy #include <misc/cxl-base.h>
3147d99948SChristophe Leroy
3247d99948SChristophe Leroy #ifdef DEBUG_LOW
3347d99948SChristophe Leroy #define DBG_LOW(fmt...) udbg_printf(fmt)
3447d99948SChristophe Leroy #else
3547d99948SChristophe Leroy #define DBG_LOW(fmt...)
3647d99948SChristophe Leroy #endif
3747d99948SChristophe Leroy
3847d99948SChristophe Leroy #ifdef __BIG_ENDIAN__
3947d99948SChristophe Leroy #define HPTE_LOCK_BIT 3
4047d99948SChristophe Leroy #else
4147d99948SChristophe Leroy #define HPTE_LOCK_BIT (56+3)
4247d99948SChristophe Leroy #endif
4347d99948SChristophe Leroy
44d667edc0SYueHaibing static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
4547d99948SChristophe Leroy
46be83d548SNicholas Piggin #ifdef CONFIG_LOCKDEP
47be83d548SNicholas Piggin static struct lockdep_map hpte_lock_map =
48be83d548SNicholas Piggin STATIC_LOCKDEP_MAP_INIT("hpte_lock", &hpte_lock_map);
49be83d548SNicholas Piggin
acquire_hpte_lock(void)50be83d548SNicholas Piggin static void acquire_hpte_lock(void)
51be83d548SNicholas Piggin {
52be83d548SNicholas Piggin lock_map_acquire(&hpte_lock_map);
53be83d548SNicholas Piggin }
54be83d548SNicholas Piggin
release_hpte_lock(void)55be83d548SNicholas Piggin static void release_hpte_lock(void)
56be83d548SNicholas Piggin {
57be83d548SNicholas Piggin lock_map_release(&hpte_lock_map);
58be83d548SNicholas Piggin }
59be83d548SNicholas Piggin #else
acquire_hpte_lock(void)60be83d548SNicholas Piggin static void acquire_hpte_lock(void)
61be83d548SNicholas Piggin {
62be83d548SNicholas Piggin }
63be83d548SNicholas Piggin
release_hpte_lock(void)64be83d548SNicholas Piggin static void release_hpte_lock(void)
65be83d548SNicholas Piggin {
66be83d548SNicholas Piggin }
67be83d548SNicholas Piggin #endif
68be83d548SNicholas Piggin
___tlbie(unsigned long vpn,int psize,int apsize,int ssize)6947d99948SChristophe Leroy static inline unsigned long ___tlbie(unsigned long vpn, int psize,
7047d99948SChristophe Leroy int apsize, int ssize)
7147d99948SChristophe Leroy {
7247d99948SChristophe Leroy unsigned long va;
7347d99948SChristophe Leroy unsigned int penc;
7447d99948SChristophe Leroy unsigned long sllp;
7547d99948SChristophe Leroy
7647d99948SChristophe Leroy /*
7747d99948SChristophe Leroy * We need 14 to 65 bits of va for a tlibe of 4K page
7847d99948SChristophe Leroy * With vpn we ignore the lower VPN_SHIFT bits already.
7947d99948SChristophe Leroy * And top two bits are already ignored because we can
8047d99948SChristophe Leroy * only accomodate 76 bits in a 64 bit vpn with a VPN_SHIFT
8147d99948SChristophe Leroy * of 12.
8247d99948SChristophe Leroy */
8347d99948SChristophe Leroy va = vpn << VPN_SHIFT;
8447d99948SChristophe Leroy /*
8547d99948SChristophe Leroy * clear top 16 bits of 64bit va, non SLS segment
8647d99948SChristophe Leroy * Older versions of the architecture (2.02 and earler) require the
8747d99948SChristophe Leroy * masking of the top 16 bits.
8847d99948SChristophe Leroy */
8947d99948SChristophe Leroy if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
9047d99948SChristophe Leroy va &= ~(0xffffULL << 48);
9147d99948SChristophe Leroy
9247d99948SChristophe Leroy switch (psize) {
9347d99948SChristophe Leroy case MMU_PAGE_4K:
9447d99948SChristophe Leroy /* clear out bits after (52) [0....52.....63] */
9547d99948SChristophe Leroy va &= ~((1ul << (64 - 52)) - 1);
9647d99948SChristophe Leroy va |= ssize << 8;
9747d99948SChristophe Leroy sllp = get_sllp_encoding(apsize);
9847d99948SChristophe Leroy va |= sllp << 5;
9947d99948SChristophe Leroy asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
10047d99948SChristophe Leroy : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
10147d99948SChristophe Leroy : "memory");
10247d99948SChristophe Leroy break;
10347d99948SChristophe Leroy default:
10447d99948SChristophe Leroy /* We need 14 to 14 + i bits of va */
10547d99948SChristophe Leroy penc = mmu_psize_defs[psize].penc[apsize];
10647d99948SChristophe Leroy va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
10747d99948SChristophe Leroy va |= penc << 12;
10847d99948SChristophe Leroy va |= ssize << 8;
10947d99948SChristophe Leroy /*
11047d99948SChristophe Leroy * AVAL bits:
11147d99948SChristophe Leroy * We don't need all the bits, but rest of the bits
11247d99948SChristophe Leroy * must be ignored by the processor.
11347d99948SChristophe Leroy * vpn cover upto 65 bits of va. (0...65) and we need
11447d99948SChristophe Leroy * 58..64 bits of va.
11547d99948SChristophe Leroy */
11647d99948SChristophe Leroy va |= (vpn & 0xfe); /* AVAL */
11747d99948SChristophe Leroy va |= 1; /* L */
11847d99948SChristophe Leroy asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
11947d99948SChristophe Leroy : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
12047d99948SChristophe Leroy : "memory");
12147d99948SChristophe Leroy break;
12247d99948SChristophe Leroy }
12347d99948SChristophe Leroy return va;
12447d99948SChristophe Leroy }
12547d99948SChristophe Leroy
fixup_tlbie_vpn(unsigned long vpn,int psize,int apsize,int ssize)126047e6575SAneesh Kumar K.V static inline void fixup_tlbie_vpn(unsigned long vpn, int psize,
127047e6575SAneesh Kumar K.V int apsize, int ssize)
12847d99948SChristophe Leroy {
129047e6575SAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
130047e6575SAneesh Kumar K.V /* Radix flush for a hash guest */
131047e6575SAneesh Kumar K.V
132047e6575SAneesh Kumar K.V unsigned long rb,rs,prs,r,ric;
133047e6575SAneesh Kumar K.V
134047e6575SAneesh Kumar K.V rb = PPC_BIT(52); /* IS = 2 */
135047e6575SAneesh Kumar K.V rs = 0; /* lpid = 0 */
136047e6575SAneesh Kumar K.V prs = 0; /* partition scoped */
137047e6575SAneesh Kumar K.V r = 1; /* radix format */
138047e6575SAneesh Kumar K.V ric = 0; /* RIC_FLSUH_TLB */
139047e6575SAneesh Kumar K.V
140047e6575SAneesh Kumar K.V /*
141047e6575SAneesh Kumar K.V * Need the extra ptesync to make sure we don't
142047e6575SAneesh Kumar K.V * re-order the tlbie
143047e6575SAneesh Kumar K.V */
144047e6575SAneesh Kumar K.V asm volatile("ptesync": : :"memory");
145047e6575SAneesh Kumar K.V asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
146047e6575SAneesh Kumar K.V : : "r"(rb), "i"(r), "i"(prs),
147047e6575SAneesh Kumar K.V "i"(ric), "r"(rs) : "memory");
148047e6575SAneesh Kumar K.V }
149047e6575SAneesh Kumar K.V
150047e6575SAneesh Kumar K.V
15109ce98caSAneesh Kumar K.V if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
15247d99948SChristophe Leroy /* Need the extra ptesync to ensure we don't reorder tlbie*/
15347d99948SChristophe Leroy asm volatile("ptesync": : :"memory");
15447d99948SChristophe Leroy ___tlbie(vpn, psize, apsize, ssize);
15547d99948SChristophe Leroy }
15647d99948SChristophe Leroy }
15747d99948SChristophe Leroy
__tlbie(unsigned long vpn,int psize,int apsize,int ssize)15847d99948SChristophe Leroy static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
15947d99948SChristophe Leroy {
16047d99948SChristophe Leroy unsigned long rb;
16147d99948SChristophe Leroy
16247d99948SChristophe Leroy rb = ___tlbie(vpn, psize, apsize, ssize);
16347d99948SChristophe Leroy trace_tlbie(0, 0, rb, 0, 0, 0, 0);
16447d99948SChristophe Leroy }
16547d99948SChristophe Leroy
__tlbiel(unsigned long vpn,int psize,int apsize,int ssize)16647d99948SChristophe Leroy static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
16747d99948SChristophe Leroy {
16847d99948SChristophe Leroy unsigned long va;
16947d99948SChristophe Leroy unsigned int penc;
17047d99948SChristophe Leroy unsigned long sllp;
17147d99948SChristophe Leroy
17247d99948SChristophe Leroy /* VPN_SHIFT can be atmost 12 */
17347d99948SChristophe Leroy va = vpn << VPN_SHIFT;
17447d99948SChristophe Leroy /*
17547d99948SChristophe Leroy * clear top 16 bits of 64 bit va, non SLS segment
17647d99948SChristophe Leroy * Older versions of the architecture (2.02 and earler) require the
17747d99948SChristophe Leroy * masking of the top 16 bits.
17847d99948SChristophe Leroy */
17947d99948SChristophe Leroy if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
18047d99948SChristophe Leroy va &= ~(0xffffULL << 48);
18147d99948SChristophe Leroy
18247d99948SChristophe Leroy switch (psize) {
18347d99948SChristophe Leroy case MMU_PAGE_4K:
18447d99948SChristophe Leroy /* clear out bits after(52) [0....52.....63] */
18547d99948SChristophe Leroy va &= ~((1ul << (64 - 52)) - 1);
18647d99948SChristophe Leroy va |= ssize << 8;
18747d99948SChristophe Leroy sllp = get_sllp_encoding(apsize);
18847d99948SChristophe Leroy va |= sllp << 5;
18962479e6eSAlexey Kardashevskiy asm volatile(ASM_FTR_IFSET("tlbiel %0", PPC_TLBIEL_v205(%0, 0), %1)
19047d99948SChristophe Leroy : : "r" (va), "i" (CPU_FTR_ARCH_206)
19147d99948SChristophe Leroy : "memory");
19247d99948SChristophe Leroy break;
19347d99948SChristophe Leroy default:
19447d99948SChristophe Leroy /* We need 14 to 14 + i bits of va */
19547d99948SChristophe Leroy penc = mmu_psize_defs[psize].penc[apsize];
19647d99948SChristophe Leroy va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
19747d99948SChristophe Leroy va |= penc << 12;
19847d99948SChristophe Leroy va |= ssize << 8;
19947d99948SChristophe Leroy /*
20047d99948SChristophe Leroy * AVAL bits:
20147d99948SChristophe Leroy * We don't need all the bits, but rest of the bits
20247d99948SChristophe Leroy * must be ignored by the processor.
20347d99948SChristophe Leroy * vpn cover upto 65 bits of va. (0...65) and we need
20447d99948SChristophe Leroy * 58..64 bits of va.
20547d99948SChristophe Leroy */
20647d99948SChristophe Leroy va |= (vpn & 0xfe);
20747d99948SChristophe Leroy va |= 1; /* L */
20862479e6eSAlexey Kardashevskiy asm volatile(ASM_FTR_IFSET("tlbiel %0", PPC_TLBIEL_v205(%0, 1), %1)
20947d99948SChristophe Leroy : : "r" (va), "i" (CPU_FTR_ARCH_206)
21047d99948SChristophe Leroy : "memory");
21147d99948SChristophe Leroy break;
21247d99948SChristophe Leroy }
21347d99948SChristophe Leroy trace_tlbie(0, 1, va, 0, 0, 0, 0);
21447d99948SChristophe Leroy
21547d99948SChristophe Leroy }
21647d99948SChristophe Leroy
tlbie(unsigned long vpn,int psize,int apsize,int ssize,int local)21747d99948SChristophe Leroy static inline void tlbie(unsigned long vpn, int psize, int apsize,
21847d99948SChristophe Leroy int ssize, int local)
21947d99948SChristophe Leroy {
22047d99948SChristophe Leroy unsigned int use_local;
22147d99948SChristophe Leroy int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
22247d99948SChristophe Leroy
22347d99948SChristophe Leroy use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && !cxl_ctx_in_use();
22447d99948SChristophe Leroy
22547d99948SChristophe Leroy if (use_local)
22647d99948SChristophe Leroy use_local = mmu_psize_defs[psize].tlbiel;
22747d99948SChristophe Leroy if (lock_tlbie && !use_local)
22847d99948SChristophe Leroy raw_spin_lock(&native_tlbie_lock);
22947d99948SChristophe Leroy asm volatile("ptesync": : :"memory");
23047d99948SChristophe Leroy if (use_local) {
23147d99948SChristophe Leroy __tlbiel(vpn, psize, apsize, ssize);
23205504b42SNicholas Piggin ppc_after_tlbiel_barrier();
23347d99948SChristophe Leroy } else {
23447d99948SChristophe Leroy __tlbie(vpn, psize, apsize, ssize);
235047e6575SAneesh Kumar K.V fixup_tlbie_vpn(vpn, psize, apsize, ssize);
23647d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync": : :"memory");
23747d99948SChristophe Leroy }
23847d99948SChristophe Leroy if (lock_tlbie && !use_local)
23947d99948SChristophe Leroy raw_spin_unlock(&native_tlbie_lock);
24047d99948SChristophe Leroy }
24147d99948SChristophe Leroy
native_lock_hpte(struct hash_pte * hptep)24247d99948SChristophe Leroy static inline void native_lock_hpte(struct hash_pte *hptep)
24347d99948SChristophe Leroy {
24447d99948SChristophe Leroy unsigned long *word = (unsigned long *)&hptep->v;
24547d99948SChristophe Leroy
246be83d548SNicholas Piggin acquire_hpte_lock();
24747d99948SChristophe Leroy while (1) {
24847d99948SChristophe Leroy if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
24947d99948SChristophe Leroy break;
25047d99948SChristophe Leroy spin_begin();
25147d99948SChristophe Leroy while(test_bit(HPTE_LOCK_BIT, word))
25247d99948SChristophe Leroy spin_cpu_relax();
25347d99948SChristophe Leroy spin_end();
25447d99948SChristophe Leroy }
25547d99948SChristophe Leroy }
25647d99948SChristophe Leroy
native_unlock_hpte(struct hash_pte * hptep)25747d99948SChristophe Leroy static inline void native_unlock_hpte(struct hash_pte *hptep)
25847d99948SChristophe Leroy {
25947d99948SChristophe Leroy unsigned long *word = (unsigned long *)&hptep->v;
26047d99948SChristophe Leroy
261be83d548SNicholas Piggin release_hpte_lock();
26247d99948SChristophe Leroy clear_bit_unlock(HPTE_LOCK_BIT, word);
26347d99948SChristophe Leroy }
26447d99948SChristophe Leroy
native_hpte_insert(unsigned long hpte_group,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags,int psize,int apsize,int ssize)26547d99948SChristophe Leroy static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
26647d99948SChristophe Leroy unsigned long pa, unsigned long rflags,
26747d99948SChristophe Leroy unsigned long vflags, int psize, int apsize, int ssize)
26847d99948SChristophe Leroy {
26947d99948SChristophe Leroy struct hash_pte *hptep = htab_address + hpte_group;
27047d99948SChristophe Leroy unsigned long hpte_v, hpte_r;
27135159b57SNicholas Piggin unsigned long flags;
27247d99948SChristophe Leroy int i;
27347d99948SChristophe Leroy
27435159b57SNicholas Piggin local_irq_save(flags);
27535159b57SNicholas Piggin
27647d99948SChristophe Leroy if (!(vflags & HPTE_V_BOLTED)) {
27747d99948SChristophe Leroy DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx,"
27847d99948SChristophe Leroy " rflags=%lx, vflags=%lx, psize=%d)\n",
27947d99948SChristophe Leroy hpte_group, vpn, pa, rflags, vflags, psize);
28047d99948SChristophe Leroy }
28147d99948SChristophe Leroy
28247d99948SChristophe Leroy for (i = 0; i < HPTES_PER_GROUP; i++) {
28347d99948SChristophe Leroy if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
28447d99948SChristophe Leroy /* retry with lock held */
28547d99948SChristophe Leroy native_lock_hpte(hptep);
28647d99948SChristophe Leroy if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID))
28747d99948SChristophe Leroy break;
28847d99948SChristophe Leroy native_unlock_hpte(hptep);
28947d99948SChristophe Leroy }
29047d99948SChristophe Leroy
29147d99948SChristophe Leroy hptep++;
29247d99948SChristophe Leroy }
29347d99948SChristophe Leroy
29435159b57SNicholas Piggin if (i == HPTES_PER_GROUP) {
29535159b57SNicholas Piggin local_irq_restore(flags);
29647d99948SChristophe Leroy return -1;
29735159b57SNicholas Piggin }
29847d99948SChristophe Leroy
29947d99948SChristophe Leroy hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
30047d99948SChristophe Leroy hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
30147d99948SChristophe Leroy
30247d99948SChristophe Leroy if (!(vflags & HPTE_V_BOLTED)) {
30347d99948SChristophe Leroy DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
30447d99948SChristophe Leroy i, hpte_v, hpte_r);
30547d99948SChristophe Leroy }
30647d99948SChristophe Leroy
30747d99948SChristophe Leroy if (cpu_has_feature(CPU_FTR_ARCH_300)) {
30847d99948SChristophe Leroy hpte_r = hpte_old_to_new_r(hpte_v, hpte_r);
30947d99948SChristophe Leroy hpte_v = hpte_old_to_new_v(hpte_v);
31047d99948SChristophe Leroy }
31147d99948SChristophe Leroy
31247d99948SChristophe Leroy hptep->r = cpu_to_be64(hpte_r);
31347d99948SChristophe Leroy /* Guarantee the second dword is visible before the valid bit */
31447d99948SChristophe Leroy eieio();
31547d99948SChristophe Leroy /*
31647d99948SChristophe Leroy * Now set the first dword including the valid bit
31747d99948SChristophe Leroy * NOTE: this also unlocks the hpte
31847d99948SChristophe Leroy */
31935159b57SNicholas Piggin release_hpte_lock();
32047d99948SChristophe Leroy hptep->v = cpu_to_be64(hpte_v);
32147d99948SChristophe Leroy
32247d99948SChristophe Leroy __asm__ __volatile__ ("ptesync" : : : "memory");
32347d99948SChristophe Leroy
32435159b57SNicholas Piggin local_irq_restore(flags);
32535159b57SNicholas Piggin
32647d99948SChristophe Leroy return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
32747d99948SChristophe Leroy }
32847d99948SChristophe Leroy
native_hpte_remove(unsigned long hpte_group)32947d99948SChristophe Leroy static long native_hpte_remove(unsigned long hpte_group)
33047d99948SChristophe Leroy {
331*8bbe9feeSMichael Ellerman unsigned long hpte_v, flags;
33247d99948SChristophe Leroy struct hash_pte *hptep;
33347d99948SChristophe Leroy int i;
33447d99948SChristophe Leroy int slot_offset;
335*8bbe9feeSMichael Ellerman
336*8bbe9feeSMichael Ellerman local_irq_save(flags);
33747d99948SChristophe Leroy
33847d99948SChristophe Leroy DBG_LOW(" remove(group=%lx)\n", hpte_group);
33947d99948SChristophe Leroy
34047d99948SChristophe Leroy /* pick a random entry to start at */
34147d99948SChristophe Leroy slot_offset = mftb() & 0x7;
34247d99948SChristophe Leroy
34347d99948SChristophe Leroy for (i = 0; i < HPTES_PER_GROUP; i++) {
34447d99948SChristophe Leroy hptep = htab_address + hpte_group + slot_offset;
34547d99948SChristophe Leroy hpte_v = be64_to_cpu(hptep->v);
34647d99948SChristophe Leroy
34747d99948SChristophe Leroy if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
34847d99948SChristophe Leroy /* retry with lock held */
34947d99948SChristophe Leroy native_lock_hpte(hptep);
35047d99948SChristophe Leroy hpte_v = be64_to_cpu(hptep->v);
35147d99948SChristophe Leroy if ((hpte_v & HPTE_V_VALID)
35247d99948SChristophe Leroy && !(hpte_v & HPTE_V_BOLTED))
35347d99948SChristophe Leroy break;
35447d99948SChristophe Leroy native_unlock_hpte(hptep);
35547d99948SChristophe Leroy }
35647d99948SChristophe Leroy
35747d99948SChristophe Leroy slot_offset++;
35847d99948SChristophe Leroy slot_offset &= 0x7;
35947d99948SChristophe Leroy }
36047d99948SChristophe Leroy
361*8bbe9feeSMichael Ellerman if (i == HPTES_PER_GROUP) {
362*8bbe9feeSMichael Ellerman i = -1;
363*8bbe9feeSMichael Ellerman goto out;
364*8bbe9feeSMichael Ellerman }
36547d99948SChristophe Leroy
36647d99948SChristophe Leroy /* Invalidate the hpte. NOTE: this also unlocks it */
367be83d548SNicholas Piggin release_hpte_lock();
36847d99948SChristophe Leroy hptep->v = 0;
369*8bbe9feeSMichael Ellerman out:
370*8bbe9feeSMichael Ellerman local_irq_restore(flags);
37147d99948SChristophe Leroy return i;
37247d99948SChristophe Leroy }
37347d99948SChristophe Leroy
native_hpte_updatepp(unsigned long slot,unsigned long newpp,unsigned long vpn,int bpsize,int apsize,int ssize,unsigned long flags)37447d99948SChristophe Leroy static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
37547d99948SChristophe Leroy unsigned long vpn, int bpsize,
37647d99948SChristophe Leroy int apsize, int ssize, unsigned long flags)
37747d99948SChristophe Leroy {
37847d99948SChristophe Leroy struct hash_pte *hptep = htab_address + slot;
37947d99948SChristophe Leroy unsigned long hpte_v, want_v;
38047d99948SChristophe Leroy int ret = 0, local = 0;
38135159b57SNicholas Piggin unsigned long irqflags;
38235159b57SNicholas Piggin
38335159b57SNicholas Piggin local_irq_save(irqflags);
38447d99948SChristophe Leroy
38547d99948SChristophe Leroy want_v = hpte_encode_avpn(vpn, bpsize, ssize);
38647d99948SChristophe Leroy
38747d99948SChristophe Leroy DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
38847d99948SChristophe Leroy vpn, want_v & HPTE_V_AVPN, slot, newpp);
38947d99948SChristophe Leroy
39047d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
39147d99948SChristophe Leroy /*
39247d99948SChristophe Leroy * We need to invalidate the TLB always because hpte_remove doesn't do
39347d99948SChristophe Leroy * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
39447d99948SChristophe Leroy * random entry from it. When we do that we don't invalidate the TLB
39547d99948SChristophe Leroy * (hpte_remove) because we assume the old translation is still
39647d99948SChristophe Leroy * technically "valid".
39747d99948SChristophe Leroy */
39847d99948SChristophe Leroy if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
39947d99948SChristophe Leroy DBG_LOW(" -> miss\n");
40047d99948SChristophe Leroy ret = -1;
40147d99948SChristophe Leroy } else {
40247d99948SChristophe Leroy native_lock_hpte(hptep);
40347d99948SChristophe Leroy /* recheck with locks held */
40447d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
40547d99948SChristophe Leroy if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) ||
40647d99948SChristophe Leroy !(hpte_v & HPTE_V_VALID))) {
40747d99948SChristophe Leroy ret = -1;
40847d99948SChristophe Leroy } else {
40947d99948SChristophe Leroy DBG_LOW(" -> hit\n");
41047d99948SChristophe Leroy /* Update the HPTE */
41147d99948SChristophe Leroy hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
41247d99948SChristophe Leroy ~(HPTE_R_PPP | HPTE_R_N)) |
41347d99948SChristophe Leroy (newpp & (HPTE_R_PPP | HPTE_R_N |
41447d99948SChristophe Leroy HPTE_R_C)));
41547d99948SChristophe Leroy }
41647d99948SChristophe Leroy native_unlock_hpte(hptep);
41747d99948SChristophe Leroy }
41847d99948SChristophe Leroy
41947d99948SChristophe Leroy if (flags & HPTE_LOCAL_UPDATE)
42047d99948SChristophe Leroy local = 1;
42147d99948SChristophe Leroy /*
42247d99948SChristophe Leroy * Ensure it is out of the tlb too if it is not a nohpte fault
42347d99948SChristophe Leroy */
42447d99948SChristophe Leroy if (!(flags & HPTE_NOHPTE_UPDATE))
42547d99948SChristophe Leroy tlbie(vpn, bpsize, apsize, ssize, local);
42647d99948SChristophe Leroy
42735159b57SNicholas Piggin local_irq_restore(irqflags);
42835159b57SNicholas Piggin
42947d99948SChristophe Leroy return ret;
43047d99948SChristophe Leroy }
43147d99948SChristophe Leroy
__native_hpte_find(unsigned long want_v,unsigned long slot)432d78d5dacSAneesh Kumar K.V static long __native_hpte_find(unsigned long want_v, unsigned long slot)
43347d99948SChristophe Leroy {
43447d99948SChristophe Leroy struct hash_pte *hptep;
435d78d5dacSAneesh Kumar K.V unsigned long hpte_v;
43647d99948SChristophe Leroy unsigned long i;
43747d99948SChristophe Leroy
43847d99948SChristophe Leroy for (i = 0; i < HPTES_PER_GROUP; i++) {
43947d99948SChristophe Leroy
44047d99948SChristophe Leroy hptep = htab_address + slot;
44147d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
44247d99948SChristophe Leroy if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
44347d99948SChristophe Leroy /* HPTE matches */
44447d99948SChristophe Leroy return slot;
44547d99948SChristophe Leroy ++slot;
44647d99948SChristophe Leroy }
44747d99948SChristophe Leroy
44847d99948SChristophe Leroy return -1;
44947d99948SChristophe Leroy }
45047d99948SChristophe Leroy
native_hpte_find(unsigned long vpn,int psize,int ssize)451d78d5dacSAneesh Kumar K.V static long native_hpte_find(unsigned long vpn, int psize, int ssize)
452d78d5dacSAneesh Kumar K.V {
453d78d5dacSAneesh Kumar K.V unsigned long hpte_group;
454d78d5dacSAneesh Kumar K.V unsigned long want_v;
455d78d5dacSAneesh Kumar K.V unsigned long hash;
456d78d5dacSAneesh Kumar K.V long slot;
457d78d5dacSAneesh Kumar K.V
458d78d5dacSAneesh Kumar K.V hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
459d78d5dacSAneesh Kumar K.V want_v = hpte_encode_avpn(vpn, psize, ssize);
460d78d5dacSAneesh Kumar K.V
461d78d5dacSAneesh Kumar K.V /*
462d78d5dacSAneesh Kumar K.V * We try to keep bolted entries always in primary hash
463d78d5dacSAneesh Kumar K.V * But in some case we can find them in secondary too.
464d78d5dacSAneesh Kumar K.V */
465d78d5dacSAneesh Kumar K.V hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
466d78d5dacSAneesh Kumar K.V slot = __native_hpte_find(want_v, hpte_group);
467d78d5dacSAneesh Kumar K.V if (slot < 0) {
468d78d5dacSAneesh Kumar K.V /* Try in secondary */
469d78d5dacSAneesh Kumar K.V hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
470d78d5dacSAneesh Kumar K.V slot = __native_hpte_find(want_v, hpte_group);
471d78d5dacSAneesh Kumar K.V if (slot < 0)
472d78d5dacSAneesh Kumar K.V return -1;
473d78d5dacSAneesh Kumar K.V }
474d78d5dacSAneesh Kumar K.V
475d78d5dacSAneesh Kumar K.V return slot;
476d78d5dacSAneesh Kumar K.V }
477d78d5dacSAneesh Kumar K.V
47847d99948SChristophe Leroy /*
47947d99948SChristophe Leroy * Update the page protection bits. Intended to be used to create
48047d99948SChristophe Leroy * guard pages for kernel data structures on pages which are bolted
48147d99948SChristophe Leroy * in the HPT. Assumes pages being operated on will not be stolen.
48247d99948SChristophe Leroy *
48347d99948SChristophe Leroy * No need to lock here because we should be the only user.
48447d99948SChristophe Leroy */
native_hpte_updateboltedpp(unsigned long newpp,unsigned long ea,int psize,int ssize)48547d99948SChristophe Leroy static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
48647d99948SChristophe Leroy int psize, int ssize)
48747d99948SChristophe Leroy {
48847d99948SChristophe Leroy unsigned long vpn;
48947d99948SChristophe Leroy unsigned long vsid;
49047d99948SChristophe Leroy long slot;
49147d99948SChristophe Leroy struct hash_pte *hptep;
49235159b57SNicholas Piggin unsigned long flags;
49335159b57SNicholas Piggin
49435159b57SNicholas Piggin local_irq_save(flags);
49547d99948SChristophe Leroy
49647d99948SChristophe Leroy vsid = get_kernel_vsid(ea, ssize);
49747d99948SChristophe Leroy vpn = hpt_vpn(ea, vsid, ssize);
49847d99948SChristophe Leroy
49947d99948SChristophe Leroy slot = native_hpte_find(vpn, psize, ssize);
50047d99948SChristophe Leroy if (slot == -1)
50147d99948SChristophe Leroy panic("could not find page to bolt\n");
50247d99948SChristophe Leroy hptep = htab_address + slot;
50347d99948SChristophe Leroy
50447d99948SChristophe Leroy /* Update the HPTE */
50547d99948SChristophe Leroy hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
50647d99948SChristophe Leroy ~(HPTE_R_PPP | HPTE_R_N)) |
50747d99948SChristophe Leroy (newpp & (HPTE_R_PPP | HPTE_R_N)));
50847d99948SChristophe Leroy /*
50947d99948SChristophe Leroy * Ensure it is out of the tlb too. Bolted entries base and
51047d99948SChristophe Leroy * actual page size will be same.
51147d99948SChristophe Leroy */
51247d99948SChristophe Leroy tlbie(vpn, psize, psize, ssize, 0);
51335159b57SNicholas Piggin
51435159b57SNicholas Piggin local_irq_restore(flags);
51547d99948SChristophe Leroy }
51647d99948SChristophe Leroy
51747d99948SChristophe Leroy /*
51847d99948SChristophe Leroy * Remove a bolted kernel entry. Memory hotplug uses this.
51947d99948SChristophe Leroy *
52047d99948SChristophe Leroy * No need to lock here because we should be the only user.
52147d99948SChristophe Leroy */
native_hpte_removebolted(unsigned long ea,int psize,int ssize)52247d99948SChristophe Leroy static int native_hpte_removebolted(unsigned long ea, int psize, int ssize)
52347d99948SChristophe Leroy {
52447d99948SChristophe Leroy unsigned long vpn;
52547d99948SChristophe Leroy unsigned long vsid;
52647d99948SChristophe Leroy long slot;
52747d99948SChristophe Leroy struct hash_pte *hptep;
52835159b57SNicholas Piggin unsigned long flags;
52935159b57SNicholas Piggin
53035159b57SNicholas Piggin local_irq_save(flags);
53147d99948SChristophe Leroy
53247d99948SChristophe Leroy vsid = get_kernel_vsid(ea, ssize);
53347d99948SChristophe Leroy vpn = hpt_vpn(ea, vsid, ssize);
53447d99948SChristophe Leroy
53547d99948SChristophe Leroy slot = native_hpte_find(vpn, psize, ssize);
53647d99948SChristophe Leroy if (slot == -1)
53747d99948SChristophe Leroy return -ENOENT;
53847d99948SChristophe Leroy
53947d99948SChristophe Leroy hptep = htab_address + slot;
54047d99948SChristophe Leroy
54147d99948SChristophe Leroy VM_WARN_ON(!(be64_to_cpu(hptep->v) & HPTE_V_BOLTED));
54247d99948SChristophe Leroy
54347d99948SChristophe Leroy /* Invalidate the hpte */
54447d99948SChristophe Leroy hptep->v = 0;
54547d99948SChristophe Leroy
54647d99948SChristophe Leroy /* Invalidate the TLB */
54747d99948SChristophe Leroy tlbie(vpn, psize, psize, ssize, 0);
54835159b57SNicholas Piggin
54935159b57SNicholas Piggin local_irq_restore(flags);
55035159b57SNicholas Piggin
55147d99948SChristophe Leroy return 0;
55247d99948SChristophe Leroy }
55347d99948SChristophe Leroy
55447d99948SChristophe Leroy
native_hpte_invalidate(unsigned long slot,unsigned long vpn,int bpsize,int apsize,int ssize,int local)55547d99948SChristophe Leroy static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
55647d99948SChristophe Leroy int bpsize, int apsize, int ssize, int local)
55747d99948SChristophe Leroy {
55847d99948SChristophe Leroy struct hash_pte *hptep = htab_address + slot;
55947d99948SChristophe Leroy unsigned long hpte_v;
56047d99948SChristophe Leroy unsigned long want_v;
56147d99948SChristophe Leroy unsigned long flags;
56247d99948SChristophe Leroy
56347d99948SChristophe Leroy local_irq_save(flags);
56447d99948SChristophe Leroy
56547d99948SChristophe Leroy DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
56647d99948SChristophe Leroy
56747d99948SChristophe Leroy want_v = hpte_encode_avpn(vpn, bpsize, ssize);
56847d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
56947d99948SChristophe Leroy
57047d99948SChristophe Leroy if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
57147d99948SChristophe Leroy native_lock_hpte(hptep);
57247d99948SChristophe Leroy /* recheck with locks held */
57347d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
57447d99948SChristophe Leroy
575be83d548SNicholas Piggin if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
57647d99948SChristophe Leroy /* Invalidate the hpte. NOTE: this also unlocks it */
577be83d548SNicholas Piggin release_hpte_lock();
57847d99948SChristophe Leroy hptep->v = 0;
579be83d548SNicholas Piggin } else
58047d99948SChristophe Leroy native_unlock_hpte(hptep);
58147d99948SChristophe Leroy }
58247d99948SChristophe Leroy /*
58347d99948SChristophe Leroy * We need to invalidate the TLB always because hpte_remove doesn't do
58447d99948SChristophe Leroy * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
58547d99948SChristophe Leroy * random entry from it. When we do that we don't invalidate the TLB
58647d99948SChristophe Leroy * (hpte_remove) because we assume the old translation is still
58747d99948SChristophe Leroy * technically "valid".
58847d99948SChristophe Leroy */
58947d99948SChristophe Leroy tlbie(vpn, bpsize, apsize, ssize, local);
59047d99948SChristophe Leroy
59147d99948SChristophe Leroy local_irq_restore(flags);
59247d99948SChristophe Leroy }
59347d99948SChristophe Leroy
59447d99948SChristophe Leroy #ifdef CONFIG_TRANSPARENT_HUGEPAGE
native_hugepage_invalidate(unsigned long vsid,unsigned long addr,unsigned char * hpte_slot_array,int psize,int ssize,int local)59547d99948SChristophe Leroy static void native_hugepage_invalidate(unsigned long vsid,
59647d99948SChristophe Leroy unsigned long addr,
59747d99948SChristophe Leroy unsigned char *hpte_slot_array,
59847d99948SChristophe Leroy int psize, int ssize, int local)
59947d99948SChristophe Leroy {
60047d99948SChristophe Leroy int i;
60147d99948SChristophe Leroy struct hash_pte *hptep;
60247d99948SChristophe Leroy int actual_psize = MMU_PAGE_16M;
60347d99948SChristophe Leroy unsigned int max_hpte_count, valid;
60447d99948SChristophe Leroy unsigned long flags, s_addr = addr;
60547d99948SChristophe Leroy unsigned long hpte_v, want_v, shift;
60647d99948SChristophe Leroy unsigned long hidx, vpn = 0, hash, slot;
60747d99948SChristophe Leroy
60847d99948SChristophe Leroy shift = mmu_psize_defs[psize].shift;
60947d99948SChristophe Leroy max_hpte_count = 1U << (PMD_SHIFT - shift);
61047d99948SChristophe Leroy
61147d99948SChristophe Leroy local_irq_save(flags);
61247d99948SChristophe Leroy for (i = 0; i < max_hpte_count; i++) {
61347d99948SChristophe Leroy valid = hpte_valid(hpte_slot_array, i);
61447d99948SChristophe Leroy if (!valid)
61547d99948SChristophe Leroy continue;
61647d99948SChristophe Leroy hidx = hpte_hash_index(hpte_slot_array, i);
61747d99948SChristophe Leroy
61847d99948SChristophe Leroy /* get the vpn */
61947d99948SChristophe Leroy addr = s_addr + (i * (1ul << shift));
62047d99948SChristophe Leroy vpn = hpt_vpn(addr, vsid, ssize);
62147d99948SChristophe Leroy hash = hpt_hash(vpn, shift, ssize);
62247d99948SChristophe Leroy if (hidx & _PTEIDX_SECONDARY)
62347d99948SChristophe Leroy hash = ~hash;
62447d99948SChristophe Leroy
62547d99948SChristophe Leroy slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
62647d99948SChristophe Leroy slot += hidx & _PTEIDX_GROUP_IX;
62747d99948SChristophe Leroy
62847d99948SChristophe Leroy hptep = htab_address + slot;
62947d99948SChristophe Leroy want_v = hpte_encode_avpn(vpn, psize, ssize);
63047d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
63147d99948SChristophe Leroy
63247d99948SChristophe Leroy /* Even if we miss, we need to invalidate the TLB */
63347d99948SChristophe Leroy if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
63447d99948SChristophe Leroy /* recheck with locks held */
63547d99948SChristophe Leroy native_lock_hpte(hptep);
63647d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
63747d99948SChristophe Leroy
63847d99948SChristophe Leroy if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
639be83d548SNicholas Piggin /* Invalidate the hpte. NOTE: this also unlocks it */
640be83d548SNicholas Piggin release_hpte_lock();
64147d99948SChristophe Leroy hptep->v = 0;
64247d99948SChristophe Leroy } else
64347d99948SChristophe Leroy native_unlock_hpte(hptep);
64447d99948SChristophe Leroy }
64547d99948SChristophe Leroy /*
64647d99948SChristophe Leroy * We need to do tlb invalidate for all the address, tlbie
64747d99948SChristophe Leroy * instruction compares entry_VA in tlb with the VA specified
64847d99948SChristophe Leroy * here
64947d99948SChristophe Leroy */
65047d99948SChristophe Leroy tlbie(vpn, psize, actual_psize, ssize, local);
65147d99948SChristophe Leroy }
65247d99948SChristophe Leroy local_irq_restore(flags);
65347d99948SChristophe Leroy }
65447d99948SChristophe Leroy #else
native_hugepage_invalidate(unsigned long vsid,unsigned long addr,unsigned char * hpte_slot_array,int psize,int ssize,int local)65547d99948SChristophe Leroy static void native_hugepage_invalidate(unsigned long vsid,
65647d99948SChristophe Leroy unsigned long addr,
65747d99948SChristophe Leroy unsigned char *hpte_slot_array,
65847d99948SChristophe Leroy int psize, int ssize, int local)
65947d99948SChristophe Leroy {
66047d99948SChristophe Leroy WARN(1, "%s called without THP support\n", __func__);
66147d99948SChristophe Leroy }
66247d99948SChristophe Leroy #endif
66347d99948SChristophe Leroy
hpte_decode(struct hash_pte * hpte,unsigned long slot,int * psize,int * apsize,int * ssize,unsigned long * vpn)66447d99948SChristophe Leroy static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
66547d99948SChristophe Leroy int *psize, int *apsize, int *ssize, unsigned long *vpn)
66647d99948SChristophe Leroy {
66747d99948SChristophe Leroy unsigned long avpn, pteg, vpi;
66847d99948SChristophe Leroy unsigned long hpte_v = be64_to_cpu(hpte->v);
66947d99948SChristophe Leroy unsigned long hpte_r = be64_to_cpu(hpte->r);
67047d99948SChristophe Leroy unsigned long vsid, seg_off;
67147d99948SChristophe Leroy int size, a_size, shift;
67247d99948SChristophe Leroy /* Look at the 8 bit LP value */
67347d99948SChristophe Leroy unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
67447d99948SChristophe Leroy
67547d99948SChristophe Leroy if (cpu_has_feature(CPU_FTR_ARCH_300)) {
67647d99948SChristophe Leroy hpte_v = hpte_new_to_old_v(hpte_v, hpte_r);
67747d99948SChristophe Leroy hpte_r = hpte_new_to_old_r(hpte_r);
67847d99948SChristophe Leroy }
67947d99948SChristophe Leroy if (!(hpte_v & HPTE_V_LARGE)) {
68047d99948SChristophe Leroy size = MMU_PAGE_4K;
68147d99948SChristophe Leroy a_size = MMU_PAGE_4K;
68247d99948SChristophe Leroy } else {
68347d99948SChristophe Leroy size = hpte_page_sizes[lp] & 0xf;
68447d99948SChristophe Leroy a_size = hpte_page_sizes[lp] >> 4;
68547d99948SChristophe Leroy }
68647d99948SChristophe Leroy /* This works for all page sizes, and for 256M and 1T segments */
68747d99948SChristophe Leroy *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
68847d99948SChristophe Leroy shift = mmu_psize_defs[size].shift;
68947d99948SChristophe Leroy
69047d99948SChristophe Leroy avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
69147d99948SChristophe Leroy pteg = slot / HPTES_PER_GROUP;
69247d99948SChristophe Leroy if (hpte_v & HPTE_V_SECONDARY)
69347d99948SChristophe Leroy pteg = ~pteg;
69447d99948SChristophe Leroy
69547d99948SChristophe Leroy switch (*ssize) {
69647d99948SChristophe Leroy case MMU_SEGSIZE_256M:
69747d99948SChristophe Leroy /* We only have 28 - 23 bits of seg_off in avpn */
69847d99948SChristophe Leroy seg_off = (avpn & 0x1f) << 23;
69947d99948SChristophe Leroy vsid = avpn >> 5;
70047d99948SChristophe Leroy /* We can find more bits from the pteg value */
70147d99948SChristophe Leroy if (shift < 23) {
70247d99948SChristophe Leroy vpi = (vsid ^ pteg) & htab_hash_mask;
70347d99948SChristophe Leroy seg_off |= vpi << shift;
70447d99948SChristophe Leroy }
70547d99948SChristophe Leroy *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT;
70647d99948SChristophe Leroy break;
70747d99948SChristophe Leroy case MMU_SEGSIZE_1T:
70847d99948SChristophe Leroy /* We only have 40 - 23 bits of seg_off in avpn */
70947d99948SChristophe Leroy seg_off = (avpn & 0x1ffff) << 23;
71047d99948SChristophe Leroy vsid = avpn >> 17;
71147d99948SChristophe Leroy if (shift < 23) {
71247d99948SChristophe Leroy vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
71347d99948SChristophe Leroy seg_off |= vpi << shift;
71447d99948SChristophe Leroy }
71547d99948SChristophe Leroy *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT;
71647d99948SChristophe Leroy break;
71747d99948SChristophe Leroy default:
71847d99948SChristophe Leroy *vpn = size = 0;
71947d99948SChristophe Leroy }
72047d99948SChristophe Leroy *psize = size;
72147d99948SChristophe Leroy *apsize = a_size;
72247d99948SChristophe Leroy }
72347d99948SChristophe Leroy
72447d99948SChristophe Leroy /*
72547d99948SChristophe Leroy * clear all mappings on kexec. All cpus are in real mode (or they will
72647d99948SChristophe Leroy * be when they isi), and we are the only one left. We rely on our kernel
72747d99948SChristophe Leroy * mapping being 0xC0's and the hardware ignoring those two real bits.
72847d99948SChristophe Leroy *
72947d99948SChristophe Leroy * This must be called with interrupts disabled.
73047d99948SChristophe Leroy *
73147d99948SChristophe Leroy * Taking the native_tlbie_lock is unsafe here due to the possibility of
73247d99948SChristophe Leroy * lockdep being on. On pre POWER5 hardware, not taking the lock could
73347d99948SChristophe Leroy * cause deadlock. POWER5 and newer not taking the lock is fine. This only
73447d99948SChristophe Leroy * gets called during boot before secondary CPUs have come up and during
73547d99948SChristophe Leroy * crashdump and all bets are off anyway.
73647d99948SChristophe Leroy *
73747d99948SChristophe Leroy * TODO: add batching support when enabled. remember, no dynamic memory here,
73847d99948SChristophe Leroy * although there is the control page available...
73947d99948SChristophe Leroy */
native_hpte_clear(void)7408119cefdSHari Bathini static notrace void native_hpte_clear(void)
74147d99948SChristophe Leroy {
74247d99948SChristophe Leroy unsigned long vpn = 0;
74347d99948SChristophe Leroy unsigned long slot, slots;
74447d99948SChristophe Leroy struct hash_pte *hptep = htab_address;
74547d99948SChristophe Leroy unsigned long hpte_v;
74647d99948SChristophe Leroy unsigned long pteg_count;
74747d99948SChristophe Leroy int psize, apsize, ssize;
74847d99948SChristophe Leroy
74947d99948SChristophe Leroy pteg_count = htab_hash_mask + 1;
75047d99948SChristophe Leroy
75147d99948SChristophe Leroy slots = pteg_count * HPTES_PER_GROUP;
75247d99948SChristophe Leroy
75347d99948SChristophe Leroy for (slot = 0; slot < slots; slot++, hptep++) {
75447d99948SChristophe Leroy /*
75547d99948SChristophe Leroy * we could lock the pte here, but we are the only cpu
75647d99948SChristophe Leroy * running, right? and for crash dump, we probably
75747d99948SChristophe Leroy * don't want to wait for a maybe bad cpu.
75847d99948SChristophe Leroy */
75947d99948SChristophe Leroy hpte_v = be64_to_cpu(hptep->v);
76047d99948SChristophe Leroy
76147d99948SChristophe Leroy /*
76247d99948SChristophe Leroy * Call __tlbie() here rather than tlbie() since we can't take the
76347d99948SChristophe Leroy * native_tlbie_lock.
76447d99948SChristophe Leroy */
76547d99948SChristophe Leroy if (hpte_v & HPTE_V_VALID) {
76647d99948SChristophe Leroy hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
76747d99948SChristophe Leroy hptep->v = 0;
76847d99948SChristophe Leroy ___tlbie(vpn, psize, apsize, ssize);
76947d99948SChristophe Leroy }
77047d99948SChristophe Leroy }
77147d99948SChristophe Leroy
77247d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync":::"memory");
77347d99948SChristophe Leroy }
77447d99948SChristophe Leroy
77547d99948SChristophe Leroy /*
77647d99948SChristophe Leroy * Batched hash table flush, we batch the tlbie's to avoid taking/releasing
77747d99948SChristophe Leroy * the lock all the time
77847d99948SChristophe Leroy */
native_flush_hash_range(unsigned long number,int local)77947d99948SChristophe Leroy static void native_flush_hash_range(unsigned long number, int local)
78047d99948SChristophe Leroy {
78147d99948SChristophe Leroy unsigned long vpn = 0;
78247d99948SChristophe Leroy unsigned long hash, index, hidx, shift, slot;
78347d99948SChristophe Leroy struct hash_pte *hptep;
78447d99948SChristophe Leroy unsigned long hpte_v;
78547d99948SChristophe Leroy unsigned long want_v;
78647d99948SChristophe Leroy unsigned long flags;
78747d99948SChristophe Leroy real_pte_t pte;
78847d99948SChristophe Leroy struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
78947d99948SChristophe Leroy unsigned long psize = batch->psize;
79047d99948SChristophe Leroy int ssize = batch->ssize;
79147d99948SChristophe Leroy int i;
79247d99948SChristophe Leroy unsigned int use_local;
79347d99948SChristophe Leroy
79447d99948SChristophe Leroy use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) &&
79547d99948SChristophe Leroy mmu_psize_defs[psize].tlbiel && !cxl_ctx_in_use();
79647d99948SChristophe Leroy
79747d99948SChristophe Leroy local_irq_save(flags);
79847d99948SChristophe Leroy
79947d99948SChristophe Leroy for (i = 0; i < number; i++) {
80047d99948SChristophe Leroy vpn = batch->vpn[i];
80147d99948SChristophe Leroy pte = batch->pte[i];
80247d99948SChristophe Leroy
80347d99948SChristophe Leroy pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
80447d99948SChristophe Leroy hash = hpt_hash(vpn, shift, ssize);
80547d99948SChristophe Leroy hidx = __rpte_to_hidx(pte, index);
80647d99948SChristophe Leroy if (hidx & _PTEIDX_SECONDARY)
80747d99948SChristophe Leroy hash = ~hash;
80847d99948SChristophe Leroy slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
80947d99948SChristophe Leroy slot += hidx & _PTEIDX_GROUP_IX;
81047d99948SChristophe Leroy hptep = htab_address + slot;
81147d99948SChristophe Leroy want_v = hpte_encode_avpn(vpn, psize, ssize);
81247d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
81347d99948SChristophe Leroy
81447d99948SChristophe Leroy if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
81547d99948SChristophe Leroy continue;
81647d99948SChristophe Leroy /* lock and try again */
81747d99948SChristophe Leroy native_lock_hpte(hptep);
81847d99948SChristophe Leroy hpte_v = hpte_get_old_v(hptep);
81947d99948SChristophe Leroy
82047d99948SChristophe Leroy if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
82147d99948SChristophe Leroy native_unlock_hpte(hptep);
822be83d548SNicholas Piggin else {
823be83d548SNicholas Piggin release_hpte_lock();
82447d99948SChristophe Leroy hptep->v = 0;
825be83d548SNicholas Piggin }
82647d99948SChristophe Leroy
82747d99948SChristophe Leroy } pte_iterate_hashed_end();
82847d99948SChristophe Leroy }
82947d99948SChristophe Leroy
83047d99948SChristophe Leroy if (use_local) {
83147d99948SChristophe Leroy asm volatile("ptesync":::"memory");
83247d99948SChristophe Leroy for (i = 0; i < number; i++) {
83347d99948SChristophe Leroy vpn = batch->vpn[i];
83447d99948SChristophe Leroy pte = batch->pte[i];
83547d99948SChristophe Leroy
83647d99948SChristophe Leroy pte_iterate_hashed_subpages(pte, psize,
83747d99948SChristophe Leroy vpn, index, shift) {
83847d99948SChristophe Leroy __tlbiel(vpn, psize, psize, ssize);
83947d99948SChristophe Leroy } pte_iterate_hashed_end();
84047d99948SChristophe Leroy }
84105504b42SNicholas Piggin ppc_after_tlbiel_barrier();
84247d99948SChristophe Leroy } else {
84347d99948SChristophe Leroy int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
84447d99948SChristophe Leroy
84547d99948SChristophe Leroy if (lock_tlbie)
84647d99948SChristophe Leroy raw_spin_lock(&native_tlbie_lock);
84747d99948SChristophe Leroy
84847d99948SChristophe Leroy asm volatile("ptesync":::"memory");
84947d99948SChristophe Leroy for (i = 0; i < number; i++) {
85047d99948SChristophe Leroy vpn = batch->vpn[i];
85147d99948SChristophe Leroy pte = batch->pte[i];
85247d99948SChristophe Leroy
85347d99948SChristophe Leroy pte_iterate_hashed_subpages(pte, psize,
85447d99948SChristophe Leroy vpn, index, shift) {
85547d99948SChristophe Leroy __tlbie(vpn, psize, psize, ssize);
85647d99948SChristophe Leroy } pte_iterate_hashed_end();
85747d99948SChristophe Leroy }
85847d99948SChristophe Leroy /*
85947d99948SChristophe Leroy * Just do one more with the last used values.
86047d99948SChristophe Leroy */
861047e6575SAneesh Kumar K.V fixup_tlbie_vpn(vpn, psize, psize, ssize);
86247d99948SChristophe Leroy asm volatile("eieio; tlbsync; ptesync":::"memory");
86347d99948SChristophe Leroy
86447d99948SChristophe Leroy if (lock_tlbie)
86547d99948SChristophe Leroy raw_spin_unlock(&native_tlbie_lock);
86647d99948SChristophe Leroy }
86747d99948SChristophe Leroy
86847d99948SChristophe Leroy local_irq_restore(flags);
86947d99948SChristophe Leroy }
87047d99948SChristophe Leroy
hpte_init_native(void)87147d99948SChristophe Leroy void __init hpte_init_native(void)
87247d99948SChristophe Leroy {
87347d99948SChristophe Leroy mmu_hash_ops.hpte_invalidate = native_hpte_invalidate;
87447d99948SChristophe Leroy mmu_hash_ops.hpte_updatepp = native_hpte_updatepp;
87547d99948SChristophe Leroy mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
87647d99948SChristophe Leroy mmu_hash_ops.hpte_removebolted = native_hpte_removebolted;
87747d99948SChristophe Leroy mmu_hash_ops.hpte_insert = native_hpte_insert;
87847d99948SChristophe Leroy mmu_hash_ops.hpte_remove = native_hpte_remove;
87947d99948SChristophe Leroy mmu_hash_ops.hpte_clear_all = native_hpte_clear;
88047d99948SChristophe Leroy mmu_hash_ops.flush_hash_range = native_flush_hash_range;
88147d99948SChristophe Leroy mmu_hash_ops.hugepage_invalidate = native_hugepage_invalidate;
88247d99948SChristophe Leroy }
883