1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
227137e52SSam Ravnborg /*
327137e52SSam Ravnborg * arch/sparc64/mm/init.c
427137e52SSam Ravnborg *
527137e52SSam Ravnborg * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
627137e52SSam Ravnborg * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
727137e52SSam Ravnborg */
827137e52SSam Ravnborg
9cdd4f4c7SPaul Gortmaker #include <linux/extable.h>
1027137e52SSam Ravnborg #include <linux/kernel.h>
1127137e52SSam Ravnborg #include <linux/sched.h>
1227137e52SSam Ravnborg #include <linux/string.h>
1327137e52SSam Ravnborg #include <linux/init.h>
1457c8a661SMike Rapoport #include <linux/memblock.h>
1527137e52SSam Ravnborg #include <linux/mm.h>
1627137e52SSam Ravnborg #include <linux/hugetlb.h>
1727137e52SSam Ravnborg #include <linux/initrd.h>
1827137e52SSam Ravnborg #include <linux/swap.h>
1927137e52SSam Ravnborg #include <linux/pagemap.h>
2027137e52SSam Ravnborg #include <linux/poison.h>
2127137e52SSam Ravnborg #include <linux/fs.h>
2227137e52SSam Ravnborg #include <linux/seq_file.h>
2327137e52SSam Ravnborg #include <linux/kprobes.h>
2427137e52SSam Ravnborg #include <linux/cache.h>
2527137e52SSam Ravnborg #include <linux/sort.h>
26f6d4fb5cSbob picco #include <linux/ioport.h>
2727137e52SSam Ravnborg #include <linux/percpu.h>
2827137e52SSam Ravnborg #include <linux/mmzone.h>
295a0e3ad6STejun Heo #include <linux/gfp.h>
30426e5c42SMuchun Song #include <linux/bootmem_info.h>
3127137e52SSam Ravnborg
3227137e52SSam Ravnborg #include <asm/head.h>
3327137e52SSam Ravnborg #include <asm/page.h>
3427137e52SSam Ravnborg #include <asm/pgalloc.h>
3527137e52SSam Ravnborg #include <asm/oplib.h>
3627137e52SSam Ravnborg #include <asm/iommu.h>
3727137e52SSam Ravnborg #include <asm/io.h>
387c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
3927137e52SSam Ravnborg #include <asm/mmu_context.h>
4027137e52SSam Ravnborg #include <asm/tlbflush.h>
4127137e52SSam Ravnborg #include <asm/dma.h>
4227137e52SSam Ravnborg #include <asm/starfire.h>
4327137e52SSam Ravnborg #include <asm/tlb.h>
4427137e52SSam Ravnborg #include <asm/spitfire.h>
4527137e52SSam Ravnborg #include <asm/sections.h>
4627137e52SSam Ravnborg #include <asm/tsb.h>
4727137e52SSam Ravnborg #include <asm/hypervisor.h>
4827137e52SSam Ravnborg #include <asm/prom.h>
4927137e52SSam Ravnborg #include <asm/mdesc.h>
5027137e52SSam Ravnborg #include <asm/cpudata.h>
5159dec13bSSam Ravnborg #include <asm/setup.h>
5227137e52SSam Ravnborg #include <asm/irq.h>
5327137e52SSam Ravnborg
5427137e52SSam Ravnborg #include "init_64.h"
5527137e52SSam Ravnborg
564f93d21dSDavid S. Miller unsigned long kern_linear_pte_xor[4] __read_mostly;
57494e5b6fSKhalid Aziz static unsigned long page_cache4v_flag;
5827137e52SSam Ravnborg
594f93d21dSDavid S. Miller /* A bitmap, two bits for every 256MB of physical memory. These two
604f93d21dSDavid S. Miller * bits determine what page size we use for kernel linear
614f93d21dSDavid S. Miller * translations. They form an index into kern_linear_pte_xor[]. The
624f93d21dSDavid S. Miller * value in the indexed slot is XOR'd with the TLB miss virtual
634f93d21dSDavid S. Miller * address to form the resulting TTE. The mapping is:
644f93d21dSDavid S. Miller *
654f93d21dSDavid S. Miller * 0 ==> 4MB
664f93d21dSDavid S. Miller * 1 ==> 256MB
674f93d21dSDavid S. Miller * 2 ==> 2GB
684f93d21dSDavid S. Miller * 3 ==> 16GB
694f93d21dSDavid S. Miller *
704f93d21dSDavid S. Miller * All sun4v chips support 256MB pages. Only SPARC-T4 and later
714f93d21dSDavid S. Miller * support 2GB pages, and hopefully future cpus will support the 16GB
724f93d21dSDavid S. Miller * pages as well. For slots 2 and 3, we encode a 256MB TTE xor there
734f93d21dSDavid S. Miller * if these larger page sizes are not supported by the cpu.
744f93d21dSDavid S. Miller *
754f93d21dSDavid S. Miller * It would be nice to determine this from the machine description
764f93d21dSDavid S. Miller * 'cpu' properties, but we need to have this table setup before the
774f93d21dSDavid S. Miller * MDESC is initialized.
7827137e52SSam Ravnborg */
7927137e52SSam Ravnborg
8027137e52SSam Ravnborg #ifndef CONFIG_DEBUG_PAGEALLOC
814f93d21dSDavid S. Miller /* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
824f93d21dSDavid S. Miller * Space is allocated for this right after the trap table in
834f93d21dSDavid S. Miller * arch/sparc64/kernel/head.S
8427137e52SSam Ravnborg */
8527137e52SSam Ravnborg extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
8627137e52SSam Ravnborg #endif
870dd5b7b0SDavid S. Miller extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
8827137e52SSam Ravnborg
89ce33fdc5SDavid S. Miller static unsigned long cpu_pgsz_mask;
90ce33fdc5SDavid S. Miller
91d195b71bSDavid S. Miller #define MAX_BANKS 1024
9227137e52SSam Ravnborg
937c9503b8SGreg Kroah-Hartman static struct linux_prom64_registers pavail[MAX_BANKS];
947c9503b8SGreg Kroah-Hartman static int pavail_ents;
9527137e52SSam Ravnborg
9652708d69SNitin Gupta u64 numa_latency[MAX_NUMNODES][MAX_NUMNODES];
9752708d69SNitin Gupta
cmp_p64(const void * a,const void * b)9827137e52SSam Ravnborg static int cmp_p64(const void *a, const void *b)
9927137e52SSam Ravnborg {
10027137e52SSam Ravnborg const struct linux_prom64_registers *x = a, *y = b;
10127137e52SSam Ravnborg
10227137e52SSam Ravnborg if (x->phys_addr > y->phys_addr)
10327137e52SSam Ravnborg return 1;
10427137e52SSam Ravnborg if (x->phys_addr < y->phys_addr)
10527137e52SSam Ravnborg return -1;
10627137e52SSam Ravnborg return 0;
10727137e52SSam Ravnborg }
10827137e52SSam Ravnborg
read_obp_memory(const char * property,struct linux_prom64_registers * regs,int * num_ents)10927137e52SSam Ravnborg static void __init read_obp_memory(const char *property,
11027137e52SSam Ravnborg struct linux_prom64_registers *regs,
11127137e52SSam Ravnborg int *num_ents)
11227137e52SSam Ravnborg {
1138d125562SAndres Salomon phandle node = prom_finddevice("/memory");
11427137e52SSam Ravnborg int prop_size = prom_getproplen(node, property);
11527137e52SSam Ravnborg int ents, ret, i;
11627137e52SSam Ravnborg
11727137e52SSam Ravnborg ents = prop_size / sizeof(struct linux_prom64_registers);
11827137e52SSam Ravnborg if (ents > MAX_BANKS) {
11927137e52SSam Ravnborg prom_printf("The machine has more %s property entries than "
12027137e52SSam Ravnborg "this kernel can support (%d).\n",
12127137e52SSam Ravnborg property, MAX_BANKS);
12227137e52SSam Ravnborg prom_halt();
12327137e52SSam Ravnborg }
12427137e52SSam Ravnborg
12527137e52SSam Ravnborg ret = prom_getproperty(node, property, (char *) regs, prop_size);
12627137e52SSam Ravnborg if (ret == -1) {
1275da444aaSAkinobu Mita prom_printf("Couldn't get %s property from /memory.\n",
1285da444aaSAkinobu Mita property);
12927137e52SSam Ravnborg prom_halt();
13027137e52SSam Ravnborg }
13127137e52SSam Ravnborg
13227137e52SSam Ravnborg /* Sanitize what we got from the firmware, by page aligning
13327137e52SSam Ravnborg * everything.
13427137e52SSam Ravnborg */
13527137e52SSam Ravnborg for (i = 0; i < ents; i++) {
13627137e52SSam Ravnborg unsigned long base, size;
13727137e52SSam Ravnborg
13827137e52SSam Ravnborg base = regs[i].phys_addr;
13927137e52SSam Ravnborg size = regs[i].reg_size;
14027137e52SSam Ravnborg
14127137e52SSam Ravnborg size &= PAGE_MASK;
14227137e52SSam Ravnborg if (base & ~PAGE_MASK) {
14327137e52SSam Ravnborg unsigned long new_base = PAGE_ALIGN(base);
14427137e52SSam Ravnborg
14527137e52SSam Ravnborg size -= new_base - base;
14627137e52SSam Ravnborg if ((long) size < 0L)
14727137e52SSam Ravnborg size = 0UL;
14827137e52SSam Ravnborg base = new_base;
14927137e52SSam Ravnborg }
15027137e52SSam Ravnborg if (size == 0UL) {
15127137e52SSam Ravnborg /* If it is empty, simply get rid of it.
15227137e52SSam Ravnborg * This simplifies the logic of the other
15327137e52SSam Ravnborg * functions that process these arrays.
15427137e52SSam Ravnborg */
15527137e52SSam Ravnborg memmove(®s[i], ®s[i + 1],
15627137e52SSam Ravnborg (ents - i - 1) * sizeof(regs[0]));
15727137e52SSam Ravnborg i--;
15827137e52SSam Ravnborg ents--;
15927137e52SSam Ravnborg continue;
16027137e52SSam Ravnborg }
16127137e52SSam Ravnborg regs[i].phys_addr = base;
16227137e52SSam Ravnborg regs[i].reg_size = size;
16327137e52SSam Ravnborg }
16427137e52SSam Ravnborg
16527137e52SSam Ravnborg *num_ents = ents;
16627137e52SSam Ravnborg
16727137e52SSam Ravnborg sort(regs, ents, sizeof(struct linux_prom64_registers),
16827137e52SSam Ravnborg cmp_p64, NULL);
16927137e52SSam Ravnborg }
17027137e52SSam Ravnborg
17127137e52SSam Ravnborg /* Kernel physical address base and size in bytes. */
17227137e52SSam Ravnborg unsigned long kern_base __read_mostly;
17327137e52SSam Ravnborg unsigned long kern_size __read_mostly;
17427137e52SSam Ravnborg
17527137e52SSam Ravnborg /* Initial ramdisk setup */
17627137e52SSam Ravnborg extern unsigned long sparc_ramdisk_image64;
17727137e52SSam Ravnborg extern unsigned int sparc_ramdisk_image;
17827137e52SSam Ravnborg extern unsigned int sparc_ramdisk_size;
17927137e52SSam Ravnborg
18027137e52SSam Ravnborg struct page *mem_map_zero __read_mostly;
18127137e52SSam Ravnborg EXPORT_SYMBOL(mem_map_zero);
18227137e52SSam Ravnborg
18327137e52SSam Ravnborg unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
18427137e52SSam Ravnborg
18527137e52SSam Ravnborg unsigned long sparc64_kern_pri_context __read_mostly;
18627137e52SSam Ravnborg unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
18727137e52SSam Ravnborg unsigned long sparc64_kern_sec_context __read_mostly;
18827137e52SSam Ravnborg
18927137e52SSam Ravnborg int num_kernel_image_mappings;
19027137e52SSam Ravnborg
19127137e52SSam Ravnborg #ifdef CONFIG_DEBUG_DCFLUSH
19227137e52SSam Ravnborg atomic_t dcpage_flushes = ATOMIC_INIT(0);
19327137e52SSam Ravnborg #ifdef CONFIG_SMP
19427137e52SSam Ravnborg atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
19527137e52SSam Ravnborg #endif
19627137e52SSam Ravnborg #endif
19727137e52SSam Ravnborg
flush_dcache_folio_impl(struct folio * folio)1981a10a44dSMatthew Wilcox (Oracle) inline void flush_dcache_folio_impl(struct folio *folio)
19927137e52SSam Ravnborg {
2001a10a44dSMatthew Wilcox (Oracle) unsigned int i, nr = folio_nr_pages(folio);
2011a10a44dSMatthew Wilcox (Oracle)
20227137e52SSam Ravnborg BUG_ON(tlb_type == hypervisor);
20327137e52SSam Ravnborg #ifdef CONFIG_DEBUG_DCFLUSH
20427137e52SSam Ravnborg atomic_inc(&dcpage_flushes);
20527137e52SSam Ravnborg #endif
20627137e52SSam Ravnborg
20727137e52SSam Ravnborg #ifdef DCACHE_ALIASING_POSSIBLE
2081a10a44dSMatthew Wilcox (Oracle) for (i = 0; i < nr; i++)
2091a10a44dSMatthew Wilcox (Oracle) __flush_dcache_page(folio_address(folio) + i * PAGE_SIZE,
21027137e52SSam Ravnborg ((tlb_type == spitfire) &&
2111a10a44dSMatthew Wilcox (Oracle) folio_flush_mapping(folio) != NULL));
21227137e52SSam Ravnborg #else
2131a10a44dSMatthew Wilcox (Oracle) if (folio_flush_mapping(folio) != NULL &&
2141a10a44dSMatthew Wilcox (Oracle) tlb_type == spitfire) {
2151a10a44dSMatthew Wilcox (Oracle) for (i = 0; i < nr; i++)
2161a10a44dSMatthew Wilcox (Oracle) __flush_icache_page((pfn + i) * PAGE_SIZE);
2171a10a44dSMatthew Wilcox (Oracle) }
21827137e52SSam Ravnborg #endif
21927137e52SSam Ravnborg }
22027137e52SSam Ravnborg
22127137e52SSam Ravnborg #define PG_dcache_dirty PG_arch_1
22227137e52SSam Ravnborg #define PG_dcache_cpu_shift 32UL
22327137e52SSam Ravnborg #define PG_dcache_cpu_mask \
22427137e52SSam Ravnborg ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
22527137e52SSam Ravnborg
2261a10a44dSMatthew Wilcox (Oracle) #define dcache_dirty_cpu(folio) \
2271a10a44dSMatthew Wilcox (Oracle) (((folio)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
22827137e52SSam Ravnborg
set_dcache_dirty(struct folio * folio,int this_cpu)2291a10a44dSMatthew Wilcox (Oracle) static inline void set_dcache_dirty(struct folio *folio, int this_cpu)
23027137e52SSam Ravnborg {
23127137e52SSam Ravnborg unsigned long mask = this_cpu;
23227137e52SSam Ravnborg unsigned long non_cpu_bits;
23327137e52SSam Ravnborg
23427137e52SSam Ravnborg non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
23527137e52SSam Ravnborg mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
23627137e52SSam Ravnborg
23727137e52SSam Ravnborg __asm__ __volatile__("1:\n\t"
23827137e52SSam Ravnborg "ldx [%2], %%g7\n\t"
23927137e52SSam Ravnborg "and %%g7, %1, %%g1\n\t"
24027137e52SSam Ravnborg "or %%g1, %0, %%g1\n\t"
24127137e52SSam Ravnborg "casx [%2], %%g7, %%g1\n\t"
24227137e52SSam Ravnborg "cmp %%g7, %%g1\n\t"
24327137e52SSam Ravnborg "bne,pn %%xcc, 1b\n\t"
24427137e52SSam Ravnborg " nop"
24527137e52SSam Ravnborg : /* no outputs */
2461a10a44dSMatthew Wilcox (Oracle) : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags)
24727137e52SSam Ravnborg : "g1", "g7");
24827137e52SSam Ravnborg }
24927137e52SSam Ravnborg
clear_dcache_dirty_cpu(struct folio * folio,unsigned long cpu)2501a10a44dSMatthew Wilcox (Oracle) static inline void clear_dcache_dirty_cpu(struct folio *folio, unsigned long cpu)
25127137e52SSam Ravnborg {
25227137e52SSam Ravnborg unsigned long mask = (1UL << PG_dcache_dirty);
25327137e52SSam Ravnborg
25427137e52SSam Ravnborg __asm__ __volatile__("! test_and_clear_dcache_dirty\n"
25527137e52SSam Ravnborg "1:\n\t"
25627137e52SSam Ravnborg "ldx [%2], %%g7\n\t"
25727137e52SSam Ravnborg "srlx %%g7, %4, %%g1\n\t"
25827137e52SSam Ravnborg "and %%g1, %3, %%g1\n\t"
25927137e52SSam Ravnborg "cmp %%g1, %0\n\t"
26027137e52SSam Ravnborg "bne,pn %%icc, 2f\n\t"
26127137e52SSam Ravnborg " andn %%g7, %1, %%g1\n\t"
26227137e52SSam Ravnborg "casx [%2], %%g7, %%g1\n\t"
26327137e52SSam Ravnborg "cmp %%g7, %%g1\n\t"
26427137e52SSam Ravnborg "bne,pn %%xcc, 1b\n\t"
26527137e52SSam Ravnborg " nop\n"
26627137e52SSam Ravnborg "2:"
26727137e52SSam Ravnborg : /* no outputs */
2681a10a44dSMatthew Wilcox (Oracle) : "r" (cpu), "r" (mask), "r" (&folio->flags),
26927137e52SSam Ravnborg "i" (PG_dcache_cpu_mask),
27027137e52SSam Ravnborg "i" (PG_dcache_cpu_shift)
27127137e52SSam Ravnborg : "g1", "g7");
27227137e52SSam Ravnborg }
27327137e52SSam Ravnborg
tsb_insert(struct tsb * ent,unsigned long tag,unsigned long pte)27427137e52SSam Ravnborg static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
27527137e52SSam Ravnborg {
27627137e52SSam Ravnborg unsigned long tsb_addr = (unsigned long) ent;
27727137e52SSam Ravnborg
27827137e52SSam Ravnborg if (tlb_type == cheetah_plus || tlb_type == hypervisor)
27927137e52SSam Ravnborg tsb_addr = __pa(tsb_addr);
28027137e52SSam Ravnborg
28127137e52SSam Ravnborg __tsb_insert(tsb_addr, tag, pte);
28227137e52SSam Ravnborg }
28327137e52SSam Ravnborg
28427137e52SSam Ravnborg unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
28527137e52SSam Ravnborg
flush_dcache(unsigned long pfn)286ff9aefbfSSam Ravnborg static void flush_dcache(unsigned long pfn)
28727137e52SSam Ravnborg {
28827137e52SSam Ravnborg struct page *page;
28927137e52SSam Ravnborg
290ff9aefbfSSam Ravnborg page = pfn_to_page(pfn);
2911a78cedbSDavid S. Miller if (page) {
2921a10a44dSMatthew Wilcox (Oracle) struct folio *folio = page_folio(page);
293ff9aefbfSSam Ravnborg unsigned long pg_flags;
294ff9aefbfSSam Ravnborg
2951a10a44dSMatthew Wilcox (Oracle) pg_flags = folio->flags;
296ff9aefbfSSam Ravnborg if (pg_flags & (1UL << PG_dcache_dirty)) {
29727137e52SSam Ravnborg int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
29827137e52SSam Ravnborg PG_dcache_cpu_mask);
29927137e52SSam Ravnborg int this_cpu = get_cpu();
30027137e52SSam Ravnborg
30127137e52SSam Ravnborg /* This is just to optimize away some function calls
30227137e52SSam Ravnborg * in the SMP case.
30327137e52SSam Ravnborg */
30427137e52SSam Ravnborg if (cpu == this_cpu)
3051a10a44dSMatthew Wilcox (Oracle) flush_dcache_folio_impl(folio);
30627137e52SSam Ravnborg else
3071a10a44dSMatthew Wilcox (Oracle) smp_flush_dcache_folio_impl(folio, cpu);
30827137e52SSam Ravnborg
3091a10a44dSMatthew Wilcox (Oracle) clear_dcache_dirty_cpu(folio, cpu);
31027137e52SSam Ravnborg
31127137e52SSam Ravnborg put_cpu();
31227137e52SSam Ravnborg }
31327137e52SSam Ravnborg }
314ff9aefbfSSam Ravnborg }
315ff9aefbfSSam Ravnborg
3169e695d2eSDavid Miller /* mm->context.lock must be held */
__update_mmu_tsb_insert(struct mm_struct * mm,unsigned long tsb_index,unsigned long tsb_hash_shift,unsigned long address,unsigned long tte)3179e695d2eSDavid Miller static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,
3189e695d2eSDavid Miller unsigned long tsb_hash_shift, unsigned long address,
3199e695d2eSDavid Miller unsigned long tte)
3209e695d2eSDavid Miller {
3219e695d2eSDavid Miller struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;
3229e695d2eSDavid Miller unsigned long tag;
3239e695d2eSDavid Miller
324bcd896baSDavid S. Miller if (unlikely(!tsb))
325bcd896baSDavid S. Miller return;
326bcd896baSDavid S. Miller
3279e695d2eSDavid Miller tsb += ((address >> tsb_hash_shift) &
3289e695d2eSDavid Miller (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
3299e695d2eSDavid Miller tag = (address >> 22UL);
3309e695d2eSDavid Miller tsb_insert(tsb, tag, tte);
3319e695d2eSDavid Miller }
3329e695d2eSDavid Miller
333c7d9f77dSNitin Gupta #ifdef CONFIG_HUGETLB_PAGE
hugetlbpage_init(void)3348399e4b8SNitin Gupta static int __init hugetlbpage_init(void)
3358399e4b8SNitin Gupta {
33638237830SMike Kravetz hugetlb_add_hstate(HPAGE_64K_SHIFT - PAGE_SHIFT);
33738237830SMike Kravetz hugetlb_add_hstate(HPAGE_SHIFT - PAGE_SHIFT);
33838237830SMike Kravetz hugetlb_add_hstate(HPAGE_256MB_SHIFT - PAGE_SHIFT);
33938237830SMike Kravetz hugetlb_add_hstate(HPAGE_2GB_SHIFT - PAGE_SHIFT);
3408399e4b8SNitin Gupta
3418399e4b8SNitin Gupta return 0;
3428399e4b8SNitin Gupta }
3438399e4b8SNitin Gupta
3448399e4b8SNitin Gupta arch_initcall(hugetlbpage_init);
3458399e4b8SNitin Gupta
pud_huge_patch(void)346df7b2155SNitin Gupta static void __init pud_huge_patch(void)
347df7b2155SNitin Gupta {
348df7b2155SNitin Gupta struct pud_huge_patch_entry *p;
349df7b2155SNitin Gupta unsigned long addr;
350df7b2155SNitin Gupta
351df7b2155SNitin Gupta p = &__pud_huge_patch;
352df7b2155SNitin Gupta addr = p->addr;
353df7b2155SNitin Gupta *(unsigned int *)addr = p->insn;
354df7b2155SNitin Gupta
355df7b2155SNitin Gupta __asm__ __volatile__("flush %0" : : "r" (addr));
356df7b2155SNitin Gupta }
357df7b2155SNitin Gupta
arch_hugetlb_valid_size(unsigned long size)358ae94da89SMike Kravetz bool __init arch_hugetlb_valid_size(unsigned long size)
359c7d9f77dSNitin Gupta {
360ae94da89SMike Kravetz unsigned int hugepage_shift = ilog2(size);
361c7d9f77dSNitin Gupta unsigned short hv_pgsz_idx;
362c7d9f77dSNitin Gupta unsigned int hv_pgsz_mask;
363c7d9f77dSNitin Gupta
364c7d9f77dSNitin Gupta switch (hugepage_shift) {
365df7b2155SNitin Gupta case HPAGE_16GB_SHIFT:
366df7b2155SNitin Gupta hv_pgsz_mask = HV_PGSZ_MASK_16GB;
367df7b2155SNitin Gupta hv_pgsz_idx = HV_PGSZ_IDX_16GB;
368df7b2155SNitin Gupta pud_huge_patch();
369df7b2155SNitin Gupta break;
37085b1da7cSNitin Gupta case HPAGE_2GB_SHIFT:
37185b1da7cSNitin Gupta hv_pgsz_mask = HV_PGSZ_MASK_2GB;
37285b1da7cSNitin Gupta hv_pgsz_idx = HV_PGSZ_IDX_2GB;
37385b1da7cSNitin Gupta break;
374c7d9f77dSNitin Gupta case HPAGE_256MB_SHIFT:
375c7d9f77dSNitin Gupta hv_pgsz_mask = HV_PGSZ_MASK_256MB;
376c7d9f77dSNitin Gupta hv_pgsz_idx = HV_PGSZ_IDX_256MB;
377c7d9f77dSNitin Gupta break;
378c7d9f77dSNitin Gupta case HPAGE_SHIFT:
379c7d9f77dSNitin Gupta hv_pgsz_mask = HV_PGSZ_MASK_4MB;
380c7d9f77dSNitin Gupta hv_pgsz_idx = HV_PGSZ_IDX_4MB;
381c7d9f77dSNitin Gupta break;
382dcd1912dSNitin Gupta case HPAGE_64K_SHIFT:
383dcd1912dSNitin Gupta hv_pgsz_mask = HV_PGSZ_MASK_64K;
384dcd1912dSNitin Gupta hv_pgsz_idx = HV_PGSZ_IDX_64K;
385dcd1912dSNitin Gupta break;
386c7d9f77dSNitin Gupta default:
387c7d9f77dSNitin Gupta hv_pgsz_mask = 0;
388c7d9f77dSNitin Gupta }
389c7d9f77dSNitin Gupta
390ae94da89SMike Kravetz if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U)
391ae94da89SMike Kravetz return false;
392ae94da89SMike Kravetz
393ae94da89SMike Kravetz return true;
394ae94da89SMike Kravetz }
395c7d9f77dSNitin Gupta #endif /* CONFIG_HUGETLB_PAGE */
396c7d9f77dSNitin Gupta
update_mmu_cache_range(struct vm_fault * vmf,struct vm_area_struct * vma,unsigned long address,pte_t * ptep,unsigned int nr)3971a10a44dSMatthew Wilcox (Oracle) void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma,
3981a10a44dSMatthew Wilcox (Oracle) unsigned long address, pte_t *ptep, unsigned int nr)
399ff9aefbfSSam Ravnborg {
400ff9aefbfSSam Ravnborg struct mm_struct *mm;
401bcd896baSDavid S. Miller unsigned long flags;
402df7b2155SNitin Gupta bool is_huge_tsb;
4034b3073e1SRussell King pte_t pte = *ptep;
4041a10a44dSMatthew Wilcox (Oracle) unsigned int i;
405ff9aefbfSSam Ravnborg
406ff9aefbfSSam Ravnborg if (tlb_type != hypervisor) {
407ff9aefbfSSam Ravnborg unsigned long pfn = pte_pfn(pte);
408ff9aefbfSSam Ravnborg
409ff9aefbfSSam Ravnborg if (pfn_valid(pfn))
410ff9aefbfSSam Ravnborg flush_dcache(pfn);
411ff9aefbfSSam Ravnborg }
41227137e52SSam Ravnborg
41327137e52SSam Ravnborg mm = vma->vm_mm;
41427137e52SSam Ravnborg
41518f38132SDavid S. Miller /* Don't insert a non-valid PTE into the TSB, we'll deadlock. */
41618f38132SDavid S. Miller if (!pte_accessible(mm, pte))
41718f38132SDavid S. Miller return;
41818f38132SDavid S. Miller
41927137e52SSam Ravnborg spin_lock_irqsave(&mm->context.lock, flags);
42027137e52SSam Ravnborg
421df7b2155SNitin Gupta is_huge_tsb = false;
4229e695d2eSDavid Miller #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
423df7b2155SNitin Gupta if (mm->context.hugetlb_pte_count || mm->context.thp_pte_count) {
424df7b2155SNitin Gupta unsigned long hugepage_size = PAGE_SIZE;
425df7b2155SNitin Gupta
426df7b2155SNitin Gupta if (is_vm_hugetlb_page(vma))
427df7b2155SNitin Gupta hugepage_size = huge_page_size(hstate_vma(vma));
428df7b2155SNitin Gupta
429df7b2155SNitin Gupta if (hugepage_size >= PUD_SIZE) {
430df7b2155SNitin Gupta unsigned long mask = 0x1ffc00000UL;
431df7b2155SNitin Gupta
432df7b2155SNitin Gupta /* Transfer bits [32:22] from address to resolve
433df7b2155SNitin Gupta * at 4M granularity.
434df7b2155SNitin Gupta */
435df7b2155SNitin Gupta pte_val(pte) &= ~mask;
436df7b2155SNitin Gupta pte_val(pte) |= (address & mask);
437df7b2155SNitin Gupta } else if (hugepage_size >= PMD_SIZE) {
438df7b2155SNitin Gupta /* We are fabricating 8MB pages using 4MB
439df7b2155SNitin Gupta * real hw pages.
440df7b2155SNitin Gupta */
4417bc3777cSNitin Gupta pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT));
442df7b2155SNitin Gupta }
443df7b2155SNitin Gupta
444df7b2155SNitin Gupta if (hugepage_size >= PMD_SIZE) {
445df7b2155SNitin Gupta __update_mmu_tsb_insert(mm, MM_TSB_HUGE,
446df7b2155SNitin Gupta REAL_HPAGE_SHIFT, address, pte_val(pte));
447df7b2155SNitin Gupta is_huge_tsb = true;
448df7b2155SNitin Gupta }
449df7b2155SNitin Gupta }
45027137e52SSam Ravnborg #endif
4511a10a44dSMatthew Wilcox (Oracle) if (!is_huge_tsb) {
4521a10a44dSMatthew Wilcox (Oracle) for (i = 0; i < nr; i++) {
453bcd896baSDavid S. Miller __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,
4549e695d2eSDavid Miller address, pte_val(pte));
4551a10a44dSMatthew Wilcox (Oracle) address += PAGE_SIZE;
4561a10a44dSMatthew Wilcox (Oracle) pte_val(pte) += PAGE_SIZE;
4571a10a44dSMatthew Wilcox (Oracle) }
4581a10a44dSMatthew Wilcox (Oracle) }
45927137e52SSam Ravnborg
46027137e52SSam Ravnborg spin_unlock_irqrestore(&mm->context.lock, flags);
46127137e52SSam Ravnborg }
46227137e52SSam Ravnborg
flush_dcache_folio(struct folio * folio)4631a10a44dSMatthew Wilcox (Oracle) void flush_dcache_folio(struct folio *folio)
46427137e52SSam Ravnborg {
4651a10a44dSMatthew Wilcox (Oracle) unsigned long pfn = folio_pfn(folio);
46627137e52SSam Ravnborg struct address_space *mapping;
46727137e52SSam Ravnborg int this_cpu;
46827137e52SSam Ravnborg
46927137e52SSam Ravnborg if (tlb_type == hypervisor)
47027137e52SSam Ravnborg return;
47127137e52SSam Ravnborg
47227137e52SSam Ravnborg /* Do not bother with the expensive D-cache flush if it
47327137e52SSam Ravnborg * is merely the zero page. The 'bigcore' testcase in GDB
47427137e52SSam Ravnborg * causes this case to run millions of times.
47527137e52SSam Ravnborg */
4761a10a44dSMatthew Wilcox (Oracle) if (is_zero_pfn(pfn))
47727137e52SSam Ravnborg return;
47827137e52SSam Ravnborg
47927137e52SSam Ravnborg this_cpu = get_cpu();
48027137e52SSam Ravnborg
4811a10a44dSMatthew Wilcox (Oracle) mapping = folio_flush_mapping(folio);
48227137e52SSam Ravnborg if (mapping && !mapping_mapped(mapping)) {
4831a10a44dSMatthew Wilcox (Oracle) bool dirty = test_bit(PG_dcache_dirty, &folio->flags);
48427137e52SSam Ravnborg if (dirty) {
4851a10a44dSMatthew Wilcox (Oracle) int dirty_cpu = dcache_dirty_cpu(folio);
48627137e52SSam Ravnborg
48727137e52SSam Ravnborg if (dirty_cpu == this_cpu)
48827137e52SSam Ravnborg goto out;
4891a10a44dSMatthew Wilcox (Oracle) smp_flush_dcache_folio_impl(folio, dirty_cpu);
49027137e52SSam Ravnborg }
4911a10a44dSMatthew Wilcox (Oracle) set_dcache_dirty(folio, this_cpu);
49227137e52SSam Ravnborg } else {
49327137e52SSam Ravnborg /* We could delay the flush for the !page_mapping
49427137e52SSam Ravnborg * case too. But that case is for exec env/arg
49527137e52SSam Ravnborg * pages and those are %99 certainly going to get
49627137e52SSam Ravnborg * faulted into the tlb (and thus flushed) anyways.
49727137e52SSam Ravnborg */
4981a10a44dSMatthew Wilcox (Oracle) flush_dcache_folio_impl(folio);
49927137e52SSam Ravnborg }
50027137e52SSam Ravnborg
50127137e52SSam Ravnborg out:
50227137e52SSam Ravnborg put_cpu();
50327137e52SSam Ravnborg }
5041a10a44dSMatthew Wilcox (Oracle) EXPORT_SYMBOL(flush_dcache_folio);
50527137e52SSam Ravnborg
flush_icache_range(unsigned long start,unsigned long end)50627137e52SSam Ravnborg void __kprobes flush_icache_range(unsigned long start, unsigned long end)
50727137e52SSam Ravnborg {
50827137e52SSam Ravnborg /* Cheetah and Hypervisor platform cpus have coherent I-cache. */
50927137e52SSam Ravnborg if (tlb_type == spitfire) {
51027137e52SSam Ravnborg unsigned long kaddr;
51127137e52SSam Ravnborg
51227137e52SSam Ravnborg /* This code only runs on Spitfire cpus so this is
51327137e52SSam Ravnborg * why we can assume _PAGE_PADDR_4U.
51427137e52SSam Ravnborg */
51527137e52SSam Ravnborg for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {
51627137e52SSam Ravnborg unsigned long paddr, mask = _PAGE_PADDR_4U;
51727137e52SSam Ravnborg
51827137e52SSam Ravnborg if (kaddr >= PAGE_OFFSET)
51927137e52SSam Ravnborg paddr = kaddr & mask;
52027137e52SSam Ravnborg else {
521e05c7b1fSMike Rapoport pte_t *ptep = virt_to_kpte(kaddr);
52227137e52SSam Ravnborg
52327137e52SSam Ravnborg paddr = pte_val(*ptep) & mask;
52427137e52SSam Ravnborg }
52527137e52SSam Ravnborg __flush_icache_page(paddr);
52627137e52SSam Ravnborg }
52727137e52SSam Ravnborg }
52827137e52SSam Ravnborg }
529917c3660SSam Ravnborg EXPORT_SYMBOL(flush_icache_range);
53027137e52SSam Ravnborg
mmu_info(struct seq_file * m)53127137e52SSam Ravnborg void mmu_info(struct seq_file *m)
53227137e52SSam Ravnborg {
533ce33fdc5SDavid S. Miller static const char *pgsz_strings[] = {
534ce33fdc5SDavid S. Miller "8K", "64K", "512K", "4MB", "32MB",
535ce33fdc5SDavid S. Miller "256MB", "2GB", "16GB",
536ce33fdc5SDavid S. Miller };
537ce33fdc5SDavid S. Miller int i, printed;
538ce33fdc5SDavid S. Miller
53927137e52SSam Ravnborg if (tlb_type == cheetah)
54027137e52SSam Ravnborg seq_printf(m, "MMU Type\t: Cheetah\n");
54127137e52SSam Ravnborg else if (tlb_type == cheetah_plus)
54227137e52SSam Ravnborg seq_printf(m, "MMU Type\t: Cheetah+\n");
54327137e52SSam Ravnborg else if (tlb_type == spitfire)
54427137e52SSam Ravnborg seq_printf(m, "MMU Type\t: Spitfire\n");
54527137e52SSam Ravnborg else if (tlb_type == hypervisor)
54627137e52SSam Ravnborg seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
54727137e52SSam Ravnborg else
54827137e52SSam Ravnborg seq_printf(m, "MMU Type\t: ???\n");
54927137e52SSam Ravnborg
550ce33fdc5SDavid S. Miller seq_printf(m, "MMU PGSZs\t: ");
551ce33fdc5SDavid S. Miller printed = 0;
552ce33fdc5SDavid S. Miller for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {
553ce33fdc5SDavid S. Miller if (cpu_pgsz_mask & (1UL << i)) {
554ce33fdc5SDavid S. Miller seq_printf(m, "%s%s",
555ce33fdc5SDavid S. Miller printed ? "," : "", pgsz_strings[i]);
556ce33fdc5SDavid S. Miller printed++;
557ce33fdc5SDavid S. Miller }
558ce33fdc5SDavid S. Miller }
559ce33fdc5SDavid S. Miller seq_putc(m, '\n');
560ce33fdc5SDavid S. Miller
56127137e52SSam Ravnborg #ifdef CONFIG_DEBUG_DCFLUSH
56227137e52SSam Ravnborg seq_printf(m, "DCPageFlushes\t: %d\n",
56327137e52SSam Ravnborg atomic_read(&dcpage_flushes));
56427137e52SSam Ravnborg #ifdef CONFIG_SMP
56527137e52SSam Ravnborg seq_printf(m, "DCPageFlushesXC\t: %d\n",
56627137e52SSam Ravnborg atomic_read(&dcpage_flushes_xcall));
56727137e52SSam Ravnborg #endif /* CONFIG_SMP */
56827137e52SSam Ravnborg #endif /* CONFIG_DEBUG_DCFLUSH */
56927137e52SSam Ravnborg }
57027137e52SSam Ravnborg
57127137e52SSam Ravnborg struct linux_prom_translation prom_trans[512] __read_mostly;
57227137e52SSam Ravnborg unsigned int prom_trans_ents __read_mostly;
57327137e52SSam Ravnborg
57427137e52SSam Ravnborg unsigned long kern_locked_tte_data;
57527137e52SSam Ravnborg
57627137e52SSam Ravnborg /* The obp translations are saved based on 8k pagesize, since obp can
57727137e52SSam Ravnborg * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
57827137e52SSam Ravnborg * HI_OBP_ADDRESS range are handled in ktlb.S.
57927137e52SSam Ravnborg */
in_obp_range(unsigned long vaddr)58027137e52SSam Ravnborg static inline int in_obp_range(unsigned long vaddr)
58127137e52SSam Ravnborg {
58227137e52SSam Ravnborg return (vaddr >= LOW_OBP_ADDRESS &&
58327137e52SSam Ravnborg vaddr < HI_OBP_ADDRESS);
58427137e52SSam Ravnborg }
58527137e52SSam Ravnborg
cmp_ptrans(const void * a,const void * b)58627137e52SSam Ravnborg static int cmp_ptrans(const void *a, const void *b)
58727137e52SSam Ravnborg {
58827137e52SSam Ravnborg const struct linux_prom_translation *x = a, *y = b;
58927137e52SSam Ravnborg
59027137e52SSam Ravnborg if (x->virt > y->virt)
59127137e52SSam Ravnborg return 1;
59227137e52SSam Ravnborg if (x->virt < y->virt)
59327137e52SSam Ravnborg return -1;
59427137e52SSam Ravnborg return 0;
59527137e52SSam Ravnborg }
59627137e52SSam Ravnborg
59727137e52SSam Ravnborg /* Read OBP translations property into 'prom_trans[]'. */
read_obp_translations(void)59827137e52SSam Ravnborg static void __init read_obp_translations(void)
59927137e52SSam Ravnborg {
60027137e52SSam Ravnborg int n, node, ents, first, last, i;
60127137e52SSam Ravnborg
60227137e52SSam Ravnborg node = prom_finddevice("/virtual-memory");
60327137e52SSam Ravnborg n = prom_getproplen(node, "translations");
60427137e52SSam Ravnborg if (unlikely(n == 0 || n == -1)) {
60527137e52SSam Ravnborg prom_printf("prom_mappings: Couldn't get size.\n");
60627137e52SSam Ravnborg prom_halt();
60727137e52SSam Ravnborg }
60827137e52SSam Ravnborg if (unlikely(n > sizeof(prom_trans))) {
6095da444aaSAkinobu Mita prom_printf("prom_mappings: Size %d is too big.\n", n);
61027137e52SSam Ravnborg prom_halt();
61127137e52SSam Ravnborg }
61227137e52SSam Ravnborg
61327137e52SSam Ravnborg if ((n = prom_getproperty(node, "translations",
61427137e52SSam Ravnborg (char *)&prom_trans[0],
61527137e52SSam Ravnborg sizeof(prom_trans))) == -1) {
61627137e52SSam Ravnborg prom_printf("prom_mappings: Couldn't get property.\n");
61727137e52SSam Ravnborg prom_halt();
61827137e52SSam Ravnborg }
61927137e52SSam Ravnborg
62027137e52SSam Ravnborg n = n / sizeof(struct linux_prom_translation);
62127137e52SSam Ravnborg
62227137e52SSam Ravnborg ents = n;
62327137e52SSam Ravnborg
62427137e52SSam Ravnborg sort(prom_trans, ents, sizeof(struct linux_prom_translation),
62527137e52SSam Ravnborg cmp_ptrans, NULL);
62627137e52SSam Ravnborg
62727137e52SSam Ravnborg /* Now kick out all the non-OBP entries. */
62827137e52SSam Ravnborg for (i = 0; i < ents; i++) {
62927137e52SSam Ravnborg if (in_obp_range(prom_trans[i].virt))
63027137e52SSam Ravnborg break;
63127137e52SSam Ravnborg }
63227137e52SSam Ravnborg first = i;
63327137e52SSam Ravnborg for (; i < ents; i++) {
63427137e52SSam Ravnborg if (!in_obp_range(prom_trans[i].virt))
63527137e52SSam Ravnborg break;
63627137e52SSam Ravnborg }
63727137e52SSam Ravnborg last = i;
63827137e52SSam Ravnborg
63927137e52SSam Ravnborg for (i = 0; i < (last - first); i++) {
64027137e52SSam Ravnborg struct linux_prom_translation *src = &prom_trans[i + first];
64127137e52SSam Ravnborg struct linux_prom_translation *dest = &prom_trans[i];
64227137e52SSam Ravnborg
64327137e52SSam Ravnborg *dest = *src;
64427137e52SSam Ravnborg }
64527137e52SSam Ravnborg for (; i < ents; i++) {
64627137e52SSam Ravnborg struct linux_prom_translation *dest = &prom_trans[i];
64727137e52SSam Ravnborg dest->virt = dest->size = dest->data = 0x0UL;
64827137e52SSam Ravnborg }
64927137e52SSam Ravnborg
65027137e52SSam Ravnborg prom_trans_ents = last - first;
65127137e52SSam Ravnborg
65227137e52SSam Ravnborg if (tlb_type == spitfire) {
65327137e52SSam Ravnborg /* Clear diag TTE bits. */
65427137e52SSam Ravnborg for (i = 0; i < prom_trans_ents; i++)
65527137e52SSam Ravnborg prom_trans[i].data &= ~0x0003fe0000000000UL;
65627137e52SSam Ravnborg }
657f4142cbaSDavid S. Miller
658f4142cbaSDavid S. Miller /* Force execute bit on. */
659f4142cbaSDavid S. Miller for (i = 0; i < prom_trans_ents; i++)
660f4142cbaSDavid S. Miller prom_trans[i].data |= (tlb_type == hypervisor ?
661f4142cbaSDavid S. Miller _PAGE_EXEC_4V : _PAGE_EXEC_4U);
66227137e52SSam Ravnborg }
66327137e52SSam Ravnborg
hypervisor_tlb_lock(unsigned long vaddr,unsigned long pte,unsigned long mmu)66427137e52SSam Ravnborg static void __init hypervisor_tlb_lock(unsigned long vaddr,
66527137e52SSam Ravnborg unsigned long pte,
66627137e52SSam Ravnborg unsigned long mmu)
66727137e52SSam Ravnborg {
66827137e52SSam Ravnborg unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);
66927137e52SSam Ravnborg
67027137e52SSam Ravnborg if (ret != 0) {
6715da444aaSAkinobu Mita prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "
67227137e52SSam Ravnborg "errors with %lx\n", vaddr, 0, pte, mmu, ret);
67327137e52SSam Ravnborg prom_halt();
67427137e52SSam Ravnborg }
67527137e52SSam Ravnborg }
67627137e52SSam Ravnborg
67727137e52SSam Ravnborg static unsigned long kern_large_tte(unsigned long paddr);
67827137e52SSam Ravnborg
remap_kernel(void)67927137e52SSam Ravnborg static void __init remap_kernel(void)
68027137e52SSam Ravnborg {
68127137e52SSam Ravnborg unsigned long phys_page, tte_vaddr, tte_data;
68227137e52SSam Ravnborg int i, tlb_ent = sparc64_highest_locked_tlbent();
68327137e52SSam Ravnborg
68427137e52SSam Ravnborg tte_vaddr = (unsigned long) KERNBASE;
6850eef331aSDavid S. Miller phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
68627137e52SSam Ravnborg tte_data = kern_large_tte(phys_page);
68727137e52SSam Ravnborg
68827137e52SSam Ravnborg kern_locked_tte_data = tte_data;
68927137e52SSam Ravnborg
69027137e52SSam Ravnborg /* Now lock us into the TLBs via Hypervisor or OBP. */
69127137e52SSam Ravnborg if (tlb_type == hypervisor) {
69227137e52SSam Ravnborg for (i = 0; i < num_kernel_image_mappings; i++) {
69327137e52SSam Ravnborg hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
69427137e52SSam Ravnborg hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
69527137e52SSam Ravnborg tte_vaddr += 0x400000;
69627137e52SSam Ravnborg tte_data += 0x400000;
69727137e52SSam Ravnborg }
69827137e52SSam Ravnborg } else {
69927137e52SSam Ravnborg for (i = 0; i < num_kernel_image_mappings; i++) {
70027137e52SSam Ravnborg prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);
70127137e52SSam Ravnborg prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);
70227137e52SSam Ravnborg tte_vaddr += 0x400000;
70327137e52SSam Ravnborg tte_data += 0x400000;
70427137e52SSam Ravnborg }
70527137e52SSam Ravnborg sparc64_highest_unlocked_tlb_ent = tlb_ent - i;
70627137e52SSam Ravnborg }
70727137e52SSam Ravnborg if (tlb_type == cheetah_plus) {
70827137e52SSam Ravnborg sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
70927137e52SSam Ravnborg CTX_CHEETAH_PLUS_NUC);
71027137e52SSam Ravnborg sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
71127137e52SSam Ravnborg sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
71227137e52SSam Ravnborg }
71327137e52SSam Ravnborg }
71427137e52SSam Ravnborg
71527137e52SSam Ravnborg
inherit_prom_mappings(void)71627137e52SSam Ravnborg static void __init inherit_prom_mappings(void)
71727137e52SSam Ravnborg {
71827137e52SSam Ravnborg /* Now fixup OBP's idea about where we really are mapped. */
71927137e52SSam Ravnborg printk("Remapping the kernel... ");
72027137e52SSam Ravnborg remap_kernel();
72127137e52SSam Ravnborg printk("done.\n");
72227137e52SSam Ravnborg }
72327137e52SSam Ravnborg
prom_world(int enter)72427137e52SSam Ravnborg void prom_world(int enter)
72527137e52SSam Ravnborg {
726a5ad8378SArnd Bergmann /*
727a5ad8378SArnd Bergmann * No need to change the address space any more, just flush
728a5ad8378SArnd Bergmann * the register windows
729a5ad8378SArnd Bergmann */
73027137e52SSam Ravnborg __asm__ __volatile__("flushw");
73127137e52SSam Ravnborg }
73227137e52SSam Ravnborg
__flush_dcache_range(unsigned long start,unsigned long end)73327137e52SSam Ravnborg void __flush_dcache_range(unsigned long start, unsigned long end)
73427137e52SSam Ravnborg {
73527137e52SSam Ravnborg unsigned long va;
73627137e52SSam Ravnborg
73727137e52SSam Ravnborg if (tlb_type == spitfire) {
73827137e52SSam Ravnborg int n = 0;
73927137e52SSam Ravnborg
74027137e52SSam Ravnborg for (va = start; va < end; va += 32) {
74127137e52SSam Ravnborg spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
74227137e52SSam Ravnborg if (++n >= 512)
74327137e52SSam Ravnborg break;
74427137e52SSam Ravnborg }
74527137e52SSam Ravnborg } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
74627137e52SSam Ravnborg start = __pa(start);
74727137e52SSam Ravnborg end = __pa(end);
74827137e52SSam Ravnborg for (va = start; va < end; va += 32)
74927137e52SSam Ravnborg __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
75027137e52SSam Ravnborg "membar #Sync"
75127137e52SSam Ravnborg : /* no outputs */
75227137e52SSam Ravnborg : "r" (va),
75327137e52SSam Ravnborg "i" (ASI_DCACHE_INVALIDATE));
75427137e52SSam Ravnborg }
75527137e52SSam Ravnborg }
756917c3660SSam Ravnborg EXPORT_SYMBOL(__flush_dcache_range);
75727137e52SSam Ravnborg
75827137e52SSam Ravnborg /* get_new_mmu_context() uses "cache + 1". */
75927137e52SSam Ravnborg DEFINE_SPINLOCK(ctx_alloc_lock);
760c4415235SPavel Tatashin unsigned long tlb_context_cache = CTX_FIRST_VERSION;
76127137e52SSam Ravnborg #define MAX_CTX_NR (1UL << CTX_NR_BITS)
76227137e52SSam Ravnborg #define CTX_BMAP_SLOTS BITS_TO_LONGS(MAX_CTX_NR)
76327137e52SSam Ravnborg DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);
7647a5b4bbfSPavel Tatashin DEFINE_PER_CPU(struct mm_struct *, per_cpu_secondary_mm) = {0};
76527137e52SSam Ravnborg
mmu_context_wrap(void)766a0582f26SPavel Tatashin static void mmu_context_wrap(void)
767a0582f26SPavel Tatashin {
768a0582f26SPavel Tatashin unsigned long old_ver = tlb_context_cache & CTX_VERSION_MASK;
769a0582f26SPavel Tatashin unsigned long new_ver, new_ctx, old_ctx;
770a0582f26SPavel Tatashin struct mm_struct *mm;
771a0582f26SPavel Tatashin int cpu;
772a0582f26SPavel Tatashin
773a0582f26SPavel Tatashin bitmap_zero(mmu_context_bmap, 1 << CTX_NR_BITS);
774a0582f26SPavel Tatashin
775a0582f26SPavel Tatashin /* Reserve kernel context */
776a0582f26SPavel Tatashin set_bit(0, mmu_context_bmap);
777a0582f26SPavel Tatashin
778a0582f26SPavel Tatashin new_ver = (tlb_context_cache & CTX_VERSION_MASK) + CTX_FIRST_VERSION;
779a0582f26SPavel Tatashin if (unlikely(new_ver == 0))
780a0582f26SPavel Tatashin new_ver = CTX_FIRST_VERSION;
781a0582f26SPavel Tatashin tlb_context_cache = new_ver;
782a0582f26SPavel Tatashin
783a0582f26SPavel Tatashin /*
784a0582f26SPavel Tatashin * Make sure that any new mm that are added into per_cpu_secondary_mm,
785a0582f26SPavel Tatashin * are going to go through get_new_mmu_context() path.
786a0582f26SPavel Tatashin */
787a0582f26SPavel Tatashin mb();
788a0582f26SPavel Tatashin
789a0582f26SPavel Tatashin /*
790a0582f26SPavel Tatashin * Updated versions to current on those CPUs that had valid secondary
791a0582f26SPavel Tatashin * contexts
792a0582f26SPavel Tatashin */
793a0582f26SPavel Tatashin for_each_online_cpu(cpu) {
794a0582f26SPavel Tatashin /*
795a0582f26SPavel Tatashin * If a new mm is stored after we took this mm from the array,
796a0582f26SPavel Tatashin * it will go into get_new_mmu_context() path, because we
797a0582f26SPavel Tatashin * already bumped the version in tlb_context_cache.
798a0582f26SPavel Tatashin */
799a0582f26SPavel Tatashin mm = per_cpu(per_cpu_secondary_mm, cpu);
800a0582f26SPavel Tatashin
801a0582f26SPavel Tatashin if (unlikely(!mm || mm == &init_mm))
802a0582f26SPavel Tatashin continue;
803a0582f26SPavel Tatashin
804a0582f26SPavel Tatashin old_ctx = mm->context.sparc64_ctx_val;
805a0582f26SPavel Tatashin if (likely((old_ctx & CTX_VERSION_MASK) == old_ver)) {
806a0582f26SPavel Tatashin new_ctx = (old_ctx & ~CTX_VERSION_MASK) | new_ver;
807a0582f26SPavel Tatashin set_bit(new_ctx & CTX_NR_MASK, mmu_context_bmap);
808a0582f26SPavel Tatashin mm->context.sparc64_ctx_val = new_ctx;
809a0582f26SPavel Tatashin }
810a0582f26SPavel Tatashin }
811a0582f26SPavel Tatashin }
812a0582f26SPavel Tatashin
81327137e52SSam Ravnborg /* Caller does TLB context flushing on local CPU if necessary.
81427137e52SSam Ravnborg * The caller also ensures that CTX_VALID(mm->context) is false.
81527137e52SSam Ravnborg *
81627137e52SSam Ravnborg * We must be careful about boundary cases so that we never
81727137e52SSam Ravnborg * let the user have CTX 0 (nucleus) or we ever use a CTX
81827137e52SSam Ravnborg * version of zero (and thus NO_CONTEXT would not be caught
81927137e52SSam Ravnborg * by version mis-match tests in mmu_context.h).
82027137e52SSam Ravnborg *
82127137e52SSam Ravnborg * Always invoked with interrupts disabled.
82227137e52SSam Ravnborg */
get_new_mmu_context(struct mm_struct * mm)82327137e52SSam Ravnborg void get_new_mmu_context(struct mm_struct *mm)
82427137e52SSam Ravnborg {
82527137e52SSam Ravnborg unsigned long ctx, new_ctx;
82627137e52SSam Ravnborg unsigned long orig_pgsz_bits;
82727137e52SSam Ravnborg
82807df8418SKirill Tkhai spin_lock(&ctx_alloc_lock);
829a0582f26SPavel Tatashin retry:
830a0582f26SPavel Tatashin /* wrap might have happened, test again if our context became valid */
831a0582f26SPavel Tatashin if (unlikely(CTX_VALID(mm->context)))
832a0582f26SPavel Tatashin goto out;
83327137e52SSam Ravnborg orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
83427137e52SSam Ravnborg ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
83527137e52SSam Ravnborg new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
83627137e52SSam Ravnborg if (new_ctx >= (1 << CTX_NR_BITS)) {
83727137e52SSam Ravnborg new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
83827137e52SSam Ravnborg if (new_ctx >= ctx) {
839a0582f26SPavel Tatashin mmu_context_wrap();
840a0582f26SPavel Tatashin goto retry;
84127137e52SSam Ravnborg }
84227137e52SSam Ravnborg }
84358897485SPavel Tatashin if (mm->context.sparc64_ctx_val)
84458897485SPavel Tatashin cpumask_clear(mm_cpumask(mm));
84527137e52SSam Ravnborg mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
84627137e52SSam Ravnborg new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
84727137e52SSam Ravnborg tlb_context_cache = new_ctx;
84827137e52SSam Ravnborg mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
849a0582f26SPavel Tatashin out:
85007df8418SKirill Tkhai spin_unlock(&ctx_alloc_lock);
85127137e52SSam Ravnborg }
85227137e52SSam Ravnborg
85327137e52SSam Ravnborg static int numa_enabled = 1;
85427137e52SSam Ravnborg static int numa_debug;
85527137e52SSam Ravnborg
early_numa(char * p)85627137e52SSam Ravnborg static int __init early_numa(char *p)
85727137e52SSam Ravnborg {
85827137e52SSam Ravnborg if (!p)
85927137e52SSam Ravnborg return 0;
86027137e52SSam Ravnborg
86127137e52SSam Ravnborg if (strstr(p, "off"))
86227137e52SSam Ravnborg numa_enabled = 0;
86327137e52SSam Ravnborg
86427137e52SSam Ravnborg if (strstr(p, "debug"))
86527137e52SSam Ravnborg numa_debug = 1;
86627137e52SSam Ravnborg
86727137e52SSam Ravnborg return 0;
86827137e52SSam Ravnborg }
86927137e52SSam Ravnborg early_param("numa", early_numa);
87027137e52SSam Ravnborg
87127137e52SSam Ravnborg #define numadbg(f, a...) \
87227137e52SSam Ravnborg do { if (numa_debug) \
87327137e52SSam Ravnborg printk(KERN_INFO f, ## a); \
87427137e52SSam Ravnborg } while (0)
87527137e52SSam Ravnborg
find_ramdisk(unsigned long phys_base)87627137e52SSam Ravnborg static void __init find_ramdisk(unsigned long phys_base)
87727137e52SSam Ravnborg {
87827137e52SSam Ravnborg #ifdef CONFIG_BLK_DEV_INITRD
87927137e52SSam Ravnborg if (sparc_ramdisk_image || sparc_ramdisk_image64) {
88027137e52SSam Ravnborg unsigned long ramdisk_image;
88127137e52SSam Ravnborg
88227137e52SSam Ravnborg /* Older versions of the bootloader only supported a
88327137e52SSam Ravnborg * 32-bit physical address for the ramdisk image
88427137e52SSam Ravnborg * location, stored at sparc_ramdisk_image. Newer
88527137e52SSam Ravnborg * SILO versions set sparc_ramdisk_image to zero and
88627137e52SSam Ravnborg * provide a full 64-bit physical address at
88727137e52SSam Ravnborg * sparc_ramdisk_image64.
88827137e52SSam Ravnborg */
88927137e52SSam Ravnborg ramdisk_image = sparc_ramdisk_image;
89027137e52SSam Ravnborg if (!ramdisk_image)
89127137e52SSam Ravnborg ramdisk_image = sparc_ramdisk_image64;
89227137e52SSam Ravnborg
89327137e52SSam Ravnborg /* Another bootloader quirk. The bootloader normalizes
89427137e52SSam Ravnborg * the physical address to KERNBASE, so we have to
89527137e52SSam Ravnborg * factor that back out and add in the lowest valid
89627137e52SSam Ravnborg * physical page address to get the true physical address.
89727137e52SSam Ravnborg */
89827137e52SSam Ravnborg ramdisk_image -= KERNBASE;
89927137e52SSam Ravnborg ramdisk_image += phys_base;
90027137e52SSam Ravnborg
90127137e52SSam Ravnborg numadbg("Found ramdisk at physical address 0x%lx, size %u\n",
90227137e52SSam Ravnborg ramdisk_image, sparc_ramdisk_size);
90327137e52SSam Ravnborg
90427137e52SSam Ravnborg initrd_start = ramdisk_image;
90527137e52SSam Ravnborg initrd_end = ramdisk_image + sparc_ramdisk_size;
90627137e52SSam Ravnborg
90795f72d1eSYinghai Lu memblock_reserve(initrd_start, sparc_ramdisk_size);
90827137e52SSam Ravnborg
90927137e52SSam Ravnborg initrd_start += PAGE_OFFSET;
91027137e52SSam Ravnborg initrd_end += PAGE_OFFSET;
91127137e52SSam Ravnborg }
91227137e52SSam Ravnborg #endif
91327137e52SSam Ravnborg }
91427137e52SSam Ravnborg
91527137e52SSam Ravnborg struct node_mem_mask {
91627137e52SSam Ravnborg unsigned long mask;
9171537b26dSPavel Tatashin unsigned long match;
91827137e52SSam Ravnborg };
91927137e52SSam Ravnborg static struct node_mem_mask node_masks[MAX_NUMNODES];
92027137e52SSam Ravnborg static int num_node_masks;
92127137e52SSam Ravnborg
922a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
92348d37216SSam Ravnborg
9241537b26dSPavel Tatashin struct mdesc_mlgroup {
9251537b26dSPavel Tatashin u64 node;
9261537b26dSPavel Tatashin u64 latency;
9271537b26dSPavel Tatashin u64 match;
9281537b26dSPavel Tatashin u64 mask;
9291537b26dSPavel Tatashin };
9301537b26dSPavel Tatashin
9311537b26dSPavel Tatashin static struct mdesc_mlgroup *mlgroups;
9321537b26dSPavel Tatashin static int num_mlgroups;
9331537b26dSPavel Tatashin
93427137e52SSam Ravnborg int numa_cpu_lookup_table[NR_CPUS];
93527137e52SSam Ravnborg cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
93627137e52SSam Ravnborg
93727137e52SSam Ravnborg struct mdesc_mblock {
93827137e52SSam Ravnborg u64 base;
93927137e52SSam Ravnborg u64 size;
94027137e52SSam Ravnborg u64 offset; /* RA-to-PA */
94127137e52SSam Ravnborg };
94227137e52SSam Ravnborg static struct mdesc_mblock *mblocks;
94327137e52SSam Ravnborg static int num_mblocks;
94427137e52SSam Ravnborg
addr_to_mblock(unsigned long addr)9451537b26dSPavel Tatashin static struct mdesc_mblock * __init addr_to_mblock(unsigned long addr)
94627137e52SSam Ravnborg {
9471537b26dSPavel Tatashin struct mdesc_mblock *m = NULL;
94827137e52SSam Ravnborg int i;
94927137e52SSam Ravnborg
95027137e52SSam Ravnborg for (i = 0; i < num_mblocks; i++) {
9511537b26dSPavel Tatashin m = &mblocks[i];
95227137e52SSam Ravnborg
95327137e52SSam Ravnborg if (addr >= m->base &&
95427137e52SSam Ravnborg addr < (m->base + m->size)) {
95527137e52SSam Ravnborg break;
95627137e52SSam Ravnborg }
95727137e52SSam Ravnborg }
9581537b26dSPavel Tatashin
9591537b26dSPavel Tatashin return m;
96027137e52SSam Ravnborg }
96127137e52SSam Ravnborg
memblock_nid_range_sun4u(u64 start,u64 end,int * nid)9621537b26dSPavel Tatashin static u64 __init memblock_nid_range_sun4u(u64 start, u64 end, int *nid)
96327137e52SSam Ravnborg {
9641537b26dSPavel Tatashin int prev_nid, new_nid;
96527137e52SSam Ravnborg
96698fa15f3SAnshuman Khandual prev_nid = NUMA_NO_NODE;
9671537b26dSPavel Tatashin for ( ; start < end; start += PAGE_SIZE) {
9681537b26dSPavel Tatashin for (new_nid = 0; new_nid < num_node_masks; new_nid++) {
9691537b26dSPavel Tatashin struct node_mem_mask *p = &node_masks[new_nid];
97027137e52SSam Ravnborg
9711537b26dSPavel Tatashin if ((start & p->mask) == p->match) {
97298fa15f3SAnshuman Khandual if (prev_nid == NUMA_NO_NODE)
9731537b26dSPavel Tatashin prev_nid = new_nid;
9741537b26dSPavel Tatashin break;
97574a5ed5cSThomas Tai }
97674a5ed5cSThomas Tai }
97774a5ed5cSThomas Tai
9781537b26dSPavel Tatashin if (new_nid == num_node_masks) {
9791537b26dSPavel Tatashin prev_nid = 0;
9801537b26dSPavel Tatashin WARN_ONCE(1, "addr[%Lx] doesn't match a NUMA node rule. Some memory will be owned by node 0.",
9811537b26dSPavel Tatashin start);
9821537b26dSPavel Tatashin break;
9831537b26dSPavel Tatashin }
9841537b26dSPavel Tatashin
9851537b26dSPavel Tatashin if (prev_nid != new_nid)
9861537b26dSPavel Tatashin break;
9871537b26dSPavel Tatashin }
9881537b26dSPavel Tatashin *nid = prev_nid;
9891537b26dSPavel Tatashin
9901537b26dSPavel Tatashin return start > end ? end : start;
99127137e52SSam Ravnborg }
99227137e52SSam Ravnborg
memblock_nid_range(u64 start,u64 end,int * nid)99387a349f9SThomas Tai static u64 __init memblock_nid_range(u64 start, u64 end, int *nid)
99427137e52SSam Ravnborg {
9951537b26dSPavel Tatashin u64 ret_end, pa_start, m_mask, m_match, m_end;
9961537b26dSPavel Tatashin struct mdesc_mblock *mblock;
9971537b26dSPavel Tatashin int _nid, i;
99827137e52SSam Ravnborg
9991537b26dSPavel Tatashin if (tlb_type != hypervisor)
10001537b26dSPavel Tatashin return memblock_nid_range_sun4u(start, end, nid);
10011537b26dSPavel Tatashin
10021537b26dSPavel Tatashin mblock = addr_to_mblock(start);
10031537b26dSPavel Tatashin if (!mblock) {
10041537b26dSPavel Tatashin WARN_ONCE(1, "memblock_nid_range: Can't find mblock addr[%Lx]",
10051537b26dSPavel Tatashin start);
10061537b26dSPavel Tatashin
10071537b26dSPavel Tatashin _nid = 0;
10081537b26dSPavel Tatashin ret_end = end;
10091537b26dSPavel Tatashin goto done;
101027137e52SSam Ravnborg }
101127137e52SSam Ravnborg
10121537b26dSPavel Tatashin pa_start = start + mblock->offset;
10131537b26dSPavel Tatashin m_match = 0;
10141537b26dSPavel Tatashin m_mask = 0;
101527137e52SSam Ravnborg
10161537b26dSPavel Tatashin for (_nid = 0; _nid < num_node_masks; _nid++) {
10171537b26dSPavel Tatashin struct node_mem_mask *const m = &node_masks[_nid];
10181537b26dSPavel Tatashin
10191537b26dSPavel Tatashin if ((pa_start & m->mask) == m->match) {
10201537b26dSPavel Tatashin m_match = m->match;
10211537b26dSPavel Tatashin m_mask = m->mask;
10221537b26dSPavel Tatashin break;
10231537b26dSPavel Tatashin }
10241537b26dSPavel Tatashin }
10251537b26dSPavel Tatashin
10261537b26dSPavel Tatashin if (num_node_masks == _nid) {
10271537b26dSPavel Tatashin /* We could not find NUMA group, so default to 0, but lets
10281537b26dSPavel Tatashin * search for latency group, so we could calculate the correct
10291537b26dSPavel Tatashin * end address that we return
10301537b26dSPavel Tatashin */
10311537b26dSPavel Tatashin _nid = 0;
10321537b26dSPavel Tatashin
10331537b26dSPavel Tatashin for (i = 0; i < num_mlgroups; i++) {
10341537b26dSPavel Tatashin struct mdesc_mlgroup *const m = &mlgroups[i];
10351537b26dSPavel Tatashin
10361537b26dSPavel Tatashin if ((pa_start & m->mask) == m->match) {
10371537b26dSPavel Tatashin m_match = m->match;
10381537b26dSPavel Tatashin m_mask = m->mask;
10391537b26dSPavel Tatashin break;
10401537b26dSPavel Tatashin }
10411537b26dSPavel Tatashin }
10421537b26dSPavel Tatashin
10431537b26dSPavel Tatashin if (i == num_mlgroups) {
10441537b26dSPavel Tatashin WARN_ONCE(1, "memblock_nid_range: Can't find latency group addr[%Lx]",
10451537b26dSPavel Tatashin start);
10461537b26dSPavel Tatashin
10471537b26dSPavel Tatashin ret_end = end;
10481537b26dSPavel Tatashin goto done;
10491537b26dSPavel Tatashin }
10501537b26dSPavel Tatashin }
10511537b26dSPavel Tatashin
10521537b26dSPavel Tatashin /*
10531537b26dSPavel Tatashin * Each latency group has match and mask, and each memory block has an
10541537b26dSPavel Tatashin * offset. An address belongs to a latency group if its address matches
10551537b26dSPavel Tatashin * the following formula: ((addr + offset) & mask) == match
10561537b26dSPavel Tatashin * It is, however, slow to check every single page if it matches a
10571537b26dSPavel Tatashin * particular latency group. As optimization we calculate end value by
10581537b26dSPavel Tatashin * using bit arithmetics.
10591537b26dSPavel Tatashin */
10601537b26dSPavel Tatashin m_end = m_match + (1ul << __ffs(m_mask)) - mblock->offset;
10611537b26dSPavel Tatashin m_end += pa_start & ~((1ul << fls64(m_mask)) - 1);
10621537b26dSPavel Tatashin ret_end = m_end > end ? end : m_end;
10631537b26dSPavel Tatashin
10641537b26dSPavel Tatashin done:
10651537b26dSPavel Tatashin *nid = _nid;
10661537b26dSPavel Tatashin return ret_end;
106727137e52SSam Ravnborg }
106827137e52SSam Ravnborg #endif
106927137e52SSam Ravnborg
107027137e52SSam Ravnborg /* This must be invoked after performing all of the necessary
10712a4814dfSTejun Heo * memblock_set_node() calls for 'nid'. We need to be able to get
107227137e52SSam Ravnborg * correct data from get_pfn_range_for_nid().
107327137e52SSam Ravnborg */
allocate_node_data(int nid)107427137e52SSam Ravnborg static void __init allocate_node_data(int nid)
107527137e52SSam Ravnborg {
107627137e52SSam Ravnborg struct pglist_data *p;
1077aa6f0790SPaul Gortmaker unsigned long start_pfn, end_pfn;
1078a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
1079aa6f0790SPaul Gortmaker
1080b63a07d6SMike Rapoport NODE_DATA(nid) = memblock_alloc_node(sizeof(struct pglist_data),
10819a8dd708SMike Rapoport SMP_CACHE_BYTES, nid);
1082b63a07d6SMike Rapoport if (!NODE_DATA(nid)) {
108327137e52SSam Ravnborg prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);
108427137e52SSam Ravnborg prom_halt();
108527137e52SSam Ravnborg }
108627137e52SSam Ravnborg
1087625d693eSDavid S. Miller NODE_DATA(nid)->node_id = nid;
108827137e52SSam Ravnborg #endif
108927137e52SSam Ravnborg
109027137e52SSam Ravnborg p = NODE_DATA(nid);
109127137e52SSam Ravnborg
109227137e52SSam Ravnborg get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
109327137e52SSam Ravnborg p->node_start_pfn = start_pfn;
109427137e52SSam Ravnborg p->node_spanned_pages = end_pfn - start_pfn;
109527137e52SSam Ravnborg }
109627137e52SSam Ravnborg
init_node_masks_nonnuma(void)109727137e52SSam Ravnborg static void init_node_masks_nonnuma(void)
109827137e52SSam Ravnborg {
1099a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
110027137e52SSam Ravnborg int i;
110148d37216SSam Ravnborg #endif
110227137e52SSam Ravnborg
110327137e52SSam Ravnborg numadbg("Initializing tables for non-numa.\n");
110427137e52SSam Ravnborg
11051537b26dSPavel Tatashin node_masks[0].mask = 0;
11061537b26dSPavel Tatashin node_masks[0].match = 0;
110727137e52SSam Ravnborg num_node_masks = 1;
110827137e52SSam Ravnborg
1109a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
111027137e52SSam Ravnborg for (i = 0; i < NR_CPUS; i++)
111127137e52SSam Ravnborg numa_cpu_lookup_table[i] = 0;
111227137e52SSam Ravnborg
1113fb1fece5SKOSAKI Motohiro cpumask_setall(&numa_cpumask_lookup_table[0]);
111448d37216SSam Ravnborg #endif
111527137e52SSam Ravnborg }
111627137e52SSam Ravnborg
1117a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
111827137e52SSam Ravnborg struct pglist_data *node_data[MAX_NUMNODES];
111927137e52SSam Ravnborg
112027137e52SSam Ravnborg EXPORT_SYMBOL(numa_cpu_lookup_table);
112127137e52SSam Ravnborg EXPORT_SYMBOL(numa_cpumask_lookup_table);
112227137e52SSam Ravnborg EXPORT_SYMBOL(node_data);
112327137e52SSam Ravnborg
scan_pio_for_cfg_handle(struct mdesc_handle * md,u64 pio,u32 cfg_handle)112427137e52SSam Ravnborg static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio,
112527137e52SSam Ravnborg u32 cfg_handle)
112627137e52SSam Ravnborg {
112727137e52SSam Ravnborg u64 arc;
112827137e52SSam Ravnborg
112927137e52SSam Ravnborg mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) {
113027137e52SSam Ravnborg u64 target = mdesc_arc_target(md, arc);
113127137e52SSam Ravnborg const u64 *val;
113227137e52SSam Ravnborg
113327137e52SSam Ravnborg val = mdesc_get_property(md, target,
113427137e52SSam Ravnborg "cfg-handle", NULL);
113527137e52SSam Ravnborg if (val && *val == cfg_handle)
113627137e52SSam Ravnborg return 0;
113727137e52SSam Ravnborg }
113827137e52SSam Ravnborg return -ENODEV;
113927137e52SSam Ravnborg }
114027137e52SSam Ravnborg
scan_arcs_for_cfg_handle(struct mdesc_handle * md,u64 grp,u32 cfg_handle)114127137e52SSam Ravnborg static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp,
114227137e52SSam Ravnborg u32 cfg_handle)
114327137e52SSam Ravnborg {
114427137e52SSam Ravnborg u64 arc, candidate, best_latency = ~(u64)0;
114527137e52SSam Ravnborg
114627137e52SSam Ravnborg candidate = MDESC_NODE_NULL;
114727137e52SSam Ravnborg mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
114827137e52SSam Ravnborg u64 target = mdesc_arc_target(md, arc);
114927137e52SSam Ravnborg const char *name = mdesc_node_name(md, target);
115027137e52SSam Ravnborg const u64 *val;
115127137e52SSam Ravnborg
115227137e52SSam Ravnborg if (strcmp(name, "pio-latency-group"))
115327137e52SSam Ravnborg continue;
115427137e52SSam Ravnborg
115527137e52SSam Ravnborg val = mdesc_get_property(md, target, "latency", NULL);
115627137e52SSam Ravnborg if (!val)
115727137e52SSam Ravnborg continue;
115827137e52SSam Ravnborg
115927137e52SSam Ravnborg if (*val < best_latency) {
116027137e52SSam Ravnborg candidate = target;
116127137e52SSam Ravnborg best_latency = *val;
116227137e52SSam Ravnborg }
116327137e52SSam Ravnborg }
116427137e52SSam Ravnborg
116527137e52SSam Ravnborg if (candidate == MDESC_NODE_NULL)
116627137e52SSam Ravnborg return -ENODEV;
116727137e52SSam Ravnborg
116827137e52SSam Ravnborg return scan_pio_for_cfg_handle(md, candidate, cfg_handle);
116927137e52SSam Ravnborg }
117027137e52SSam Ravnborg
of_node_to_nid(struct device_node * dp)117127137e52SSam Ravnborg int of_node_to_nid(struct device_node *dp)
117227137e52SSam Ravnborg {
117327137e52SSam Ravnborg const struct linux_prom64_registers *regs;
117427137e52SSam Ravnborg struct mdesc_handle *md;
117527137e52SSam Ravnborg u32 cfg_handle;
117627137e52SSam Ravnborg int count, nid;
117727137e52SSam Ravnborg u64 grp;
117827137e52SSam Ravnborg
117927137e52SSam Ravnborg /* This is the right thing to do on currently supported
118027137e52SSam Ravnborg * SUN4U NUMA platforms as well, as the PCI controller does
118127137e52SSam Ravnborg * not sit behind any particular memory controller.
118227137e52SSam Ravnborg */
118327137e52SSam Ravnborg if (!mlgroups)
118427137e52SSam Ravnborg return -1;
118527137e52SSam Ravnborg
118627137e52SSam Ravnborg regs = of_get_property(dp, "reg", NULL);
118727137e52SSam Ravnborg if (!regs)
118827137e52SSam Ravnborg return -1;
118927137e52SSam Ravnborg
119027137e52SSam Ravnborg cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff;
119127137e52SSam Ravnborg
119227137e52SSam Ravnborg md = mdesc_grab();
119327137e52SSam Ravnborg
119427137e52SSam Ravnborg count = 0;
119598fa15f3SAnshuman Khandual nid = NUMA_NO_NODE;
119627137e52SSam Ravnborg mdesc_for_each_node_by_name(md, grp, "group") {
119727137e52SSam Ravnborg if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {
119827137e52SSam Ravnborg nid = count;
119927137e52SSam Ravnborg break;
120027137e52SSam Ravnborg }
120127137e52SSam Ravnborg count++;
120227137e52SSam Ravnborg }
120327137e52SSam Ravnborg
120427137e52SSam Ravnborg mdesc_release(md);
120527137e52SSam Ravnborg
120627137e52SSam Ravnborg return nid;
120727137e52SSam Ravnborg }
120827137e52SSam Ravnborg
add_node_ranges(void)120901c45381SDavid S. Miller static void __init add_node_ranges(void)
121027137e52SSam Ravnborg {
1211b10d6bcaSMike Rapoport phys_addr_t start, end;
1212cd429ce2SPavel Tatashin unsigned long prev_max;
1213b10d6bcaSMike Rapoport u64 i;
1214cd429ce2SPavel Tatashin
1215cd429ce2SPavel Tatashin memblock_resized:
1216cd429ce2SPavel Tatashin prev_max = memblock.memory.max;
121727137e52SSam Ravnborg
1218b10d6bcaSMike Rapoport for_each_mem_range(i, &start, &end) {
121927137e52SSam Ravnborg while (start < end) {
122027137e52SSam Ravnborg unsigned long this_end;
122127137e52SSam Ravnborg int nid;
122227137e52SSam Ravnborg
122335a1f0bdSBenjamin Herrenschmidt this_end = memblock_nid_range(start, end, &nid);
122427137e52SSam Ravnborg
12252a4814dfSTejun Heo numadbg("Setting memblock NUMA node nid[%d] "
1226b10d6bcaSMike Rapoport "start[%llx] end[%lx]\n",
122727137e52SSam Ravnborg nid, start, this_end);
122827137e52SSam Ravnborg
1229e7e8de59STang Chen memblock_set_node(start, this_end - start,
1230e7e8de59STang Chen &memblock.memory, nid);
1231cd429ce2SPavel Tatashin if (memblock.memory.max != prev_max)
1232cd429ce2SPavel Tatashin goto memblock_resized;
123327137e52SSam Ravnborg start = this_end;
123427137e52SSam Ravnborg }
123527137e52SSam Ravnborg }
123627137e52SSam Ravnborg }
123727137e52SSam Ravnborg
grab_mlgroups(struct mdesc_handle * md)123827137e52SSam Ravnborg static int __init grab_mlgroups(struct mdesc_handle *md)
123927137e52SSam Ravnborg {
124027137e52SSam Ravnborg unsigned long paddr;
124127137e52SSam Ravnborg int count = 0;
124227137e52SSam Ravnborg u64 node;
124327137e52SSam Ravnborg
124427137e52SSam Ravnborg mdesc_for_each_node_by_name(md, node, "memory-latency-group")
124527137e52SSam Ravnborg count++;
124627137e52SSam Ravnborg if (!count)
124727137e52SSam Ravnborg return -ENOENT;
124827137e52SSam Ravnborg
12499a8dd708SMike Rapoport paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mlgroup),
125027137e52SSam Ravnborg SMP_CACHE_BYTES);
125127137e52SSam Ravnborg if (!paddr)
125227137e52SSam Ravnborg return -ENOMEM;
125327137e52SSam Ravnborg
125427137e52SSam Ravnborg mlgroups = __va(paddr);
125527137e52SSam Ravnborg num_mlgroups = count;
125627137e52SSam Ravnborg
125727137e52SSam Ravnborg count = 0;
125827137e52SSam Ravnborg mdesc_for_each_node_by_name(md, node, "memory-latency-group") {
125927137e52SSam Ravnborg struct mdesc_mlgroup *m = &mlgroups[count++];
126027137e52SSam Ravnborg const u64 *val;
126127137e52SSam Ravnborg
126227137e52SSam Ravnborg m->node = node;
126327137e52SSam Ravnborg
126427137e52SSam Ravnborg val = mdesc_get_property(md, node, "latency", NULL);
126527137e52SSam Ravnborg m->latency = *val;
126627137e52SSam Ravnborg val = mdesc_get_property(md, node, "address-match", NULL);
126727137e52SSam Ravnborg m->match = *val;
126827137e52SSam Ravnborg val = mdesc_get_property(md, node, "address-mask", NULL);
126927137e52SSam Ravnborg m->mask = *val;
127027137e52SSam Ravnborg
127190181136SSam Ravnborg numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "
127290181136SSam Ravnborg "match[%llx] mask[%llx]\n",
127327137e52SSam Ravnborg count - 1, m->node, m->latency, m->match, m->mask);
127427137e52SSam Ravnborg }
127527137e52SSam Ravnborg
127627137e52SSam Ravnborg return 0;
127727137e52SSam Ravnborg }
127827137e52SSam Ravnborg
grab_mblocks(struct mdesc_handle * md)127927137e52SSam Ravnborg static int __init grab_mblocks(struct mdesc_handle *md)
128027137e52SSam Ravnborg {
128127137e52SSam Ravnborg unsigned long paddr;
128227137e52SSam Ravnborg int count = 0;
128327137e52SSam Ravnborg u64 node;
128427137e52SSam Ravnborg
128527137e52SSam Ravnborg mdesc_for_each_node_by_name(md, node, "mblock")
128627137e52SSam Ravnborg count++;
128727137e52SSam Ravnborg if (!count)
128827137e52SSam Ravnborg return -ENOENT;
128927137e52SSam Ravnborg
12909a8dd708SMike Rapoport paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mblock),
129127137e52SSam Ravnborg SMP_CACHE_BYTES);
129227137e52SSam Ravnborg if (!paddr)
129327137e52SSam Ravnborg return -ENOMEM;
129427137e52SSam Ravnborg
129527137e52SSam Ravnborg mblocks = __va(paddr);
129627137e52SSam Ravnborg num_mblocks = count;
129727137e52SSam Ravnborg
129827137e52SSam Ravnborg count = 0;
129927137e52SSam Ravnborg mdesc_for_each_node_by_name(md, node, "mblock") {
130027137e52SSam Ravnborg struct mdesc_mblock *m = &mblocks[count++];
130127137e52SSam Ravnborg const u64 *val;
130227137e52SSam Ravnborg
130327137e52SSam Ravnborg val = mdesc_get_property(md, node, "base", NULL);
130427137e52SSam Ravnborg m->base = *val;
130527137e52SSam Ravnborg val = mdesc_get_property(md, node, "size", NULL);
130627137e52SSam Ravnborg m->size = *val;
130727137e52SSam Ravnborg val = mdesc_get_property(md, node,
130827137e52SSam Ravnborg "address-congruence-offset", NULL);
1309771a37ffSbob picco
1310771a37ffSbob picco /* The address-congruence-offset property is optional.
1311771a37ffSbob picco * Explicity zero it be identifty this.
1312771a37ffSbob picco */
1313771a37ffSbob picco if (val)
131427137e52SSam Ravnborg m->offset = *val;
1315771a37ffSbob picco else
1316771a37ffSbob picco m->offset = 0UL;
131727137e52SSam Ravnborg
131890181136SSam Ravnborg numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",
131927137e52SSam Ravnborg count - 1, m->base, m->size, m->offset);
132027137e52SSam Ravnborg }
132127137e52SSam Ravnborg
132227137e52SSam Ravnborg return 0;
132327137e52SSam Ravnborg }
132427137e52SSam Ravnborg
numa_parse_mdesc_group_cpus(struct mdesc_handle * md,u64 grp,cpumask_t * mask)132527137e52SSam Ravnborg static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md,
132627137e52SSam Ravnborg u64 grp, cpumask_t *mask)
132727137e52SSam Ravnborg {
132827137e52SSam Ravnborg u64 arc;
132927137e52SSam Ravnborg
1330fb1fece5SKOSAKI Motohiro cpumask_clear(mask);
133127137e52SSam Ravnborg
133227137e52SSam Ravnborg mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) {
133327137e52SSam Ravnborg u64 target = mdesc_arc_target(md, arc);
133427137e52SSam Ravnborg const char *name = mdesc_node_name(md, target);
133527137e52SSam Ravnborg const u64 *id;
133627137e52SSam Ravnborg
133727137e52SSam Ravnborg if (strcmp(name, "cpu"))
133827137e52SSam Ravnborg continue;
133927137e52SSam Ravnborg id = mdesc_get_property(md, target, "id", NULL);
1340e305cb8fSRusty Russell if (*id < nr_cpu_ids)
1341fb1fece5SKOSAKI Motohiro cpumask_set_cpu(*id, mask);
134227137e52SSam Ravnborg }
134327137e52SSam Ravnborg }
134427137e52SSam Ravnborg
find_mlgroup(u64 node)134527137e52SSam Ravnborg static struct mdesc_mlgroup * __init find_mlgroup(u64 node)
134627137e52SSam Ravnborg {
134727137e52SSam Ravnborg int i;
134827137e52SSam Ravnborg
134927137e52SSam Ravnborg for (i = 0; i < num_mlgroups; i++) {
135027137e52SSam Ravnborg struct mdesc_mlgroup *m = &mlgroups[i];
135127137e52SSam Ravnborg if (m->node == node)
135227137e52SSam Ravnborg return m;
135327137e52SSam Ravnborg }
135427137e52SSam Ravnborg return NULL;
135527137e52SSam Ravnborg }
135627137e52SSam Ravnborg
__node_distance(int from,int to)135752708d69SNitin Gupta int __node_distance(int from, int to)
135852708d69SNitin Gupta {
135952708d69SNitin Gupta if ((from >= MAX_NUMNODES) || (to >= MAX_NUMNODES)) {
136052708d69SNitin Gupta pr_warn("Returning default NUMA distance value for %d->%d\n",
136152708d69SNitin Gupta from, to);
136252708d69SNitin Gupta return (from == to) ? LOCAL_DISTANCE : REMOTE_DISTANCE;
136352708d69SNitin Gupta }
136452708d69SNitin Gupta return numa_latency[from][to];
136552708d69SNitin Gupta }
13662b4792eaSDavid S. Miller EXPORT_SYMBOL(__node_distance);
136752708d69SNitin Gupta
find_best_numa_node_for_mlgroup(struct mdesc_mlgroup * grp)1368bdf2f59eSPaul Gortmaker static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp)
136952708d69SNitin Gupta {
137052708d69SNitin Gupta int i;
137152708d69SNitin Gupta
137252708d69SNitin Gupta for (i = 0; i < MAX_NUMNODES; i++) {
137352708d69SNitin Gupta struct node_mem_mask *n = &node_masks[i];
137452708d69SNitin Gupta
13751537b26dSPavel Tatashin if ((grp->mask == n->mask) && (grp->match == n->match))
137652708d69SNitin Gupta break;
137752708d69SNitin Gupta }
137852708d69SNitin Gupta return i;
137952708d69SNitin Gupta }
138052708d69SNitin Gupta
find_numa_latencies_for_group(struct mdesc_handle * md,u64 grp,int index)1381bdf2f59eSPaul Gortmaker static void __init find_numa_latencies_for_group(struct mdesc_handle *md,
1382bdf2f59eSPaul Gortmaker u64 grp, int index)
138352708d69SNitin Gupta {
138452708d69SNitin Gupta u64 arc;
138552708d69SNitin Gupta
138652708d69SNitin Gupta mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
138752708d69SNitin Gupta int tnode;
138852708d69SNitin Gupta u64 target = mdesc_arc_target(md, arc);
138952708d69SNitin Gupta struct mdesc_mlgroup *m = find_mlgroup(target);
139052708d69SNitin Gupta
139152708d69SNitin Gupta if (!m)
139252708d69SNitin Gupta continue;
139352708d69SNitin Gupta tnode = find_best_numa_node_for_mlgroup(m);
139452708d69SNitin Gupta if (tnode == MAX_NUMNODES)
139552708d69SNitin Gupta continue;
139652708d69SNitin Gupta numa_latency[index][tnode] = m->latency;
139752708d69SNitin Gupta }
139852708d69SNitin Gupta }
139952708d69SNitin Gupta
numa_attach_mlgroup(struct mdesc_handle * md,u64 grp,int index)140027137e52SSam Ravnborg static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp,
140127137e52SSam Ravnborg int index)
140227137e52SSam Ravnborg {
140327137e52SSam Ravnborg struct mdesc_mlgroup *candidate = NULL;
140427137e52SSam Ravnborg u64 arc, best_latency = ~(u64)0;
140527137e52SSam Ravnborg struct node_mem_mask *n;
140627137e52SSam Ravnborg
140727137e52SSam Ravnborg mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
140827137e52SSam Ravnborg u64 target = mdesc_arc_target(md, arc);
140927137e52SSam Ravnborg struct mdesc_mlgroup *m = find_mlgroup(target);
141027137e52SSam Ravnborg if (!m)
141127137e52SSam Ravnborg continue;
141227137e52SSam Ravnborg if (m->latency < best_latency) {
141327137e52SSam Ravnborg candidate = m;
141427137e52SSam Ravnborg best_latency = m->latency;
141527137e52SSam Ravnborg }
141627137e52SSam Ravnborg }
141727137e52SSam Ravnborg if (!candidate)
141827137e52SSam Ravnborg return -ENOENT;
141927137e52SSam Ravnborg
142027137e52SSam Ravnborg if (num_node_masks != index) {
142127137e52SSam Ravnborg printk(KERN_ERR "Inconsistent NUMA state, "
142227137e52SSam Ravnborg "index[%d] != num_node_masks[%d]\n",
142327137e52SSam Ravnborg index, num_node_masks);
142427137e52SSam Ravnborg return -EINVAL;
142527137e52SSam Ravnborg }
142627137e52SSam Ravnborg
142727137e52SSam Ravnborg n = &node_masks[num_node_masks++];
142827137e52SSam Ravnborg
142927137e52SSam Ravnborg n->mask = candidate->mask;
14301537b26dSPavel Tatashin n->match = candidate->match;
143127137e52SSam Ravnborg
14321537b26dSPavel Tatashin numadbg("NUMA NODE[%d]: mask[%lx] match[%lx] (latency[%llx])\n",
14331537b26dSPavel Tatashin index, n->mask, n->match, candidate->latency);
143427137e52SSam Ravnborg
143527137e52SSam Ravnborg return 0;
143627137e52SSam Ravnborg }
143727137e52SSam Ravnborg
numa_parse_mdesc_group(struct mdesc_handle * md,u64 grp,int index)143827137e52SSam Ravnborg static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp,
143927137e52SSam Ravnborg int index)
144027137e52SSam Ravnborg {
144127137e52SSam Ravnborg cpumask_t mask;
144227137e52SSam Ravnborg int cpu;
144327137e52SSam Ravnborg
144427137e52SSam Ravnborg numa_parse_mdesc_group_cpus(md, grp, &mask);
144527137e52SSam Ravnborg
1446fb1fece5SKOSAKI Motohiro for_each_cpu(cpu, &mask)
144727137e52SSam Ravnborg numa_cpu_lookup_table[cpu] = index;
1448fb1fece5SKOSAKI Motohiro cpumask_copy(&numa_cpumask_lookup_table[index], &mask);
144927137e52SSam Ravnborg
145027137e52SSam Ravnborg if (numa_debug) {
145127137e52SSam Ravnborg printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index);
1452fb1fece5SKOSAKI Motohiro for_each_cpu(cpu, &mask)
145327137e52SSam Ravnborg printk("%d ", cpu);
145427137e52SSam Ravnborg printk("]\n");
145527137e52SSam Ravnborg }
145627137e52SSam Ravnborg
145727137e52SSam Ravnborg return numa_attach_mlgroup(md, grp, index);
145827137e52SSam Ravnborg }
145927137e52SSam Ravnborg
numa_parse_mdesc(void)146027137e52SSam Ravnborg static int __init numa_parse_mdesc(void)
146127137e52SSam Ravnborg {
146227137e52SSam Ravnborg struct mdesc_handle *md = mdesc_grab();
146352708d69SNitin Gupta int i, j, err, count;
146427137e52SSam Ravnborg u64 node;
146527137e52SSam Ravnborg
146627137e52SSam Ravnborg node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups");
146727137e52SSam Ravnborg if (node == MDESC_NODE_NULL) {
146827137e52SSam Ravnborg mdesc_release(md);
146927137e52SSam Ravnborg return -ENOENT;
147027137e52SSam Ravnborg }
147127137e52SSam Ravnborg
147227137e52SSam Ravnborg err = grab_mblocks(md);
147327137e52SSam Ravnborg if (err < 0)
147427137e52SSam Ravnborg goto out;
147527137e52SSam Ravnborg
147627137e52SSam Ravnborg err = grab_mlgroups(md);
147727137e52SSam Ravnborg if (err < 0)
147827137e52SSam Ravnborg goto out;
147927137e52SSam Ravnborg
148027137e52SSam Ravnborg count = 0;
148127137e52SSam Ravnborg mdesc_for_each_node_by_name(md, node, "group") {
148227137e52SSam Ravnborg err = numa_parse_mdesc_group(md, node, count);
148327137e52SSam Ravnborg if (err < 0)
148427137e52SSam Ravnborg break;
148527137e52SSam Ravnborg count++;
148627137e52SSam Ravnborg }
148727137e52SSam Ravnborg
148852708d69SNitin Gupta count = 0;
148952708d69SNitin Gupta mdesc_for_each_node_by_name(md, node, "group") {
149052708d69SNitin Gupta find_numa_latencies_for_group(md, node, count);
149152708d69SNitin Gupta count++;
149252708d69SNitin Gupta }
149352708d69SNitin Gupta
149452708d69SNitin Gupta /* Normalize numa latency matrix according to ACPI SLIT spec. */
149552708d69SNitin Gupta for (i = 0; i < MAX_NUMNODES; i++) {
149652708d69SNitin Gupta u64 self_latency = numa_latency[i][i];
149752708d69SNitin Gupta
149852708d69SNitin Gupta for (j = 0; j < MAX_NUMNODES; j++) {
149952708d69SNitin Gupta numa_latency[i][j] =
150052708d69SNitin Gupta (numa_latency[i][j] * LOCAL_DISTANCE) /
150152708d69SNitin Gupta self_latency;
150252708d69SNitin Gupta }
150352708d69SNitin Gupta }
150452708d69SNitin Gupta
150527137e52SSam Ravnborg add_node_ranges();
150627137e52SSam Ravnborg
150727137e52SSam Ravnborg for (i = 0; i < num_node_masks; i++) {
150827137e52SSam Ravnborg allocate_node_data(i);
150927137e52SSam Ravnborg node_set_online(i);
151027137e52SSam Ravnborg }
151127137e52SSam Ravnborg
151227137e52SSam Ravnborg err = 0;
151327137e52SSam Ravnborg out:
151427137e52SSam Ravnborg mdesc_release(md);
151527137e52SSam Ravnborg return err;
151627137e52SSam Ravnborg }
151727137e52SSam Ravnborg
numa_parse_jbus(void)151827137e52SSam Ravnborg static int __init numa_parse_jbus(void)
151927137e52SSam Ravnborg {
152027137e52SSam Ravnborg unsigned long cpu, index;
152127137e52SSam Ravnborg
152227137e52SSam Ravnborg /* NUMA node id is encoded in bits 36 and higher, and there is
152327137e52SSam Ravnborg * a 1-to-1 mapping from CPU ID to NUMA node ID.
152427137e52SSam Ravnborg */
152527137e52SSam Ravnborg index = 0;
152627137e52SSam Ravnborg for_each_present_cpu(cpu) {
152727137e52SSam Ravnborg numa_cpu_lookup_table[cpu] = index;
1528fb1fece5SKOSAKI Motohiro cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu));
152927137e52SSam Ravnborg node_masks[index].mask = ~((1UL << 36UL) - 1UL);
15301537b26dSPavel Tatashin node_masks[index].match = cpu << 36UL;
153127137e52SSam Ravnborg
153227137e52SSam Ravnborg index++;
153327137e52SSam Ravnborg }
153427137e52SSam Ravnborg num_node_masks = index;
153527137e52SSam Ravnborg
153627137e52SSam Ravnborg add_node_ranges();
153727137e52SSam Ravnborg
153827137e52SSam Ravnborg for (index = 0; index < num_node_masks; index++) {
153927137e52SSam Ravnborg allocate_node_data(index);
154027137e52SSam Ravnborg node_set_online(index);
154127137e52SSam Ravnborg }
154227137e52SSam Ravnborg
154327137e52SSam Ravnborg return 0;
154427137e52SSam Ravnborg }
154527137e52SSam Ravnborg
numa_parse_sun4u(void)154627137e52SSam Ravnborg static int __init numa_parse_sun4u(void)
154727137e52SSam Ravnborg {
154827137e52SSam Ravnborg if (tlb_type == cheetah || tlb_type == cheetah_plus) {
154927137e52SSam Ravnborg unsigned long ver;
155027137e52SSam Ravnborg
155127137e52SSam Ravnborg __asm__ ("rdpr %%ver, %0" : "=r" (ver));
155227137e52SSam Ravnborg if ((ver >> 32UL) == __JALAPENO_ID ||
155327137e52SSam Ravnborg (ver >> 32UL) == __SERRANO_ID)
155427137e52SSam Ravnborg return numa_parse_jbus();
155527137e52SSam Ravnborg }
155627137e52SSam Ravnborg return -1;
155727137e52SSam Ravnborg }
155827137e52SSam Ravnborg
bootmem_init_numa(void)155927137e52SSam Ravnborg static int __init bootmem_init_numa(void)
156027137e52SSam Ravnborg {
156136beca65SNitin Gupta int i, j;
156227137e52SSam Ravnborg int err = -1;
156327137e52SSam Ravnborg
156427137e52SSam Ravnborg numadbg("bootmem_init_numa()\n");
156527137e52SSam Ravnborg
156636beca65SNitin Gupta /* Some sane defaults for numa latency values */
156736beca65SNitin Gupta for (i = 0; i < MAX_NUMNODES; i++) {
156836beca65SNitin Gupta for (j = 0; j < MAX_NUMNODES; j++)
156936beca65SNitin Gupta numa_latency[i][j] = (i == j) ?
157036beca65SNitin Gupta LOCAL_DISTANCE : REMOTE_DISTANCE;
157136beca65SNitin Gupta }
157236beca65SNitin Gupta
157327137e52SSam Ravnborg if (numa_enabled) {
157427137e52SSam Ravnborg if (tlb_type == hypervisor)
157527137e52SSam Ravnborg err = numa_parse_mdesc();
157627137e52SSam Ravnborg else
157727137e52SSam Ravnborg err = numa_parse_sun4u();
157827137e52SSam Ravnborg }
157927137e52SSam Ravnborg return err;
158027137e52SSam Ravnborg }
158127137e52SSam Ravnborg
158227137e52SSam Ravnborg #else
158327137e52SSam Ravnborg
bootmem_init_numa(void)158427137e52SSam Ravnborg static int bootmem_init_numa(void)
158527137e52SSam Ravnborg {
158627137e52SSam Ravnborg return -1;
158727137e52SSam Ravnborg }
158827137e52SSam Ravnborg
158927137e52SSam Ravnborg #endif
159027137e52SSam Ravnborg
bootmem_init_nonnuma(void)159127137e52SSam Ravnborg static void __init bootmem_init_nonnuma(void)
159227137e52SSam Ravnborg {
159395f72d1eSYinghai Lu unsigned long top_of_ram = memblock_end_of_DRAM();
159495f72d1eSYinghai Lu unsigned long total_ram = memblock_phys_mem_size();
159527137e52SSam Ravnborg
159627137e52SSam Ravnborg numadbg("bootmem_init_nonnuma()\n");
159727137e52SSam Ravnborg
159827137e52SSam Ravnborg printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
159927137e52SSam Ravnborg top_of_ram, total_ram);
160027137e52SSam Ravnborg printk(KERN_INFO "Memory hole size: %ldMB\n",
160127137e52SSam Ravnborg (top_of_ram - total_ram) >> 20);
160227137e52SSam Ravnborg
160327137e52SSam Ravnborg init_node_masks_nonnuma();
1604d7dc899aSStefan Agner memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0);
160527137e52SSam Ravnborg allocate_node_data(0);
160627137e52SSam Ravnborg node_set_online(0);
160727137e52SSam Ravnborg }
160827137e52SSam Ravnborg
bootmem_init(unsigned long phys_base)160927137e52SSam Ravnborg static unsigned long __init bootmem_init(unsigned long phys_base)
161027137e52SSam Ravnborg {
161127137e52SSam Ravnborg unsigned long end_pfn;
161227137e52SSam Ravnborg
161395f72d1eSYinghai Lu end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
161427137e52SSam Ravnborg max_pfn = max_low_pfn = end_pfn;
161527137e52SSam Ravnborg min_low_pfn = (phys_base >> PAGE_SHIFT);
161627137e52SSam Ravnborg
161727137e52SSam Ravnborg if (bootmem_init_numa() < 0)
161827137e52SSam Ravnborg bootmem_init_nonnuma();
161927137e52SSam Ravnborg
1620625d693eSDavid S. Miller /* Dump memblock with node info. */
1621625d693eSDavid S. Miller memblock_dump_all();
1622625d693eSDavid S. Miller
162327137e52SSam Ravnborg /* XXX cpu notifier XXX */
162427137e52SSam Ravnborg
162527137e52SSam Ravnborg sparse_init();
162627137e52SSam Ravnborg
162727137e52SSam Ravnborg return end_pfn;
162827137e52SSam Ravnborg }
162927137e52SSam Ravnborg
163027137e52SSam Ravnborg static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
163127137e52SSam Ravnborg static int pall_ents __initdata;
163227137e52SSam Ravnborg
16330dd5b7b0SDavid S. Miller static unsigned long max_phys_bits = 40;
16340dd5b7b0SDavid S. Miller
kern_addr_valid(unsigned long addr)16350dd5b7b0SDavid S. Miller bool kern_addr_valid(unsigned long addr)
16360dd5b7b0SDavid S. Miller {
16370dd5b7b0SDavid S. Miller pgd_t *pgd;
16385637bc50SMike Rapoport p4d_t *p4d;
16390dd5b7b0SDavid S. Miller pud_t *pud;
16400dd5b7b0SDavid S. Miller pmd_t *pmd;
16410dd5b7b0SDavid S. Miller pte_t *pte;
16420dd5b7b0SDavid S. Miller
1643bb4e6e85SDavid S. Miller if ((long)addr < 0L) {
1644bb4e6e85SDavid S. Miller unsigned long pa = __pa(addr);
1645bb4e6e85SDavid S. Miller
1646adfae8a5Sbob picco if ((pa >> max_phys_bits) != 0UL)
16470dd5b7b0SDavid S. Miller return false;
16480dd5b7b0SDavid S. Miller
1649bb4e6e85SDavid S. Miller return pfn_valid(pa >> PAGE_SHIFT);
1650bb4e6e85SDavid S. Miller }
1651bb4e6e85SDavid S. Miller
16520dd5b7b0SDavid S. Miller if (addr >= (unsigned long) KERNBASE &&
16530dd5b7b0SDavid S. Miller addr < (unsigned long)&_end)
16540dd5b7b0SDavid S. Miller return true;
16550dd5b7b0SDavid S. Miller
16560dd5b7b0SDavid S. Miller pgd = pgd_offset_k(addr);
16570dd5b7b0SDavid S. Miller if (pgd_none(*pgd))
165857829ea4SJason Yan return false;
16590dd5b7b0SDavid S. Miller
16605637bc50SMike Rapoport p4d = p4d_offset(pgd, addr);
16615637bc50SMike Rapoport if (p4d_none(*p4d))
166257829ea4SJason Yan return false;
16635637bc50SMike Rapoport
16645637bc50SMike Rapoport pud = pud_offset(p4d, addr);
16650dd5b7b0SDavid S. Miller if (pud_none(*pud))
166657829ea4SJason Yan return false;
16670dd5b7b0SDavid S. Miller
1668*907835e6SPeter Xu if (pud_leaf(*pud))
16690dd5b7b0SDavid S. Miller return pfn_valid(pud_pfn(*pud));
16700dd5b7b0SDavid S. Miller
16710dd5b7b0SDavid S. Miller pmd = pmd_offset(pud, addr);
16720dd5b7b0SDavid S. Miller if (pmd_none(*pmd))
167357829ea4SJason Yan return false;
16740dd5b7b0SDavid S. Miller
16750dd5b7b0SDavid S. Miller if (pmd_large(*pmd))
16760dd5b7b0SDavid S. Miller return pfn_valid(pmd_pfn(*pmd));
16770dd5b7b0SDavid S. Miller
16780dd5b7b0SDavid S. Miller pte = pte_offset_kernel(pmd, addr);
16790dd5b7b0SDavid S. Miller if (pte_none(*pte))
168057829ea4SJason Yan return false;
16810dd5b7b0SDavid S. Miller
16820dd5b7b0SDavid S. Miller return pfn_valid(pte_pfn(*pte));
16830dd5b7b0SDavid S. Miller }
16840dd5b7b0SDavid S. Miller
kernel_map_hugepud(unsigned long vstart,unsigned long vend,pud_t * pud)16850dd5b7b0SDavid S. Miller static unsigned long __ref kernel_map_hugepud(unsigned long vstart,
16860dd5b7b0SDavid S. Miller unsigned long vend,
16870dd5b7b0SDavid S. Miller pud_t *pud)
16880dd5b7b0SDavid S. Miller {
16890dd5b7b0SDavid S. Miller const unsigned long mask16gb = (1UL << 34) - 1UL;
16900dd5b7b0SDavid S. Miller u64 pte_val = vstart;
16910dd5b7b0SDavid S. Miller
16920dd5b7b0SDavid S. Miller /* Each PUD is 8GB */
16930dd5b7b0SDavid S. Miller if ((vstart & mask16gb) ||
16940dd5b7b0SDavid S. Miller (vend - vstart <= mask16gb)) {
16950dd5b7b0SDavid S. Miller pte_val ^= kern_linear_pte_xor[2];
16960dd5b7b0SDavid S. Miller pud_val(*pud) = pte_val | _PAGE_PUD_HUGE;
16970dd5b7b0SDavid S. Miller
16980dd5b7b0SDavid S. Miller return vstart + PUD_SIZE;
16990dd5b7b0SDavid S. Miller }
17000dd5b7b0SDavid S. Miller
17010dd5b7b0SDavid S. Miller pte_val ^= kern_linear_pte_xor[3];
17020dd5b7b0SDavid S. Miller pte_val |= _PAGE_PUD_HUGE;
17030dd5b7b0SDavid S. Miller
17040dd5b7b0SDavid S. Miller vend = vstart + mask16gb + 1UL;
17050dd5b7b0SDavid S. Miller while (vstart < vend) {
17060dd5b7b0SDavid S. Miller pud_val(*pud) = pte_val;
17070dd5b7b0SDavid S. Miller
17080dd5b7b0SDavid S. Miller pte_val += PUD_SIZE;
17090dd5b7b0SDavid S. Miller vstart += PUD_SIZE;
17100dd5b7b0SDavid S. Miller pud++;
17110dd5b7b0SDavid S. Miller }
17120dd5b7b0SDavid S. Miller return vstart;
17130dd5b7b0SDavid S. Miller }
17140dd5b7b0SDavid S. Miller
kernel_can_map_hugepud(unsigned long vstart,unsigned long vend,bool guard)17150dd5b7b0SDavid S. Miller static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend,
17160dd5b7b0SDavid S. Miller bool guard)
17170dd5b7b0SDavid S. Miller {
17180dd5b7b0SDavid S. Miller if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE)
17190dd5b7b0SDavid S. Miller return true;
17200dd5b7b0SDavid S. Miller
17210dd5b7b0SDavid S. Miller return false;
17220dd5b7b0SDavid S. Miller }
17230dd5b7b0SDavid S. Miller
kernel_map_hugepmd(unsigned long vstart,unsigned long vend,pmd_t * pmd)17240dd5b7b0SDavid S. Miller static unsigned long __ref kernel_map_hugepmd(unsigned long vstart,
17250dd5b7b0SDavid S. Miller unsigned long vend,
17260dd5b7b0SDavid S. Miller pmd_t *pmd)
17270dd5b7b0SDavid S. Miller {
17280dd5b7b0SDavid S. Miller const unsigned long mask256mb = (1UL << 28) - 1UL;
17290dd5b7b0SDavid S. Miller const unsigned long mask2gb = (1UL << 31) - 1UL;
17300dd5b7b0SDavid S. Miller u64 pte_val = vstart;
17310dd5b7b0SDavid S. Miller
17320dd5b7b0SDavid S. Miller /* Each PMD is 8MB */
17330dd5b7b0SDavid S. Miller if ((vstart & mask256mb) ||
17340dd5b7b0SDavid S. Miller (vend - vstart <= mask256mb)) {
17350dd5b7b0SDavid S. Miller pte_val ^= kern_linear_pte_xor[0];
17360dd5b7b0SDavid S. Miller pmd_val(*pmd) = pte_val | _PAGE_PMD_HUGE;
17370dd5b7b0SDavid S. Miller
17380dd5b7b0SDavid S. Miller return vstart + PMD_SIZE;
17390dd5b7b0SDavid S. Miller }
17400dd5b7b0SDavid S. Miller
17410dd5b7b0SDavid S. Miller if ((vstart & mask2gb) ||
17420dd5b7b0SDavid S. Miller (vend - vstart <= mask2gb)) {
17430dd5b7b0SDavid S. Miller pte_val ^= kern_linear_pte_xor[1];
17440dd5b7b0SDavid S. Miller pte_val |= _PAGE_PMD_HUGE;
17450dd5b7b0SDavid S. Miller vend = vstart + mask256mb + 1UL;
17460dd5b7b0SDavid S. Miller } else {
17470dd5b7b0SDavid S. Miller pte_val ^= kern_linear_pte_xor[2];
17480dd5b7b0SDavid S. Miller pte_val |= _PAGE_PMD_HUGE;
17490dd5b7b0SDavid S. Miller vend = vstart + mask2gb + 1UL;
17500dd5b7b0SDavid S. Miller }
17510dd5b7b0SDavid S. Miller
17520dd5b7b0SDavid S. Miller while (vstart < vend) {
17530dd5b7b0SDavid S. Miller pmd_val(*pmd) = pte_val;
17540dd5b7b0SDavid S. Miller
17550dd5b7b0SDavid S. Miller pte_val += PMD_SIZE;
17560dd5b7b0SDavid S. Miller vstart += PMD_SIZE;
17570dd5b7b0SDavid S. Miller pmd++;
17580dd5b7b0SDavid S. Miller }
17590dd5b7b0SDavid S. Miller
17600dd5b7b0SDavid S. Miller return vstart;
17610dd5b7b0SDavid S. Miller }
17620dd5b7b0SDavid S. Miller
kernel_can_map_hugepmd(unsigned long vstart,unsigned long vend,bool guard)17630dd5b7b0SDavid S. Miller static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend,
17640dd5b7b0SDavid S. Miller bool guard)
17650dd5b7b0SDavid S. Miller {
17660dd5b7b0SDavid S. Miller if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE)
17670dd5b7b0SDavid S. Miller return true;
17680dd5b7b0SDavid S. Miller
17690dd5b7b0SDavid S. Miller return false;
17700dd5b7b0SDavid S. Miller }
17710dd5b7b0SDavid S. Miller
kernel_map_range(unsigned long pstart,unsigned long pend,pgprot_t prot,bool use_huge)177227137e52SSam Ravnborg static unsigned long __ref kernel_map_range(unsigned long pstart,
17730dd5b7b0SDavid S. Miller unsigned long pend, pgprot_t prot,
17740dd5b7b0SDavid S. Miller bool use_huge)
177527137e52SSam Ravnborg {
177627137e52SSam Ravnborg unsigned long vstart = PAGE_OFFSET + pstart;
177727137e52SSam Ravnborg unsigned long vend = PAGE_OFFSET + pend;
177827137e52SSam Ravnborg unsigned long alloc_bytes = 0UL;
177927137e52SSam Ravnborg
178027137e52SSam Ravnborg if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
178127137e52SSam Ravnborg prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
178227137e52SSam Ravnborg vstart, vend);
178327137e52SSam Ravnborg prom_halt();
178427137e52SSam Ravnborg }
178527137e52SSam Ravnborg
178627137e52SSam Ravnborg while (vstart < vend) {
178727137e52SSam Ravnborg unsigned long this_end, paddr = __pa(vstart);
178827137e52SSam Ravnborg pgd_t *pgd = pgd_offset_k(vstart);
17895637bc50SMike Rapoport p4d_t *p4d;
179027137e52SSam Ravnborg pud_t *pud;
179127137e52SSam Ravnborg pmd_t *pmd;
179227137e52SSam Ravnborg pte_t *pte;
179327137e52SSam Ravnborg
1794ac55c768SDavid S. Miller if (pgd_none(*pgd)) {
1795ac55c768SDavid S. Miller pud_t *new;
1796ac55c768SDavid S. Miller
17974fc4a09eSMike Rapoport new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
17984fc4a09eSMike Rapoport PAGE_SIZE);
1799b1e1c869SMike Rapoport if (!new)
1800b1e1c869SMike Rapoport goto err_alloc;
1801ac55c768SDavid S. Miller alloc_bytes += PAGE_SIZE;
1802ac55c768SDavid S. Miller pgd_populate(&init_mm, pgd, new);
1803ac55c768SDavid S. Miller }
18045637bc50SMike Rapoport
18055637bc50SMike Rapoport p4d = p4d_offset(pgd, vstart);
18065637bc50SMike Rapoport if (p4d_none(*p4d)) {
18075637bc50SMike Rapoport pud_t *new;
18085637bc50SMike Rapoport
18095637bc50SMike Rapoport new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
18105637bc50SMike Rapoport PAGE_SIZE);
18115637bc50SMike Rapoport if (!new)
18125637bc50SMike Rapoport goto err_alloc;
18135637bc50SMike Rapoport alloc_bytes += PAGE_SIZE;
18145637bc50SMike Rapoport p4d_populate(&init_mm, p4d, new);
18155637bc50SMike Rapoport }
18165637bc50SMike Rapoport
18175637bc50SMike Rapoport pud = pud_offset(p4d, vstart);
181827137e52SSam Ravnborg if (pud_none(*pud)) {
181927137e52SSam Ravnborg pmd_t *new;
182027137e52SSam Ravnborg
18210dd5b7b0SDavid S. Miller if (kernel_can_map_hugepud(vstart, vend, use_huge)) {
18220dd5b7b0SDavid S. Miller vstart = kernel_map_hugepud(vstart, vend, pud);
18230dd5b7b0SDavid S. Miller continue;
18240dd5b7b0SDavid S. Miller }
18254fc4a09eSMike Rapoport new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
18264fc4a09eSMike Rapoport PAGE_SIZE);
1827b1e1c869SMike Rapoport if (!new)
1828b1e1c869SMike Rapoport goto err_alloc;
182927137e52SSam Ravnborg alloc_bytes += PAGE_SIZE;
183027137e52SSam Ravnborg pud_populate(&init_mm, pud, new);
183127137e52SSam Ravnborg }
183227137e52SSam Ravnborg
183327137e52SSam Ravnborg pmd = pmd_offset(pud, vstart);
18340dd5b7b0SDavid S. Miller if (pmd_none(*pmd)) {
183527137e52SSam Ravnborg pte_t *new;
183627137e52SSam Ravnborg
18370dd5b7b0SDavid S. Miller if (kernel_can_map_hugepmd(vstart, vend, use_huge)) {
18380dd5b7b0SDavid S. Miller vstart = kernel_map_hugepmd(vstart, vend, pmd);
18390dd5b7b0SDavid S. Miller continue;
18400dd5b7b0SDavid S. Miller }
18414fc4a09eSMike Rapoport new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
18424fc4a09eSMike Rapoport PAGE_SIZE);
1843b1e1c869SMike Rapoport if (!new)
1844b1e1c869SMike Rapoport goto err_alloc;
184527137e52SSam Ravnborg alloc_bytes += PAGE_SIZE;
184627137e52SSam Ravnborg pmd_populate_kernel(&init_mm, pmd, new);
184727137e52SSam Ravnborg }
184827137e52SSam Ravnborg
184927137e52SSam Ravnborg pte = pte_offset_kernel(pmd, vstart);
185027137e52SSam Ravnborg this_end = (vstart + PMD_SIZE) & PMD_MASK;
185127137e52SSam Ravnborg if (this_end > vend)
185227137e52SSam Ravnborg this_end = vend;
185327137e52SSam Ravnborg
185427137e52SSam Ravnborg while (vstart < this_end) {
185527137e52SSam Ravnborg pte_val(*pte) = (paddr | pgprot_val(prot));
185627137e52SSam Ravnborg
185727137e52SSam Ravnborg vstart += PAGE_SIZE;
185827137e52SSam Ravnborg paddr += PAGE_SIZE;
185927137e52SSam Ravnborg pte++;
186027137e52SSam Ravnborg }
186127137e52SSam Ravnborg }
186227137e52SSam Ravnborg
186327137e52SSam Ravnborg return alloc_bytes;
1864b1e1c869SMike Rapoport
1865b1e1c869SMike Rapoport err_alloc:
1866b1e1c869SMike Rapoport panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",
1867b1e1c869SMike Rapoport __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1868b1e1c869SMike Rapoport return -ENOMEM;
186927137e52SSam Ravnborg }
187027137e52SSam Ravnborg
flush_all_kernel_tsbs(void)18710dd5b7b0SDavid S. Miller static void __init flush_all_kernel_tsbs(void)
18720dd5b7b0SDavid S. Miller {
18730dd5b7b0SDavid S. Miller int i;
18740dd5b7b0SDavid S. Miller
18750dd5b7b0SDavid S. Miller for (i = 0; i < KERNEL_TSB_NENTRIES; i++) {
18760dd5b7b0SDavid S. Miller struct tsb *ent = &swapper_tsb[i];
18770dd5b7b0SDavid S. Miller
18780dd5b7b0SDavid S. Miller ent->tag = (1UL << TSB_TAG_INVALID_BIT);
18790dd5b7b0SDavid S. Miller }
18800dd5b7b0SDavid S. Miller #ifndef CONFIG_DEBUG_PAGEALLOC
18810dd5b7b0SDavid S. Miller for (i = 0; i < KERNEL_TSB4M_NENTRIES; i++) {
18820dd5b7b0SDavid S. Miller struct tsb *ent = &swapper_4m_tsb[i];
18830dd5b7b0SDavid S. Miller
18840dd5b7b0SDavid S. Miller ent->tag = (1UL << TSB_TAG_INVALID_BIT);
18850dd5b7b0SDavid S. Miller }
18860dd5b7b0SDavid S. Miller #endif
18870dd5b7b0SDavid S. Miller }
18880dd5b7b0SDavid S. Miller
188927137e52SSam Ravnborg extern unsigned int kvmap_linear_patch[1];
189027137e52SSam Ravnborg
kernel_physical_mapping_init(void)189127137e52SSam Ravnborg static void __init kernel_physical_mapping_init(void)
189227137e52SSam Ravnborg {
189327137e52SSam Ravnborg unsigned long i, mem_alloced = 0UL;
18940dd5b7b0SDavid S. Miller bool use_huge = true;
189527137e52SSam Ravnborg
18960dd5b7b0SDavid S. Miller #ifdef CONFIG_DEBUG_PAGEALLOC
18970dd5b7b0SDavid S. Miller use_huge = false;
18980dd5b7b0SDavid S. Miller #endif
189927137e52SSam Ravnborg for (i = 0; i < pall_ents; i++) {
190027137e52SSam Ravnborg unsigned long phys_start, phys_end;
190127137e52SSam Ravnborg
190227137e52SSam Ravnborg phys_start = pall[i].phys_addr;
190327137e52SSam Ravnborg phys_end = phys_start + pall[i].reg_size;
190427137e52SSam Ravnborg
190527137e52SSam Ravnborg mem_alloced += kernel_map_range(phys_start, phys_end,
19060dd5b7b0SDavid S. Miller PAGE_KERNEL, use_huge);
190727137e52SSam Ravnborg }
190827137e52SSam Ravnborg
190927137e52SSam Ravnborg printk("Allocated %ld bytes for kernel page tables.\n",
191027137e52SSam Ravnborg mem_alloced);
191127137e52SSam Ravnborg
191227137e52SSam Ravnborg kvmap_linear_patch[0] = 0x01000000; /* nop */
191327137e52SSam Ravnborg flushi(&kvmap_linear_patch[0]);
191427137e52SSam Ravnborg
19150dd5b7b0SDavid S. Miller flush_all_kernel_tsbs();
19160dd5b7b0SDavid S. Miller
191727137e52SSam Ravnborg __flush_tlb_all();
191827137e52SSam Ravnborg }
191927137e52SSam Ravnborg
192027137e52SSam Ravnborg #ifdef CONFIG_DEBUG_PAGEALLOC
__kernel_map_pages(struct page * page,int numpages,int enable)1921031bc574SJoonsoo Kim void __kernel_map_pages(struct page *page, int numpages, int enable)
192227137e52SSam Ravnborg {
192327137e52SSam Ravnborg unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
192427137e52SSam Ravnborg unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
192527137e52SSam Ravnborg
192627137e52SSam Ravnborg kernel_map_range(phys_start, phys_end,
19270dd5b7b0SDavid S. Miller (enable ? PAGE_KERNEL : __pgprot(0)), false);
192827137e52SSam Ravnborg
192927137e52SSam Ravnborg flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
193027137e52SSam Ravnborg PAGE_OFFSET + phys_end);
193127137e52SSam Ravnborg
193227137e52SSam Ravnborg /* we should perform an IPI and flush all tlbs,
193327137e52SSam Ravnborg * but that can deadlock->flush only current cpu.
193427137e52SSam Ravnborg */
193527137e52SSam Ravnborg __flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
193627137e52SSam Ravnborg PAGE_OFFSET + phys_end);
193727137e52SSam Ravnborg }
193827137e52SSam Ravnborg #endif
193927137e52SSam Ravnborg
find_ecache_flush_span(unsigned long size)194027137e52SSam Ravnborg unsigned long __init find_ecache_flush_span(unsigned long size)
194127137e52SSam Ravnborg {
194227137e52SSam Ravnborg int i;
194327137e52SSam Ravnborg
194427137e52SSam Ravnborg for (i = 0; i < pavail_ents; i++) {
194527137e52SSam Ravnborg if (pavail[i].reg_size >= size)
194627137e52SSam Ravnborg return pavail[i].phys_addr;
194727137e52SSam Ravnborg }
194827137e52SSam Ravnborg
194927137e52SSam Ravnborg return ~0UL;
195027137e52SSam Ravnborg }
195127137e52SSam Ravnborg
1952b2d43834SDavid S. Miller unsigned long PAGE_OFFSET;
1953b2d43834SDavid S. Miller EXPORT_SYMBOL(PAGE_OFFSET);
1954b2d43834SDavid S. Miller
1955bb4e6e85SDavid S. Miller unsigned long VMALLOC_END = 0x0000010000000000UL;
1956bb4e6e85SDavid S. Miller EXPORT_SYMBOL(VMALLOC_END);
1957bb4e6e85SDavid S. Miller
19584397bed0SDavid S. Miller unsigned long sparc64_va_hole_top = 0xfffff80000000000UL;
19594397bed0SDavid S. Miller unsigned long sparc64_va_hole_bottom = 0x0000080000000000UL;
19604397bed0SDavid S. Miller
setup_page_offset(void)1961b2d43834SDavid S. Miller static void __init setup_page_offset(void)
1962b2d43834SDavid S. Miller {
1963b2d43834SDavid S. Miller if (tlb_type == cheetah || tlb_type == cheetah_plus) {
19644397bed0SDavid S. Miller /* Cheetah/Panther support a full 64-bit virtual
19654397bed0SDavid S. Miller * address, so we can use all that our page tables
19664397bed0SDavid S. Miller * support.
19674397bed0SDavid S. Miller */
19684397bed0SDavid S. Miller sparc64_va_hole_top = 0xfff0000000000000UL;
19694397bed0SDavid S. Miller sparc64_va_hole_bottom = 0x0010000000000000UL;
19704397bed0SDavid S. Miller
1971b2d43834SDavid S. Miller max_phys_bits = 42;
1972b2d43834SDavid S. Miller } else if (tlb_type == hypervisor) {
1973b2d43834SDavid S. Miller switch (sun4v_chip_type) {
1974b2d43834SDavid S. Miller case SUN4V_CHIP_NIAGARA1:
1975b2d43834SDavid S. Miller case SUN4V_CHIP_NIAGARA2:
19764397bed0SDavid S. Miller /* T1 and T2 support 48-bit virtual addresses. */
19774397bed0SDavid S. Miller sparc64_va_hole_top = 0xffff800000000000UL;
19784397bed0SDavid S. Miller sparc64_va_hole_bottom = 0x0000800000000000UL;
19794397bed0SDavid S. Miller
1980b2d43834SDavid S. Miller max_phys_bits = 39;
1981b2d43834SDavid S. Miller break;
1982b2d43834SDavid S. Miller case SUN4V_CHIP_NIAGARA3:
19834397bed0SDavid S. Miller /* T3 supports 48-bit virtual addresses. */
19844397bed0SDavid S. Miller sparc64_va_hole_top = 0xffff800000000000UL;
19854397bed0SDavid S. Miller sparc64_va_hole_bottom = 0x0000800000000000UL;
19864397bed0SDavid S. Miller
1987b2d43834SDavid S. Miller max_phys_bits = 43;
1988b2d43834SDavid S. Miller break;
1989b2d43834SDavid S. Miller case SUN4V_CHIP_NIAGARA4:
1990b2d43834SDavid S. Miller case SUN4V_CHIP_NIAGARA5:
1991b2d43834SDavid S. Miller case SUN4V_CHIP_SPARC64X:
19927c0fa0f2SDavid S. Miller case SUN4V_CHIP_SPARC_M6:
19934397bed0SDavid S. Miller /* T4 and later support 52-bit virtual addresses. */
19944397bed0SDavid S. Miller sparc64_va_hole_top = 0xfff8000000000000UL;
19954397bed0SDavid S. Miller sparc64_va_hole_bottom = 0x0008000000000000UL;
1996b2d43834SDavid S. Miller max_phys_bits = 47;
1997b2d43834SDavid S. Miller break;
19987c0fa0f2SDavid S. Miller case SUN4V_CHIP_SPARC_M7:
1999c5b8b5beSKhalid Aziz case SUN4V_CHIP_SPARC_SN:
20007c0fa0f2SDavid S. Miller /* M7 and later support 52-bit virtual addresses. */
20017c0fa0f2SDavid S. Miller sparc64_va_hole_top = 0xfff8000000000000UL;
20027c0fa0f2SDavid S. Miller sparc64_va_hole_bottom = 0x0008000000000000UL;
20037c0fa0f2SDavid S. Miller max_phys_bits = 49;
20047c0fa0f2SDavid S. Miller break;
2005fdaccf74SVijay Kumar case SUN4V_CHIP_SPARC_M8:
2006fdaccf74SVijay Kumar default:
2007fdaccf74SVijay Kumar /* M8 and later support 54-bit virtual addresses.
2008fdaccf74SVijay Kumar * However, restricting M8 and above VA bits to 53
2009fdaccf74SVijay Kumar * as 4-level page table cannot support more than
2010fdaccf74SVijay Kumar * 53 VA bits.
2011fdaccf74SVijay Kumar */
2012fdaccf74SVijay Kumar sparc64_va_hole_top = 0xfff0000000000000UL;
2013fdaccf74SVijay Kumar sparc64_va_hole_bottom = 0x0010000000000000UL;
2014fdaccf74SVijay Kumar max_phys_bits = 51;
2015fdaccf74SVijay Kumar break;
2016b2d43834SDavid S. Miller }
2017b2d43834SDavid S. Miller }
2018b2d43834SDavid S. Miller
2019b2d43834SDavid S. Miller if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {
2020b2d43834SDavid S. Miller prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",
2021b2d43834SDavid S. Miller max_phys_bits);
2022b2d43834SDavid S. Miller prom_halt();
2023b2d43834SDavid S. Miller }
2024b2d43834SDavid S. Miller
2025bb4e6e85SDavid S. Miller PAGE_OFFSET = sparc64_va_hole_top;
2026bb4e6e85SDavid S. Miller VMALLOC_END = ((sparc64_va_hole_bottom >> 1) +
2027bb4e6e85SDavid S. Miller (sparc64_va_hole_bottom >> 2));
2028b2d43834SDavid S. Miller
2029bb4e6e85SDavid S. Miller pr_info("MM: PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
2030b2d43834SDavid S. Miller PAGE_OFFSET, max_phys_bits);
2031bb4e6e85SDavid S. Miller pr_info("MM: VMALLOC [0x%016lx --> 0x%016lx]\n",
2032bb4e6e85SDavid S. Miller VMALLOC_START, VMALLOC_END);
2033bb4e6e85SDavid S. Miller pr_info("MM: VMEMMAP [0x%016lx --> 0x%016lx]\n",
2034bb4e6e85SDavid S. Miller VMEMMAP_BASE, VMEMMAP_BASE << 1);
2035b2d43834SDavid S. Miller }
2036b2d43834SDavid S. Miller
tsb_phys_patch(void)203727137e52SSam Ravnborg static void __init tsb_phys_patch(void)
203827137e52SSam Ravnborg {
203927137e52SSam Ravnborg struct tsb_ldquad_phys_patch_entry *pquad;
204027137e52SSam Ravnborg struct tsb_phys_patch_entry *p;
204127137e52SSam Ravnborg
204227137e52SSam Ravnborg pquad = &__tsb_ldquad_phys_patch;
204327137e52SSam Ravnborg while (pquad < &__tsb_ldquad_phys_patch_end) {
204427137e52SSam Ravnborg unsigned long addr = pquad->addr;
204527137e52SSam Ravnborg
204627137e52SSam Ravnborg if (tlb_type == hypervisor)
204727137e52SSam Ravnborg *(unsigned int *) addr = pquad->sun4v_insn;
204827137e52SSam Ravnborg else
204927137e52SSam Ravnborg *(unsigned int *) addr = pquad->sun4u_insn;
205027137e52SSam Ravnborg wmb();
205127137e52SSam Ravnborg __asm__ __volatile__("flush %0"
205227137e52SSam Ravnborg : /* no outputs */
205327137e52SSam Ravnborg : "r" (addr));
205427137e52SSam Ravnborg
205527137e52SSam Ravnborg pquad++;
205627137e52SSam Ravnborg }
205727137e52SSam Ravnborg
205827137e52SSam Ravnborg p = &__tsb_phys_patch;
205927137e52SSam Ravnborg while (p < &__tsb_phys_patch_end) {
206027137e52SSam Ravnborg unsigned long addr = p->addr;
206127137e52SSam Ravnborg
206227137e52SSam Ravnborg *(unsigned int *) addr = p->insn;
206327137e52SSam Ravnborg wmb();
206427137e52SSam Ravnborg __asm__ __volatile__("flush %0"
206527137e52SSam Ravnborg : /* no outputs */
206627137e52SSam Ravnborg : "r" (addr));
206727137e52SSam Ravnborg
206827137e52SSam Ravnborg p++;
206927137e52SSam Ravnborg }
207027137e52SSam Ravnborg }
207127137e52SSam Ravnborg
207227137e52SSam Ravnborg /* Don't mark as init, we give this to the Hypervisor. */
207327137e52SSam Ravnborg #ifndef CONFIG_DEBUG_PAGEALLOC
207427137e52SSam Ravnborg #define NUM_KTSB_DESCR 2
207527137e52SSam Ravnborg #else
207627137e52SSam Ravnborg #define NUM_KTSB_DESCR 1
207727137e52SSam Ravnborg #endif
207827137e52SSam Ravnborg static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
207927137e52SSam Ravnborg
20808c82dc0eSDavid S. Miller /* The swapper TSBs are loaded with a base sequence of:
20818c82dc0eSDavid S. Miller *
20828c82dc0eSDavid S. Miller * sethi %uhi(SYMBOL), REG1
20838c82dc0eSDavid S. Miller * sethi %hi(SYMBOL), REG2
20848c82dc0eSDavid S. Miller * or REG1, %ulo(SYMBOL), REG1
20858c82dc0eSDavid S. Miller * or REG2, %lo(SYMBOL), REG2
20868c82dc0eSDavid S. Miller * sllx REG1, 32, REG1
20878c82dc0eSDavid S. Miller * or REG1, REG2, REG1
20888c82dc0eSDavid S. Miller *
20898c82dc0eSDavid S. Miller * When we use physical addressing for the TSB accesses, we patch the
20908c82dc0eSDavid S. Miller * first four instructions in the above sequence.
20918c82dc0eSDavid S. Miller */
20928c82dc0eSDavid S. Miller
patch_one_ktsb_phys(unsigned int * start,unsigned int * end,unsigned long pa)20939076d0e7SDavid S. Miller static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
20949076d0e7SDavid S. Miller {
20958c82dc0eSDavid S. Miller unsigned long high_bits, low_bits;
20968c82dc0eSDavid S. Miller
20978c82dc0eSDavid S. Miller high_bits = (pa >> 32) & 0xffffffff;
20988c82dc0eSDavid S. Miller low_bits = (pa >> 0) & 0xffffffff;
20999076d0e7SDavid S. Miller
21009076d0e7SDavid S. Miller while (start < end) {
21019076d0e7SDavid S. Miller unsigned int *ia = (unsigned int *)(unsigned long)*start;
21029076d0e7SDavid S. Miller
21038c82dc0eSDavid S. Miller ia[0] = (ia[0] & ~0x3fffff) | (high_bits >> 10);
21049076d0e7SDavid S. Miller __asm__ __volatile__("flush %0" : : "r" (ia));
21059076d0e7SDavid S. Miller
21068c82dc0eSDavid S. Miller ia[1] = (ia[1] & ~0x3fffff) | (low_bits >> 10);
21079076d0e7SDavid S. Miller __asm__ __volatile__("flush %0" : : "r" (ia + 1));
21089076d0e7SDavid S. Miller
21098c82dc0eSDavid S. Miller ia[2] = (ia[2] & ~0x1fff) | (high_bits & 0x3ff);
21108c82dc0eSDavid S. Miller __asm__ __volatile__("flush %0" : : "r" (ia + 2));
21118c82dc0eSDavid S. Miller
21128c82dc0eSDavid S. Miller ia[3] = (ia[3] & ~0x1fff) | (low_bits & 0x3ff);
21138c82dc0eSDavid S. Miller __asm__ __volatile__("flush %0" : : "r" (ia + 3));
21148c82dc0eSDavid S. Miller
21159076d0e7SDavid S. Miller start++;
21169076d0e7SDavid S. Miller }
21179076d0e7SDavid S. Miller }
21189076d0e7SDavid S. Miller
ktsb_phys_patch(void)21199076d0e7SDavid S. Miller static void ktsb_phys_patch(void)
21209076d0e7SDavid S. Miller {
21219076d0e7SDavid S. Miller extern unsigned int __swapper_tsb_phys_patch;
21229076d0e7SDavid S. Miller extern unsigned int __swapper_tsb_phys_patch_end;
21239076d0e7SDavid S. Miller unsigned long ktsb_pa;
21249076d0e7SDavid S. Miller
21259076d0e7SDavid S. Miller ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
21269076d0e7SDavid S. Miller patch_one_ktsb_phys(&__swapper_tsb_phys_patch,
21279076d0e7SDavid S. Miller &__swapper_tsb_phys_patch_end, ktsb_pa);
21289076d0e7SDavid S. Miller #ifndef CONFIG_DEBUG_PAGEALLOC
21290785a8e8SDavid S. Miller {
21300785a8e8SDavid S. Miller extern unsigned int __swapper_4m_tsb_phys_patch;
21310785a8e8SDavid S. Miller extern unsigned int __swapper_4m_tsb_phys_patch_end;
21329076d0e7SDavid S. Miller ktsb_pa = (kern_base +
21339076d0e7SDavid S. Miller ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
21349076d0e7SDavid S. Miller patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,
21359076d0e7SDavid S. Miller &__swapper_4m_tsb_phys_patch_end, ktsb_pa);
21360785a8e8SDavid S. Miller }
21379076d0e7SDavid S. Miller #endif
21389076d0e7SDavid S. Miller }
21399076d0e7SDavid S. Miller
sun4v_ktsb_init(void)214027137e52SSam Ravnborg static void __init sun4v_ktsb_init(void)
214127137e52SSam Ravnborg {
214227137e52SSam Ravnborg unsigned long ktsb_pa;
214327137e52SSam Ravnborg
214427137e52SSam Ravnborg /* First KTSB for PAGE_SIZE mappings. */
214527137e52SSam Ravnborg ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
214627137e52SSam Ravnborg
214727137e52SSam Ravnborg switch (PAGE_SIZE) {
214827137e52SSam Ravnborg case 8 * 1024:
214927137e52SSam Ravnborg default:
215027137e52SSam Ravnborg ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
215127137e52SSam Ravnborg ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
215227137e52SSam Ravnborg break;
215327137e52SSam Ravnborg
215427137e52SSam Ravnborg case 64 * 1024:
215527137e52SSam Ravnborg ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
215627137e52SSam Ravnborg ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
215727137e52SSam Ravnborg break;
215827137e52SSam Ravnborg
215927137e52SSam Ravnborg case 512 * 1024:
216027137e52SSam Ravnborg ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
216127137e52SSam Ravnborg ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
216227137e52SSam Ravnborg break;
216327137e52SSam Ravnborg
216427137e52SSam Ravnborg case 4 * 1024 * 1024:
216527137e52SSam Ravnborg ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
216627137e52SSam Ravnborg ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
216727137e52SSam Ravnborg break;
21686cb79b3fSJoe Perches }
216927137e52SSam Ravnborg
217027137e52SSam Ravnborg ktsb_descr[0].assoc = 1;
217127137e52SSam Ravnborg ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
217227137e52SSam Ravnborg ktsb_descr[0].ctx_idx = 0;
217327137e52SSam Ravnborg ktsb_descr[0].tsb_base = ktsb_pa;
217427137e52SSam Ravnborg ktsb_descr[0].resv = 0;
217527137e52SSam Ravnborg
217627137e52SSam Ravnborg #ifndef CONFIG_DEBUG_PAGEALLOC
21774f93d21dSDavid S. Miller /* Second KTSB for 4MB/256MB/2GB/16GB mappings. */
217827137e52SSam Ravnborg ktsb_pa = (kern_base +
217927137e52SSam Ravnborg ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
218027137e52SSam Ravnborg
218127137e52SSam Ravnborg ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
2182c69ad0a3SDavid S. Miller ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB |
2183c69ad0a3SDavid S. Miller HV_PGSZ_MASK_256MB |
2184c69ad0a3SDavid S. Miller HV_PGSZ_MASK_2GB |
2185c69ad0a3SDavid S. Miller HV_PGSZ_MASK_16GB) &
2186c69ad0a3SDavid S. Miller cpu_pgsz_mask);
218727137e52SSam Ravnborg ktsb_descr[1].assoc = 1;
218827137e52SSam Ravnborg ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
218927137e52SSam Ravnborg ktsb_descr[1].ctx_idx = 0;
219027137e52SSam Ravnborg ktsb_descr[1].tsb_base = ktsb_pa;
219127137e52SSam Ravnborg ktsb_descr[1].resv = 0;
219227137e52SSam Ravnborg #endif
219327137e52SSam Ravnborg }
219427137e52SSam Ravnborg
sun4v_ktsb_register(void)21952066aaddSPaul Gortmaker void sun4v_ktsb_register(void)
219627137e52SSam Ravnborg {
219727137e52SSam Ravnborg unsigned long pa, ret;
219827137e52SSam Ravnborg
219927137e52SSam Ravnborg pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);
220027137e52SSam Ravnborg
220127137e52SSam Ravnborg ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);
220227137e52SSam Ravnborg if (ret != 0) {
220327137e52SSam Ravnborg prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "
220427137e52SSam Ravnborg "errors with %lx\n", pa, ret);
220527137e52SSam Ravnborg prom_halt();
220627137e52SSam Ravnborg }
220727137e52SSam Ravnborg }
220827137e52SSam Ravnborg
sun4u_linear_pte_xor_finalize(void)2209c69ad0a3SDavid S. Miller static void __init sun4u_linear_pte_xor_finalize(void)
2210c69ad0a3SDavid S. Miller {
2211c69ad0a3SDavid S. Miller #ifndef CONFIG_DEBUG_PAGEALLOC
2212c69ad0a3SDavid S. Miller /* This is where we would add Panther support for
2213c69ad0a3SDavid S. Miller * 32MB and 256MB pages.
2214c69ad0a3SDavid S. Miller */
2215c69ad0a3SDavid S. Miller #endif
2216c69ad0a3SDavid S. Miller }
2217c69ad0a3SDavid S. Miller
sun4v_linear_pte_xor_finalize(void)2218c69ad0a3SDavid S. Miller static void __init sun4v_linear_pte_xor_finalize(void)
2219c69ad0a3SDavid S. Miller {
2220494e5b6fSKhalid Aziz unsigned long pagecv_flag;
2221494e5b6fSKhalid Aziz
2222494e5b6fSKhalid Aziz /* Bit 9 of TTE is no longer CV bit on M7 processor and it instead
2223494e5b6fSKhalid Aziz * enables MCD error. Do not set bit 9 on M7 processor.
2224494e5b6fSKhalid Aziz */
2225494e5b6fSKhalid Aziz switch (sun4v_chip_type) {
2226494e5b6fSKhalid Aziz case SUN4V_CHIP_SPARC_M7:
22277d484acbSAllen Pais case SUN4V_CHIP_SPARC_M8:
2228c5b8b5beSKhalid Aziz case SUN4V_CHIP_SPARC_SN:
2229494e5b6fSKhalid Aziz pagecv_flag = 0x00;
2230494e5b6fSKhalid Aziz break;
2231494e5b6fSKhalid Aziz default:
2232494e5b6fSKhalid Aziz pagecv_flag = _PAGE_CV_4V;
2233494e5b6fSKhalid Aziz break;
2234494e5b6fSKhalid Aziz }
2235c69ad0a3SDavid S. Miller #ifndef CONFIG_DEBUG_PAGEALLOC
2236c69ad0a3SDavid S. Miller if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
2237c69ad0a3SDavid S. Miller kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
2238922631b9SDavid S. Miller PAGE_OFFSET;
2239494e5b6fSKhalid Aziz kern_linear_pte_xor[1] |= (_PAGE_CP_4V | pagecv_flag |
2240c69ad0a3SDavid S. Miller _PAGE_P_4V | _PAGE_W_4V);
2241c69ad0a3SDavid S. Miller } else {
2242c69ad0a3SDavid S. Miller kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
2243c69ad0a3SDavid S. Miller }
2244c69ad0a3SDavid S. Miller
2245c69ad0a3SDavid S. Miller if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
2246c69ad0a3SDavid S. Miller kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
2247922631b9SDavid S. Miller PAGE_OFFSET;
2248494e5b6fSKhalid Aziz kern_linear_pte_xor[2] |= (_PAGE_CP_4V | pagecv_flag |
2249c69ad0a3SDavid S. Miller _PAGE_P_4V | _PAGE_W_4V);
2250c69ad0a3SDavid S. Miller } else {
2251c69ad0a3SDavid S. Miller kern_linear_pte_xor[2] = kern_linear_pte_xor[1];
2252c69ad0a3SDavid S. Miller }
2253c69ad0a3SDavid S. Miller
2254c69ad0a3SDavid S. Miller if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
2255c69ad0a3SDavid S. Miller kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
2256922631b9SDavid S. Miller PAGE_OFFSET;
2257494e5b6fSKhalid Aziz kern_linear_pte_xor[3] |= (_PAGE_CP_4V | pagecv_flag |
2258c69ad0a3SDavid S. Miller _PAGE_P_4V | _PAGE_W_4V);
2259c69ad0a3SDavid S. Miller } else {
2260c69ad0a3SDavid S. Miller kern_linear_pte_xor[3] = kern_linear_pte_xor[2];
2261c69ad0a3SDavid S. Miller }
2262c69ad0a3SDavid S. Miller #endif
2263c69ad0a3SDavid S. Miller }
2264c69ad0a3SDavid S. Miller
226527137e52SSam Ravnborg /* paging_init() sets up the page tables */
226627137e52SSam Ravnborg
226727137e52SSam Ravnborg static unsigned long last_valid_pfn;
2268ac55c768SDavid S. Miller
226927137e52SSam Ravnborg static void sun4u_pgprot_init(void);
227027137e52SSam Ravnborg static void sun4v_pgprot_init(void);
227127137e52SSam Ravnborg
2272494e5b6fSKhalid Aziz #define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U)
2273494e5b6fSKhalid Aziz #define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V)
2274494e5b6fSKhalid Aziz #define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
2275494e5b6fSKhalid Aziz #define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
2276494e5b6fSKhalid Aziz #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
2277494e5b6fSKhalid Aziz #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
2278494e5b6fSKhalid Aziz
22797c21d533Sbob picco /* We need to exclude reserved regions. This exclusion will include
22807c21d533Sbob picco * vmlinux and initrd. To be more precise the initrd size could be used to
22817c21d533Sbob picco * compute a new lower limit because it is freed later during initialization.
22827c21d533Sbob picco */
reduce_memory(phys_addr_t limit_ram)22837c21d533Sbob picco static void __init reduce_memory(phys_addr_t limit_ram)
22847c21d533Sbob picco {
2285f4d9a23dSMike Rapoport limit_ram += memblock_reserved_size();
2286f4d9a23dSMike Rapoport memblock_enforce_memory_limit(limit_ram);
22877c21d533Sbob picco }
22887c21d533Sbob picco
paging_init(void)228927137e52SSam Ravnborg void __init paging_init(void)
229027137e52SSam Ravnborg {
229127137e52SSam Ravnborg unsigned long end_pfn, shift, phys_base;
229227137e52SSam Ravnborg unsigned long real_end, i;
229327137e52SSam Ravnborg
2294b2d43834SDavid S. Miller setup_page_offset();
2295b2d43834SDavid S. Miller
229627137e52SSam Ravnborg /* These build time checkes make sure that the dcache_dirty_cpu()
22971a10a44dSMatthew Wilcox (Oracle) * folio->flags usage will work.
229827137e52SSam Ravnborg *
229927137e52SSam Ravnborg * When a page gets marked as dcache-dirty, we store the
23001a10a44dSMatthew Wilcox (Oracle) * cpu number starting at bit 32 in the folio->flags. Also,
230127137e52SSam Ravnborg * functions like clear_dcache_dirty_cpu use the cpu mask
230227137e52SSam Ravnborg * in 13-bit signed-immediate instruction fields.
230327137e52SSam Ravnborg */
230427137e52SSam Ravnborg
230527137e52SSam Ravnborg /*
230627137e52SSam Ravnborg * Page flags must not reach into upper 32 bits that are used
230727137e52SSam Ravnborg * for the cpu number
230827137e52SSam Ravnborg */
230927137e52SSam Ravnborg BUILD_BUG_ON(NR_PAGEFLAGS > 32);
231027137e52SSam Ravnborg
231127137e52SSam Ravnborg /*
231227137e52SSam Ravnborg * The bit fields placed in the high range must not reach below
231327137e52SSam Ravnborg * the 32 bit boundary. Otherwise we cannot place the cpu field
231427137e52SSam Ravnborg * at the 32 bit boundary.
231527137e52SSam Ravnborg */
231627137e52SSam Ravnborg BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +
231727137e52SSam Ravnborg ilog2(roundup_pow_of_two(NR_CPUS)) > 32);
231827137e52SSam Ravnborg
231927137e52SSam Ravnborg BUILD_BUG_ON(NR_CPUS > 4096);
232027137e52SSam Ravnborg
23210eef331aSDavid S. Miller kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
232227137e52SSam Ravnborg kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
232327137e52SSam Ravnborg
232427137e52SSam Ravnborg /* Invalidate both kernel TSBs. */
232527137e52SSam Ravnborg memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
232627137e52SSam Ravnborg #ifndef CONFIG_DEBUG_PAGEALLOC
232727137e52SSam Ravnborg memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
232827137e52SSam Ravnborg #endif
232927137e52SSam Ravnborg
2330494e5b6fSKhalid Aziz /* TTE.cv bit on sparc v9 occupies the same position as TTE.mcde
2331494e5b6fSKhalid Aziz * bit on M7 processor. This is a conflicting usage of the same
2332494e5b6fSKhalid Aziz * bit. Enabling TTE.cv on M7 would turn on Memory Corruption
2333494e5b6fSKhalid Aziz * Detection error on all pages and this will lead to problems
2334494e5b6fSKhalid Aziz * later. Kernel does not run with MCD enabled and hence rest
2335494e5b6fSKhalid Aziz * of the required steps to fully configure memory corruption
2336494e5b6fSKhalid Aziz * detection are not taken. We need to ensure TTE.mcde is not
2337494e5b6fSKhalid Aziz * set on M7 processor. Compute the value of cacheability
2338494e5b6fSKhalid Aziz * flag for use later taking this into consideration.
2339494e5b6fSKhalid Aziz */
2340494e5b6fSKhalid Aziz switch (sun4v_chip_type) {
2341494e5b6fSKhalid Aziz case SUN4V_CHIP_SPARC_M7:
23427d484acbSAllen Pais case SUN4V_CHIP_SPARC_M8:
2343c5b8b5beSKhalid Aziz case SUN4V_CHIP_SPARC_SN:
2344494e5b6fSKhalid Aziz page_cache4v_flag = _PAGE_CP_4V;
2345494e5b6fSKhalid Aziz break;
2346494e5b6fSKhalid Aziz default:
2347494e5b6fSKhalid Aziz page_cache4v_flag = _PAGE_CACHE_4V;
2348494e5b6fSKhalid Aziz break;
2349494e5b6fSKhalid Aziz }
2350494e5b6fSKhalid Aziz
235127137e52SSam Ravnborg if (tlb_type == hypervisor)
235227137e52SSam Ravnborg sun4v_pgprot_init();
235327137e52SSam Ravnborg else
235427137e52SSam Ravnborg sun4u_pgprot_init();
235527137e52SSam Ravnborg
235627137e52SSam Ravnborg if (tlb_type == cheetah_plus ||
23579076d0e7SDavid S. Miller tlb_type == hypervisor) {
235827137e52SSam Ravnborg tsb_phys_patch();
23599076d0e7SDavid S. Miller ktsb_phys_patch();
23609076d0e7SDavid S. Miller }
236127137e52SSam Ravnborg
2362c69ad0a3SDavid S. Miller if (tlb_type == hypervisor)
236327137e52SSam Ravnborg sun4v_patch_tlb_handlers();
236427137e52SSam Ravnborg
236527137e52SSam Ravnborg /* Find available physical memory...
236627137e52SSam Ravnborg *
236727137e52SSam Ravnborg * Read it twice in order to work around a bug in openfirmware.
236827137e52SSam Ravnborg * The call to grab this table itself can cause openfirmware to
236927137e52SSam Ravnborg * allocate memory, which in turn can take away some space from
237027137e52SSam Ravnborg * the list of available memory. Reading it twice makes sure
237127137e52SSam Ravnborg * we really do get the final value.
237227137e52SSam Ravnborg */
237327137e52SSam Ravnborg read_obp_translations();
237427137e52SSam Ravnborg read_obp_memory("reg", &pall[0], &pall_ents);
237527137e52SSam Ravnborg read_obp_memory("available", &pavail[0], &pavail_ents);
237627137e52SSam Ravnborg read_obp_memory("available", &pavail[0], &pavail_ents);
237727137e52SSam Ravnborg
237827137e52SSam Ravnborg phys_base = 0xffffffffffffffffUL;
237927137e52SSam Ravnborg for (i = 0; i < pavail_ents; i++) {
238027137e52SSam Ravnborg phys_base = min(phys_base, pavail[i].phys_addr);
238195f72d1eSYinghai Lu memblock_add(pavail[i].phys_addr, pavail[i].reg_size);
238227137e52SSam Ravnborg }
238327137e52SSam Ravnborg
238495f72d1eSYinghai Lu memblock_reserve(kern_base, kern_size);
238527137e52SSam Ravnborg
238627137e52SSam Ravnborg find_ramdisk(phys_base);
238727137e52SSam Ravnborg
23887c21d533Sbob picco if (cmdline_memory_size)
23897c21d533Sbob picco reduce_memory(cmdline_memory_size);
239027137e52SSam Ravnborg
23911aadc056STejun Heo memblock_allow_resize();
239295f72d1eSYinghai Lu memblock_dump_all();
239327137e52SSam Ravnborg
239427137e52SSam Ravnborg set_bit(0, mmu_context_bmap);
239527137e52SSam Ravnborg
239627137e52SSam Ravnborg shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
239727137e52SSam Ravnborg
239827137e52SSam Ravnborg real_end = (unsigned long)_end;
23990eef331aSDavid S. Miller num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB);
240027137e52SSam Ravnborg printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
240127137e52SSam Ravnborg num_kernel_image_mappings);
240227137e52SSam Ravnborg
240327137e52SSam Ravnborg /* Set kernel pgd to upper alias so physical page computations
240427137e52SSam Ravnborg * work.
240527137e52SSam Ravnborg */
240627137e52SSam Ravnborg init_mm.pgd += ((shift) / (sizeof(pgd_t)));
240727137e52SSam Ravnborg
2408d195b71bSDavid S. Miller memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
240927137e52SSam Ravnborg
241027137e52SSam Ravnborg inherit_prom_mappings();
241127137e52SSam Ravnborg
241227137e52SSam Ravnborg /* Ok, we can use our TLB miss and window trap handlers safely. */
241327137e52SSam Ravnborg setup_tba();
241427137e52SSam Ravnborg
241527137e52SSam Ravnborg __flush_tlb_all();
241627137e52SSam Ravnborg
241727137e52SSam Ravnborg prom_build_devicetree();
2418b696fdc2SDavid S. Miller of_populate_present_mask();
2419b99c6ebeSDavid S. Miller #ifndef CONFIG_SMP
2420b99c6ebeSDavid S. Miller of_fill_in_cpu_data();
2421b99c6ebeSDavid S. Miller #endif
242227137e52SSam Ravnborg
2423890db403SDavid S. Miller if (tlb_type == hypervisor) {
242427137e52SSam Ravnborg sun4v_mdesc_init();
24256ac5c610SStephen Rothwell mdesc_populate_present_mask(cpu_all_mask);
2426b99c6ebeSDavid S. Miller #ifndef CONFIG_SMP
2427b99c6ebeSDavid S. Miller mdesc_fill_in_cpu_data(cpu_all_mask);
2428b99c6ebeSDavid S. Miller #endif
2429ce33fdc5SDavid S. Miller mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);
2430c69ad0a3SDavid S. Miller
2431c69ad0a3SDavid S. Miller sun4v_linear_pte_xor_finalize();
2432c69ad0a3SDavid S. Miller
2433c69ad0a3SDavid S. Miller sun4v_ktsb_init();
2434c69ad0a3SDavid S. Miller sun4v_ktsb_register();
2435ce33fdc5SDavid S. Miller } else {
2436ce33fdc5SDavid S. Miller unsigned long impl, ver;
2437ce33fdc5SDavid S. Miller
2438ce33fdc5SDavid S. Miller cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |
2439ce33fdc5SDavid S. Miller HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);
2440ce33fdc5SDavid S. Miller
2441ce33fdc5SDavid S. Miller __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));
2442ce33fdc5SDavid S. Miller impl = ((ver >> 32) & 0xffff);
2443ce33fdc5SDavid S. Miller if (impl == PANTHER_IMPL)
2444ce33fdc5SDavid S. Miller cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |
2445ce33fdc5SDavid S. Miller HV_PGSZ_MASK_256MB);
2446c69ad0a3SDavid S. Miller
2447c69ad0a3SDavid S. Miller sun4u_linear_pte_xor_finalize();
2448890db403SDavid S. Miller }
244927137e52SSam Ravnborg
2450c69ad0a3SDavid S. Miller /* Flush the TLBs and the 4M TSB so that the updated linear
2451c69ad0a3SDavid S. Miller * pte XOR settings are realized for all mappings.
2452c69ad0a3SDavid S. Miller */
2453c69ad0a3SDavid S. Miller __flush_tlb_all();
2454c69ad0a3SDavid S. Miller #ifndef CONFIG_DEBUG_PAGEALLOC
2455c69ad0a3SDavid S. Miller memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
2456c69ad0a3SDavid S. Miller #endif
2457c69ad0a3SDavid S. Miller __flush_tlb_all();
2458c69ad0a3SDavid S. Miller
245927137e52SSam Ravnborg /* Setup bootmem... */
246027137e52SSam Ravnborg last_valid_pfn = end_pfn = bootmem_init(phys_base);
246127137e52SSam Ravnborg
246227137e52SSam Ravnborg kernel_physical_mapping_init();
246327137e52SSam Ravnborg
246427137e52SSam Ravnborg {
246527137e52SSam Ravnborg unsigned long max_zone_pfns[MAX_NR_ZONES];
246627137e52SSam Ravnborg
246727137e52SSam Ravnborg memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
246827137e52SSam Ravnborg
246927137e52SSam Ravnborg max_zone_pfns[ZONE_NORMAL] = end_pfn;
247027137e52SSam Ravnborg
24719691a071SMike Rapoport free_area_init(max_zone_pfns);
247227137e52SSam Ravnborg }
247327137e52SSam Ravnborg
247427137e52SSam Ravnborg printk("Booting Linux...\n");
247527137e52SSam Ravnborg }
247627137e52SSam Ravnborg
page_in_phys_avail(unsigned long paddr)24777c9503b8SGreg Kroah-Hartman int page_in_phys_avail(unsigned long paddr)
247827137e52SSam Ravnborg {
247927137e52SSam Ravnborg int i;
248027137e52SSam Ravnborg
248127137e52SSam Ravnborg paddr &= PAGE_MASK;
248227137e52SSam Ravnborg
248327137e52SSam Ravnborg for (i = 0; i < pavail_ents; i++) {
248427137e52SSam Ravnborg unsigned long start, end;
248527137e52SSam Ravnborg
248627137e52SSam Ravnborg start = pavail[i].phys_addr;
248727137e52SSam Ravnborg end = start + pavail[i].reg_size;
248827137e52SSam Ravnborg
248927137e52SSam Ravnborg if (paddr >= start && paddr < end)
249027137e52SSam Ravnborg return 1;
249127137e52SSam Ravnborg }
249227137e52SSam Ravnborg if (paddr >= kern_base && paddr < (kern_base + kern_size))
249327137e52SSam Ravnborg return 1;
249427137e52SSam Ravnborg #ifdef CONFIG_BLK_DEV_INITRD
249527137e52SSam Ravnborg if (paddr >= __pa(initrd_start) &&
249627137e52SSam Ravnborg paddr < __pa(PAGE_ALIGN(initrd_end)))
249727137e52SSam Ravnborg return 1;
249827137e52SSam Ravnborg #endif
249927137e52SSam Ravnborg
250027137e52SSam Ravnborg return 0;
250127137e52SSam Ravnborg }
250227137e52SSam Ravnborg
register_page_bootmem_info(void)2503961f8fa0SYinghai Lu static void __init register_page_bootmem_info(void)
2504961f8fa0SYinghai Lu {
2505a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
2506961f8fa0SYinghai Lu int i;
2507961f8fa0SYinghai Lu
2508961f8fa0SYinghai Lu for_each_online_node(i)
2509961f8fa0SYinghai Lu if (NODE_DATA(i)->node_spanned_pages)
2510961f8fa0SYinghai Lu register_page_bootmem_info_node(NODE_DATA(i));
2511961f8fa0SYinghai Lu #endif
2512961f8fa0SYinghai Lu }
mem_init(void)251327137e52SSam Ravnborg void __init mem_init(void)
251427137e52SSam Ravnborg {
251527137e52SSam Ravnborg high_memory = __va(last_valid_pfn << PAGE_SHIFT);
251627137e52SSam Ravnborg
2517c6ffc5caSMike Rapoport memblock_free_all();
251827137e52SSam Ravnborg
251927137e52SSam Ravnborg /*
25202a20aa17SPavel Tatashin * Must be done after boot memory is put on freelist, because here we
25212a20aa17SPavel Tatashin * might set fields in deferred struct pages that have not yet been
2522c6ffc5caSMike Rapoport * initialized, and memblock_free_all() initializes all the reserved
25232a20aa17SPavel Tatashin * deferred pages for us.
25242a20aa17SPavel Tatashin */
25252a20aa17SPavel Tatashin register_page_bootmem_info();
25262a20aa17SPavel Tatashin
25272a20aa17SPavel Tatashin /*
252827137e52SSam Ravnborg * Set up the zero page, mark it reserved, so that page count
252927137e52SSam Ravnborg * is not manipulated when freeing the page from user ptes.
253027137e52SSam Ravnborg */
253127137e52SSam Ravnborg mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
253227137e52SSam Ravnborg if (mem_map_zero == NULL) {
253327137e52SSam Ravnborg prom_printf("paging_init: Cannot alloc zero page.\n");
253427137e52SSam Ravnborg prom_halt();
253527137e52SSam Ravnborg }
253670affe45SJiang Liu mark_page_reserved(mem_map_zero);
253727137e52SSam Ravnborg
253827137e52SSam Ravnborg
253927137e52SSam Ravnborg if (tlb_type == cheetah || tlb_type == cheetah_plus)
254027137e52SSam Ravnborg cheetah_ecache_flush_init();
254127137e52SSam Ravnborg }
254227137e52SSam Ravnborg
free_initmem(void)254327137e52SSam Ravnborg void free_initmem(void)
254427137e52SSam Ravnborg {
254527137e52SSam Ravnborg unsigned long addr, initend;
254627137e52SSam Ravnborg int do_free = 1;
254727137e52SSam Ravnborg
254827137e52SSam Ravnborg /* If the physical memory maps were trimmed by kernel command
254927137e52SSam Ravnborg * line options, don't even try freeing this initmem stuff up.
255027137e52SSam Ravnborg * The kernel image could have been in the trimmed out region
255127137e52SSam Ravnborg * and if so the freeing below will free invalid page structs.
255227137e52SSam Ravnborg */
255327137e52SSam Ravnborg if (cmdline_memory_size)
255427137e52SSam Ravnborg do_free = 0;
255527137e52SSam Ravnborg
255627137e52SSam Ravnborg /*
255727137e52SSam Ravnborg * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
255827137e52SSam Ravnborg */
255927137e52SSam Ravnborg addr = PAGE_ALIGN((unsigned long)(__init_begin));
256027137e52SSam Ravnborg initend = (unsigned long)(__init_end) & PAGE_MASK;
256127137e52SSam Ravnborg for (; addr < initend; addr += PAGE_SIZE) {
256227137e52SSam Ravnborg unsigned long page;
256327137e52SSam Ravnborg
256427137e52SSam Ravnborg page = (addr +
256527137e52SSam Ravnborg ((unsigned long) __va(kern_base)) -
256627137e52SSam Ravnborg ((unsigned long) KERNBASE));
256727137e52SSam Ravnborg memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
256827137e52SSam Ravnborg
256970affe45SJiang Liu if (do_free)
257070affe45SJiang Liu free_reserved_page(virt_to_page(page));
257127137e52SSam Ravnborg }
257227137e52SSam Ravnborg }
257327137e52SSam Ravnborg
257427137e52SSam Ravnborg pgprot_t PAGE_KERNEL __read_mostly;
257527137e52SSam Ravnborg EXPORT_SYMBOL(PAGE_KERNEL);
257627137e52SSam Ravnborg
257727137e52SSam Ravnborg pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
257827137e52SSam Ravnborg pgprot_t PAGE_COPY __read_mostly;
257927137e52SSam Ravnborg
258027137e52SSam Ravnborg pgprot_t PAGE_SHARED __read_mostly;
258127137e52SSam Ravnborg EXPORT_SYMBOL(PAGE_SHARED);
258227137e52SSam Ravnborg
258327137e52SSam Ravnborg unsigned long pg_iobits __read_mostly;
258427137e52SSam Ravnborg
258527137e52SSam Ravnborg unsigned long _PAGE_IE __read_mostly;
258627137e52SSam Ravnborg EXPORT_SYMBOL(_PAGE_IE);
258727137e52SSam Ravnborg
258827137e52SSam Ravnborg unsigned long _PAGE_E __read_mostly;
258927137e52SSam Ravnborg EXPORT_SYMBOL(_PAGE_E);
259027137e52SSam Ravnborg
259127137e52SSam Ravnborg unsigned long _PAGE_CACHE __read_mostly;
259227137e52SSam Ravnborg EXPORT_SYMBOL(_PAGE_CACHE);
259327137e52SSam Ravnborg
259427137e52SSam Ravnborg #ifdef CONFIG_SPARSEMEM_VMEMMAP
vmemmap_populate(unsigned long vstart,unsigned long vend,int node,struct vmem_altmap * altmap)25950aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
25967b73d978SChristoph Hellwig int node, struct vmem_altmap *altmap)
259727137e52SSam Ravnborg {
259827137e52SSam Ravnborg unsigned long pte_base;
259927137e52SSam Ravnborg
260027137e52SSam Ravnborg pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
260127137e52SSam Ravnborg _PAGE_CP_4U | _PAGE_CV_4U |
260227137e52SSam Ravnborg _PAGE_P_4U | _PAGE_W_4U);
260327137e52SSam Ravnborg if (tlb_type == hypervisor)
260427137e52SSam Ravnborg pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |
2605494e5b6fSKhalid Aziz page_cache4v_flag | _PAGE_P_4V | _PAGE_W_4V);
260627137e52SSam Ravnborg
2607c06240c7SDavid S. Miller pte_base |= _PAGE_PMD_HUGE;
260827137e52SSam Ravnborg
2609c06240c7SDavid S. Miller vstart = vstart & PMD_MASK;
2610c06240c7SDavid S. Miller vend = ALIGN(vend, PMD_SIZE);
2611c06240c7SDavid S. Miller for (; vstart < vend; vstart += PMD_SIZE) {
2612df8ee578SPavel Tatashin pgd_t *pgd = vmemmap_pgd_populate(vstart, node);
2613c06240c7SDavid S. Miller unsigned long pte;
26145637bc50SMike Rapoport p4d_t *p4d;
2615c06240c7SDavid S. Miller pud_t *pud;
2616c06240c7SDavid S. Miller pmd_t *pmd;
2617c06240c7SDavid S. Miller
2618df8ee578SPavel Tatashin if (!pgd)
2619c06240c7SDavid S. Miller return -ENOMEM;
2620c06240c7SDavid S. Miller
26215637bc50SMike Rapoport p4d = vmemmap_p4d_populate(pgd, vstart, node);
26225637bc50SMike Rapoport if (!p4d)
26235637bc50SMike Rapoport return -ENOMEM;
26245637bc50SMike Rapoport
26255637bc50SMike Rapoport pud = vmemmap_pud_populate(p4d, vstart, node);
2626df8ee578SPavel Tatashin if (!pud)
2627c06240c7SDavid S. Miller return -ENOMEM;
2628c06240c7SDavid S. Miller
2629c06240c7SDavid S. Miller pmd = pmd_offset(pud, vstart);
2630c06240c7SDavid S. Miller pte = pmd_val(*pmd);
2631c06240c7SDavid S. Miller if (!(pte & _PAGE_VALID)) {
2632c06240c7SDavid S. Miller void *block = vmemmap_alloc_block(PMD_SIZE, node);
2633c06240c7SDavid S. Miller
263427137e52SSam Ravnborg if (!block)
263527137e52SSam Ravnborg return -ENOMEM;
263627137e52SSam Ravnborg
2637c06240c7SDavid S. Miller pmd_val(*pmd) = pte_base | __pa(block);
2638c06240c7SDavid S. Miller }
2639c06240c7SDavid S. Miller }
264027137e52SSam Ravnborg
264127137e52SSam Ravnborg return 0;
264227137e52SSam Ravnborg }
26432856cc2eSDavid S. Miller
vmemmap_free(unsigned long start,unsigned long end,struct vmem_altmap * altmap)264424b6d416SChristoph Hellwig void vmemmap_free(unsigned long start, unsigned long end,
264524b6d416SChristoph Hellwig struct vmem_altmap *altmap)
26460197518cSTang Chen {
26470197518cSTang Chen }
264827137e52SSam Ravnborg #endif /* CONFIG_SPARSEMEM_VMEMMAP */
264927137e52SSam Ravnborg
265025740d31SAnshuman Khandual /* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */
265125740d31SAnshuman Khandual static pgprot_t protection_map[16] __ro_after_init;
265225740d31SAnshuman Khandual
prot_init_common(unsigned long page_none,unsigned long page_shared,unsigned long page_copy,unsigned long page_readonly,unsigned long page_exec_bit)265327137e52SSam Ravnborg static void prot_init_common(unsigned long page_none,
265427137e52SSam Ravnborg unsigned long page_shared,
265527137e52SSam Ravnborg unsigned long page_copy,
265627137e52SSam Ravnborg unsigned long page_readonly,
265727137e52SSam Ravnborg unsigned long page_exec_bit)
265827137e52SSam Ravnborg {
265927137e52SSam Ravnborg PAGE_COPY = __pgprot(page_copy);
266027137e52SSam Ravnborg PAGE_SHARED = __pgprot(page_shared);
266127137e52SSam Ravnborg
266227137e52SSam Ravnborg protection_map[0x0] = __pgprot(page_none);
266327137e52SSam Ravnborg protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
266427137e52SSam Ravnborg protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
266527137e52SSam Ravnborg protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
266627137e52SSam Ravnborg protection_map[0x4] = __pgprot(page_readonly);
266727137e52SSam Ravnborg protection_map[0x5] = __pgprot(page_readonly);
266827137e52SSam Ravnborg protection_map[0x6] = __pgprot(page_copy);
266927137e52SSam Ravnborg protection_map[0x7] = __pgprot(page_copy);
267027137e52SSam Ravnborg protection_map[0x8] = __pgprot(page_none);
267127137e52SSam Ravnborg protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
267227137e52SSam Ravnborg protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
267327137e52SSam Ravnborg protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
267427137e52SSam Ravnborg protection_map[0xc] = __pgprot(page_readonly);
267527137e52SSam Ravnborg protection_map[0xd] = __pgprot(page_readonly);
267627137e52SSam Ravnborg protection_map[0xe] = __pgprot(page_shared);
267727137e52SSam Ravnborg protection_map[0xf] = __pgprot(page_shared);
267827137e52SSam Ravnborg }
267927137e52SSam Ravnborg
sun4u_pgprot_init(void)268027137e52SSam Ravnborg static void __init sun4u_pgprot_init(void)
268127137e52SSam Ravnborg {
268227137e52SSam Ravnborg unsigned long page_none, page_shared, page_copy, page_readonly;
268327137e52SSam Ravnborg unsigned long page_exec_bit;
26844f93d21dSDavid S. Miller int i;
268527137e52SSam Ravnborg
268627137e52SSam Ravnborg PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
268727137e52SSam Ravnborg _PAGE_CACHE_4U | _PAGE_P_4U |
268827137e52SSam Ravnborg __ACCESS_BITS_4U | __DIRTY_BITS_4U |
268927137e52SSam Ravnborg _PAGE_EXEC_4U);
269027137e52SSam Ravnborg PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
269127137e52SSam Ravnborg _PAGE_CACHE_4U | _PAGE_P_4U |
269227137e52SSam Ravnborg __ACCESS_BITS_4U | __DIRTY_BITS_4U |
269327137e52SSam Ravnborg _PAGE_EXEC_4U | _PAGE_L_4U);
269427137e52SSam Ravnborg
269527137e52SSam Ravnborg _PAGE_IE = _PAGE_IE_4U;
269627137e52SSam Ravnborg _PAGE_E = _PAGE_E_4U;
269727137e52SSam Ravnborg _PAGE_CACHE = _PAGE_CACHE_4U;
269827137e52SSam Ravnborg
269927137e52SSam Ravnborg pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
270027137e52SSam Ravnborg __ACCESS_BITS_4U | _PAGE_E_4U);
270127137e52SSam Ravnborg
270227137e52SSam Ravnborg #ifdef CONFIG_DEBUG_PAGEALLOC
2703922631b9SDavid S. Miller kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
270427137e52SSam Ravnborg #else
270527137e52SSam Ravnborg kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
2706922631b9SDavid S. Miller PAGE_OFFSET;
270727137e52SSam Ravnborg #endif
270827137e52SSam Ravnborg kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
270927137e52SSam Ravnborg _PAGE_P_4U | _PAGE_W_4U);
271027137e52SSam Ravnborg
27114f93d21dSDavid S. Miller for (i = 1; i < 4; i++)
27124f93d21dSDavid S. Miller kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
271327137e52SSam Ravnborg
271427137e52SSam Ravnborg _PAGE_ALL_SZ_BITS = (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
271527137e52SSam Ravnborg _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
271627137e52SSam Ravnborg _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
271727137e52SSam Ravnborg
271827137e52SSam Ravnborg
271927137e52SSam Ravnborg page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
272027137e52SSam Ravnborg page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
272127137e52SSam Ravnborg __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
272227137e52SSam Ravnborg page_copy = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
272327137e52SSam Ravnborg __ACCESS_BITS_4U | _PAGE_EXEC_4U);
272427137e52SSam Ravnborg page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
272527137e52SSam Ravnborg __ACCESS_BITS_4U | _PAGE_EXEC_4U);
272627137e52SSam Ravnborg
272727137e52SSam Ravnborg page_exec_bit = _PAGE_EXEC_4U;
272827137e52SSam Ravnborg
272927137e52SSam Ravnborg prot_init_common(page_none, page_shared, page_copy, page_readonly,
273027137e52SSam Ravnborg page_exec_bit);
273127137e52SSam Ravnborg }
273227137e52SSam Ravnborg
sun4v_pgprot_init(void)273327137e52SSam Ravnborg static void __init sun4v_pgprot_init(void)
273427137e52SSam Ravnborg {
273527137e52SSam Ravnborg unsigned long page_none, page_shared, page_copy, page_readonly;
273627137e52SSam Ravnborg unsigned long page_exec_bit;
27374f93d21dSDavid S. Miller int i;
273827137e52SSam Ravnborg
273927137e52SSam Ravnborg PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
2740494e5b6fSKhalid Aziz page_cache4v_flag | _PAGE_P_4V |
274127137e52SSam Ravnborg __ACCESS_BITS_4V | __DIRTY_BITS_4V |
274227137e52SSam Ravnborg _PAGE_EXEC_4V);
274327137e52SSam Ravnborg PAGE_KERNEL_LOCKED = PAGE_KERNEL;
274427137e52SSam Ravnborg
274527137e52SSam Ravnborg _PAGE_IE = _PAGE_IE_4V;
274627137e52SSam Ravnborg _PAGE_E = _PAGE_E_4V;
2747494e5b6fSKhalid Aziz _PAGE_CACHE = page_cache4v_flag;
274827137e52SSam Ravnborg
274927137e52SSam Ravnborg #ifdef CONFIG_DEBUG_PAGEALLOC
2750922631b9SDavid S. Miller kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
275127137e52SSam Ravnborg #else
275227137e52SSam Ravnborg kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
2753922631b9SDavid S. Miller PAGE_OFFSET;
275427137e52SSam Ravnborg #endif
2755494e5b6fSKhalid Aziz kern_linear_pte_xor[0] |= (page_cache4v_flag | _PAGE_P_4V |
2756494e5b6fSKhalid Aziz _PAGE_W_4V);
275727137e52SSam Ravnborg
2758c69ad0a3SDavid S. Miller for (i = 1; i < 4; i++)
2759c69ad0a3SDavid S. Miller kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
27604f93d21dSDavid S. Miller
276127137e52SSam Ravnborg pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
276227137e52SSam Ravnborg __ACCESS_BITS_4V | _PAGE_E_4V);
276327137e52SSam Ravnborg
276427137e52SSam Ravnborg _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
276527137e52SSam Ravnborg _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
276627137e52SSam Ravnborg _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
276727137e52SSam Ravnborg _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);
276827137e52SSam Ravnborg
2769494e5b6fSKhalid Aziz page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | page_cache4v_flag;
2770494e5b6fSKhalid Aziz page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
277127137e52SSam Ravnborg __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
2772494e5b6fSKhalid Aziz page_copy = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
277327137e52SSam Ravnborg __ACCESS_BITS_4V | _PAGE_EXEC_4V);
2774494e5b6fSKhalid Aziz page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
277527137e52SSam Ravnborg __ACCESS_BITS_4V | _PAGE_EXEC_4V);
277627137e52SSam Ravnborg
277727137e52SSam Ravnborg page_exec_bit = _PAGE_EXEC_4V;
277827137e52SSam Ravnborg
277927137e52SSam Ravnborg prot_init_common(page_none, page_shared, page_copy, page_readonly,
278027137e52SSam Ravnborg page_exec_bit);
278127137e52SSam Ravnborg }
278227137e52SSam Ravnborg
pte_sz_bits(unsigned long sz)278327137e52SSam Ravnborg unsigned long pte_sz_bits(unsigned long sz)
278427137e52SSam Ravnborg {
278527137e52SSam Ravnborg if (tlb_type == hypervisor) {
278627137e52SSam Ravnborg switch (sz) {
278727137e52SSam Ravnborg case 8 * 1024:
278827137e52SSam Ravnborg default:
278927137e52SSam Ravnborg return _PAGE_SZ8K_4V;
279027137e52SSam Ravnborg case 64 * 1024:
279127137e52SSam Ravnborg return _PAGE_SZ64K_4V;
279227137e52SSam Ravnborg case 512 * 1024:
279327137e52SSam Ravnborg return _PAGE_SZ512K_4V;
279427137e52SSam Ravnborg case 4 * 1024 * 1024:
279527137e52SSam Ravnborg return _PAGE_SZ4MB_4V;
27966cb79b3fSJoe Perches }
279727137e52SSam Ravnborg } else {
279827137e52SSam Ravnborg switch (sz) {
279927137e52SSam Ravnborg case 8 * 1024:
280027137e52SSam Ravnborg default:
280127137e52SSam Ravnborg return _PAGE_SZ8K_4U;
280227137e52SSam Ravnborg case 64 * 1024:
280327137e52SSam Ravnborg return _PAGE_SZ64K_4U;
280427137e52SSam Ravnborg case 512 * 1024:
280527137e52SSam Ravnborg return _PAGE_SZ512K_4U;
280627137e52SSam Ravnborg case 4 * 1024 * 1024:
280727137e52SSam Ravnborg return _PAGE_SZ4MB_4U;
28086cb79b3fSJoe Perches }
280927137e52SSam Ravnborg }
281027137e52SSam Ravnborg }
281127137e52SSam Ravnborg
mk_pte_io(unsigned long page,pgprot_t prot,int space,unsigned long page_size)281227137e52SSam Ravnborg pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
281327137e52SSam Ravnborg {
281427137e52SSam Ravnborg pte_t pte;
281527137e52SSam Ravnborg
281627137e52SSam Ravnborg pte_val(pte) = page | pgprot_val(pgprot_noncached(prot));
281727137e52SSam Ravnborg pte_val(pte) |= (((unsigned long)space) << 32);
281827137e52SSam Ravnborg pte_val(pte) |= pte_sz_bits(page_size);
281927137e52SSam Ravnborg
282027137e52SSam Ravnborg return pte;
282127137e52SSam Ravnborg }
282227137e52SSam Ravnborg
kern_large_tte(unsigned long paddr)282327137e52SSam Ravnborg static unsigned long kern_large_tte(unsigned long paddr)
282427137e52SSam Ravnborg {
282527137e52SSam Ravnborg unsigned long val;
282627137e52SSam Ravnborg
282727137e52SSam Ravnborg val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
282827137e52SSam Ravnborg _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
282927137e52SSam Ravnborg _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
283027137e52SSam Ravnborg if (tlb_type == hypervisor)
283127137e52SSam Ravnborg val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
2832494e5b6fSKhalid Aziz page_cache4v_flag | _PAGE_P_4V |
283327137e52SSam Ravnborg _PAGE_EXEC_4V | _PAGE_W_4V);
283427137e52SSam Ravnborg
283527137e52SSam Ravnborg return val | paddr;
283627137e52SSam Ravnborg }
283727137e52SSam Ravnborg
283827137e52SSam Ravnborg /* If not locked, zap it. */
__flush_tlb_all(void)283927137e52SSam Ravnborg void __flush_tlb_all(void)
284027137e52SSam Ravnborg {
284127137e52SSam Ravnborg unsigned long pstate;
284227137e52SSam Ravnborg int i;
284327137e52SSam Ravnborg
284427137e52SSam Ravnborg __asm__ __volatile__("flushw\n\t"
284527137e52SSam Ravnborg "rdpr %%pstate, %0\n\t"
284627137e52SSam Ravnborg "wrpr %0, %1, %%pstate"
284727137e52SSam Ravnborg : "=r" (pstate)
284827137e52SSam Ravnborg : "i" (PSTATE_IE));
284927137e52SSam Ravnborg if (tlb_type == hypervisor) {
285027137e52SSam Ravnborg sun4v_mmu_demap_all();
285127137e52SSam Ravnborg } else if (tlb_type == spitfire) {
285227137e52SSam Ravnborg for (i = 0; i < 64; i++) {
285327137e52SSam Ravnborg /* Spitfire Errata #32 workaround */
285427137e52SSam Ravnborg /* NOTE: Always runs on spitfire, so no
285527137e52SSam Ravnborg * cheetah+ page size encodings.
285627137e52SSam Ravnborg */
285727137e52SSam Ravnborg __asm__ __volatile__("stxa %0, [%1] %2\n\t"
285827137e52SSam Ravnborg "flush %%g6"
285927137e52SSam Ravnborg : /* No outputs */
286027137e52SSam Ravnborg : "r" (0),
286127137e52SSam Ravnborg "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
286227137e52SSam Ravnborg
286327137e52SSam Ravnborg if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
286427137e52SSam Ravnborg __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
286527137e52SSam Ravnborg "membar #Sync"
286627137e52SSam Ravnborg : /* no outputs */
286727137e52SSam Ravnborg : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
286827137e52SSam Ravnborg spitfire_put_dtlb_data(i, 0x0UL);
286927137e52SSam Ravnborg }
287027137e52SSam Ravnborg
287127137e52SSam Ravnborg /* Spitfire Errata #32 workaround */
287227137e52SSam Ravnborg /* NOTE: Always runs on spitfire, so no
287327137e52SSam Ravnborg * cheetah+ page size encodings.
287427137e52SSam Ravnborg */
287527137e52SSam Ravnborg __asm__ __volatile__("stxa %0, [%1] %2\n\t"
287627137e52SSam Ravnborg "flush %%g6"
287727137e52SSam Ravnborg : /* No outputs */
287827137e52SSam Ravnborg : "r" (0),
287927137e52SSam Ravnborg "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
288027137e52SSam Ravnborg
288127137e52SSam Ravnborg if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
288227137e52SSam Ravnborg __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
288327137e52SSam Ravnborg "membar #Sync"
288427137e52SSam Ravnborg : /* no outputs */
288527137e52SSam Ravnborg : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
288627137e52SSam Ravnborg spitfire_put_itlb_data(i, 0x0UL);
288727137e52SSam Ravnborg }
288827137e52SSam Ravnborg }
288927137e52SSam Ravnborg } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
289027137e52SSam Ravnborg cheetah_flush_dtlb_all();
289127137e52SSam Ravnborg cheetah_flush_itlb_all();
289227137e52SSam Ravnborg }
289327137e52SSam Ravnborg __asm__ __volatile__("wrpr %0, 0, %%pstate"
289427137e52SSam Ravnborg : : "r" (pstate));
289527137e52SSam Ravnborg }
2896c460bec7SDavid Miller
pte_alloc_one_kernel(struct mm_struct * mm)28974cf58924SJoel Fernandes (Google) pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
2898c460bec7SDavid Miller {
289975f296d9SLevin, Alexander (Sasha Levin) struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
290037b3a8ffSDavid S. Miller pte_t *pte = NULL;
2901c460bec7SDavid Miller
2902c460bec7SDavid Miller if (page)
2903c460bec7SDavid Miller pte = (pte_t *) page_address(page);
2904c460bec7SDavid Miller
2905c460bec7SDavid Miller return pte;
2906c460bec7SDavid Miller }
2907c460bec7SDavid Miller
pte_alloc_one(struct mm_struct * mm)29084cf58924SJoel Fernandes (Google) pgtable_t pte_alloc_one(struct mm_struct *mm)
2909c460bec7SDavid Miller {
2910b3311d70SVishal Moola (Oracle) struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0);
2911b3311d70SVishal Moola (Oracle)
2912b3311d70SVishal Moola (Oracle) if (!ptdesc)
29131ae9ae5fSKirill A. Shutemov return NULL;
2914b3311d70SVishal Moola (Oracle) if (!pagetable_pte_ctor(ptdesc)) {
2915b3311d70SVishal Moola (Oracle) pagetable_free(ptdesc);
29161ae9ae5fSKirill A. Shutemov return NULL;
2917c460bec7SDavid Miller }
2918b3311d70SVishal Moola (Oracle) return ptdesc_address(ptdesc);
2919c460bec7SDavid Miller }
2920c460bec7SDavid Miller
pte_free_kernel(struct mm_struct * mm,pte_t * pte)2921c460bec7SDavid Miller void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
2922c460bec7SDavid Miller {
292337b3a8ffSDavid S. Miller free_page((unsigned long)pte);
2924c460bec7SDavid Miller }
2925c460bec7SDavid Miller
__pte_free(pgtable_t pte)2926c460bec7SDavid Miller static void __pte_free(pgtable_t pte)
2927c460bec7SDavid Miller {
2928b3311d70SVishal Moola (Oracle) struct ptdesc *ptdesc = virt_to_ptdesc(pte);
292937b3a8ffSDavid S. Miller
2930b3311d70SVishal Moola (Oracle) pagetable_pte_dtor(ptdesc);
2931b3311d70SVishal Moola (Oracle) pagetable_free(ptdesc);
2932c460bec7SDavid Miller }
2933c460bec7SDavid Miller
pte_free(struct mm_struct * mm,pgtable_t pte)2934c460bec7SDavid Miller void pte_free(struct mm_struct *mm, pgtable_t pte)
2935c460bec7SDavid Miller {
2936c460bec7SDavid Miller __pte_free(pte);
2937c460bec7SDavid Miller }
2938c460bec7SDavid Miller
pgtable_free(void * table,bool is_page)2939c460bec7SDavid Miller void pgtable_free(void *table, bool is_page)
2940c460bec7SDavid Miller {
2941c460bec7SDavid Miller if (is_page)
2942c460bec7SDavid Miller __pte_free(table);
2943c460bec7SDavid Miller else
2944c460bec7SDavid Miller kmem_cache_free(pgtable_cache, table);
2945c460bec7SDavid Miller }
29469e695d2eSDavid Miller
29479e695d2eSDavid Miller #ifdef CONFIG_TRANSPARENT_HUGEPAGE
pte_free_now(struct rcu_head * head)2948ad1ac8d9SHugh Dickins static void pte_free_now(struct rcu_head *head)
2949ad1ac8d9SHugh Dickins {
2950ad1ac8d9SHugh Dickins struct page *page;
2951ad1ac8d9SHugh Dickins
2952ad1ac8d9SHugh Dickins page = container_of(head, struct page, rcu_head);
2953ad1ac8d9SHugh Dickins __pte_free((pgtable_t)page_address(page));
2954ad1ac8d9SHugh Dickins }
2955ad1ac8d9SHugh Dickins
pte_free_defer(struct mm_struct * mm,pgtable_t pgtable)2956ad1ac8d9SHugh Dickins void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
2957ad1ac8d9SHugh Dickins {
2958ad1ac8d9SHugh Dickins struct page *page;
2959ad1ac8d9SHugh Dickins
2960ad1ac8d9SHugh Dickins page = virt_to_page(pgtable);
2961ad1ac8d9SHugh Dickins call_rcu(&page->rcu_head, pte_free_now);
2962ad1ac8d9SHugh Dickins }
2963ad1ac8d9SHugh Dickins
update_mmu_cache_pmd(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmd)29649e695d2eSDavid Miller void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
29659e695d2eSDavid Miller pmd_t *pmd)
29669e695d2eSDavid Miller {
29679e695d2eSDavid Miller unsigned long pte, flags;
29689e695d2eSDavid Miller struct mm_struct *mm;
29699e695d2eSDavid Miller pmd_t entry = *pmd;
29709e695d2eSDavid Miller
29719e695d2eSDavid Miller if (!pmd_large(entry) || !pmd_young(entry))
29729e695d2eSDavid Miller return;
29739e695d2eSDavid Miller
2974a7b9403fSDavid S. Miller pte = pmd_val(entry);
29759e695d2eSDavid Miller
297618f38132SDavid S. Miller /* Don't insert a non-valid PMD into the TSB, we'll deadlock. */
297718f38132SDavid S. Miller if (!(pte & _PAGE_VALID))
297818f38132SDavid S. Miller return;
297918f38132SDavid S. Miller
298037b3a8ffSDavid S. Miller /* We are fabricating 8MB pages using 4MB real hw pages. */
298137b3a8ffSDavid S. Miller pte |= (addr & (1UL << REAL_HPAGE_SHIFT));
29829e695d2eSDavid Miller
29839e695d2eSDavid Miller mm = vma->vm_mm;
29849e695d2eSDavid Miller
29859e695d2eSDavid Miller spin_lock_irqsave(&mm->context.lock, flags);
29869e695d2eSDavid Miller
29879e695d2eSDavid Miller if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
298837b3a8ffSDavid S. Miller __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
29899e695d2eSDavid Miller addr, pte);
29909e695d2eSDavid Miller
29919e695d2eSDavid Miller spin_unlock_irqrestore(&mm->context.lock, flags);
29929e695d2eSDavid Miller }
29939e695d2eSDavid Miller #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
29949e695d2eSDavid Miller
29959e695d2eSDavid Miller #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
context_reload(void * __data)29969e695d2eSDavid Miller static void context_reload(void *__data)
29979e695d2eSDavid Miller {
29989e695d2eSDavid Miller struct mm_struct *mm = __data;
29999e695d2eSDavid Miller
30009e695d2eSDavid Miller if (mm == current->mm)
30019e695d2eSDavid Miller load_secondary_context(mm);
30029e695d2eSDavid Miller }
30039e695d2eSDavid Miller
hugetlb_setup(struct pt_regs * regs)30040fbebed6SDavid S. Miller void hugetlb_setup(struct pt_regs *regs)
30059e695d2eSDavid Miller {
30060fbebed6SDavid S. Miller struct mm_struct *mm = current->mm;
30070fbebed6SDavid S. Miller struct tsb_config *tp;
30089e695d2eSDavid Miller
300970ffdb93SDavid Hildenbrand if (faulthandler_disabled() || !mm) {
30100fbebed6SDavid S. Miller const struct exception_table_entry *entry;
30110fbebed6SDavid S. Miller
30120fbebed6SDavid S. Miller entry = search_exception_tables(regs->tpc);
30130fbebed6SDavid S. Miller if (entry) {
30140fbebed6SDavid S. Miller regs->tpc = entry->fixup;
30150fbebed6SDavid S. Miller regs->tnpc = regs->tpc + 4;
30169e695d2eSDavid Miller return;
30170fbebed6SDavid S. Miller }
30180fbebed6SDavid S. Miller pr_alert("Unexpected HugeTLB setup in atomic context.\n");
30190fbebed6SDavid S. Miller die_if_kernel("HugeTSB in atomic", regs);
30200fbebed6SDavid S. Miller }
30219e695d2eSDavid Miller
30220fbebed6SDavid S. Miller tp = &mm->context.tsb_block[MM_TSB_HUGE];
30230fbebed6SDavid S. Miller if (likely(tp->tsb == NULL))
30249e695d2eSDavid Miller tsb_grow(mm, MM_TSB_HUGE, 0);
30250fbebed6SDavid S. Miller
30269e695d2eSDavid Miller tsb_context_switch(mm);
30279e695d2eSDavid Miller smp_tsb_sync(mm);
30289e695d2eSDavid Miller
30299e695d2eSDavid Miller /* On UltraSPARC-III+ and later, configure the second half of
30309e695d2eSDavid Miller * the Data-TLB for huge pages.
30319e695d2eSDavid Miller */
30329e695d2eSDavid Miller if (tlb_type == cheetah_plus) {
30339ea46abeSDavid S. Miller bool need_context_reload = false;
30349e695d2eSDavid Miller unsigned long ctx;
30359e695d2eSDavid Miller
30369ea46abeSDavid S. Miller spin_lock_irq(&ctx_alloc_lock);
30379e695d2eSDavid Miller ctx = mm->context.sparc64_ctx_val;
30389e695d2eSDavid Miller ctx &= ~CTX_PGSZ_MASK;
30399e695d2eSDavid Miller ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;
30409e695d2eSDavid Miller ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;
30419e695d2eSDavid Miller
30429e695d2eSDavid Miller if (ctx != mm->context.sparc64_ctx_val) {
30439e695d2eSDavid Miller /* When changing the page size fields, we
30449e695d2eSDavid Miller * must perform a context flush so that no
30459e695d2eSDavid Miller * stale entries match. This flush must
30469e695d2eSDavid Miller * occur with the original context register
30479e695d2eSDavid Miller * settings.
30489e695d2eSDavid Miller */
30499e695d2eSDavid Miller do_flush_tlb_mm(mm);
30509e695d2eSDavid Miller
30519e695d2eSDavid Miller /* Reload the context register of all processors
30529e695d2eSDavid Miller * also executing in this address space.
30539e695d2eSDavid Miller */
30549e695d2eSDavid Miller mm->context.sparc64_ctx_val = ctx;
30559ea46abeSDavid S. Miller need_context_reload = true;
30569e695d2eSDavid Miller }
30579ea46abeSDavid S. Miller spin_unlock_irq(&ctx_alloc_lock);
30589ea46abeSDavid S. Miller
30599ea46abeSDavid S. Miller if (need_context_reload)
30609ea46abeSDavid S. Miller on_each_cpu(context_reload, mm, 0);
30619e695d2eSDavid Miller }
30629e695d2eSDavid Miller }
30639e695d2eSDavid Miller #endif
3064f6d4fb5cSbob picco
3065f6d4fb5cSbob picco static struct resource code_resource = {
3066f6d4fb5cSbob picco .name = "Kernel code",
306735d98e93SToshi Kani .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
3068f6d4fb5cSbob picco };
3069f6d4fb5cSbob picco
3070f6d4fb5cSbob picco static struct resource data_resource = {
3071f6d4fb5cSbob picco .name = "Kernel data",
307235d98e93SToshi Kani .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
3073f6d4fb5cSbob picco };
3074f6d4fb5cSbob picco
3075f6d4fb5cSbob picco static struct resource bss_resource = {
3076f6d4fb5cSbob picco .name = "Kernel bss",
307735d98e93SToshi Kani .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
3078f6d4fb5cSbob picco };
3079f6d4fb5cSbob picco
compute_kern_paddr(void * addr)3080f6d4fb5cSbob picco static inline resource_size_t compute_kern_paddr(void *addr)
3081f6d4fb5cSbob picco {
3082f6d4fb5cSbob picco return (resource_size_t) (addr - KERNBASE + kern_base);
3083f6d4fb5cSbob picco }
3084f6d4fb5cSbob picco
kernel_lds_init(void)3085f6d4fb5cSbob picco static void __init kernel_lds_init(void)
3086f6d4fb5cSbob picco {
3087f6d4fb5cSbob picco code_resource.start = compute_kern_paddr(_text);
3088f6d4fb5cSbob picco code_resource.end = compute_kern_paddr(_etext - 1);
3089f6d4fb5cSbob picco data_resource.start = compute_kern_paddr(_etext);
3090f6d4fb5cSbob picco data_resource.end = compute_kern_paddr(_edata - 1);
3091f6d4fb5cSbob picco bss_resource.start = compute_kern_paddr(__bss_start);
3092f6d4fb5cSbob picco bss_resource.end = compute_kern_paddr(_end - 1);
3093f6d4fb5cSbob picco }
3094f6d4fb5cSbob picco
report_memory(void)3095f6d4fb5cSbob picco static int __init report_memory(void)
3096f6d4fb5cSbob picco {
3097f6d4fb5cSbob picco int i;
3098f6d4fb5cSbob picco struct resource *res;
3099f6d4fb5cSbob picco
3100f6d4fb5cSbob picco kernel_lds_init();
3101f6d4fb5cSbob picco
3102f6d4fb5cSbob picco for (i = 0; i < pavail_ents; i++) {
3103f6d4fb5cSbob picco res = kzalloc(sizeof(struct resource), GFP_KERNEL);
3104f6d4fb5cSbob picco
3105f6d4fb5cSbob picco if (!res) {
3106f6d4fb5cSbob picco pr_warn("Failed to allocate source.\n");
3107f6d4fb5cSbob picco break;
3108f6d4fb5cSbob picco }
3109f6d4fb5cSbob picco
3110f6d4fb5cSbob picco res->name = "System RAM";
3111f6d4fb5cSbob picco res->start = pavail[i].phys_addr;
3112f6d4fb5cSbob picco res->end = pavail[i].phys_addr + pavail[i].reg_size - 1;
311335d98e93SToshi Kani res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM;
3114f6d4fb5cSbob picco
3115f6d4fb5cSbob picco if (insert_resource(&iomem_resource, res) < 0) {
3116f6d4fb5cSbob picco pr_warn("Resource insertion failed.\n");
3117f6d4fb5cSbob picco break;
3118f6d4fb5cSbob picco }
3119f6d4fb5cSbob picco
3120f6d4fb5cSbob picco insert_resource(res, &code_resource);
3121f6d4fb5cSbob picco insert_resource(res, &data_resource);
3122f6d4fb5cSbob picco insert_resource(res, &bss_resource);
3123f6d4fb5cSbob picco }
3124f6d4fb5cSbob picco
3125f6d4fb5cSbob picco return 0;
3126f6d4fb5cSbob picco }
31273c08158eSDavid S. Miller arch_initcall(report_memory);
3128e9011d08SDavid S. Miller
31294ca9a237SDavid S. Miller #ifdef CONFIG_SMP
31304ca9a237SDavid S. Miller #define do_flush_tlb_kernel_range smp_flush_tlb_kernel_range
31314ca9a237SDavid S. Miller #else
31324ca9a237SDavid S. Miller #define do_flush_tlb_kernel_range __flush_tlb_kernel_range
31334ca9a237SDavid S. Miller #endif
31344ca9a237SDavid S. Miller
flush_tlb_kernel_range(unsigned long start,unsigned long end)31354ca9a237SDavid S. Miller void flush_tlb_kernel_range(unsigned long start, unsigned long end)
31364ca9a237SDavid S. Miller {
31374ca9a237SDavid S. Miller if (start < HI_OBP_ADDRESS && end > LOW_OBP_ADDRESS) {
31384ca9a237SDavid S. Miller if (start < LOW_OBP_ADDRESS) {
31394ca9a237SDavid S. Miller flush_tsb_kernel_range(start, LOW_OBP_ADDRESS);
31404ca9a237SDavid S. Miller do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS);
31414ca9a237SDavid S. Miller }
31424ca9a237SDavid S. Miller if (end > HI_OBP_ADDRESS) {
3143473ad7f4SDavid S. Miller flush_tsb_kernel_range(HI_OBP_ADDRESS, end);
3144473ad7f4SDavid S. Miller do_flush_tlb_kernel_range(HI_OBP_ADDRESS, end);
31454ca9a237SDavid S. Miller }
31464ca9a237SDavid S. Miller } else {
31474ca9a237SDavid S. Miller flush_tsb_kernel_range(start, end);
31484ca9a237SDavid S. Miller do_flush_tlb_kernel_range(start, end);
31494ca9a237SDavid S. Miller }
31504ca9a237SDavid S. Miller }
315174a04967SKhalid Aziz
copy_user_highpage(struct page * to,struct page * from,unsigned long vaddr,struct vm_area_struct * vma)315274a04967SKhalid Aziz void copy_user_highpage(struct page *to, struct page *from,
315374a04967SKhalid Aziz unsigned long vaddr, struct vm_area_struct *vma)
315474a04967SKhalid Aziz {
315574a04967SKhalid Aziz char *vfrom, *vto;
315674a04967SKhalid Aziz
315774a04967SKhalid Aziz vfrom = kmap_atomic(from);
315874a04967SKhalid Aziz vto = kmap_atomic(to);
315974a04967SKhalid Aziz copy_user_page(vto, vfrom, vaddr, to);
316074a04967SKhalid Aziz kunmap_atomic(vto);
316174a04967SKhalid Aziz kunmap_atomic(vfrom);
316274a04967SKhalid Aziz
316374a04967SKhalid Aziz /* If this page has ADI enabled, copy over any ADI tags
316474a04967SKhalid Aziz * as well
316574a04967SKhalid Aziz */
316674a04967SKhalid Aziz if (vma->vm_flags & VM_SPARC_ADI) {
316774a04967SKhalid Aziz unsigned long pfrom, pto, i, adi_tag;
316874a04967SKhalid Aziz
316974a04967SKhalid Aziz pfrom = page_to_phys(from);
317074a04967SKhalid Aziz pto = page_to_phys(to);
317174a04967SKhalid Aziz
317274a04967SKhalid Aziz for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) {
317374a04967SKhalid Aziz asm volatile("ldxa [%1] %2, %0\n\t"
317474a04967SKhalid Aziz : "=r" (adi_tag)
317574a04967SKhalid Aziz : "r" (i), "i" (ASI_MCD_REAL));
317674a04967SKhalid Aziz asm volatile("stxa %0, [%1] %2\n\t"
317774a04967SKhalid Aziz :
317874a04967SKhalid Aziz : "r" (adi_tag), "r" (pto),
317974a04967SKhalid Aziz "i" (ASI_MCD_REAL));
318074a04967SKhalid Aziz pto += adi_blksize();
318174a04967SKhalid Aziz }
318274a04967SKhalid Aziz asm volatile("membar #Sync\n\t");
318374a04967SKhalid Aziz }
318474a04967SKhalid Aziz }
318574a04967SKhalid Aziz EXPORT_SYMBOL(copy_user_highpage);
318674a04967SKhalid Aziz
copy_highpage(struct page * to,struct page * from)318774a04967SKhalid Aziz void copy_highpage(struct page *to, struct page *from)
318874a04967SKhalid Aziz {
318974a04967SKhalid Aziz char *vfrom, *vto;
319074a04967SKhalid Aziz
319174a04967SKhalid Aziz vfrom = kmap_atomic(from);
319274a04967SKhalid Aziz vto = kmap_atomic(to);
319374a04967SKhalid Aziz copy_page(vto, vfrom);
319474a04967SKhalid Aziz kunmap_atomic(vto);
319574a04967SKhalid Aziz kunmap_atomic(vfrom);
319674a04967SKhalid Aziz
319774a04967SKhalid Aziz /* If this platform is ADI enabled, copy any ADI tags
319874a04967SKhalid Aziz * as well
319974a04967SKhalid Aziz */
320074a04967SKhalid Aziz if (adi_capable()) {
320174a04967SKhalid Aziz unsigned long pfrom, pto, i, adi_tag;
320274a04967SKhalid Aziz
320374a04967SKhalid Aziz pfrom = page_to_phys(from);
320474a04967SKhalid Aziz pto = page_to_phys(to);
320574a04967SKhalid Aziz
320674a04967SKhalid Aziz for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) {
320774a04967SKhalid Aziz asm volatile("ldxa [%1] %2, %0\n\t"
320874a04967SKhalid Aziz : "=r" (adi_tag)
320974a04967SKhalid Aziz : "r" (i), "i" (ASI_MCD_REAL));
321074a04967SKhalid Aziz asm volatile("stxa %0, [%1] %2\n\t"
321174a04967SKhalid Aziz :
321274a04967SKhalid Aziz : "r" (adi_tag), "r" (pto),
321374a04967SKhalid Aziz "i" (ASI_MCD_REAL));
321474a04967SKhalid Aziz pto += adi_blksize();
321574a04967SKhalid Aziz }
321674a04967SKhalid Aziz asm volatile("membar #Sync\n\t");
321774a04967SKhalid Aziz }
321874a04967SKhalid Aziz }
321974a04967SKhalid Aziz EXPORT_SYMBOL(copy_highpage);
322091d4ce98SAnshuman Khandual
vm_get_page_prot(unsigned long vm_flags)322191d4ce98SAnshuman Khandual pgprot_t vm_get_page_prot(unsigned long vm_flags)
322291d4ce98SAnshuman Khandual {
322391d4ce98SAnshuman Khandual unsigned long prot = pgprot_val(protection_map[vm_flags &
322491d4ce98SAnshuman Khandual (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]);
322591d4ce98SAnshuman Khandual
322691d4ce98SAnshuman Khandual if (vm_flags & VM_SPARC_ADI)
322791d4ce98SAnshuman Khandual prot |= _PAGE_MCD_4V;
322891d4ce98SAnshuman Khandual
322991d4ce98SAnshuman Khandual return __pgprot(prot);
323091d4ce98SAnshuman Khandual }
323191d4ce98SAnshuman Khandual EXPORT_SYMBOL(vm_get_page_prot);
3232