11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * TLB support routines. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co 51da177e4SLinus Torvalds * David Mosberger-Tang <davidm@hpl.hp.com> 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * 08/02/00 A. Mallick <asit.k.mallick@intel.com> 81da177e4SLinus Torvalds * Modified RID allocation for SMP 91da177e4SLinus Torvalds * Goutham Rao <goutham.rao@intel.com> 101da177e4SLinus Torvalds * IPI based ptc implementation and A-step IPI implementation. 11dcc17d1bSPeter Keilty * Rohit Seth <rohit.seth@intel.com> 12dcc17d1bSPeter Keilty * Ken Chen <kenneth.w.chen@intel.com> 13aec103bfSde Dinechin, Christophe (Integrity VM) * Christophe de Dinechin <ddd@hp.com>: Avoid ptc.e on memory allocation 142046b94eSFenghua Yu * Copyright (C) 2007 Intel Corp 152046b94eSFenghua Yu * Fenghua Yu <fenghua.yu@intel.com> 162046b94eSFenghua Yu * Add multiple ptc.g/ptc.ga instruction support in global tlb purge. 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds #include <linux/module.h> 191da177e4SLinus Torvalds #include <linux/init.h> 201da177e4SLinus Torvalds #include <linux/kernel.h> 211da177e4SLinus Torvalds #include <linux/sched.h> 221da177e4SLinus Torvalds #include <linux/smp.h> 231da177e4SLinus Torvalds #include <linux/mm.h> 24dcc17d1bSPeter Keilty #include <linux/bootmem.h> 251da177e4SLinus Torvalds 261da177e4SLinus Torvalds #include <asm/delay.h> 271da177e4SLinus Torvalds #include <asm/mmu_context.h> 281da177e4SLinus Torvalds #include <asm/pgalloc.h> 291da177e4SLinus Torvalds #include <asm/pal.h> 301da177e4SLinus Torvalds #include <asm/tlbflush.h> 31dcc17d1bSPeter Keilty #include <asm/dma.h> 322046b94eSFenghua Yu #include <asm/sal.h> 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds static struct { 351da177e4SLinus Torvalds unsigned long mask; /* mask of supported purge page-sizes */ 3658cd9082SChen, Kenneth W unsigned long max_bits; /* log2 of largest supported purge page-size */ 371da177e4SLinus Torvalds } purge; 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds struct ia64_ctx ia64_ctx = { 408737d595SMilind Arun Choudhary .lock = __SPIN_LOCK_UNLOCKED(ia64_ctx.lock), 411da177e4SLinus Torvalds .next = 1, 421da177e4SLinus Torvalds .max_ctx = ~0U 431da177e4SLinus Torvalds }; 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds DEFINE_PER_CPU(u8, ia64_need_tlb_flush); 461da177e4SLinus Torvalds 471da177e4SLinus Torvalds /* 48dcc17d1bSPeter Keilty * Initializes the ia64_ctx.bitmap array based on max_ctx+1. 49dcc17d1bSPeter Keilty * Called after cpu_init() has setup ia64_ctx.max_ctx based on 50dcc17d1bSPeter Keilty * maximum RID that is supported by boot CPU. 51dcc17d1bSPeter Keilty */ 52dcc17d1bSPeter Keilty void __init 53dcc17d1bSPeter Keilty mmu_context_init (void) 54dcc17d1bSPeter Keilty { 55dcc17d1bSPeter Keilty ia64_ctx.bitmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 56dcc17d1bSPeter Keilty ia64_ctx.flushmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 57dcc17d1bSPeter Keilty } 58dcc17d1bSPeter Keilty 59dcc17d1bSPeter Keilty /* 601da177e4SLinus Torvalds * Acquire the ia64_ctx.lock before calling this function! 611da177e4SLinus Torvalds */ 621da177e4SLinus Torvalds void 631da177e4SLinus Torvalds wrap_mmu_context (struct mm_struct *mm) 641da177e4SLinus Torvalds { 6558cd9082SChen, Kenneth W int i, cpu; 66dcc17d1bSPeter Keilty unsigned long flush_bit; 671da177e4SLinus Torvalds 68dcc17d1bSPeter Keilty for (i=0; i <= ia64_ctx.max_ctx / BITS_PER_LONG; i++) { 69dcc17d1bSPeter Keilty flush_bit = xchg(&ia64_ctx.flushmap[i], 0); 70dcc17d1bSPeter Keilty ia64_ctx.bitmap[i] ^= flush_bit; 71dcc17d1bSPeter Keilty } 721da177e4SLinus Torvalds 73dcc17d1bSPeter Keilty /* use offset at 300 to skip daemons */ 74dcc17d1bSPeter Keilty ia64_ctx.next = find_next_zero_bit(ia64_ctx.bitmap, 75dcc17d1bSPeter Keilty ia64_ctx.max_ctx, 300); 76dcc17d1bSPeter Keilty ia64_ctx.limit = find_next_bit(ia64_ctx.bitmap, 77dcc17d1bSPeter Keilty ia64_ctx.max_ctx, ia64_ctx.next); 781da177e4SLinus Torvalds 7958cd9082SChen, Kenneth W /* 8058cd9082SChen, Kenneth W * can't call flush_tlb_all() here because of race condition 8158cd9082SChen, Kenneth W * with O(1) scheduler [EF] 8258cd9082SChen, Kenneth W */ 8358cd9082SChen, Kenneth W cpu = get_cpu(); /* prevent preemption/migration */ 8458cd9082SChen, Kenneth W for_each_online_cpu(i) 85dc565b52Shawkes@sgi.com if (i != cpu) 861da177e4SLinus Torvalds per_cpu(ia64_need_tlb_flush, i) = 1; 871da177e4SLinus Torvalds put_cpu(); 881da177e4SLinus Torvalds local_flush_tlb_all(); 891da177e4SLinus Torvalds } 901da177e4SLinus Torvalds 912046b94eSFenghua Yu /* 922046b94eSFenghua Yu * Implement "spinaphores" ... like counting semaphores, but they 932046b94eSFenghua Yu * spin instead of sleeping. If there are ever any other users for 942046b94eSFenghua Yu * this primitive it can be moved up to a spinaphore.h header. 952046b94eSFenghua Yu */ 962046b94eSFenghua Yu struct spinaphore { 972046b94eSFenghua Yu atomic_t cur; 982046b94eSFenghua Yu }; 992046b94eSFenghua Yu 1002046b94eSFenghua Yu static inline void spinaphore_init(struct spinaphore *ss, int val) 1012046b94eSFenghua Yu { 1022046b94eSFenghua Yu atomic_set(&ss->cur, val); 1032046b94eSFenghua Yu } 1042046b94eSFenghua Yu 1052046b94eSFenghua Yu static inline void down_spin(struct spinaphore *ss) 1062046b94eSFenghua Yu { 1072046b94eSFenghua Yu while (unlikely(!atomic_add_unless(&ss->cur, -1, 0))) 1082046b94eSFenghua Yu while (atomic_read(&ss->cur) == 0) 1092046b94eSFenghua Yu cpu_relax(); 1102046b94eSFenghua Yu } 1112046b94eSFenghua Yu 1122046b94eSFenghua Yu static inline void up_spin(struct spinaphore *ss) 1132046b94eSFenghua Yu { 1142046b94eSFenghua Yu atomic_add(1, &ss->cur); 1152046b94eSFenghua Yu } 1162046b94eSFenghua Yu 1172046b94eSFenghua Yu static struct spinaphore ptcg_sem; 1182046b94eSFenghua Yu static u16 nptcg = 1; 1192046b94eSFenghua Yu static int need_ptcg_sem = 1; 1202046b94eSFenghua Yu static int toolatetochangeptcgsem = 0; 1212046b94eSFenghua Yu 1222046b94eSFenghua Yu /* 1232046b94eSFenghua Yu * Maximum number of simultaneous ptc.g purges in the system can 1242046b94eSFenghua Yu * be defined by PAL_VM_SUMMARY (in which case we should take 1252046b94eSFenghua Yu * the smallest value for any cpu in the system) or by the PAL 1262046b94eSFenghua Yu * override table (in which case we should ignore the value from 1272046b94eSFenghua Yu * PAL_VM_SUMMARY). 1282046b94eSFenghua Yu * 1292046b94eSFenghua Yu * Complicating the logic here is the fact that num_possible_cpus() 1302046b94eSFenghua Yu * isn't fully setup until we start bringing cpus online. 1312046b94eSFenghua Yu */ 1322046b94eSFenghua Yu void 1332046b94eSFenghua Yu setup_ptcg_sem(int max_purges, int from_palo) 1342046b94eSFenghua Yu { 1352046b94eSFenghua Yu static int have_palo; 1362046b94eSFenghua Yu static int firstcpu = 1; 1372046b94eSFenghua Yu 1382046b94eSFenghua Yu if (toolatetochangeptcgsem) { 1392046b94eSFenghua Yu BUG_ON(max_purges < nptcg); 1402046b94eSFenghua Yu return; 1412046b94eSFenghua Yu } 1422046b94eSFenghua Yu 1432046b94eSFenghua Yu if (from_palo) { 1442046b94eSFenghua Yu have_palo = 1; 1452046b94eSFenghua Yu 1462046b94eSFenghua Yu /* In PALO max_purges == 0 really means it! */ 1472046b94eSFenghua Yu if (max_purges == 0) 1482046b94eSFenghua Yu panic("Whoa! Platform does not support global TLB purges.\n"); 1492046b94eSFenghua Yu nptcg = max_purges; 1502046b94eSFenghua Yu if (nptcg == PALO_MAX_TLB_PURGES) { 1512046b94eSFenghua Yu need_ptcg_sem = 0; 1522046b94eSFenghua Yu return; 1532046b94eSFenghua Yu } 1542046b94eSFenghua Yu goto resetsema; 1552046b94eSFenghua Yu } 1562046b94eSFenghua Yu if (have_palo) { 1572046b94eSFenghua Yu if (nptcg != PALO_MAX_TLB_PURGES) 1582046b94eSFenghua Yu need_ptcg_sem = (num_possible_cpus() > nptcg); 1592046b94eSFenghua Yu return; 1602046b94eSFenghua Yu } 1612046b94eSFenghua Yu 1622046b94eSFenghua Yu /* In PAL_VM_SUMMARY max_purges == 0 actually means 1 */ 1632046b94eSFenghua Yu if (max_purges == 0) max_purges = 1; 1642046b94eSFenghua Yu 1652046b94eSFenghua Yu if (firstcpu) { 1662046b94eSFenghua Yu nptcg = max_purges; 1672046b94eSFenghua Yu firstcpu = 0; 1682046b94eSFenghua Yu } 1692046b94eSFenghua Yu if (max_purges < nptcg) 1702046b94eSFenghua Yu nptcg = max_purges; 1712046b94eSFenghua Yu if (nptcg == PAL_MAX_PURGES) { 1722046b94eSFenghua Yu need_ptcg_sem = 0; 1732046b94eSFenghua Yu return; 1742046b94eSFenghua Yu } else 1752046b94eSFenghua Yu need_ptcg_sem = (num_possible_cpus() > nptcg); 1762046b94eSFenghua Yu 1772046b94eSFenghua Yu resetsema: 1782046b94eSFenghua Yu spinaphore_init(&ptcg_sem, max_purges); 1792046b94eSFenghua Yu } 1802046b94eSFenghua Yu 1811da177e4SLinus Torvalds void 18258cd9082SChen, Kenneth W ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start, 18358cd9082SChen, Kenneth W unsigned long end, unsigned long nbits) 1841da177e4SLinus Torvalds { 185aec103bfSde Dinechin, Christophe (Integrity VM) struct mm_struct *active_mm = current->active_mm; 186aec103bfSde Dinechin, Christophe (Integrity VM) 1872046b94eSFenghua Yu toolatetochangeptcgsem = 1; 1882046b94eSFenghua Yu 189aec103bfSde Dinechin, Christophe (Integrity VM) if (mm != active_mm) { 190aec103bfSde Dinechin, Christophe (Integrity VM) /* Restore region IDs for mm */ 191aec103bfSde Dinechin, Christophe (Integrity VM) if (mm && active_mm) { 192aec103bfSde Dinechin, Christophe (Integrity VM) activate_context(mm); 193aec103bfSde Dinechin, Christophe (Integrity VM) } else { 194c1902aaeSDean Roe flush_tlb_all(); 195c1902aaeSDean Roe return; 196c1902aaeSDean Roe } 197aec103bfSde Dinechin, Christophe (Integrity VM) } 198c1902aaeSDean Roe 1992046b94eSFenghua Yu if (need_ptcg_sem) 2002046b94eSFenghua Yu down_spin(&ptcg_sem); 2012046b94eSFenghua Yu 2021da177e4SLinus Torvalds do { 2031da177e4SLinus Torvalds /* 2041da177e4SLinus Torvalds * Flush ALAT entries also. 2051da177e4SLinus Torvalds */ 2061da177e4SLinus Torvalds ia64_ptcga(start, (nbits << 2)); 2071da177e4SLinus Torvalds ia64_srlz_i(); 2081da177e4SLinus Torvalds start += (1UL << nbits); 2091da177e4SLinus Torvalds } while (start < end); 2102046b94eSFenghua Yu 2112046b94eSFenghua Yu if (need_ptcg_sem) 2122046b94eSFenghua Yu up_spin(&ptcg_sem); 213aec103bfSde Dinechin, Christophe (Integrity VM) 214aec103bfSde Dinechin, Christophe (Integrity VM) if (mm != active_mm) { 215aec103bfSde Dinechin, Christophe (Integrity VM) activate_context(active_mm); 216aec103bfSde Dinechin, Christophe (Integrity VM) } 2171da177e4SLinus Torvalds } 2181da177e4SLinus Torvalds 2191da177e4SLinus Torvalds void 2201da177e4SLinus Torvalds local_flush_tlb_all (void) 2211da177e4SLinus Torvalds { 2221da177e4SLinus Torvalds unsigned long i, j, flags, count0, count1, stride0, stride1, addr; 2231da177e4SLinus Torvalds 2241da177e4SLinus Torvalds addr = local_cpu_data->ptce_base; 2251da177e4SLinus Torvalds count0 = local_cpu_data->ptce_count[0]; 2261da177e4SLinus Torvalds count1 = local_cpu_data->ptce_count[1]; 2271da177e4SLinus Torvalds stride0 = local_cpu_data->ptce_stride[0]; 2281da177e4SLinus Torvalds stride1 = local_cpu_data->ptce_stride[1]; 2291da177e4SLinus Torvalds 2301da177e4SLinus Torvalds local_irq_save(flags); 2311da177e4SLinus Torvalds for (i = 0; i < count0; ++i) { 2321da177e4SLinus Torvalds for (j = 0; j < count1; ++j) { 2331da177e4SLinus Torvalds ia64_ptce(addr); 2341da177e4SLinus Torvalds addr += stride1; 2351da177e4SLinus Torvalds } 2361da177e4SLinus Torvalds addr += stride0; 2371da177e4SLinus Torvalds } 2381da177e4SLinus Torvalds local_irq_restore(flags); 2391da177e4SLinus Torvalds ia64_srlz_i(); /* srlz.i implies srlz.d */ 2401da177e4SLinus Torvalds } 2411da177e4SLinus Torvalds 2421da177e4SLinus Torvalds void 24358cd9082SChen, Kenneth W flush_tlb_range (struct vm_area_struct *vma, unsigned long start, 24458cd9082SChen, Kenneth W unsigned long end) 2451da177e4SLinus Torvalds { 2461da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 2471da177e4SLinus Torvalds unsigned long size = end - start; 2481da177e4SLinus Torvalds unsigned long nbits; 2491da177e4SLinus Torvalds 250c1902aaeSDean Roe #ifndef CONFIG_SMP 2511da177e4SLinus Torvalds if (mm != current->active_mm) { 2521da177e4SLinus Torvalds mm->context = 0; 2531da177e4SLinus Torvalds return; 2541da177e4SLinus Torvalds } 255c1902aaeSDean Roe #endif 2561da177e4SLinus Torvalds 2571da177e4SLinus Torvalds nbits = ia64_fls(size + 0xfff); 25858cd9082SChen, Kenneth W while (unlikely (((1UL << nbits) & purge.mask) == 0) && 25958cd9082SChen, Kenneth W (nbits < purge.max_bits)) 2601da177e4SLinus Torvalds ++nbits; 2611da177e4SLinus Torvalds if (nbits > purge.max_bits) 2621da177e4SLinus Torvalds nbits = purge.max_bits; 2631da177e4SLinus Torvalds start &= ~((1UL << nbits) - 1); 2641da177e4SLinus Torvalds 265663b97f7SHugh Dickins preempt_disable(); 266ce9eed5aSChen, Kenneth W #ifdef CONFIG_SMP 267ce9eed5aSChen, Kenneth W if (mm != current->active_mm || cpus_weight(mm->cpu_vm_mask) != 1) { 268ce9eed5aSChen, Kenneth W platform_global_tlb_purge(mm, start, end, nbits); 269ce9eed5aSChen, Kenneth W preempt_enable(); 270ce9eed5aSChen, Kenneth W return; 271ce9eed5aSChen, Kenneth W } 272ce9eed5aSChen, Kenneth W #endif 2731da177e4SLinus Torvalds do { 2741da177e4SLinus Torvalds ia64_ptcl(start, (nbits<<2)); 2751da177e4SLinus Torvalds start += (1UL << nbits); 2761da177e4SLinus Torvalds } while (start < end); 277663b97f7SHugh Dickins preempt_enable(); 2781da177e4SLinus Torvalds ia64_srlz_i(); /* srlz.i implies srlz.d */ 2791da177e4SLinus Torvalds } 2801da177e4SLinus Torvalds EXPORT_SYMBOL(flush_tlb_range); 2811da177e4SLinus Torvalds 2821da177e4SLinus Torvalds void __devinit 2831da177e4SLinus Torvalds ia64_tlb_init (void) 2841da177e4SLinus Torvalds { 285256a7e09SJes Sorensen ia64_ptce_info_t uninitialized_var(ptce_info); /* GCC be quiet */ 2861da177e4SLinus Torvalds unsigned long tr_pgbits; 2871da177e4SLinus Torvalds long status; 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { 2901da177e4SLinus Torvalds printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld; " 2911da177e4SLinus Torvalds "defaulting to architected purge page-sizes.\n", status); 2921da177e4SLinus Torvalds purge.mask = 0x115557000UL; 2931da177e4SLinus Torvalds } 2941da177e4SLinus Torvalds purge.max_bits = ia64_fls(purge.mask); 2951da177e4SLinus Torvalds 2961da177e4SLinus Torvalds ia64_get_ptce(&ptce_info); 2971da177e4SLinus Torvalds local_cpu_data->ptce_base = ptce_info.base; 2981da177e4SLinus Torvalds local_cpu_data->ptce_count[0] = ptce_info.count[0]; 2991da177e4SLinus Torvalds local_cpu_data->ptce_count[1] = ptce_info.count[1]; 3001da177e4SLinus Torvalds local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; 3011da177e4SLinus Torvalds local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; 3021da177e4SLinus Torvalds 3031da177e4SLinus Torvalds local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ 3041da177e4SLinus Torvalds } 305