11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * TLB support routines. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co 51da177e4SLinus Torvalds * David Mosberger-Tang <davidm@hpl.hp.com> 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * 08/02/00 A. Mallick <asit.k.mallick@intel.com> 81da177e4SLinus Torvalds * Modified RID allocation for SMP 91da177e4SLinus Torvalds * Goutham Rao <goutham.rao@intel.com> 101da177e4SLinus Torvalds * IPI based ptc implementation and A-step IPI implementation. 11dcc17d1bSPeter Keilty * Rohit Seth <rohit.seth@intel.com> 12dcc17d1bSPeter Keilty * Ken Chen <kenneth.w.chen@intel.com> 131da177e4SLinus Torvalds */ 141da177e4SLinus Torvalds #include <linux/config.h> 151da177e4SLinus Torvalds #include <linux/module.h> 161da177e4SLinus Torvalds #include <linux/init.h> 171da177e4SLinus Torvalds #include <linux/kernel.h> 181da177e4SLinus Torvalds #include <linux/sched.h> 191da177e4SLinus Torvalds #include <linux/smp.h> 201da177e4SLinus Torvalds #include <linux/mm.h> 21dcc17d1bSPeter Keilty #include <linux/bootmem.h> 221da177e4SLinus Torvalds 231da177e4SLinus Torvalds #include <asm/delay.h> 241da177e4SLinus Torvalds #include <asm/mmu_context.h> 251da177e4SLinus Torvalds #include <asm/pgalloc.h> 261da177e4SLinus Torvalds #include <asm/pal.h> 271da177e4SLinus Torvalds #include <asm/tlbflush.h> 28dcc17d1bSPeter Keilty #include <asm/dma.h> 291da177e4SLinus Torvalds 301da177e4SLinus Torvalds static struct { 311da177e4SLinus Torvalds unsigned long mask; /* mask of supported purge page-sizes */ 3258cd9082SChen, Kenneth W unsigned long max_bits; /* log2 of largest supported purge page-size */ 331da177e4SLinus Torvalds } purge; 341da177e4SLinus Torvalds 351da177e4SLinus Torvalds struct ia64_ctx ia64_ctx = { 361da177e4SLinus Torvalds .lock = SPIN_LOCK_UNLOCKED, 371da177e4SLinus Torvalds .next = 1, 381da177e4SLinus Torvalds .max_ctx = ~0U 391da177e4SLinus Torvalds }; 401da177e4SLinus Torvalds 411da177e4SLinus Torvalds DEFINE_PER_CPU(u8, ia64_need_tlb_flush); 421da177e4SLinus Torvalds 431da177e4SLinus Torvalds /* 44dcc17d1bSPeter Keilty * Initializes the ia64_ctx.bitmap array based on max_ctx+1. 45dcc17d1bSPeter Keilty * Called after cpu_init() has setup ia64_ctx.max_ctx based on 46dcc17d1bSPeter Keilty * maximum RID that is supported by boot CPU. 47dcc17d1bSPeter Keilty */ 48dcc17d1bSPeter Keilty void __init 49dcc17d1bSPeter Keilty mmu_context_init (void) 50dcc17d1bSPeter Keilty { 51dcc17d1bSPeter Keilty ia64_ctx.bitmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 52dcc17d1bSPeter Keilty ia64_ctx.flushmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 53dcc17d1bSPeter Keilty } 54dcc17d1bSPeter Keilty 55dcc17d1bSPeter Keilty /* 561da177e4SLinus Torvalds * Acquire the ia64_ctx.lock before calling this function! 571da177e4SLinus Torvalds */ 581da177e4SLinus Torvalds void 591da177e4SLinus Torvalds wrap_mmu_context (struct mm_struct *mm) 601da177e4SLinus Torvalds { 6158cd9082SChen, Kenneth W int i, cpu; 62dcc17d1bSPeter Keilty unsigned long flush_bit; 631da177e4SLinus Torvalds 64dcc17d1bSPeter Keilty for (i=0; i <= ia64_ctx.max_ctx / BITS_PER_LONG; i++) { 65dcc17d1bSPeter Keilty flush_bit = xchg(&ia64_ctx.flushmap[i], 0); 66dcc17d1bSPeter Keilty ia64_ctx.bitmap[i] ^= flush_bit; 67dcc17d1bSPeter Keilty } 681da177e4SLinus Torvalds 69dcc17d1bSPeter Keilty /* use offset at 300 to skip daemons */ 70dcc17d1bSPeter Keilty ia64_ctx.next = find_next_zero_bit(ia64_ctx.bitmap, 71dcc17d1bSPeter Keilty ia64_ctx.max_ctx, 300); 72dcc17d1bSPeter Keilty ia64_ctx.limit = find_next_bit(ia64_ctx.bitmap, 73dcc17d1bSPeter Keilty ia64_ctx.max_ctx, ia64_ctx.next); 741da177e4SLinus Torvalds 7558cd9082SChen, Kenneth W /* 7658cd9082SChen, Kenneth W * can't call flush_tlb_all() here because of race condition 7758cd9082SChen, Kenneth W * with O(1) scheduler [EF] 7858cd9082SChen, Kenneth W */ 7958cd9082SChen, Kenneth W cpu = get_cpu(); /* prevent preemption/migration */ 8058cd9082SChen, Kenneth W for_each_online_cpu(i) 81dc565b52Shawkes@sgi.com if (i != cpu) 821da177e4SLinus Torvalds per_cpu(ia64_need_tlb_flush, i) = 1; 831da177e4SLinus Torvalds put_cpu(); 841da177e4SLinus Torvalds local_flush_tlb_all(); 851da177e4SLinus Torvalds } 861da177e4SLinus Torvalds 871da177e4SLinus Torvalds void 8858cd9082SChen, Kenneth W ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start, 8958cd9082SChen, Kenneth W unsigned long end, unsigned long nbits) 901da177e4SLinus Torvalds { 911da177e4SLinus Torvalds static DEFINE_SPINLOCK(ptcg_lock); 921da177e4SLinus Torvalds 93cfbb1426SJack Steiner if (mm != current->active_mm || !current->mm) { 94c1902aaeSDean Roe flush_tlb_all(); 95c1902aaeSDean Roe return; 96c1902aaeSDean Roe } 97c1902aaeSDean Roe 981da177e4SLinus Torvalds /* HW requires global serialization of ptc.ga. */ 991da177e4SLinus Torvalds spin_lock(&ptcg_lock); 1001da177e4SLinus Torvalds { 1011da177e4SLinus Torvalds do { 1021da177e4SLinus Torvalds /* 1031da177e4SLinus Torvalds * Flush ALAT entries also. 1041da177e4SLinus Torvalds */ 1051da177e4SLinus Torvalds ia64_ptcga(start, (nbits<<2)); 1061da177e4SLinus Torvalds ia64_srlz_i(); 1071da177e4SLinus Torvalds start += (1UL << nbits); 1081da177e4SLinus Torvalds } while (start < end); 1091da177e4SLinus Torvalds } 1101da177e4SLinus Torvalds spin_unlock(&ptcg_lock); 1111da177e4SLinus Torvalds } 1121da177e4SLinus Torvalds 1131da177e4SLinus Torvalds void 1141da177e4SLinus Torvalds local_flush_tlb_all (void) 1151da177e4SLinus Torvalds { 1161da177e4SLinus Torvalds unsigned long i, j, flags, count0, count1, stride0, stride1, addr; 1171da177e4SLinus Torvalds 1181da177e4SLinus Torvalds addr = local_cpu_data->ptce_base; 1191da177e4SLinus Torvalds count0 = local_cpu_data->ptce_count[0]; 1201da177e4SLinus Torvalds count1 = local_cpu_data->ptce_count[1]; 1211da177e4SLinus Torvalds stride0 = local_cpu_data->ptce_stride[0]; 1221da177e4SLinus Torvalds stride1 = local_cpu_data->ptce_stride[1]; 1231da177e4SLinus Torvalds 1241da177e4SLinus Torvalds local_irq_save(flags); 1251da177e4SLinus Torvalds for (i = 0; i < count0; ++i) { 1261da177e4SLinus Torvalds for (j = 0; j < count1; ++j) { 1271da177e4SLinus Torvalds ia64_ptce(addr); 1281da177e4SLinus Torvalds addr += stride1; 1291da177e4SLinus Torvalds } 1301da177e4SLinus Torvalds addr += stride0; 1311da177e4SLinus Torvalds } 1321da177e4SLinus Torvalds local_irq_restore(flags); 1331da177e4SLinus Torvalds ia64_srlz_i(); /* srlz.i implies srlz.d */ 1341da177e4SLinus Torvalds } 1351da177e4SLinus Torvalds 1361da177e4SLinus Torvalds void 13758cd9082SChen, Kenneth W flush_tlb_range (struct vm_area_struct *vma, unsigned long start, 13858cd9082SChen, Kenneth W unsigned long end) 1391da177e4SLinus Torvalds { 1401da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 1411da177e4SLinus Torvalds unsigned long size = end - start; 1421da177e4SLinus Torvalds unsigned long nbits; 1431da177e4SLinus Torvalds 144c1902aaeSDean Roe #ifndef CONFIG_SMP 1451da177e4SLinus Torvalds if (mm != current->active_mm) { 1461da177e4SLinus Torvalds mm->context = 0; 1471da177e4SLinus Torvalds return; 1481da177e4SLinus Torvalds } 149c1902aaeSDean Roe #endif 1501da177e4SLinus Torvalds 1511da177e4SLinus Torvalds nbits = ia64_fls(size + 0xfff); 15258cd9082SChen, Kenneth W while (unlikely (((1UL << nbits) & purge.mask) == 0) && 15358cd9082SChen, Kenneth W (nbits < purge.max_bits)) 1541da177e4SLinus Torvalds ++nbits; 1551da177e4SLinus Torvalds if (nbits > purge.max_bits) 1561da177e4SLinus Torvalds nbits = purge.max_bits; 1571da177e4SLinus Torvalds start &= ~((1UL << nbits) - 1); 1581da177e4SLinus Torvalds 1591da177e4SLinus Torvalds # ifdef CONFIG_SMP 160c1902aaeSDean Roe platform_global_tlb_purge(mm, start, end, nbits); 1611da177e4SLinus Torvalds # else 162663b97f7SHugh Dickins preempt_disable(); 1631da177e4SLinus Torvalds do { 1641da177e4SLinus Torvalds ia64_ptcl(start, (nbits<<2)); 1651da177e4SLinus Torvalds start += (1UL << nbits); 1661da177e4SLinus Torvalds } while (start < end); 167663b97f7SHugh Dickins preempt_enable(); 1681da177e4SLinus Torvalds # endif 1691da177e4SLinus Torvalds 1701da177e4SLinus Torvalds ia64_srlz_i(); /* srlz.i implies srlz.d */ 1711da177e4SLinus Torvalds } 1721da177e4SLinus Torvalds EXPORT_SYMBOL(flush_tlb_range); 1731da177e4SLinus Torvalds 1741da177e4SLinus Torvalds void __devinit 1751da177e4SLinus Torvalds ia64_tlb_init (void) 1761da177e4SLinus Torvalds { 1771da177e4SLinus Torvalds ia64_ptce_info_t ptce_info; 1781da177e4SLinus Torvalds unsigned long tr_pgbits; 1791da177e4SLinus Torvalds long status; 1801da177e4SLinus Torvalds 1811da177e4SLinus Torvalds if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { 1821da177e4SLinus Torvalds printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld;" 1831da177e4SLinus Torvalds "defaulting to architected purge page-sizes.\n", status); 1841da177e4SLinus Torvalds purge.mask = 0x115557000UL; 1851da177e4SLinus Torvalds } 1861da177e4SLinus Torvalds purge.max_bits = ia64_fls(purge.mask); 1871da177e4SLinus Torvalds 1881da177e4SLinus Torvalds ia64_get_ptce(&ptce_info); 1891da177e4SLinus Torvalds local_cpu_data->ptce_base = ptce_info.base; 1901da177e4SLinus Torvalds local_cpu_data->ptce_count[0] = ptce_info.count[0]; 1911da177e4SLinus Torvalds local_cpu_data->ptce_count[1] = ptce_info.count[1]; 1921da177e4SLinus Torvalds local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; 1931da177e4SLinus Torvalds local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ 1961da177e4SLinus Torvalds } 197