11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * TLB support routines. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co 51da177e4SLinus Torvalds * David Mosberger-Tang <davidm@hpl.hp.com> 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * 08/02/00 A. Mallick <asit.k.mallick@intel.com> 81da177e4SLinus Torvalds * Modified RID allocation for SMP 91da177e4SLinus Torvalds * Goutham Rao <goutham.rao@intel.com> 101da177e4SLinus Torvalds * IPI based ptc implementation and A-step IPI implementation. 11dcc17d1bSPeter Keilty * Rohit Seth <rohit.seth@intel.com> 12dcc17d1bSPeter Keilty * Ken Chen <kenneth.w.chen@intel.com> 131da177e4SLinus Torvalds */ 141da177e4SLinus Torvalds #include <linux/module.h> 151da177e4SLinus Torvalds #include <linux/init.h> 161da177e4SLinus Torvalds #include <linux/kernel.h> 171da177e4SLinus Torvalds #include <linux/sched.h> 181da177e4SLinus Torvalds #include <linux/smp.h> 191da177e4SLinus Torvalds #include <linux/mm.h> 20dcc17d1bSPeter Keilty #include <linux/bootmem.h> 211da177e4SLinus Torvalds 221da177e4SLinus Torvalds #include <asm/delay.h> 231da177e4SLinus Torvalds #include <asm/mmu_context.h> 241da177e4SLinus Torvalds #include <asm/pgalloc.h> 251da177e4SLinus Torvalds #include <asm/pal.h> 261da177e4SLinus Torvalds #include <asm/tlbflush.h> 27dcc17d1bSPeter Keilty #include <asm/dma.h> 281da177e4SLinus Torvalds 291da177e4SLinus Torvalds static struct { 301da177e4SLinus Torvalds unsigned long mask; /* mask of supported purge page-sizes */ 3158cd9082SChen, Kenneth W unsigned long max_bits; /* log2 of largest supported purge page-size */ 321da177e4SLinus Torvalds } purge; 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds struct ia64_ctx ia64_ctx = { 358737d595SMilind Arun Choudhary .lock = __SPIN_LOCK_UNLOCKED(ia64_ctx.lock), 361da177e4SLinus Torvalds .next = 1, 371da177e4SLinus Torvalds .max_ctx = ~0U 381da177e4SLinus Torvalds }; 391da177e4SLinus Torvalds 401da177e4SLinus Torvalds DEFINE_PER_CPU(u8, ia64_need_tlb_flush); 411da177e4SLinus Torvalds 421da177e4SLinus Torvalds /* 43dcc17d1bSPeter Keilty * Initializes the ia64_ctx.bitmap array based on max_ctx+1. 44dcc17d1bSPeter Keilty * Called after cpu_init() has setup ia64_ctx.max_ctx based on 45dcc17d1bSPeter Keilty * maximum RID that is supported by boot CPU. 46dcc17d1bSPeter Keilty */ 47dcc17d1bSPeter Keilty void __init 48dcc17d1bSPeter Keilty mmu_context_init (void) 49dcc17d1bSPeter Keilty { 50dcc17d1bSPeter Keilty ia64_ctx.bitmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 51dcc17d1bSPeter Keilty ia64_ctx.flushmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 52dcc17d1bSPeter Keilty } 53dcc17d1bSPeter Keilty 54dcc17d1bSPeter Keilty /* 551da177e4SLinus Torvalds * Acquire the ia64_ctx.lock before calling this function! 561da177e4SLinus Torvalds */ 571da177e4SLinus Torvalds void 581da177e4SLinus Torvalds wrap_mmu_context (struct mm_struct *mm) 591da177e4SLinus Torvalds { 6058cd9082SChen, Kenneth W int i, cpu; 61dcc17d1bSPeter Keilty unsigned long flush_bit; 621da177e4SLinus Torvalds 63dcc17d1bSPeter Keilty for (i=0; i <= ia64_ctx.max_ctx / BITS_PER_LONG; i++) { 64dcc17d1bSPeter Keilty flush_bit = xchg(&ia64_ctx.flushmap[i], 0); 65dcc17d1bSPeter Keilty ia64_ctx.bitmap[i] ^= flush_bit; 66dcc17d1bSPeter Keilty } 671da177e4SLinus Torvalds 68dcc17d1bSPeter Keilty /* use offset at 300 to skip daemons */ 69dcc17d1bSPeter Keilty ia64_ctx.next = find_next_zero_bit(ia64_ctx.bitmap, 70dcc17d1bSPeter Keilty ia64_ctx.max_ctx, 300); 71dcc17d1bSPeter Keilty ia64_ctx.limit = find_next_bit(ia64_ctx.bitmap, 72dcc17d1bSPeter Keilty ia64_ctx.max_ctx, ia64_ctx.next); 731da177e4SLinus Torvalds 7458cd9082SChen, Kenneth W /* 7558cd9082SChen, Kenneth W * can't call flush_tlb_all() here because of race condition 7658cd9082SChen, Kenneth W * with O(1) scheduler [EF] 7758cd9082SChen, Kenneth W */ 7858cd9082SChen, Kenneth W cpu = get_cpu(); /* prevent preemption/migration */ 7958cd9082SChen, Kenneth W for_each_online_cpu(i) 80dc565b52Shawkes@sgi.com if (i != cpu) 811da177e4SLinus Torvalds per_cpu(ia64_need_tlb_flush, i) = 1; 821da177e4SLinus Torvalds put_cpu(); 831da177e4SLinus Torvalds local_flush_tlb_all(); 841da177e4SLinus Torvalds } 851da177e4SLinus Torvalds 861da177e4SLinus Torvalds void 8758cd9082SChen, Kenneth W ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start, 8858cd9082SChen, Kenneth W unsigned long end, unsigned long nbits) 891da177e4SLinus Torvalds { 901da177e4SLinus Torvalds static DEFINE_SPINLOCK(ptcg_lock); 911da177e4SLinus Torvalds 92cfbb1426SJack Steiner if (mm != current->active_mm || !current->mm) { 93c1902aaeSDean Roe flush_tlb_all(); 94c1902aaeSDean Roe return; 95c1902aaeSDean Roe } 96c1902aaeSDean Roe 971da177e4SLinus Torvalds /* HW requires global serialization of ptc.ga. */ 981da177e4SLinus Torvalds spin_lock(&ptcg_lock); 991da177e4SLinus Torvalds { 1001da177e4SLinus Torvalds do { 1011da177e4SLinus Torvalds /* 1021da177e4SLinus Torvalds * Flush ALAT entries also. 1031da177e4SLinus Torvalds */ 1041da177e4SLinus Torvalds ia64_ptcga(start, (nbits<<2)); 1051da177e4SLinus Torvalds ia64_srlz_i(); 1061da177e4SLinus Torvalds start += (1UL << nbits); 1071da177e4SLinus Torvalds } while (start < end); 1081da177e4SLinus Torvalds } 1091da177e4SLinus Torvalds spin_unlock(&ptcg_lock); 1101da177e4SLinus Torvalds } 1111da177e4SLinus Torvalds 1121da177e4SLinus Torvalds void 1131da177e4SLinus Torvalds local_flush_tlb_all (void) 1141da177e4SLinus Torvalds { 1151da177e4SLinus Torvalds unsigned long i, j, flags, count0, count1, stride0, stride1, addr; 1161da177e4SLinus Torvalds 1171da177e4SLinus Torvalds addr = local_cpu_data->ptce_base; 1181da177e4SLinus Torvalds count0 = local_cpu_data->ptce_count[0]; 1191da177e4SLinus Torvalds count1 = local_cpu_data->ptce_count[1]; 1201da177e4SLinus Torvalds stride0 = local_cpu_data->ptce_stride[0]; 1211da177e4SLinus Torvalds stride1 = local_cpu_data->ptce_stride[1]; 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds local_irq_save(flags); 1241da177e4SLinus Torvalds for (i = 0; i < count0; ++i) { 1251da177e4SLinus Torvalds for (j = 0; j < count1; ++j) { 1261da177e4SLinus Torvalds ia64_ptce(addr); 1271da177e4SLinus Torvalds addr += stride1; 1281da177e4SLinus Torvalds } 1291da177e4SLinus Torvalds addr += stride0; 1301da177e4SLinus Torvalds } 1311da177e4SLinus Torvalds local_irq_restore(flags); 1321da177e4SLinus Torvalds ia64_srlz_i(); /* srlz.i implies srlz.d */ 1331da177e4SLinus Torvalds } 1341da177e4SLinus Torvalds 1351da177e4SLinus Torvalds void 13658cd9082SChen, Kenneth W flush_tlb_range (struct vm_area_struct *vma, unsigned long start, 13758cd9082SChen, Kenneth W unsigned long end) 1381da177e4SLinus Torvalds { 1391da177e4SLinus Torvalds struct mm_struct *mm = vma->vm_mm; 1401da177e4SLinus Torvalds unsigned long size = end - start; 1411da177e4SLinus Torvalds unsigned long nbits; 1421da177e4SLinus Torvalds 143c1902aaeSDean Roe #ifndef CONFIG_SMP 1441da177e4SLinus Torvalds if (mm != current->active_mm) { 1451da177e4SLinus Torvalds mm->context = 0; 1461da177e4SLinus Torvalds return; 1471da177e4SLinus Torvalds } 148c1902aaeSDean Roe #endif 1491da177e4SLinus Torvalds 1501da177e4SLinus Torvalds nbits = ia64_fls(size + 0xfff); 15158cd9082SChen, Kenneth W while (unlikely (((1UL << nbits) & purge.mask) == 0) && 15258cd9082SChen, Kenneth W (nbits < purge.max_bits)) 1531da177e4SLinus Torvalds ++nbits; 1541da177e4SLinus Torvalds if (nbits > purge.max_bits) 1551da177e4SLinus Torvalds nbits = purge.max_bits; 1561da177e4SLinus Torvalds start &= ~((1UL << nbits) - 1); 1571da177e4SLinus Torvalds 158663b97f7SHugh Dickins preempt_disable(); 159ce9eed5aSChen, Kenneth W #ifdef CONFIG_SMP 160ce9eed5aSChen, Kenneth W if (mm != current->active_mm || cpus_weight(mm->cpu_vm_mask) != 1) { 161ce9eed5aSChen, Kenneth W platform_global_tlb_purge(mm, start, end, nbits); 162ce9eed5aSChen, Kenneth W preempt_enable(); 163ce9eed5aSChen, Kenneth W return; 164ce9eed5aSChen, Kenneth W } 165ce9eed5aSChen, Kenneth W #endif 1661da177e4SLinus Torvalds do { 1671da177e4SLinus Torvalds ia64_ptcl(start, (nbits<<2)); 1681da177e4SLinus Torvalds start += (1UL << nbits); 1691da177e4SLinus Torvalds } while (start < end); 170663b97f7SHugh Dickins preempt_enable(); 1711da177e4SLinus Torvalds ia64_srlz_i(); /* srlz.i implies srlz.d */ 1721da177e4SLinus Torvalds } 1731da177e4SLinus Torvalds EXPORT_SYMBOL(flush_tlb_range); 1741da177e4SLinus Torvalds 1751da177e4SLinus Torvalds void __devinit 1761da177e4SLinus Torvalds ia64_tlb_init (void) 1771da177e4SLinus Torvalds { 178256a7e09SJes Sorensen ia64_ptce_info_t uninitialized_var(ptce_info); /* GCC be quiet */ 1791da177e4SLinus Torvalds unsigned long tr_pgbits; 1801da177e4SLinus Torvalds long status; 1811da177e4SLinus Torvalds 1821da177e4SLinus Torvalds if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { 1831da177e4SLinus Torvalds printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld;" 1841da177e4SLinus Torvalds "defaulting to architected purge page-sizes.\n", status); 1851da177e4SLinus Torvalds purge.mask = 0x115557000UL; 1861da177e4SLinus Torvalds } 1871da177e4SLinus Torvalds purge.max_bits = ia64_fls(purge.mask); 1881da177e4SLinus Torvalds 1891da177e4SLinus Torvalds ia64_get_ptce(&ptce_info); 1901da177e4SLinus Torvalds local_cpu_data->ptce_base = ptce_info.base; 1911da177e4SLinus Torvalds local_cpu_data->ptce_count[0] = ptce_info.count[0]; 1921da177e4SLinus Torvalds local_cpu_data->ptce_count[1] = ptce_info.count[1]; 1931da177e4SLinus Torvalds local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; 1941da177e4SLinus Torvalds local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; 1951da177e4SLinus Torvalds 1961da177e4SLinus Torvalds local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ 1971da177e4SLinus Torvalds } 198