1 /* 2 * TLB support routines. 3 * 4 * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co 5 * David Mosberger-Tang <davidm@hpl.hp.com> 6 * 7 * 08/02/00 A. Mallick <asit.k.mallick@intel.com> 8 * Modified RID allocation for SMP 9 * Goutham Rao <goutham.rao@intel.com> 10 * IPI based ptc implementation and A-step IPI implementation. 11 * Rohit Seth <rohit.seth@intel.com> 12 * Ken Chen <kenneth.w.chen@intel.com> 13 */ 14 #include <linux/module.h> 15 #include <linux/init.h> 16 #include <linux/kernel.h> 17 #include <linux/sched.h> 18 #include <linux/smp.h> 19 #include <linux/mm.h> 20 #include <linux/bootmem.h> 21 22 #include <asm/delay.h> 23 #include <asm/mmu_context.h> 24 #include <asm/pgalloc.h> 25 #include <asm/pal.h> 26 #include <asm/tlbflush.h> 27 #include <asm/dma.h> 28 29 static struct { 30 unsigned long mask; /* mask of supported purge page-sizes */ 31 unsigned long max_bits; /* log2 of largest supported purge page-size */ 32 } purge; 33 34 struct ia64_ctx ia64_ctx = { 35 .lock = __SPIN_LOCK_UNLOCKED(ia64_ctx.lock), 36 .next = 1, 37 .max_ctx = ~0U 38 }; 39 40 DEFINE_PER_CPU(u8, ia64_need_tlb_flush); 41 42 /* 43 * Initializes the ia64_ctx.bitmap array based on max_ctx+1. 44 * Called after cpu_init() has setup ia64_ctx.max_ctx based on 45 * maximum RID that is supported by boot CPU. 46 */ 47 void __init 48 mmu_context_init (void) 49 { 50 ia64_ctx.bitmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 51 ia64_ctx.flushmap = alloc_bootmem((ia64_ctx.max_ctx+1)>>3); 52 } 53 54 /* 55 * Acquire the ia64_ctx.lock before calling this function! 56 */ 57 void 58 wrap_mmu_context (struct mm_struct *mm) 59 { 60 int i, cpu; 61 unsigned long flush_bit; 62 63 for (i=0; i <= ia64_ctx.max_ctx / BITS_PER_LONG; i++) { 64 flush_bit = xchg(&ia64_ctx.flushmap[i], 0); 65 ia64_ctx.bitmap[i] ^= flush_bit; 66 } 67 68 /* use offset at 300 to skip daemons */ 69 ia64_ctx.next = find_next_zero_bit(ia64_ctx.bitmap, 70 ia64_ctx.max_ctx, 300); 71 ia64_ctx.limit = find_next_bit(ia64_ctx.bitmap, 72 ia64_ctx.max_ctx, ia64_ctx.next); 73 74 /* 75 * can't call flush_tlb_all() here because of race condition 76 * with O(1) scheduler [EF] 77 */ 78 cpu = get_cpu(); /* prevent preemption/migration */ 79 for_each_online_cpu(i) 80 if (i != cpu) 81 per_cpu(ia64_need_tlb_flush, i) = 1; 82 put_cpu(); 83 local_flush_tlb_all(); 84 } 85 86 void 87 ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start, 88 unsigned long end, unsigned long nbits) 89 { 90 static DEFINE_SPINLOCK(ptcg_lock); 91 92 if (mm != current->active_mm || !current->mm) { 93 flush_tlb_all(); 94 return; 95 } 96 97 /* HW requires global serialization of ptc.ga. */ 98 spin_lock(&ptcg_lock); 99 { 100 do { 101 /* 102 * Flush ALAT entries also. 103 */ 104 ia64_ptcga(start, (nbits<<2)); 105 ia64_srlz_i(); 106 start += (1UL << nbits); 107 } while (start < end); 108 } 109 spin_unlock(&ptcg_lock); 110 } 111 112 void 113 local_flush_tlb_all (void) 114 { 115 unsigned long i, j, flags, count0, count1, stride0, stride1, addr; 116 117 addr = local_cpu_data->ptce_base; 118 count0 = local_cpu_data->ptce_count[0]; 119 count1 = local_cpu_data->ptce_count[1]; 120 stride0 = local_cpu_data->ptce_stride[0]; 121 stride1 = local_cpu_data->ptce_stride[1]; 122 123 local_irq_save(flags); 124 for (i = 0; i < count0; ++i) { 125 for (j = 0; j < count1; ++j) { 126 ia64_ptce(addr); 127 addr += stride1; 128 } 129 addr += stride0; 130 } 131 local_irq_restore(flags); 132 ia64_srlz_i(); /* srlz.i implies srlz.d */ 133 } 134 135 void 136 flush_tlb_range (struct vm_area_struct *vma, unsigned long start, 137 unsigned long end) 138 { 139 struct mm_struct *mm = vma->vm_mm; 140 unsigned long size = end - start; 141 unsigned long nbits; 142 143 #ifndef CONFIG_SMP 144 if (mm != current->active_mm) { 145 mm->context = 0; 146 return; 147 } 148 #endif 149 150 nbits = ia64_fls(size + 0xfff); 151 while (unlikely (((1UL << nbits) & purge.mask) == 0) && 152 (nbits < purge.max_bits)) 153 ++nbits; 154 if (nbits > purge.max_bits) 155 nbits = purge.max_bits; 156 start &= ~((1UL << nbits) - 1); 157 158 preempt_disable(); 159 #ifdef CONFIG_SMP 160 if (mm != current->active_mm || cpus_weight(mm->cpu_vm_mask) != 1) { 161 platform_global_tlb_purge(mm, start, end, nbits); 162 preempt_enable(); 163 return; 164 } 165 #endif 166 do { 167 ia64_ptcl(start, (nbits<<2)); 168 start += (1UL << nbits); 169 } while (start < end); 170 preempt_enable(); 171 ia64_srlz_i(); /* srlz.i implies srlz.d */ 172 } 173 EXPORT_SYMBOL(flush_tlb_range); 174 175 void __devinit 176 ia64_tlb_init (void) 177 { 178 ia64_ptce_info_t uninitialized_var(ptce_info); /* GCC be quiet */ 179 unsigned long tr_pgbits; 180 long status; 181 182 if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { 183 printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld;" 184 "defaulting to architected purge page-sizes.\n", status); 185 purge.mask = 0x115557000UL; 186 } 187 purge.max_bits = ia64_fls(purge.mask); 188 189 ia64_get_ptce(&ptce_info); 190 local_cpu_data->ptce_base = ptce_info.base; 191 local_cpu_data->ptce_count[0] = ptce_info.count[0]; 192 local_cpu_data->ptce_count[1] = ptce_info.count[1]; 193 local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; 194 local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; 195 196 local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ 197 } 198