1 #ifndef _S390_TLBFLUSH_H 2 #define _S390_TLBFLUSH_H 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 #include <asm/processor.h> 7 #include <asm/pgalloc.h> 8 9 /* 10 * Flush all TLB entries on the local CPU. 11 */ 12 static inline void __tlb_flush_local(void) 13 { 14 asm volatile("ptlb" : : : "memory"); 15 } 16 17 /* 18 * Flush TLB entries for a specific ASCE on all CPUs 19 */ 20 static inline void __tlb_flush_idte(unsigned long asce) 21 { 22 /* Global TLB flush for the mm */ 23 asm volatile( 24 " .insn rrf,0xb98e0000,0,%0,%1,0" 25 : : "a" (2048), "a" (asce) : "cc"); 26 } 27 28 /* 29 * Flush TLB entries for a specific ASCE on the local CPU 30 */ 31 static inline void __tlb_flush_idte_local(unsigned long asce) 32 { 33 /* Local TLB flush for the mm */ 34 asm volatile( 35 " .insn rrf,0xb98e0000,0,%0,%1,1" 36 : : "a" (2048), "a" (asce) : "cc"); 37 } 38 39 #ifdef CONFIG_SMP 40 void smp_ptlb_all(void); 41 42 /* 43 * Flush all TLB entries on all CPUs. 44 */ 45 static inline void __tlb_flush_global(void) 46 { 47 register unsigned long reg2 asm("2"); 48 register unsigned long reg3 asm("3"); 49 register unsigned long reg4 asm("4"); 50 long dummy; 51 52 dummy = 0; 53 reg2 = reg3 = 0; 54 reg4 = ((unsigned long) &dummy) + 1; 55 asm volatile( 56 " csp %0,%2" 57 : : "d" (reg2), "d" (reg3), "d" (reg4), "m" (dummy) : "cc" ); 58 } 59 60 /* 61 * Flush TLB entries for a specific mm on all CPUs (in case gmap is used 62 * this implicates multiple ASCEs!). 63 */ 64 static inline void __tlb_flush_full(struct mm_struct *mm) 65 { 66 preempt_disable(); 67 atomic_add(0x10000, &mm->context.attach_count); 68 if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { 69 /* Local TLB flush */ 70 __tlb_flush_local(); 71 } else { 72 /* Global TLB flush */ 73 __tlb_flush_global(); 74 /* Reset TLB flush mask */ 75 if (MACHINE_HAS_TLB_LC) 76 cpumask_copy(mm_cpumask(mm), 77 &mm->context.cpu_attach_mask); 78 } 79 atomic_sub(0x10000, &mm->context.attach_count); 80 preempt_enable(); 81 } 82 83 /* 84 * Flush TLB entries for a specific ASCE on all CPUs. 85 */ 86 static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) 87 { 88 int active, count; 89 90 preempt_disable(); 91 active = (mm == current->active_mm) ? 1 : 0; 92 count = atomic_add_return(0x10000, &mm->context.attach_count); 93 if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active && 94 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { 95 __tlb_flush_idte_local(asce); 96 } else { 97 if (MACHINE_HAS_IDTE) 98 __tlb_flush_idte(asce); 99 else 100 __tlb_flush_global(); 101 /* Reset TLB flush mask */ 102 if (MACHINE_HAS_TLB_LC) 103 cpumask_copy(mm_cpumask(mm), 104 &mm->context.cpu_attach_mask); 105 } 106 atomic_sub(0x10000, &mm->context.attach_count); 107 preempt_enable(); 108 } 109 110 static inline void __tlb_flush_kernel(void) 111 { 112 if (MACHINE_HAS_IDTE) 113 __tlb_flush_idte(init_mm.context.asce); 114 else 115 __tlb_flush_global(); 116 } 117 #else 118 #define __tlb_flush_global() __tlb_flush_local() 119 #define __tlb_flush_full(mm) __tlb_flush_local() 120 121 /* 122 * Flush TLB entries for a specific ASCE on all CPUs. 123 */ 124 static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) 125 { 126 if (MACHINE_HAS_TLB_LC) 127 __tlb_flush_idte_local(asce); 128 else 129 __tlb_flush_local(); 130 } 131 132 static inline void __tlb_flush_kernel(void) 133 { 134 if (MACHINE_HAS_TLB_LC) 135 __tlb_flush_idte_local(init_mm.context.asce); 136 else 137 __tlb_flush_local(); 138 } 139 #endif 140 141 static inline void __tlb_flush_mm(struct mm_struct * mm) 142 { 143 /* 144 * If the machine has IDTE we prefer to do a per mm flush 145 * on all cpus instead of doing a local flush if the mm 146 * only ran on the local cpu. 147 */ 148 if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) 149 __tlb_flush_asce(mm, mm->context.asce); 150 else 151 __tlb_flush_full(mm); 152 } 153 154 static inline void __tlb_flush_mm_lazy(struct mm_struct * mm) 155 { 156 if (mm->context.flush_mm) { 157 __tlb_flush_mm(mm); 158 mm->context.flush_mm = 0; 159 } 160 } 161 162 /* 163 * TLB flushing: 164 * flush_tlb() - flushes the current mm struct TLBs 165 * flush_tlb_all() - flushes all processes TLBs 166 * flush_tlb_mm(mm) - flushes the specified mm context TLB's 167 * flush_tlb_page(vma, vmaddr) - flushes one page 168 * flush_tlb_range(vma, start, end) - flushes a range of pages 169 * flush_tlb_kernel_range(start, end) - flushes a range of kernel pages 170 */ 171 172 /* 173 * flush_tlb_mm goes together with ptep_set_wrprotect for the 174 * copy_page_range operation and flush_tlb_range is related to 175 * ptep_get_and_clear for change_protection. ptep_set_wrprotect and 176 * ptep_get_and_clear do not flush the TLBs directly if the mm has 177 * only one user. At the end of the update the flush_tlb_mm and 178 * flush_tlb_range functions need to do the flush. 179 */ 180 #define flush_tlb() do { } while (0) 181 #define flush_tlb_all() do { } while (0) 182 #define flush_tlb_page(vma, addr) do { } while (0) 183 184 static inline void flush_tlb_mm(struct mm_struct *mm) 185 { 186 __tlb_flush_mm_lazy(mm); 187 } 188 189 static inline void flush_tlb_range(struct vm_area_struct *vma, 190 unsigned long start, unsigned long end) 191 { 192 __tlb_flush_mm_lazy(vma->vm_mm); 193 } 194 195 static inline void flush_tlb_kernel_range(unsigned long start, 196 unsigned long end) 197 { 198 __tlb_flush_kernel(); 199 } 200 201 #endif /* _S390_TLBFLUSH_H */ 202