1 /* 2 * linux/arch/arm/kernel/smp_tlb.c 3 * 4 * Copyright (C) 2002 ARM Limited, All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/preempt.h> 11 #include <linux/smp.h> 12 #include <linux/uaccess.h> 13 14 #include <asm/smp_plat.h> 15 #include <asm/tlbflush.h> 16 #include <asm/mmu_context.h> 17 18 /**********************************************************************/ 19 20 /* 21 * TLB operations 22 */ 23 struct tlb_args { 24 struct vm_area_struct *ta_vma; 25 unsigned long ta_start; 26 unsigned long ta_end; 27 }; 28 29 static inline void ipi_flush_tlb_all(void *ignored) 30 { 31 local_flush_tlb_all(); 32 } 33 34 static inline void ipi_flush_tlb_mm(void *arg) 35 { 36 struct mm_struct *mm = (struct mm_struct *)arg; 37 38 local_flush_tlb_mm(mm); 39 } 40 41 static inline void ipi_flush_tlb_page(void *arg) 42 { 43 struct tlb_args *ta = (struct tlb_args *)arg; 44 unsigned int __ua_flags = uaccess_save_and_enable(); 45 46 local_flush_tlb_page(ta->ta_vma, ta->ta_start); 47 48 uaccess_restore(__ua_flags); 49 } 50 51 static inline void ipi_flush_tlb_kernel_page(void *arg) 52 { 53 struct tlb_args *ta = (struct tlb_args *)arg; 54 55 local_flush_tlb_kernel_page(ta->ta_start); 56 } 57 58 static inline void ipi_flush_tlb_range(void *arg) 59 { 60 struct tlb_args *ta = (struct tlb_args *)arg; 61 unsigned int __ua_flags = uaccess_save_and_enable(); 62 63 local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end); 64 65 uaccess_restore(__ua_flags); 66 } 67 68 static inline void ipi_flush_tlb_kernel_range(void *arg) 69 { 70 struct tlb_args *ta = (struct tlb_args *)arg; 71 72 local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end); 73 } 74 75 static inline void ipi_flush_bp_all(void *ignored) 76 { 77 local_flush_bp_all(); 78 } 79 80 #ifdef CONFIG_ARM_ERRATA_798181 81 bool (*erratum_a15_798181_handler)(void); 82 83 static bool erratum_a15_798181_partial(void) 84 { 85 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0)); 86 dsb(ish); 87 return false; 88 } 89 90 static bool erratum_a15_798181_broadcast(void) 91 { 92 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0)); 93 dsb(ish); 94 return true; 95 } 96 97 void erratum_a15_798181_init(void) 98 { 99 unsigned int midr = read_cpuid_id(); 100 unsigned int revidr = read_cpuid(CPUID_REVIDR); 101 102 /* Brahma-B15 r0p0..r0p2 affected 103 * Cortex-A15 r0p0..r3p3 w/o ECO fix affected 104 * Fixes applied to A15 with respect to the revision and revidr are: 105 * 106 * r0p0-r2p1: No fixes applied 107 * r2p2,r2p3: 108 * REVIDR[4]: 798181 Moving a virtual page that is being accessed 109 * by an active process can lead to unexpected behavior 110 * REVIDR[9]: Not defined 111 * r2p4,r3p0,r3p1,r3p2: 112 * REVIDR[4]: 798181 Moving a virtual page that is being accessed 113 * by an active process can lead to unexpected behavior 114 * REVIDR[9]: 798181 Moving a virtual page that is being accessed 115 * by an active process can lead to unexpected behavior 116 * - This is an update to a previously released ECO. 117 * r3p3: 118 * REVIDR[4]: Reserved 119 * REVIDR[9]: 798181 Moving a virtual page that is being accessed 120 * by an active process can lead to unexpected behavior 121 * - This is an update to a previously released ECO. 122 * 123 * Handling: 124 * REVIDR[9] set -> No WA 125 * REVIDR[4] set, REVIDR[9] cleared -> Partial WA 126 * Both cleared -> Full WA 127 */ 128 if ((midr & 0xff0ffff0) == 0x420f00f0 && midr <= 0x420f00f2) { 129 erratum_a15_798181_handler = erratum_a15_798181_broadcast; 130 } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f2) { 131 erratum_a15_798181_handler = erratum_a15_798181_broadcast; 132 } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f4) { 133 if (revidr & 0x10) 134 erratum_a15_798181_handler = 135 erratum_a15_798181_partial; 136 else 137 erratum_a15_798181_handler = 138 erratum_a15_798181_broadcast; 139 } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x413fc0f3) { 140 if ((revidr & 0x210) == 0) 141 erratum_a15_798181_handler = 142 erratum_a15_798181_broadcast; 143 else if (revidr & 0x10) 144 erratum_a15_798181_handler = 145 erratum_a15_798181_partial; 146 } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x414fc0f0) { 147 if ((revidr & 0x200) == 0) 148 erratum_a15_798181_handler = 149 erratum_a15_798181_partial; 150 } 151 } 152 #endif 153 154 static void ipi_flush_tlb_a15_erratum(void *arg) 155 { 156 dmb(); 157 } 158 159 static void broadcast_tlb_a15_erratum(void) 160 { 161 if (!erratum_a15_798181()) 162 return; 163 164 smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1); 165 } 166 167 static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm) 168 { 169 int this_cpu; 170 cpumask_t mask = { CPU_BITS_NONE }; 171 172 if (!erratum_a15_798181()) 173 return; 174 175 this_cpu = get_cpu(); 176 a15_erratum_get_cpumask(this_cpu, mm, &mask); 177 smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1); 178 put_cpu(); 179 } 180 181 void flush_tlb_all(void) 182 { 183 if (tlb_ops_need_broadcast()) 184 on_each_cpu(ipi_flush_tlb_all, NULL, 1); 185 else 186 __flush_tlb_all(); 187 broadcast_tlb_a15_erratum(); 188 } 189 190 void flush_tlb_mm(struct mm_struct *mm) 191 { 192 if (tlb_ops_need_broadcast()) 193 on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1); 194 else 195 __flush_tlb_mm(mm); 196 broadcast_tlb_mm_a15_erratum(mm); 197 } 198 199 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) 200 { 201 if (tlb_ops_need_broadcast()) { 202 struct tlb_args ta; 203 ta.ta_vma = vma; 204 ta.ta_start = uaddr; 205 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page, 206 &ta, 1); 207 } else 208 __flush_tlb_page(vma, uaddr); 209 broadcast_tlb_mm_a15_erratum(vma->vm_mm); 210 } 211 212 void flush_tlb_kernel_page(unsigned long kaddr) 213 { 214 if (tlb_ops_need_broadcast()) { 215 struct tlb_args ta; 216 ta.ta_start = kaddr; 217 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1); 218 } else 219 __flush_tlb_kernel_page(kaddr); 220 broadcast_tlb_a15_erratum(); 221 } 222 223 void flush_tlb_range(struct vm_area_struct *vma, 224 unsigned long start, unsigned long end) 225 { 226 if (tlb_ops_need_broadcast()) { 227 struct tlb_args ta; 228 ta.ta_vma = vma; 229 ta.ta_start = start; 230 ta.ta_end = end; 231 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range, 232 &ta, 1); 233 } else 234 local_flush_tlb_range(vma, start, end); 235 broadcast_tlb_mm_a15_erratum(vma->vm_mm); 236 } 237 238 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 239 { 240 if (tlb_ops_need_broadcast()) { 241 struct tlb_args ta; 242 ta.ta_start = start; 243 ta.ta_end = end; 244 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); 245 } else 246 local_flush_tlb_kernel_range(start, end); 247 broadcast_tlb_a15_erratum(); 248 } 249 250 void flush_bp_all(void) 251 { 252 if (tlb_ops_need_broadcast()) 253 on_each_cpu(ipi_flush_bp_all, NULL, 1); 254 else 255 __flush_bp_all(); 256 } 257