1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * This file contains the routines for handling the MMU on those 4 * PowerPC implementations where the MMU is not using the hash 5 * table, such as 8xx, 4xx, BookE's etc... 6 * 7 * Copyright 2008 Ben Herrenschmidt <benh@kernel.crashing.org> 8 * IBM Corp. 9 * 10 * Derived from previous arch/powerpc/mm/mmu_context.c 11 * and arch/powerpc/include/asm/mmu_context.h 12 * 13 * TODO: 14 * 15 * - The global context lock will not scale very well 16 * - The maps should be dynamically allocated to allow for processors 17 * that support more PID bits at runtime 18 * - Implement flush_tlb_mm() by making the context stale and picking 19 * a new one 20 * - More aggressively clear stale map bits and maybe find some way to 21 * also clear mm->cpu_vm_mask bits when processes are migrated 22 */ 23 24 #include <linux/kernel.h> 25 #include <linux/mm.h> 26 #include <linux/init.h> 27 #include <linux/spinlock.h> 28 #include <linux/memblock.h> 29 #include <linux/notifier.h> 30 #include <linux/cpu.h> 31 #include <linux/slab.h> 32 33 #include <asm/mmu_context.h> 34 #include <asm/tlbflush.h> 35 #include <asm/smp.h> 36 37 #include <mm/mmu_decl.h> 38 39 /* 40 * Room for two PTE table pointers, usually the kernel and current user 41 * pointer to their respective root page table (pgdir). 42 */ 43 void *abatron_pteptrs[2]; 44 45 /* 46 * The MPC8xx has only 16 contexts. We rotate through them on each task switch. 47 * A better way would be to keep track of tasks that own contexts, and implement 48 * an LRU usage. That way very active tasks don't always have to pay the TLB 49 * reload overhead. The kernel pages are mapped shared, so the kernel can run on 50 * behalf of any task that makes a kernel entry. Shared does not mean they are 51 * not protected, just that the ASID comparison is not performed. -- Dan 52 * 53 * The IBM4xx has 256 contexts, so we can just rotate through these as a way of 54 * "switching" contexts. If the TID of the TLB is zero, the PID/TID comparison 55 * is disabled, so we can use a TID of zero to represent all kernel pages as 56 * shared among all contexts. -- Dan 57 * 58 * The IBM 47x core supports 16-bit PIDs, thus 65535 contexts. We should 59 * normally never have to steal though the facility is present if needed. 60 * -- BenH 61 */ 62 #define FIRST_CONTEXT 1 63 #if defined(CONFIG_PPC_8xx) 64 #define LAST_CONTEXT 16 65 #elif defined(CONFIG_PPC_47x) 66 #define LAST_CONTEXT 65535 67 #else 68 #define LAST_CONTEXT 255 69 #endif 70 71 static unsigned int next_context, nr_free_contexts; 72 static unsigned long *context_map; 73 static unsigned long *stale_map[NR_CPUS]; 74 static struct mm_struct **context_mm; 75 static DEFINE_RAW_SPINLOCK(context_lock); 76 77 #define CTX_MAP_SIZE \ 78 (sizeof(unsigned long) * (LAST_CONTEXT / BITS_PER_LONG + 1)) 79 80 81 /* Steal a context from a task that has one at the moment. 82 * 83 * This is used when we are running out of available PID numbers 84 * on the processors. 85 * 86 * This isn't an LRU system, it just frees up each context in 87 * turn (sort-of pseudo-random replacement :). This would be the 88 * place to implement an LRU scheme if anyone was motivated to do it. 89 * -- paulus 90 * 91 * For context stealing, we use a slightly different approach for 92 * SMP and UP. Basically, the UP one is simpler and doesn't use 93 * the stale map as we can just flush the local CPU 94 * -- benh 95 */ 96 static unsigned int steal_context_smp(unsigned int id) 97 { 98 struct mm_struct *mm; 99 unsigned int cpu, max, i; 100 101 max = LAST_CONTEXT - FIRST_CONTEXT; 102 103 /* Attempt to free next_context first and then loop until we manage */ 104 while (max--) { 105 /* Pick up the victim mm */ 106 mm = context_mm[id]; 107 108 /* We have a candidate victim, check if it's active, on SMP 109 * we cannot steal active contexts 110 */ 111 if (mm->context.active) { 112 id++; 113 if (id > LAST_CONTEXT) 114 id = FIRST_CONTEXT; 115 continue; 116 } 117 118 /* Mark this mm has having no context anymore */ 119 mm->context.id = MMU_NO_CONTEXT; 120 121 /* Mark it stale on all CPUs that used this mm. For threaded 122 * implementations, we set it on all threads on each core 123 * represented in the mask. A future implementation will use 124 * a core map instead but this will do for now. 125 */ 126 for_each_cpu(cpu, mm_cpumask(mm)) { 127 for (i = cpu_first_thread_sibling(cpu); 128 i <= cpu_last_thread_sibling(cpu); i++) { 129 if (stale_map[i]) 130 __set_bit(id, stale_map[i]); 131 } 132 cpu = i - 1; 133 } 134 return id; 135 } 136 137 /* This will happen if you have more CPUs than available contexts, 138 * all we can do here is wait a bit and try again 139 */ 140 raw_spin_unlock(&context_lock); 141 cpu_relax(); 142 raw_spin_lock(&context_lock); 143 144 /* This will cause the caller to try again */ 145 return MMU_NO_CONTEXT; 146 } 147 148 static unsigned int steal_all_contexts(void) 149 { 150 struct mm_struct *mm; 151 int cpu = smp_processor_id(); 152 unsigned int id; 153 154 for (id = FIRST_CONTEXT; id <= LAST_CONTEXT; id++) { 155 /* Pick up the victim mm */ 156 mm = context_mm[id]; 157 158 /* Mark this mm as having no context anymore */ 159 mm->context.id = MMU_NO_CONTEXT; 160 if (id != FIRST_CONTEXT) { 161 context_mm[id] = NULL; 162 __clear_bit(id, context_map); 163 } 164 if (IS_ENABLED(CONFIG_SMP)) 165 __clear_bit(id, stale_map[cpu]); 166 } 167 168 /* Flush the TLB for all contexts (not to be used on SMP) */ 169 _tlbil_all(); 170 171 nr_free_contexts = LAST_CONTEXT - FIRST_CONTEXT; 172 173 return FIRST_CONTEXT; 174 } 175 176 /* Note that this will also be called on SMP if all other CPUs are 177 * offlined, which means that it may be called for cpu != 0. For 178 * this to work, we somewhat assume that CPUs that are onlined 179 * come up with a fully clean TLB (or are cleaned when offlined) 180 */ 181 static unsigned int steal_context_up(unsigned int id) 182 { 183 struct mm_struct *mm; 184 int cpu = smp_processor_id(); 185 186 /* Pick up the victim mm */ 187 mm = context_mm[id]; 188 189 /* Flush the TLB for that context */ 190 local_flush_tlb_mm(mm); 191 192 /* Mark this mm has having no context anymore */ 193 mm->context.id = MMU_NO_CONTEXT; 194 195 /* XXX This clear should ultimately be part of local_flush_tlb_mm */ 196 if (IS_ENABLED(CONFIG_SMP)) 197 __clear_bit(id, stale_map[cpu]); 198 199 return id; 200 } 201 202 static void set_context(unsigned long id, pgd_t *pgd) 203 { 204 if (IS_ENABLED(CONFIG_PPC_8xx)) { 205 s16 offset = (s16)(__pa(swapper_pg_dir)); 206 207 /* 208 * Register M_TWB will contain base address of level 1 table minus the 209 * lower part of the kernel PGDIR base address, so that all accesses to 210 * level 1 table are done relative to lower part of kernel PGDIR base 211 * address. 212 */ 213 mtspr(SPRN_M_TWB, __pa(pgd) - offset); 214 215 /* Update context */ 216 mtspr(SPRN_M_CASID, id - 1); 217 218 /* sync */ 219 mb(); 220 } else { 221 if (IS_ENABLED(CONFIG_40x)) 222 mb(); /* sync */ 223 224 mtspr(SPRN_PID, id); 225 isync(); 226 } 227 } 228 229 void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, 230 struct task_struct *tsk) 231 { 232 unsigned int id; 233 unsigned int i, cpu = smp_processor_id(); 234 unsigned long *map; 235 236 /* No lockless fast path .. yet */ 237 raw_spin_lock(&context_lock); 238 239 if (IS_ENABLED(CONFIG_SMP)) { 240 /* Mark us active and the previous one not anymore */ 241 next->context.active++; 242 if (prev) { 243 WARN_ON(prev->context.active < 1); 244 prev->context.active--; 245 } 246 } 247 248 again: 249 250 /* If we already have a valid assigned context, skip all that */ 251 id = next->context.id; 252 if (likely(id != MMU_NO_CONTEXT)) 253 goto ctxt_ok; 254 255 /* We really don't have a context, let's try to acquire one */ 256 id = next_context; 257 if (id > LAST_CONTEXT) 258 id = FIRST_CONTEXT; 259 map = context_map; 260 261 /* No more free contexts, let's try to steal one */ 262 if (nr_free_contexts == 0) { 263 if (num_online_cpus() > 1) { 264 id = steal_context_smp(id); 265 if (id == MMU_NO_CONTEXT) 266 goto again; 267 goto stolen; 268 } 269 if (IS_ENABLED(CONFIG_PPC_8xx)) 270 id = steal_all_contexts(); 271 else 272 id = steal_context_up(id); 273 goto stolen; 274 } 275 nr_free_contexts--; 276 277 /* We know there's at least one free context, try to find it */ 278 while (__test_and_set_bit(id, map)) { 279 id = find_next_zero_bit(map, LAST_CONTEXT+1, id); 280 if (id > LAST_CONTEXT) 281 id = FIRST_CONTEXT; 282 } 283 stolen: 284 next_context = id + 1; 285 context_mm[id] = next; 286 next->context.id = id; 287 288 ctxt_ok: 289 290 /* If that context got marked stale on this CPU, then flush the 291 * local TLB for it and unmark it before we use it 292 */ 293 if (IS_ENABLED(CONFIG_SMP) && test_bit(id, stale_map[cpu])) { 294 local_flush_tlb_mm(next); 295 296 /* XXX This clear should ultimately be part of local_flush_tlb_mm */ 297 for (i = cpu_first_thread_sibling(cpu); 298 i <= cpu_last_thread_sibling(cpu); i++) { 299 if (stale_map[i]) 300 __clear_bit(id, stale_map[i]); 301 } 302 } 303 304 /* Flick the MMU and release lock */ 305 if (IS_ENABLED(CONFIG_BDI_SWITCH)) 306 abatron_pteptrs[1] = next->pgd; 307 set_context(id, next->pgd); 308 raw_spin_unlock(&context_lock); 309 } 310 311 /* 312 * Set up the context for a new address space. 313 */ 314 int init_new_context(struct task_struct *t, struct mm_struct *mm) 315 { 316 /* 317 * We have MMU_NO_CONTEXT set to be ~0. Hence check 318 * explicitly against context.id == 0. This ensures that we properly 319 * initialize context slice details for newly allocated mm's (which will 320 * have id == 0) and don't alter context slice inherited via fork (which 321 * will have id != 0). 322 */ 323 if (mm->context.id == 0) 324 slice_init_new_context_exec(mm); 325 mm->context.id = MMU_NO_CONTEXT; 326 mm->context.active = 0; 327 pte_frag_set(&mm->context, NULL); 328 return 0; 329 } 330 331 /* 332 * We're finished using the context for an address space. 333 */ 334 void destroy_context(struct mm_struct *mm) 335 { 336 unsigned long flags; 337 unsigned int id; 338 339 if (mm->context.id == MMU_NO_CONTEXT) 340 return; 341 342 WARN_ON(mm->context.active != 0); 343 344 raw_spin_lock_irqsave(&context_lock, flags); 345 id = mm->context.id; 346 if (id != MMU_NO_CONTEXT) { 347 __clear_bit(id, context_map); 348 mm->context.id = MMU_NO_CONTEXT; 349 context_mm[id] = NULL; 350 nr_free_contexts++; 351 } 352 raw_spin_unlock_irqrestore(&context_lock, flags); 353 } 354 355 static int mmu_ctx_cpu_prepare(unsigned int cpu) 356 { 357 /* We don't touch CPU 0 map, it's allocated at aboot and kept 358 * around forever 359 */ 360 if (cpu == boot_cpuid) 361 return 0; 362 363 stale_map[cpu] = kzalloc(CTX_MAP_SIZE, GFP_KERNEL); 364 return 0; 365 } 366 367 static int mmu_ctx_cpu_dead(unsigned int cpu) 368 { 369 #ifdef CONFIG_HOTPLUG_CPU 370 if (cpu == boot_cpuid) 371 return 0; 372 373 kfree(stale_map[cpu]); 374 stale_map[cpu] = NULL; 375 376 /* We also clear the cpu_vm_mask bits of CPUs going away */ 377 clear_tasks_mm_cpumask(cpu); 378 #endif 379 return 0; 380 } 381 382 /* 383 * Initialize the context management stuff. 384 */ 385 void __init mmu_context_init(void) 386 { 387 /* Mark init_mm as being active on all possible CPUs since 388 * we'll get called with prev == init_mm the first time 389 * we schedule on a given CPU 390 */ 391 init_mm.context.active = NR_CPUS; 392 393 /* 394 * Allocate the maps used by context management 395 */ 396 context_map = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); 397 if (!context_map) 398 panic("%s: Failed to allocate %zu bytes\n", __func__, 399 CTX_MAP_SIZE); 400 context_mm = memblock_alloc(sizeof(void *) * (LAST_CONTEXT + 1), 401 SMP_CACHE_BYTES); 402 if (!context_mm) 403 panic("%s: Failed to allocate %zu bytes\n", __func__, 404 sizeof(void *) * (LAST_CONTEXT + 1)); 405 if (IS_ENABLED(CONFIG_SMP)) { 406 stale_map[boot_cpuid] = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); 407 if (!stale_map[boot_cpuid]) 408 panic("%s: Failed to allocate %zu bytes\n", __func__, 409 CTX_MAP_SIZE); 410 411 cpuhp_setup_state_nocalls(CPUHP_POWERPC_MMU_CTX_PREPARE, 412 "powerpc/mmu/ctx:prepare", 413 mmu_ctx_cpu_prepare, mmu_ctx_cpu_dead); 414 } 415 416 printk(KERN_INFO 417 "MMU: Allocated %zu bytes of context maps for %d contexts\n", 418 2 * CTX_MAP_SIZE + (sizeof(void *) * (LAST_CONTEXT + 1)), 419 LAST_CONTEXT - FIRST_CONTEXT + 1); 420 421 /* 422 * Some processors have too few contexts to reserve one for 423 * init_mm, and require using context 0 for a normal task. 424 * Other processors reserve the use of context zero for the kernel. 425 * This code assumes FIRST_CONTEXT < 32. 426 */ 427 context_map[0] = (1 << FIRST_CONTEXT) - 1; 428 next_context = FIRST_CONTEXT; 429 nr_free_contexts = LAST_CONTEXT - FIRST_CONTEXT + 1; 430 } 431