1 #include <linux/init.h> 2 3 #include <linux/mm.h> 4 #include <linux/spinlock.h> 5 #include <linux/smp.h> 6 #include <linux/interrupt.h> 7 #include <linux/export.h> 8 #include <linux/cpu.h> 9 #include <linux/debugfs.h> 10 11 #include <asm/tlbflush.h> 12 #include <asm/mmu_context.h> 13 #include <asm/nospec-branch.h> 14 #include <asm/cache.h> 15 #include <asm/apic.h> 16 #include <asm/uv/uv.h> 17 18 /* 19 * TLB flushing, formerly SMP-only 20 * c/o Linus Torvalds. 21 * 22 * These mean you can really definitely utterly forget about 23 * writing to user space from interrupts. (Its not allowed anyway). 24 * 25 * Optimizations Manfred Spraul <manfred@colorfullife.com> 26 * 27 * More scalable flush, from Andi Kleen 28 * 29 * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi 30 */ 31 32 /* 33 * We get here when we do something requiring a TLB invalidation 34 * but could not go invalidate all of the contexts. We do the 35 * necessary invalidation by clearing out the 'ctx_id' which 36 * forces a TLB flush when the context is loaded. 37 */ 38 static void clear_asid_other(void) 39 { 40 u16 asid; 41 42 /* 43 * This is only expected to be set if we have disabled 44 * kernel _PAGE_GLOBAL pages. 45 */ 46 if (!static_cpu_has(X86_FEATURE_PTI)) { 47 WARN_ON_ONCE(1); 48 return; 49 } 50 51 for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) { 52 /* Do not need to flush the current asid */ 53 if (asid == this_cpu_read(cpu_tlbstate.loaded_mm_asid)) 54 continue; 55 /* 56 * Make sure the next time we go to switch to 57 * this asid, we do a flush: 58 */ 59 this_cpu_write(cpu_tlbstate.ctxs[asid].ctx_id, 0); 60 } 61 this_cpu_write(cpu_tlbstate.invalidate_other, false); 62 } 63 64 atomic64_t last_mm_ctx_id = ATOMIC64_INIT(1); 65 66 67 static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, 68 u16 *new_asid, bool *need_flush) 69 { 70 u16 asid; 71 72 if (!static_cpu_has(X86_FEATURE_PCID)) { 73 *new_asid = 0; 74 *need_flush = true; 75 return; 76 } 77 78 if (this_cpu_read(cpu_tlbstate.invalidate_other)) 79 clear_asid_other(); 80 81 for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) { 82 if (this_cpu_read(cpu_tlbstate.ctxs[asid].ctx_id) != 83 next->context.ctx_id) 84 continue; 85 86 *new_asid = asid; 87 *need_flush = (this_cpu_read(cpu_tlbstate.ctxs[asid].tlb_gen) < 88 next_tlb_gen); 89 return; 90 } 91 92 /* 93 * We don't currently own an ASID slot on this CPU. 94 * Allocate a slot. 95 */ 96 *new_asid = this_cpu_add_return(cpu_tlbstate.next_asid, 1) - 1; 97 if (*new_asid >= TLB_NR_DYN_ASIDS) { 98 *new_asid = 0; 99 this_cpu_write(cpu_tlbstate.next_asid, 1); 100 } 101 *need_flush = true; 102 } 103 104 static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush) 105 { 106 unsigned long new_mm_cr3; 107 108 if (need_flush) { 109 invalidate_user_asid(new_asid); 110 new_mm_cr3 = build_cr3(pgdir, new_asid); 111 } else { 112 new_mm_cr3 = build_cr3_noflush(pgdir, new_asid); 113 } 114 115 /* 116 * Caution: many callers of this function expect 117 * that load_cr3() is serializing and orders TLB 118 * fills with respect to the mm_cpumask writes. 119 */ 120 write_cr3(new_mm_cr3); 121 } 122 123 void leave_mm(int cpu) 124 { 125 struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm); 126 127 /* 128 * It's plausible that we're in lazy TLB mode while our mm is init_mm. 129 * If so, our callers still expect us to flush the TLB, but there 130 * aren't any user TLB entries in init_mm to worry about. 131 * 132 * This needs to happen before any other sanity checks due to 133 * intel_idle's shenanigans. 134 */ 135 if (loaded_mm == &init_mm) 136 return; 137 138 /* Warn if we're not lazy. */ 139 WARN_ON(!this_cpu_read(cpu_tlbstate.is_lazy)); 140 141 switch_mm(NULL, &init_mm, NULL); 142 } 143 EXPORT_SYMBOL_GPL(leave_mm); 144 145 void switch_mm(struct mm_struct *prev, struct mm_struct *next, 146 struct task_struct *tsk) 147 { 148 unsigned long flags; 149 150 local_irq_save(flags); 151 switch_mm_irqs_off(prev, next, tsk); 152 local_irq_restore(flags); 153 } 154 155 static void sync_current_stack_to_mm(struct mm_struct *mm) 156 { 157 unsigned long sp = current_stack_pointer; 158 pgd_t *pgd = pgd_offset(mm, sp); 159 160 if (pgtable_l5_enabled()) { 161 if (unlikely(pgd_none(*pgd))) { 162 pgd_t *pgd_ref = pgd_offset_k(sp); 163 164 set_pgd(pgd, *pgd_ref); 165 } 166 } else { 167 /* 168 * "pgd" is faked. The top level entries are "p4d"s, so sync 169 * the p4d. This compiles to approximately the same code as 170 * the 5-level case. 171 */ 172 p4d_t *p4d = p4d_offset(pgd, sp); 173 174 if (unlikely(p4d_none(*p4d))) { 175 pgd_t *pgd_ref = pgd_offset_k(sp); 176 p4d_t *p4d_ref = p4d_offset(pgd_ref, sp); 177 178 set_p4d(p4d, *p4d_ref); 179 } 180 } 181 } 182 183 void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, 184 struct task_struct *tsk) 185 { 186 struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm); 187 u16 prev_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); 188 unsigned cpu = smp_processor_id(); 189 u64 next_tlb_gen; 190 191 /* 192 * NB: The scheduler will call us with prev == next when switching 193 * from lazy TLB mode to normal mode if active_mm isn't changing. 194 * When this happens, we don't assume that CR3 (and hence 195 * cpu_tlbstate.loaded_mm) matches next. 196 * 197 * NB: leave_mm() calls us with prev == NULL and tsk == NULL. 198 */ 199 200 /* We don't want flush_tlb_func_* to run concurrently with us. */ 201 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) 202 WARN_ON_ONCE(!irqs_disabled()); 203 204 /* 205 * Verify that CR3 is what we think it is. This will catch 206 * hypothetical buggy code that directly switches to swapper_pg_dir 207 * without going through leave_mm() / switch_mm_irqs_off() or that 208 * does something like write_cr3(read_cr3_pa()). 209 * 210 * Only do this check if CONFIG_DEBUG_VM=y because __read_cr3() 211 * isn't free. 212 */ 213 #ifdef CONFIG_DEBUG_VM 214 if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid))) { 215 /* 216 * If we were to BUG here, we'd be very likely to kill 217 * the system so hard that we don't see the call trace. 218 * Try to recover instead by ignoring the error and doing 219 * a global flush to minimize the chance of corruption. 220 * 221 * (This is far from being a fully correct recovery. 222 * Architecturally, the CPU could prefetch something 223 * back into an incorrect ASID slot and leave it there 224 * to cause trouble down the road. It's better than 225 * nothing, though.) 226 */ 227 __flush_tlb_all(); 228 } 229 #endif 230 this_cpu_write(cpu_tlbstate.is_lazy, false); 231 232 /* 233 * The membarrier system call requires a full memory barrier and 234 * core serialization before returning to user-space, after 235 * storing to rq->curr. Writing to CR3 provides that full 236 * memory barrier and core serializing instruction. 237 */ 238 if (real_prev == next) { 239 VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) != 240 next->context.ctx_id); 241 242 /* 243 * We don't currently support having a real mm loaded without 244 * our cpu set in mm_cpumask(). We have all the bookkeeping 245 * in place to figure out whether we would need to flush 246 * if our cpu were cleared in mm_cpumask(), but we don't 247 * currently use it. 248 */ 249 if (WARN_ON_ONCE(real_prev != &init_mm && 250 !cpumask_test_cpu(cpu, mm_cpumask(next)))) 251 cpumask_set_cpu(cpu, mm_cpumask(next)); 252 253 return; 254 } else { 255 u16 new_asid; 256 bool need_flush; 257 u64 last_ctx_id = this_cpu_read(cpu_tlbstate.last_ctx_id); 258 259 /* 260 * Avoid user/user BTB poisoning by flushing the branch 261 * predictor when switching between processes. This stops 262 * one process from doing Spectre-v2 attacks on another. 263 * 264 * As an optimization, flush indirect branches only when 265 * switching into processes that disable dumping. This 266 * protects high value processes like gpg, without having 267 * too high performance overhead. IBPB is *expensive*! 268 * 269 * This will not flush branches when switching into kernel 270 * threads. It will also not flush if we switch to idle 271 * thread and back to the same process. It will flush if we 272 * switch to a different non-dumpable process. 273 */ 274 if (tsk && tsk->mm && 275 tsk->mm->context.ctx_id != last_ctx_id && 276 get_dumpable(tsk->mm) != SUID_DUMP_USER) 277 indirect_branch_prediction_barrier(); 278 279 if (IS_ENABLED(CONFIG_VMAP_STACK)) { 280 /* 281 * If our current stack is in vmalloc space and isn't 282 * mapped in the new pgd, we'll double-fault. Forcibly 283 * map it. 284 */ 285 sync_current_stack_to_mm(next); 286 } 287 288 /* 289 * Stop remote flushes for the previous mm. 290 * Skip kernel threads; we never send init_mm TLB flushing IPIs, 291 * but the bitmap manipulation can cause cache line contention. 292 */ 293 if (real_prev != &init_mm) { 294 VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu, 295 mm_cpumask(real_prev))); 296 cpumask_clear_cpu(cpu, mm_cpumask(real_prev)); 297 } 298 299 /* 300 * Start remote flushes and then read tlb_gen. 301 */ 302 if (next != &init_mm) 303 cpumask_set_cpu(cpu, mm_cpumask(next)); 304 next_tlb_gen = atomic64_read(&next->context.tlb_gen); 305 306 choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); 307 308 /* Let nmi_uaccess_okay() know that we're changing CR3. */ 309 this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING); 310 barrier(); 311 312 if (need_flush) { 313 this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); 314 this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); 315 load_new_mm_cr3(next->pgd, new_asid, true); 316 317 /* 318 * NB: This gets called via leave_mm() in the idle path 319 * where RCU functions differently. Tracing normally 320 * uses RCU, so we need to use the _rcuidle variant. 321 * 322 * (There is no good reason for this. The idle code should 323 * be rearranged to call this before rcu_idle_enter().) 324 */ 325 trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); 326 } else { 327 /* The new ASID is already up to date. */ 328 load_new_mm_cr3(next->pgd, new_asid, false); 329 330 /* See above wrt _rcuidle. */ 331 trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0); 332 } 333 334 /* 335 * Record last user mm's context id, so we can avoid 336 * flushing branch buffer with IBPB if we switch back 337 * to the same user. 338 */ 339 if (next != &init_mm) 340 this_cpu_write(cpu_tlbstate.last_ctx_id, next->context.ctx_id); 341 342 /* Make sure we write CR3 before loaded_mm. */ 343 barrier(); 344 345 this_cpu_write(cpu_tlbstate.loaded_mm, next); 346 this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid); 347 } 348 349 load_mm_cr4(next); 350 switch_ldt(real_prev, next); 351 } 352 353 /* 354 * Please ignore the name of this function. It should be called 355 * switch_to_kernel_thread(). 356 * 357 * enter_lazy_tlb() is a hint from the scheduler that we are entering a 358 * kernel thread or other context without an mm. Acceptable implementations 359 * include doing nothing whatsoever, switching to init_mm, or various clever 360 * lazy tricks to try to minimize TLB flushes. 361 * 362 * The scheduler reserves the right to call enter_lazy_tlb() several times 363 * in a row. It will notify us that we're going back to a real mm by 364 * calling switch_mm_irqs_off(). 365 */ 366 void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 367 { 368 if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm) 369 return; 370 371 if (tlb_defer_switch_to_init_mm()) { 372 /* 373 * There's a significant optimization that may be possible 374 * here. We have accurate enough TLB flush tracking that we 375 * don't need to maintain coherence of TLB per se when we're 376 * lazy. We do, however, need to maintain coherence of 377 * paging-structure caches. We could, in principle, leave our 378 * old mm loaded and only switch to init_mm when 379 * tlb_remove_page() happens. 380 */ 381 this_cpu_write(cpu_tlbstate.is_lazy, true); 382 } else { 383 switch_mm(NULL, &init_mm, NULL); 384 } 385 } 386 387 /* 388 * Call this when reinitializing a CPU. It fixes the following potential 389 * problems: 390 * 391 * - The ASID changed from what cpu_tlbstate thinks it is (most likely 392 * because the CPU was taken down and came back up with CR3's PCID 393 * bits clear. CPU hotplug can do this. 394 * 395 * - The TLB contains junk in slots corresponding to inactive ASIDs. 396 * 397 * - The CPU went so far out to lunch that it may have missed a TLB 398 * flush. 399 */ 400 void initialize_tlbstate_and_flush(void) 401 { 402 int i; 403 struct mm_struct *mm = this_cpu_read(cpu_tlbstate.loaded_mm); 404 u64 tlb_gen = atomic64_read(&init_mm.context.tlb_gen); 405 unsigned long cr3 = __read_cr3(); 406 407 /* Assert that CR3 already references the right mm. */ 408 WARN_ON((cr3 & CR3_ADDR_MASK) != __pa(mm->pgd)); 409 410 /* 411 * Assert that CR4.PCIDE is set if needed. (CR4.PCIDE initialization 412 * doesn't work like other CR4 bits because it can only be set from 413 * long mode.) 414 */ 415 WARN_ON(boot_cpu_has(X86_FEATURE_PCID) && 416 !(cr4_read_shadow() & X86_CR4_PCIDE)); 417 418 /* Force ASID 0 and force a TLB flush. */ 419 write_cr3(build_cr3(mm->pgd, 0)); 420 421 /* Reinitialize tlbstate. */ 422 this_cpu_write(cpu_tlbstate.last_ctx_id, mm->context.ctx_id); 423 this_cpu_write(cpu_tlbstate.loaded_mm_asid, 0); 424 this_cpu_write(cpu_tlbstate.next_asid, 1); 425 this_cpu_write(cpu_tlbstate.ctxs[0].ctx_id, mm->context.ctx_id); 426 this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen, tlb_gen); 427 428 for (i = 1; i < TLB_NR_DYN_ASIDS; i++) 429 this_cpu_write(cpu_tlbstate.ctxs[i].ctx_id, 0); 430 } 431 432 /* 433 * flush_tlb_func_common()'s memory ordering requirement is that any 434 * TLB fills that happen after we flush the TLB are ordered after we 435 * read active_mm's tlb_gen. We don't need any explicit barriers 436 * because all x86 flush operations are serializing and the 437 * atomic64_read operation won't be reordered by the compiler. 438 */ 439 static void flush_tlb_func_common(const struct flush_tlb_info *f, 440 bool local, enum tlb_flush_reason reason) 441 { 442 /* 443 * We have three different tlb_gen values in here. They are: 444 * 445 * - mm_tlb_gen: the latest generation. 446 * - local_tlb_gen: the generation that this CPU has already caught 447 * up to. 448 * - f->new_tlb_gen: the generation that the requester of the flush 449 * wants us to catch up to. 450 */ 451 struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm); 452 u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); 453 u64 mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen); 454 u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen); 455 456 /* This code cannot presently handle being reentered. */ 457 VM_WARN_ON(!irqs_disabled()); 458 459 if (unlikely(loaded_mm == &init_mm)) 460 return; 461 462 VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].ctx_id) != 463 loaded_mm->context.ctx_id); 464 465 if (this_cpu_read(cpu_tlbstate.is_lazy)) { 466 /* 467 * We're in lazy mode. We need to at least flush our 468 * paging-structure cache to avoid speculatively reading 469 * garbage into our TLB. Since switching to init_mm is barely 470 * slower than a minimal flush, just switch to init_mm. 471 */ 472 switch_mm_irqs_off(NULL, &init_mm, NULL); 473 return; 474 } 475 476 if (unlikely(local_tlb_gen == mm_tlb_gen)) { 477 /* 478 * There's nothing to do: we're already up to date. This can 479 * happen if two concurrent flushes happen -- the first flush to 480 * be handled can catch us all the way up, leaving no work for 481 * the second flush. 482 */ 483 trace_tlb_flush(reason, 0); 484 return; 485 } 486 487 WARN_ON_ONCE(local_tlb_gen > mm_tlb_gen); 488 WARN_ON_ONCE(f->new_tlb_gen > mm_tlb_gen); 489 490 /* 491 * If we get to this point, we know that our TLB is out of date. 492 * This does not strictly imply that we need to flush (it's 493 * possible that f->new_tlb_gen <= local_tlb_gen), but we're 494 * going to need to flush in the very near future, so we might 495 * as well get it over with. 496 * 497 * The only question is whether to do a full or partial flush. 498 * 499 * We do a partial flush if requested and two extra conditions 500 * are met: 501 * 502 * 1. f->new_tlb_gen == local_tlb_gen + 1. We have an invariant that 503 * we've always done all needed flushes to catch up to 504 * local_tlb_gen. If, for example, local_tlb_gen == 2 and 505 * f->new_tlb_gen == 3, then we know that the flush needed to bring 506 * us up to date for tlb_gen 3 is the partial flush we're 507 * processing. 508 * 509 * As an example of why this check is needed, suppose that there 510 * are two concurrent flushes. The first is a full flush that 511 * changes context.tlb_gen from 1 to 2. The second is a partial 512 * flush that changes context.tlb_gen from 2 to 3. If they get 513 * processed on this CPU in reverse order, we'll see 514 * local_tlb_gen == 1, mm_tlb_gen == 3, and end != TLB_FLUSH_ALL. 515 * If we were to use __flush_tlb_one_user() and set local_tlb_gen to 516 * 3, we'd be break the invariant: we'd update local_tlb_gen above 517 * 1 without the full flush that's needed for tlb_gen 2. 518 * 519 * 2. f->new_tlb_gen == mm_tlb_gen. This is purely an optimiation. 520 * Partial TLB flushes are not all that much cheaper than full TLB 521 * flushes, so it seems unlikely that it would be a performance win 522 * to do a partial flush if that won't bring our TLB fully up to 523 * date. By doing a full flush instead, we can increase 524 * local_tlb_gen all the way to mm_tlb_gen and we can probably 525 * avoid another flush in the very near future. 526 */ 527 if (f->end != TLB_FLUSH_ALL && 528 f->new_tlb_gen == local_tlb_gen + 1 && 529 f->new_tlb_gen == mm_tlb_gen) { 530 /* Partial flush */ 531 unsigned long addr; 532 unsigned long nr_pages = (f->end - f->start) >> PAGE_SHIFT; 533 534 addr = f->start; 535 while (addr < f->end) { 536 __flush_tlb_one_user(addr); 537 addr += PAGE_SIZE; 538 } 539 if (local) 540 count_vm_tlb_events(NR_TLB_LOCAL_FLUSH_ONE, nr_pages); 541 trace_tlb_flush(reason, nr_pages); 542 } else { 543 /* Full flush. */ 544 local_flush_tlb(); 545 if (local) 546 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); 547 trace_tlb_flush(reason, TLB_FLUSH_ALL); 548 } 549 550 /* Both paths above update our state to mm_tlb_gen. */ 551 this_cpu_write(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen, mm_tlb_gen); 552 } 553 554 static void flush_tlb_func_local(void *info, enum tlb_flush_reason reason) 555 { 556 const struct flush_tlb_info *f = info; 557 558 flush_tlb_func_common(f, true, reason); 559 } 560 561 static void flush_tlb_func_remote(void *info) 562 { 563 const struct flush_tlb_info *f = info; 564 565 inc_irq_stat(irq_tlb_count); 566 567 if (f->mm && f->mm != this_cpu_read(cpu_tlbstate.loaded_mm)) 568 return; 569 570 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); 571 flush_tlb_func_common(f, false, TLB_REMOTE_SHOOTDOWN); 572 } 573 574 void native_flush_tlb_others(const struct cpumask *cpumask, 575 const struct flush_tlb_info *info) 576 { 577 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); 578 if (info->end == TLB_FLUSH_ALL) 579 trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL); 580 else 581 trace_tlb_flush(TLB_REMOTE_SEND_IPI, 582 (info->end - info->start) >> PAGE_SHIFT); 583 584 if (is_uv_system()) { 585 /* 586 * This whole special case is confused. UV has a "Broadcast 587 * Assist Unit", which seems to be a fancy way to send IPIs. 588 * Back when x86 used an explicit TLB flush IPI, UV was 589 * optimized to use its own mechanism. These days, x86 uses 590 * smp_call_function_many(), but UV still uses a manual IPI, 591 * and that IPI's action is out of date -- it does a manual 592 * flush instead of calling flush_tlb_func_remote(). This 593 * means that the percpu tlb_gen variables won't be updated 594 * and we'll do pointless flushes on future context switches. 595 * 596 * Rather than hooking native_flush_tlb_others() here, I think 597 * that UV should be updated so that smp_call_function_many(), 598 * etc, are optimal on UV. 599 */ 600 unsigned int cpu; 601 602 cpu = smp_processor_id(); 603 cpumask = uv_flush_tlb_others(cpumask, info); 604 if (cpumask) 605 smp_call_function_many(cpumask, flush_tlb_func_remote, 606 (void *)info, 1); 607 return; 608 } 609 smp_call_function_many(cpumask, flush_tlb_func_remote, 610 (void *)info, 1); 611 } 612 613 /* 614 * See Documentation/x86/tlb.txt for details. We choose 33 615 * because it is large enough to cover the vast majority (at 616 * least 95%) of allocations, and is small enough that we are 617 * confident it will not cause too much overhead. Each single 618 * flush is about 100 ns, so this caps the maximum overhead at 619 * _about_ 3,000 ns. 620 * 621 * This is in units of pages. 622 */ 623 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; 624 625 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, 626 unsigned long end, unsigned long vmflag) 627 { 628 int cpu; 629 630 struct flush_tlb_info info __aligned(SMP_CACHE_BYTES) = { 631 .mm = mm, 632 }; 633 634 cpu = get_cpu(); 635 636 /* This is also a barrier that synchronizes with switch_mm(). */ 637 info.new_tlb_gen = inc_mm_tlb_gen(mm); 638 639 /* Should we flush just the requested range? */ 640 if ((end != TLB_FLUSH_ALL) && 641 !(vmflag & VM_HUGETLB) && 642 ((end - start) >> PAGE_SHIFT) <= tlb_single_page_flush_ceiling) { 643 info.start = start; 644 info.end = end; 645 } else { 646 info.start = 0UL; 647 info.end = TLB_FLUSH_ALL; 648 } 649 650 if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { 651 VM_WARN_ON(irqs_disabled()); 652 local_irq_disable(); 653 flush_tlb_func_local(&info, TLB_LOCAL_MM_SHOOTDOWN); 654 local_irq_enable(); 655 } 656 657 if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) 658 flush_tlb_others(mm_cpumask(mm), &info); 659 660 put_cpu(); 661 } 662 663 664 static void do_flush_tlb_all(void *info) 665 { 666 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); 667 __flush_tlb_all(); 668 } 669 670 void flush_tlb_all(void) 671 { 672 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); 673 on_each_cpu(do_flush_tlb_all, NULL, 1); 674 } 675 676 static void do_kernel_range_flush(void *info) 677 { 678 struct flush_tlb_info *f = info; 679 unsigned long addr; 680 681 /* flush range by one by one 'invlpg' */ 682 for (addr = f->start; addr < f->end; addr += PAGE_SIZE) 683 __flush_tlb_one_kernel(addr); 684 } 685 686 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 687 { 688 689 /* Balance as user space task's flush, a bit conservative */ 690 if (end == TLB_FLUSH_ALL || 691 (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) { 692 on_each_cpu(do_flush_tlb_all, NULL, 1); 693 } else { 694 struct flush_tlb_info info; 695 info.start = start; 696 info.end = end; 697 on_each_cpu(do_kernel_range_flush, &info, 1); 698 } 699 } 700 701 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) 702 { 703 struct flush_tlb_info info = { 704 .mm = NULL, 705 .start = 0UL, 706 .end = TLB_FLUSH_ALL, 707 }; 708 709 int cpu = get_cpu(); 710 711 if (cpumask_test_cpu(cpu, &batch->cpumask)) { 712 VM_WARN_ON(irqs_disabled()); 713 local_irq_disable(); 714 flush_tlb_func_local(&info, TLB_LOCAL_SHOOTDOWN); 715 local_irq_enable(); 716 } 717 718 if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) 719 flush_tlb_others(&batch->cpumask, &info); 720 721 cpumask_clear(&batch->cpumask); 722 723 put_cpu(); 724 } 725 726 static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf, 727 size_t count, loff_t *ppos) 728 { 729 char buf[32]; 730 unsigned int len; 731 732 len = sprintf(buf, "%ld\n", tlb_single_page_flush_ceiling); 733 return simple_read_from_buffer(user_buf, count, ppos, buf, len); 734 } 735 736 static ssize_t tlbflush_write_file(struct file *file, 737 const char __user *user_buf, size_t count, loff_t *ppos) 738 { 739 char buf[32]; 740 ssize_t len; 741 int ceiling; 742 743 len = min(count, sizeof(buf) - 1); 744 if (copy_from_user(buf, user_buf, len)) 745 return -EFAULT; 746 747 buf[len] = '\0'; 748 if (kstrtoint(buf, 0, &ceiling)) 749 return -EINVAL; 750 751 if (ceiling < 0) 752 return -EINVAL; 753 754 tlb_single_page_flush_ceiling = ceiling; 755 return count; 756 } 757 758 static const struct file_operations fops_tlbflush = { 759 .read = tlbflush_read_file, 760 .write = tlbflush_write_file, 761 .llseek = default_llseek, 762 }; 763 764 static int __init create_tlb_single_page_flush_ceiling(void) 765 { 766 debugfs_create_file("tlb_single_page_flush_ceiling", S_IRUSR | S_IWUSR, 767 arch_debugfs_dir, NULL, &fops_tlbflush); 768 return 0; 769 } 770 late_initcall(create_tlb_single_page_flush_ceiling); 771