xref: /openbmc/linux/arch/x86/mm/tlb.c (revision 96de2506)
1 #include <linux/init.h>
2 
3 #include <linux/mm.h>
4 #include <linux/spinlock.h>
5 #include <linux/smp.h>
6 #include <linux/interrupt.h>
7 #include <linux/export.h>
8 #include <linux/cpu.h>
9 #include <linux/debugfs.h>
10 
11 #include <asm/tlbflush.h>
12 #include <asm/mmu_context.h>
13 #include <asm/nospec-branch.h>
14 #include <asm/cache.h>
15 #include <asm/apic.h>
16 #include <asm/uv/uv.h>
17 
18 /*
19  *	TLB flushing, formerly SMP-only
20  *		c/o Linus Torvalds.
21  *
22  *	These mean you can really definitely utterly forget about
23  *	writing to user space from interrupts. (Its not allowed anyway).
24  *
25  *	Optimizations Manfred Spraul <manfred@colorfullife.com>
26  *
27  *	More scalable flush, from Andi Kleen
28  *
29  *	Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
30  */
31 
32 /*
33  * We get here when we do something requiring a TLB invalidation
34  * but could not go invalidate all of the contexts.  We do the
35  * necessary invalidation by clearing out the 'ctx_id' which
36  * forces a TLB flush when the context is loaded.
37  */
38 static void clear_asid_other(void)
39 {
40 	u16 asid;
41 
42 	/*
43 	 * This is only expected to be set if we have disabled
44 	 * kernel _PAGE_GLOBAL pages.
45 	 */
46 	if (!static_cpu_has(X86_FEATURE_PTI)) {
47 		WARN_ON_ONCE(1);
48 		return;
49 	}
50 
51 	for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) {
52 		/* Do not need to flush the current asid */
53 		if (asid == this_cpu_read(cpu_tlbstate.loaded_mm_asid))
54 			continue;
55 		/*
56 		 * Make sure the next time we go to switch to
57 		 * this asid, we do a flush:
58 		 */
59 		this_cpu_write(cpu_tlbstate.ctxs[asid].ctx_id, 0);
60 	}
61 	this_cpu_write(cpu_tlbstate.invalidate_other, false);
62 }
63 
64 atomic64_t last_mm_ctx_id = ATOMIC64_INIT(1);
65 
66 
67 static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen,
68 			    u16 *new_asid, bool *need_flush)
69 {
70 	u16 asid;
71 
72 	if (!static_cpu_has(X86_FEATURE_PCID)) {
73 		*new_asid = 0;
74 		*need_flush = true;
75 		return;
76 	}
77 
78 	if (this_cpu_read(cpu_tlbstate.invalidate_other))
79 		clear_asid_other();
80 
81 	for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) {
82 		if (this_cpu_read(cpu_tlbstate.ctxs[asid].ctx_id) !=
83 		    next->context.ctx_id)
84 			continue;
85 
86 		*new_asid = asid;
87 		*need_flush = (this_cpu_read(cpu_tlbstate.ctxs[asid].tlb_gen) <
88 			       next_tlb_gen);
89 		return;
90 	}
91 
92 	/*
93 	 * We don't currently own an ASID slot on this CPU.
94 	 * Allocate a slot.
95 	 */
96 	*new_asid = this_cpu_add_return(cpu_tlbstate.next_asid, 1) - 1;
97 	if (*new_asid >= TLB_NR_DYN_ASIDS) {
98 		*new_asid = 0;
99 		this_cpu_write(cpu_tlbstate.next_asid, 1);
100 	}
101 	*need_flush = true;
102 }
103 
104 static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
105 {
106 	unsigned long new_mm_cr3;
107 
108 	if (need_flush) {
109 		invalidate_user_asid(new_asid);
110 		new_mm_cr3 = build_cr3(pgdir, new_asid);
111 	} else {
112 		new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);
113 	}
114 
115 	/*
116 	 * Caution: many callers of this function expect
117 	 * that load_cr3() is serializing and orders TLB
118 	 * fills with respect to the mm_cpumask writes.
119 	 */
120 	write_cr3(new_mm_cr3);
121 }
122 
123 void leave_mm(int cpu)
124 {
125 	struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
126 
127 	/*
128 	 * It's plausible that we're in lazy TLB mode while our mm is init_mm.
129 	 * If so, our callers still expect us to flush the TLB, but there
130 	 * aren't any user TLB entries in init_mm to worry about.
131 	 *
132 	 * This needs to happen before any other sanity checks due to
133 	 * intel_idle's shenanigans.
134 	 */
135 	if (loaded_mm == &init_mm)
136 		return;
137 
138 	/* Warn if we're not lazy. */
139 	WARN_ON(!this_cpu_read(cpu_tlbstate.is_lazy));
140 
141 	switch_mm(NULL, &init_mm, NULL);
142 }
143 EXPORT_SYMBOL_GPL(leave_mm);
144 
145 void switch_mm(struct mm_struct *prev, struct mm_struct *next,
146 	       struct task_struct *tsk)
147 {
148 	unsigned long flags;
149 
150 	local_irq_save(flags);
151 	switch_mm_irqs_off(prev, next, tsk);
152 	local_irq_restore(flags);
153 }
154 
155 static void sync_current_stack_to_mm(struct mm_struct *mm)
156 {
157 	unsigned long sp = current_stack_pointer;
158 	pgd_t *pgd = pgd_offset(mm, sp);
159 
160 	if (pgtable_l5_enabled()) {
161 		if (unlikely(pgd_none(*pgd))) {
162 			pgd_t *pgd_ref = pgd_offset_k(sp);
163 
164 			set_pgd(pgd, *pgd_ref);
165 		}
166 	} else {
167 		/*
168 		 * "pgd" is faked.  The top level entries are "p4d"s, so sync
169 		 * the p4d.  This compiles to approximately the same code as
170 		 * the 5-level case.
171 		 */
172 		p4d_t *p4d = p4d_offset(pgd, sp);
173 
174 		if (unlikely(p4d_none(*p4d))) {
175 			pgd_t *pgd_ref = pgd_offset_k(sp);
176 			p4d_t *p4d_ref = p4d_offset(pgd_ref, sp);
177 
178 			set_p4d(p4d, *p4d_ref);
179 		}
180 	}
181 }
182 
183 void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
184 			struct task_struct *tsk)
185 {
186 	struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm);
187 	u16 prev_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
188 	unsigned cpu = smp_processor_id();
189 	u64 next_tlb_gen;
190 
191 	/*
192 	 * NB: The scheduler will call us with prev == next when switching
193 	 * from lazy TLB mode to normal mode if active_mm isn't changing.
194 	 * When this happens, we don't assume that CR3 (and hence
195 	 * cpu_tlbstate.loaded_mm) matches next.
196 	 *
197 	 * NB: leave_mm() calls us with prev == NULL and tsk == NULL.
198 	 */
199 
200 	/* We don't want flush_tlb_func_* to run concurrently with us. */
201 	if (IS_ENABLED(CONFIG_PROVE_LOCKING))
202 		WARN_ON_ONCE(!irqs_disabled());
203 
204 	/*
205 	 * Verify that CR3 is what we think it is.  This will catch
206 	 * hypothetical buggy code that directly switches to swapper_pg_dir
207 	 * without going through leave_mm() / switch_mm_irqs_off() or that
208 	 * does something like write_cr3(read_cr3_pa()).
209 	 *
210 	 * Only do this check if CONFIG_DEBUG_VM=y because __read_cr3()
211 	 * isn't free.
212 	 */
213 #ifdef CONFIG_DEBUG_VM
214 	if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid))) {
215 		/*
216 		 * If we were to BUG here, we'd be very likely to kill
217 		 * the system so hard that we don't see the call trace.
218 		 * Try to recover instead by ignoring the error and doing
219 		 * a global flush to minimize the chance of corruption.
220 		 *
221 		 * (This is far from being a fully correct recovery.
222 		 *  Architecturally, the CPU could prefetch something
223 		 *  back into an incorrect ASID slot and leave it there
224 		 *  to cause trouble down the road.  It's better than
225 		 *  nothing, though.)
226 		 */
227 		__flush_tlb_all();
228 	}
229 #endif
230 	this_cpu_write(cpu_tlbstate.is_lazy, false);
231 
232 	/*
233 	 * The membarrier system call requires a full memory barrier and
234 	 * core serialization before returning to user-space, after
235 	 * storing to rq->curr. Writing to CR3 provides that full
236 	 * memory barrier and core serializing instruction.
237 	 */
238 	if (real_prev == next) {
239 		VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) !=
240 			   next->context.ctx_id);
241 
242 		/*
243 		 * We don't currently support having a real mm loaded without
244 		 * our cpu set in mm_cpumask().  We have all the bookkeeping
245 		 * in place to figure out whether we would need to flush
246 		 * if our cpu were cleared in mm_cpumask(), but we don't
247 		 * currently use it.
248 		 */
249 		if (WARN_ON_ONCE(real_prev != &init_mm &&
250 				 !cpumask_test_cpu(cpu, mm_cpumask(next))))
251 			cpumask_set_cpu(cpu, mm_cpumask(next));
252 
253 		return;
254 	} else {
255 		u16 new_asid;
256 		bool need_flush;
257 		u64 last_ctx_id = this_cpu_read(cpu_tlbstate.last_ctx_id);
258 
259 		/*
260 		 * Avoid user/user BTB poisoning by flushing the branch
261 		 * predictor when switching between processes. This stops
262 		 * one process from doing Spectre-v2 attacks on another.
263 		 *
264 		 * As an optimization, flush indirect branches only when
265 		 * switching into processes that disable dumping. This
266 		 * protects high value processes like gpg, without having
267 		 * too high performance overhead. IBPB is *expensive*!
268 		 *
269 		 * This will not flush branches when switching into kernel
270 		 * threads. It will also not flush if we switch to idle
271 		 * thread and back to the same process. It will flush if we
272 		 * switch to a different non-dumpable process.
273 		 */
274 		if (tsk && tsk->mm &&
275 		    tsk->mm->context.ctx_id != last_ctx_id &&
276 		    get_dumpable(tsk->mm) != SUID_DUMP_USER)
277 			indirect_branch_prediction_barrier();
278 
279 		if (IS_ENABLED(CONFIG_VMAP_STACK)) {
280 			/*
281 			 * If our current stack is in vmalloc space and isn't
282 			 * mapped in the new pgd, we'll double-fault.  Forcibly
283 			 * map it.
284 			 */
285 			sync_current_stack_to_mm(next);
286 		}
287 
288 		/*
289 		 * Stop remote flushes for the previous mm.
290 		 * Skip kernel threads; we never send init_mm TLB flushing IPIs,
291 		 * but the bitmap manipulation can cause cache line contention.
292 		 */
293 		if (real_prev != &init_mm) {
294 			VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu,
295 						mm_cpumask(real_prev)));
296 			cpumask_clear_cpu(cpu, mm_cpumask(real_prev));
297 		}
298 
299 		/*
300 		 * Start remote flushes and then read tlb_gen.
301 		 */
302 		if (next != &init_mm)
303 			cpumask_set_cpu(cpu, mm_cpumask(next));
304 		next_tlb_gen = atomic64_read(&next->context.tlb_gen);
305 
306 		choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush);
307 
308 		/* Let nmi_uaccess_okay() know that we're changing CR3. */
309 		this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING);
310 		barrier();
311 
312 		if (need_flush) {
313 			this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id);
314 			this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);
315 			load_new_mm_cr3(next->pgd, new_asid, true);
316 
317 			/*
318 			 * NB: This gets called via leave_mm() in the idle path
319 			 * where RCU functions differently.  Tracing normally
320 			 * uses RCU, so we need to use the _rcuidle variant.
321 			 *
322 			 * (There is no good reason for this.  The idle code should
323 			 *  be rearranged to call this before rcu_idle_enter().)
324 			 */
325 			trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
326 		} else {
327 			/* The new ASID is already up to date. */
328 			load_new_mm_cr3(next->pgd, new_asid, false);
329 
330 			/* See above wrt _rcuidle. */
331 			trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0);
332 		}
333 
334 		/*
335 		 * Record last user mm's context id, so we can avoid
336 		 * flushing branch buffer with IBPB if we switch back
337 		 * to the same user.
338 		 */
339 		if (next != &init_mm)
340 			this_cpu_write(cpu_tlbstate.last_ctx_id, next->context.ctx_id);
341 
342 		/* Make sure we write CR3 before loaded_mm. */
343 		barrier();
344 
345 		this_cpu_write(cpu_tlbstate.loaded_mm, next);
346 		this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid);
347 	}
348 
349 	load_mm_cr4(next);
350 	switch_ldt(real_prev, next);
351 }
352 
353 /*
354  * Please ignore the name of this function.  It should be called
355  * switch_to_kernel_thread().
356  *
357  * enter_lazy_tlb() is a hint from the scheduler that we are entering a
358  * kernel thread or other context without an mm.  Acceptable implementations
359  * include doing nothing whatsoever, switching to init_mm, or various clever
360  * lazy tricks to try to minimize TLB flushes.
361  *
362  * The scheduler reserves the right to call enter_lazy_tlb() several times
363  * in a row.  It will notify us that we're going back to a real mm by
364  * calling switch_mm_irqs_off().
365  */
366 void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
367 {
368 	if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm)
369 		return;
370 
371 	if (tlb_defer_switch_to_init_mm()) {
372 		/*
373 		 * There's a significant optimization that may be possible
374 		 * here.  We have accurate enough TLB flush tracking that we
375 		 * don't need to maintain coherence of TLB per se when we're
376 		 * lazy.  We do, however, need to maintain coherence of
377 		 * paging-structure caches.  We could, in principle, leave our
378 		 * old mm loaded and only switch to init_mm when
379 		 * tlb_remove_page() happens.
380 		 */
381 		this_cpu_write(cpu_tlbstate.is_lazy, true);
382 	} else {
383 		switch_mm(NULL, &init_mm, NULL);
384 	}
385 }
386 
387 /*
388  * Call this when reinitializing a CPU.  It fixes the following potential
389  * problems:
390  *
391  * - The ASID changed from what cpu_tlbstate thinks it is (most likely
392  *   because the CPU was taken down and came back up with CR3's PCID
393  *   bits clear.  CPU hotplug can do this.
394  *
395  * - The TLB contains junk in slots corresponding to inactive ASIDs.
396  *
397  * - The CPU went so far out to lunch that it may have missed a TLB
398  *   flush.
399  */
400 void initialize_tlbstate_and_flush(void)
401 {
402 	int i;
403 	struct mm_struct *mm = this_cpu_read(cpu_tlbstate.loaded_mm);
404 	u64 tlb_gen = atomic64_read(&init_mm.context.tlb_gen);
405 	unsigned long cr3 = __read_cr3();
406 
407 	/* Assert that CR3 already references the right mm. */
408 	WARN_ON((cr3 & CR3_ADDR_MASK) != __pa(mm->pgd));
409 
410 	/*
411 	 * Assert that CR4.PCIDE is set if needed.  (CR4.PCIDE initialization
412 	 * doesn't work like other CR4 bits because it can only be set from
413 	 * long mode.)
414 	 */
415 	WARN_ON(boot_cpu_has(X86_FEATURE_PCID) &&
416 		!(cr4_read_shadow() & X86_CR4_PCIDE));
417 
418 	/* Force ASID 0 and force a TLB flush. */
419 	write_cr3(build_cr3(mm->pgd, 0));
420 
421 	/* Reinitialize tlbstate. */
422 	this_cpu_write(cpu_tlbstate.last_ctx_id, mm->context.ctx_id);
423 	this_cpu_write(cpu_tlbstate.loaded_mm_asid, 0);
424 	this_cpu_write(cpu_tlbstate.next_asid, 1);
425 	this_cpu_write(cpu_tlbstate.ctxs[0].ctx_id, mm->context.ctx_id);
426 	this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen, tlb_gen);
427 
428 	for (i = 1; i < TLB_NR_DYN_ASIDS; i++)
429 		this_cpu_write(cpu_tlbstate.ctxs[i].ctx_id, 0);
430 }
431 
432 /*
433  * flush_tlb_func_common()'s memory ordering requirement is that any
434  * TLB fills that happen after we flush the TLB are ordered after we
435  * read active_mm's tlb_gen.  We don't need any explicit barriers
436  * because all x86 flush operations are serializing and the
437  * atomic64_read operation won't be reordered by the compiler.
438  */
439 static void flush_tlb_func_common(const struct flush_tlb_info *f,
440 				  bool local, enum tlb_flush_reason reason)
441 {
442 	/*
443 	 * We have three different tlb_gen values in here.  They are:
444 	 *
445 	 * - mm_tlb_gen:     the latest generation.
446 	 * - local_tlb_gen:  the generation that this CPU has already caught
447 	 *                   up to.
448 	 * - f->new_tlb_gen: the generation that the requester of the flush
449 	 *                   wants us to catch up to.
450 	 */
451 	struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
452 	u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
453 	u64 mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen);
454 	u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
455 
456 	/* This code cannot presently handle being reentered. */
457 	VM_WARN_ON(!irqs_disabled());
458 
459 	if (unlikely(loaded_mm == &init_mm))
460 		return;
461 
462 	VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].ctx_id) !=
463 		   loaded_mm->context.ctx_id);
464 
465 	if (this_cpu_read(cpu_tlbstate.is_lazy)) {
466 		/*
467 		 * We're in lazy mode.  We need to at least flush our
468 		 * paging-structure cache to avoid speculatively reading
469 		 * garbage into our TLB.  Since switching to init_mm is barely
470 		 * slower than a minimal flush, just switch to init_mm.
471 		 */
472 		switch_mm_irqs_off(NULL, &init_mm, NULL);
473 		return;
474 	}
475 
476 	if (unlikely(local_tlb_gen == mm_tlb_gen)) {
477 		/*
478 		 * There's nothing to do: we're already up to date.  This can
479 		 * happen if two concurrent flushes happen -- the first flush to
480 		 * be handled can catch us all the way up, leaving no work for
481 		 * the second flush.
482 		 */
483 		trace_tlb_flush(reason, 0);
484 		return;
485 	}
486 
487 	WARN_ON_ONCE(local_tlb_gen > mm_tlb_gen);
488 	WARN_ON_ONCE(f->new_tlb_gen > mm_tlb_gen);
489 
490 	/*
491 	 * If we get to this point, we know that our TLB is out of date.
492 	 * This does not strictly imply that we need to flush (it's
493 	 * possible that f->new_tlb_gen <= local_tlb_gen), but we're
494 	 * going to need to flush in the very near future, so we might
495 	 * as well get it over with.
496 	 *
497 	 * The only question is whether to do a full or partial flush.
498 	 *
499 	 * We do a partial flush if requested and two extra conditions
500 	 * are met:
501 	 *
502 	 * 1. f->new_tlb_gen == local_tlb_gen + 1.  We have an invariant that
503 	 *    we've always done all needed flushes to catch up to
504 	 *    local_tlb_gen.  If, for example, local_tlb_gen == 2 and
505 	 *    f->new_tlb_gen == 3, then we know that the flush needed to bring
506 	 *    us up to date for tlb_gen 3 is the partial flush we're
507 	 *    processing.
508 	 *
509 	 *    As an example of why this check is needed, suppose that there
510 	 *    are two concurrent flushes.  The first is a full flush that
511 	 *    changes context.tlb_gen from 1 to 2.  The second is a partial
512 	 *    flush that changes context.tlb_gen from 2 to 3.  If they get
513 	 *    processed on this CPU in reverse order, we'll see
514 	 *     local_tlb_gen == 1, mm_tlb_gen == 3, and end != TLB_FLUSH_ALL.
515 	 *    If we were to use __flush_tlb_one_user() and set local_tlb_gen to
516 	 *    3, we'd be break the invariant: we'd update local_tlb_gen above
517 	 *    1 without the full flush that's needed for tlb_gen 2.
518 	 *
519 	 * 2. f->new_tlb_gen == mm_tlb_gen.  This is purely an optimiation.
520 	 *    Partial TLB flushes are not all that much cheaper than full TLB
521 	 *    flushes, so it seems unlikely that it would be a performance win
522 	 *    to do a partial flush if that won't bring our TLB fully up to
523 	 *    date.  By doing a full flush instead, we can increase
524 	 *    local_tlb_gen all the way to mm_tlb_gen and we can probably
525 	 *    avoid another flush in the very near future.
526 	 */
527 	if (f->end != TLB_FLUSH_ALL &&
528 	    f->new_tlb_gen == local_tlb_gen + 1 &&
529 	    f->new_tlb_gen == mm_tlb_gen) {
530 		/* Partial flush */
531 		unsigned long addr;
532 		unsigned long nr_pages = (f->end - f->start) >> PAGE_SHIFT;
533 
534 		addr = f->start;
535 		while (addr < f->end) {
536 			__flush_tlb_one_user(addr);
537 			addr += PAGE_SIZE;
538 		}
539 		if (local)
540 			count_vm_tlb_events(NR_TLB_LOCAL_FLUSH_ONE, nr_pages);
541 		trace_tlb_flush(reason, nr_pages);
542 	} else {
543 		/* Full flush. */
544 		local_flush_tlb();
545 		if (local)
546 			count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
547 		trace_tlb_flush(reason, TLB_FLUSH_ALL);
548 	}
549 
550 	/* Both paths above update our state to mm_tlb_gen. */
551 	this_cpu_write(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen, mm_tlb_gen);
552 }
553 
554 static void flush_tlb_func_local(void *info, enum tlb_flush_reason reason)
555 {
556 	const struct flush_tlb_info *f = info;
557 
558 	flush_tlb_func_common(f, true, reason);
559 }
560 
561 static void flush_tlb_func_remote(void *info)
562 {
563 	const struct flush_tlb_info *f = info;
564 
565 	inc_irq_stat(irq_tlb_count);
566 
567 	if (f->mm && f->mm != this_cpu_read(cpu_tlbstate.loaded_mm))
568 		return;
569 
570 	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
571 	flush_tlb_func_common(f, false, TLB_REMOTE_SHOOTDOWN);
572 }
573 
574 void native_flush_tlb_others(const struct cpumask *cpumask,
575 			     const struct flush_tlb_info *info)
576 {
577 	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
578 	if (info->end == TLB_FLUSH_ALL)
579 		trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL);
580 	else
581 		trace_tlb_flush(TLB_REMOTE_SEND_IPI,
582 				(info->end - info->start) >> PAGE_SHIFT);
583 
584 	if (is_uv_system()) {
585 		/*
586 		 * This whole special case is confused.  UV has a "Broadcast
587 		 * Assist Unit", which seems to be a fancy way to send IPIs.
588 		 * Back when x86 used an explicit TLB flush IPI, UV was
589 		 * optimized to use its own mechanism.  These days, x86 uses
590 		 * smp_call_function_many(), but UV still uses a manual IPI,
591 		 * and that IPI's action is out of date -- it does a manual
592 		 * flush instead of calling flush_tlb_func_remote().  This
593 		 * means that the percpu tlb_gen variables won't be updated
594 		 * and we'll do pointless flushes on future context switches.
595 		 *
596 		 * Rather than hooking native_flush_tlb_others() here, I think
597 		 * that UV should be updated so that smp_call_function_many(),
598 		 * etc, are optimal on UV.
599 		 */
600 		unsigned int cpu;
601 
602 		cpu = smp_processor_id();
603 		cpumask = uv_flush_tlb_others(cpumask, info);
604 		if (cpumask)
605 			smp_call_function_many(cpumask, flush_tlb_func_remote,
606 					       (void *)info, 1);
607 		return;
608 	}
609 	smp_call_function_many(cpumask, flush_tlb_func_remote,
610 			       (void *)info, 1);
611 }
612 
613 /*
614  * See Documentation/x86/tlb.txt for details.  We choose 33
615  * because it is large enough to cover the vast majority (at
616  * least 95%) of allocations, and is small enough that we are
617  * confident it will not cause too much overhead.  Each single
618  * flush is about 100 ns, so this caps the maximum overhead at
619  * _about_ 3,000 ns.
620  *
621  * This is in units of pages.
622  */
623 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
624 
625 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
626 				unsigned long end, unsigned long vmflag)
627 {
628 	int cpu;
629 
630 	struct flush_tlb_info info __aligned(SMP_CACHE_BYTES) = {
631 		.mm = mm,
632 	};
633 
634 	cpu = get_cpu();
635 
636 	/* This is also a barrier that synchronizes with switch_mm(). */
637 	info.new_tlb_gen = inc_mm_tlb_gen(mm);
638 
639 	/* Should we flush just the requested range? */
640 	if ((end != TLB_FLUSH_ALL) &&
641 	    !(vmflag & VM_HUGETLB) &&
642 	    ((end - start) >> PAGE_SHIFT) <= tlb_single_page_flush_ceiling) {
643 		info.start = start;
644 		info.end = end;
645 	} else {
646 		info.start = 0UL;
647 		info.end = TLB_FLUSH_ALL;
648 	}
649 
650 	if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) {
651 		VM_WARN_ON(irqs_disabled());
652 		local_irq_disable();
653 		flush_tlb_func_local(&info, TLB_LOCAL_MM_SHOOTDOWN);
654 		local_irq_enable();
655 	}
656 
657 	if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids)
658 		flush_tlb_others(mm_cpumask(mm), &info);
659 
660 	put_cpu();
661 }
662 
663 
664 static void do_flush_tlb_all(void *info)
665 {
666 	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
667 	__flush_tlb_all();
668 }
669 
670 void flush_tlb_all(void)
671 {
672 	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
673 	on_each_cpu(do_flush_tlb_all, NULL, 1);
674 }
675 
676 static void do_kernel_range_flush(void *info)
677 {
678 	struct flush_tlb_info *f = info;
679 	unsigned long addr;
680 
681 	/* flush range by one by one 'invlpg' */
682 	for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
683 		__flush_tlb_one_kernel(addr);
684 }
685 
686 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
687 {
688 
689 	/* Balance as user space task's flush, a bit conservative */
690 	if (end == TLB_FLUSH_ALL ||
691 	    (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) {
692 		on_each_cpu(do_flush_tlb_all, NULL, 1);
693 	} else {
694 		struct flush_tlb_info info;
695 		info.start = start;
696 		info.end = end;
697 		on_each_cpu(do_kernel_range_flush, &info, 1);
698 	}
699 }
700 
701 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
702 {
703 	struct flush_tlb_info info = {
704 		.mm = NULL,
705 		.start = 0UL,
706 		.end = TLB_FLUSH_ALL,
707 	};
708 
709 	int cpu = get_cpu();
710 
711 	if (cpumask_test_cpu(cpu, &batch->cpumask)) {
712 		VM_WARN_ON(irqs_disabled());
713 		local_irq_disable();
714 		flush_tlb_func_local(&info, TLB_LOCAL_SHOOTDOWN);
715 		local_irq_enable();
716 	}
717 
718 	if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
719 		flush_tlb_others(&batch->cpumask, &info);
720 
721 	cpumask_clear(&batch->cpumask);
722 
723 	put_cpu();
724 }
725 
726 static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
727 			     size_t count, loff_t *ppos)
728 {
729 	char buf[32];
730 	unsigned int len;
731 
732 	len = sprintf(buf, "%ld\n", tlb_single_page_flush_ceiling);
733 	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
734 }
735 
736 static ssize_t tlbflush_write_file(struct file *file,
737 		 const char __user *user_buf, size_t count, loff_t *ppos)
738 {
739 	char buf[32];
740 	ssize_t len;
741 	int ceiling;
742 
743 	len = min(count, sizeof(buf) - 1);
744 	if (copy_from_user(buf, user_buf, len))
745 		return -EFAULT;
746 
747 	buf[len] = '\0';
748 	if (kstrtoint(buf, 0, &ceiling))
749 		return -EINVAL;
750 
751 	if (ceiling < 0)
752 		return -EINVAL;
753 
754 	tlb_single_page_flush_ceiling = ceiling;
755 	return count;
756 }
757 
758 static const struct file_operations fops_tlbflush = {
759 	.read = tlbflush_read_file,
760 	.write = tlbflush_write_file,
761 	.llseek = default_llseek,
762 };
763 
764 static int __init create_tlb_single_page_flush_ceiling(void)
765 {
766 	debugfs_create_file("tlb_single_page_flush_ceiling", S_IRUSR | S_IWUSR,
767 			    arch_debugfs_dir, NULL, &fops_tlbflush);
768 	return 0;
769 }
770 late_initcall(create_tlb_single_page_flush_ceiling);
771