1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mm.h> 4 #include <linux/smp.h> 5 #include <linux/sched.h> 6 #include <asm/sbi.h> 7 #include <asm/mmu_context.h> 8 9 static inline void local_flush_tlb_all_asid(unsigned long asid) 10 { 11 __asm__ __volatile__ ("sfence.vma x0, %0" 12 : 13 : "r" (asid) 14 : "memory"); 15 } 16 17 static inline void local_flush_tlb_page_asid(unsigned long addr, 18 unsigned long asid) 19 { 20 __asm__ __volatile__ ("sfence.vma %0, %1" 21 : 22 : "r" (addr), "r" (asid) 23 : "memory"); 24 } 25 26 static inline void local_flush_tlb_range(unsigned long start, 27 unsigned long size, unsigned long stride) 28 { 29 if (size <= stride) 30 local_flush_tlb_page(start); 31 else 32 local_flush_tlb_all(); 33 } 34 35 static inline void local_flush_tlb_range_asid(unsigned long start, 36 unsigned long size, unsigned long stride, unsigned long asid) 37 { 38 if (size <= stride) 39 local_flush_tlb_page_asid(start, asid); 40 else 41 local_flush_tlb_all_asid(asid); 42 } 43 44 static void __ipi_flush_tlb_all(void *info) 45 { 46 local_flush_tlb_all(); 47 } 48 49 void flush_tlb_all(void) 50 { 51 if (riscv_use_ipi_for_rfence()) 52 on_each_cpu(__ipi_flush_tlb_all, NULL, 1); 53 else 54 sbi_remote_sfence_vma(NULL, 0, -1); 55 } 56 57 struct flush_tlb_range_data { 58 unsigned long asid; 59 unsigned long start; 60 unsigned long size; 61 unsigned long stride; 62 }; 63 64 static void __ipi_flush_tlb_range_asid(void *info) 65 { 66 struct flush_tlb_range_data *d = info; 67 68 local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); 69 } 70 71 static void __ipi_flush_tlb_range(void *info) 72 { 73 struct flush_tlb_range_data *d = info; 74 75 local_flush_tlb_range(d->start, d->size, d->stride); 76 } 77 78 static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, 79 unsigned long size, unsigned long stride) 80 { 81 struct flush_tlb_range_data ftd; 82 struct cpumask *cmask = mm_cpumask(mm); 83 unsigned int cpuid; 84 bool broadcast; 85 86 if (cpumask_empty(cmask)) 87 return; 88 89 cpuid = get_cpu(); 90 /* check if the tlbflush needs to be sent to other CPUs */ 91 broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; 92 if (static_branch_unlikely(&use_asid_allocator)) { 93 unsigned long asid = atomic_long_read(&mm->context.id) & asid_mask; 94 95 if (broadcast) { 96 if (riscv_use_ipi_for_rfence()) { 97 ftd.asid = asid; 98 ftd.start = start; 99 ftd.size = size; 100 ftd.stride = stride; 101 on_each_cpu_mask(cmask, 102 __ipi_flush_tlb_range_asid, 103 &ftd, 1); 104 } else 105 sbi_remote_sfence_vma_asid(cmask, 106 start, size, asid); 107 } else { 108 local_flush_tlb_range_asid(start, size, stride, asid); 109 } 110 } else { 111 if (broadcast) { 112 if (riscv_use_ipi_for_rfence()) { 113 ftd.asid = 0; 114 ftd.start = start; 115 ftd.size = size; 116 ftd.stride = stride; 117 on_each_cpu_mask(cmask, 118 __ipi_flush_tlb_range, 119 &ftd, 1); 120 } else 121 sbi_remote_sfence_vma(cmask, start, size); 122 } else { 123 local_flush_tlb_range(start, size, stride); 124 } 125 } 126 127 put_cpu(); 128 } 129 130 void flush_tlb_mm(struct mm_struct *mm) 131 { 132 __flush_tlb_range(mm, 0, -1, PAGE_SIZE); 133 } 134 135 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 136 { 137 __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); 138 } 139 140 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 141 unsigned long end) 142 { 143 __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE); 144 } 145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 146 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, 147 unsigned long end) 148 { 149 __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE); 150 } 151 #endif 152