1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mm.h> 4 #include <linux/smp.h> 5 #include <linux/sched.h> 6 #include <asm/sbi.h> 7 #include <asm/mmu_context.h> 8 9 static inline void local_flush_tlb_all_asid(unsigned long asid) 10 { 11 if (asid != FLUSH_TLB_NO_ASID) 12 __asm__ __volatile__ ("sfence.vma x0, %0" 13 : 14 : "r" (asid) 15 : "memory"); 16 else 17 local_flush_tlb_all(); 18 } 19 20 static inline void local_flush_tlb_page_asid(unsigned long addr, 21 unsigned long asid) 22 { 23 if (asid != FLUSH_TLB_NO_ASID) 24 __asm__ __volatile__ ("sfence.vma %0, %1" 25 : 26 : "r" (addr), "r" (asid) 27 : "memory"); 28 else 29 local_flush_tlb_page(addr); 30 } 31 32 /* 33 * Flush entire TLB if number of entries to be flushed is greater 34 * than the threshold below. 35 */ 36 static unsigned long tlb_flush_all_threshold __read_mostly = 64; 37 38 static void local_flush_tlb_range_threshold_asid(unsigned long start, 39 unsigned long size, 40 unsigned long stride, 41 unsigned long asid) 42 { 43 unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride); 44 int i; 45 46 if (nr_ptes_in_range > tlb_flush_all_threshold) { 47 local_flush_tlb_all_asid(asid); 48 return; 49 } 50 51 for (i = 0; i < nr_ptes_in_range; ++i) { 52 local_flush_tlb_page_asid(start, asid); 53 start += stride; 54 } 55 } 56 57 static inline void local_flush_tlb_range_asid(unsigned long start, 58 unsigned long size, unsigned long stride, unsigned long asid) 59 { 60 if (size <= stride) 61 local_flush_tlb_page_asid(start, asid); 62 else if (size == FLUSH_TLB_MAX_SIZE) 63 local_flush_tlb_all_asid(asid); 64 else 65 local_flush_tlb_range_threshold_asid(start, size, stride, asid); 66 } 67 68 /* Flush a range of kernel pages without broadcasting */ 69 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 70 { 71 local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID); 72 } 73 74 static void __ipi_flush_tlb_all(void *info) 75 { 76 local_flush_tlb_all(); 77 } 78 79 void flush_tlb_all(void) 80 { 81 if (riscv_use_ipi_for_rfence()) 82 on_each_cpu(__ipi_flush_tlb_all, NULL, 1); 83 else 84 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID); 85 } 86 87 struct flush_tlb_range_data { 88 unsigned long asid; 89 unsigned long start; 90 unsigned long size; 91 unsigned long stride; 92 }; 93 94 static void __ipi_flush_tlb_range_asid(void *info) 95 { 96 struct flush_tlb_range_data *d = info; 97 98 local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); 99 } 100 101 static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, 102 unsigned long size, unsigned long stride) 103 { 104 struct flush_tlb_range_data ftd; 105 const struct cpumask *cmask; 106 unsigned long asid = FLUSH_TLB_NO_ASID; 107 bool broadcast; 108 109 if (mm) { 110 unsigned int cpuid; 111 112 cmask = mm_cpumask(mm); 113 if (cpumask_empty(cmask)) 114 return; 115 116 cpuid = get_cpu(); 117 /* check if the tlbflush needs to be sent to other CPUs */ 118 broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; 119 120 if (static_branch_unlikely(&use_asid_allocator)) 121 asid = atomic_long_read(&mm->context.id) & asid_mask; 122 } else { 123 cmask = cpu_online_mask; 124 broadcast = true; 125 } 126 127 if (broadcast) { 128 if (riscv_use_ipi_for_rfence()) { 129 ftd.asid = asid; 130 ftd.start = start; 131 ftd.size = size; 132 ftd.stride = stride; 133 on_each_cpu_mask(cmask, 134 __ipi_flush_tlb_range_asid, 135 &ftd, 1); 136 } else 137 sbi_remote_sfence_vma_asid(cmask, 138 start, size, asid); 139 } else { 140 local_flush_tlb_range_asid(start, size, stride, asid); 141 } 142 143 if (mm) 144 put_cpu(); 145 } 146 147 void flush_tlb_mm(struct mm_struct *mm) 148 { 149 __flush_tlb_range(mm, 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); 150 } 151 152 void flush_tlb_mm_range(struct mm_struct *mm, 153 unsigned long start, unsigned long end, 154 unsigned int page_size) 155 { 156 __flush_tlb_range(mm, start, end - start, page_size); 157 } 158 159 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 160 { 161 __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); 162 } 163 164 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 165 unsigned long end) 166 { 167 __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE); 168 } 169 170 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 171 { 172 __flush_tlb_range(NULL, start, end - start, PAGE_SIZE); 173 } 174 175 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 176 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, 177 unsigned long end) 178 { 179 __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE); 180 } 181 #endif 182