1 #define pr_fmt(fmt) "Hyper-V: " fmt 2 3 #include <linux/hyperv.h> 4 #include <linux/log2.h> 5 #include <linux/slab.h> 6 #include <linux/types.h> 7 8 #include <asm/fpu/api.h> 9 #include <asm/mshyperv.h> 10 #include <asm/msr.h> 11 #include <asm/tlbflush.h> 12 13 #define CREATE_TRACE_POINTS 14 #include <asm/trace/hyperv.h> 15 16 /* Each gva in gva_list encodes up to 4096 pages to flush */ 17 #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE) 18 19 20 /* 21 * Fills in gva_list starting from offset. Returns the number of items added. 22 */ 23 static inline int fill_gva_list(u64 gva_list[], int offset, 24 unsigned long start, unsigned long end) 25 { 26 int gva_n = offset; 27 unsigned long cur = start, diff; 28 29 do { 30 diff = end > cur ? end - cur : 0; 31 32 gva_list[gva_n] = cur & PAGE_MASK; 33 /* 34 * Lower 12 bits encode the number of additional 35 * pages to flush (in addition to the 'cur' page). 36 */ 37 if (diff >= HV_TLB_FLUSH_UNIT) 38 gva_list[gva_n] |= ~PAGE_MASK; 39 else if (diff) 40 gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT; 41 42 cur += HV_TLB_FLUSH_UNIT; 43 gva_n++; 44 45 } while (cur < end); 46 47 return gva_n - offset; 48 } 49 50 static void hyperv_flush_tlb_others(const struct cpumask *cpus, 51 const struct flush_tlb_info *info) 52 { 53 int cpu, vcpu, gva_n, max_gvas; 54 struct hv_tlb_flush **flush_pcpu; 55 struct hv_tlb_flush *flush; 56 u64 status = U64_MAX; 57 unsigned long flags; 58 59 trace_hyperv_mmu_flush_tlb_others(cpus, info); 60 61 if (!hv_hypercall_pg) 62 goto do_native; 63 64 if (cpumask_empty(cpus)) 65 return; 66 67 local_irq_save(flags); 68 69 flush_pcpu = (struct hv_tlb_flush **) 70 this_cpu_ptr(hyperv_pcpu_input_arg); 71 72 flush = *flush_pcpu; 73 74 if (unlikely(!flush)) { 75 local_irq_restore(flags); 76 goto do_native; 77 } 78 79 if (info->mm) { 80 /* 81 * AddressSpace argument must match the CR3 with PCID bits 82 * stripped out. 83 */ 84 flush->address_space = virt_to_phys(info->mm->pgd); 85 flush->address_space &= CR3_ADDR_MASK; 86 flush->flags = 0; 87 } else { 88 flush->address_space = 0; 89 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; 90 } 91 92 flush->processor_mask = 0; 93 if (cpumask_equal(cpus, cpu_present_mask)) { 94 flush->flags |= HV_FLUSH_ALL_PROCESSORS; 95 } else { 96 for_each_cpu(cpu, cpus) { 97 vcpu = hv_cpu_number_to_vp_number(cpu); 98 if (vcpu >= 64) 99 goto do_native; 100 101 __set_bit(vcpu, (unsigned long *) 102 &flush->processor_mask); 103 } 104 } 105 106 /* 107 * We can flush not more than max_gvas with one hypercall. Flush the 108 * whole address space if we were asked to do more. 109 */ 110 max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]); 111 112 if (info->end == TLB_FLUSH_ALL) { 113 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; 114 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, 115 flush, NULL); 116 } else if (info->end && 117 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) { 118 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, 119 flush, NULL); 120 } else { 121 gva_n = fill_gva_list(flush->gva_list, 0, 122 info->start, info->end); 123 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST, 124 gva_n, 0, flush, NULL); 125 } 126 127 local_irq_restore(flags); 128 129 if (!(status & HV_HYPERCALL_RESULT_MASK)) 130 return; 131 do_native: 132 native_flush_tlb_others(cpus, info); 133 } 134 135 static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus, 136 const struct flush_tlb_info *info) 137 { 138 int nr_bank = 0, max_gvas, gva_n; 139 struct hv_tlb_flush_ex **flush_pcpu; 140 struct hv_tlb_flush_ex *flush; 141 u64 status = U64_MAX; 142 unsigned long flags; 143 144 trace_hyperv_mmu_flush_tlb_others(cpus, info); 145 146 if (!hv_hypercall_pg) 147 goto do_native; 148 149 if (cpumask_empty(cpus)) 150 return; 151 152 local_irq_save(flags); 153 154 flush_pcpu = (struct hv_tlb_flush_ex **) 155 this_cpu_ptr(hyperv_pcpu_input_arg); 156 157 flush = *flush_pcpu; 158 159 if (unlikely(!flush)) { 160 local_irq_restore(flags); 161 goto do_native; 162 } 163 164 if (info->mm) { 165 /* 166 * AddressSpace argument must match the CR3 with PCID bits 167 * stripped out. 168 */ 169 flush->address_space = virt_to_phys(info->mm->pgd); 170 flush->address_space &= CR3_ADDR_MASK; 171 flush->flags = 0; 172 } else { 173 flush->address_space = 0; 174 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; 175 } 176 177 flush->hv_vp_set.valid_bank_mask = 0; 178 179 if (!cpumask_equal(cpus, cpu_present_mask)) { 180 flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K; 181 nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus); 182 } 183 184 if (!nr_bank) { 185 flush->hv_vp_set.format = HV_GENERIC_SET_ALL; 186 flush->flags |= HV_FLUSH_ALL_PROCESSORS; 187 } 188 189 /* 190 * We can flush not more than max_gvas with one hypercall. Flush the 191 * whole address space if we were asked to do more. 192 */ 193 max_gvas = 194 (PAGE_SIZE - sizeof(*flush) - nr_bank * 195 sizeof(flush->hv_vp_set.bank_contents[0])) / 196 sizeof(flush->gva_list[0]); 197 198 if (info->end == TLB_FLUSH_ALL) { 199 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; 200 status = hv_do_rep_hypercall( 201 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX, 202 0, nr_bank, flush, NULL); 203 } else if (info->end && 204 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) { 205 status = hv_do_rep_hypercall( 206 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX, 207 0, nr_bank, flush, NULL); 208 } else { 209 gva_n = fill_gva_list(flush->gva_list, nr_bank, 210 info->start, info->end); 211 status = hv_do_rep_hypercall( 212 HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX, 213 gva_n, nr_bank, flush, NULL); 214 } 215 216 local_irq_restore(flags); 217 218 if (!(status & HV_HYPERCALL_RESULT_MASK)) 219 return; 220 do_native: 221 native_flush_tlb_others(cpus, info); 222 } 223 224 void hyperv_setup_mmu_ops(void) 225 { 226 if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED)) 227 return; 228 229 if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) { 230 pr_info("Using hypercall for remote TLB flush\n"); 231 pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others; 232 } else { 233 pr_info("Using ext hypercall for remote TLB flush\n"); 234 pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others_ex; 235 } 236 } 237