1 #define pr_fmt(fmt) "Hyper-V: " fmt 2 3 #include <linux/hyperv.h> 4 #include <linux/log2.h> 5 #include <linux/slab.h> 6 #include <linux/types.h> 7 8 #include <asm/fpu/api.h> 9 #include <asm/mshyperv.h> 10 #include <asm/msr.h> 11 #include <asm/tlbflush.h> 12 #include <asm/tlb.h> 13 14 #define CREATE_TRACE_POINTS 15 #include <asm/trace/hyperv.h> 16 17 /* Each gva in gva_list encodes up to 4096 pages to flush */ 18 #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE) 19 20 static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus, 21 const struct flush_tlb_info *info); 22 23 /* 24 * Fills in gva_list starting from offset. Returns the number of items added. 25 */ 26 static inline int fill_gva_list(u64 gva_list[], int offset, 27 unsigned long start, unsigned long end) 28 { 29 int gva_n = offset; 30 unsigned long cur = start, diff; 31 32 do { 33 diff = end > cur ? end - cur : 0; 34 35 gva_list[gva_n] = cur & PAGE_MASK; 36 /* 37 * Lower 12 bits encode the number of additional 38 * pages to flush (in addition to the 'cur' page). 39 */ 40 if (diff >= HV_TLB_FLUSH_UNIT) 41 gva_list[gva_n] |= ~PAGE_MASK; 42 else if (diff) 43 gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT; 44 45 cur += HV_TLB_FLUSH_UNIT; 46 gva_n++; 47 48 } while (cur < end); 49 50 return gva_n - offset; 51 } 52 53 static void hyperv_flush_tlb_others(const struct cpumask *cpus, 54 const struct flush_tlb_info *info) 55 { 56 int cpu, vcpu, gva_n, max_gvas; 57 struct hv_tlb_flush **flush_pcpu; 58 struct hv_tlb_flush *flush; 59 u64 status = U64_MAX; 60 unsigned long flags; 61 62 trace_hyperv_mmu_flush_tlb_others(cpus, info); 63 64 if (!hv_hypercall_pg) 65 goto do_native; 66 67 if (cpumask_empty(cpus)) 68 return; 69 70 local_irq_save(flags); 71 72 flush_pcpu = (struct hv_tlb_flush **) 73 this_cpu_ptr(hyperv_pcpu_input_arg); 74 75 flush = *flush_pcpu; 76 77 if (unlikely(!flush)) { 78 local_irq_restore(flags); 79 goto do_native; 80 } 81 82 if (info->mm) { 83 /* 84 * AddressSpace argument must match the CR3 with PCID bits 85 * stripped out. 86 */ 87 flush->address_space = virt_to_phys(info->mm->pgd); 88 flush->address_space &= CR3_ADDR_MASK; 89 flush->flags = 0; 90 } else { 91 flush->address_space = 0; 92 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; 93 } 94 95 flush->processor_mask = 0; 96 if (cpumask_equal(cpus, cpu_present_mask)) { 97 flush->flags |= HV_FLUSH_ALL_PROCESSORS; 98 } else { 99 /* 100 * From the supplied CPU set we need to figure out if we can get 101 * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE} 102 * hypercalls. This is possible when the highest VP number in 103 * the set is < 64. As VP numbers are usually in ascending order 104 * and match Linux CPU ids, here is an optimization: we check 105 * the VP number for the highest bit in the supplied set first 106 * so we can quickly find out if using *_EX hypercalls is a 107 * must. We will also check all VP numbers when walking the 108 * supplied CPU set to remain correct in all cases. 109 */ 110 if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64) 111 goto do_ex_hypercall; 112 113 for_each_cpu(cpu, cpus) { 114 vcpu = hv_cpu_number_to_vp_number(cpu); 115 if (vcpu == VP_INVAL) { 116 local_irq_restore(flags); 117 goto do_native; 118 } 119 120 if (vcpu >= 64) 121 goto do_ex_hypercall; 122 123 __set_bit(vcpu, (unsigned long *) 124 &flush->processor_mask); 125 } 126 } 127 128 /* 129 * We can flush not more than max_gvas with one hypercall. Flush the 130 * whole address space if we were asked to do more. 131 */ 132 max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]); 133 134 if (info->end == TLB_FLUSH_ALL) { 135 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; 136 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, 137 flush, NULL); 138 } else if (info->end && 139 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) { 140 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, 141 flush, NULL); 142 } else { 143 gva_n = fill_gva_list(flush->gva_list, 0, 144 info->start, info->end); 145 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST, 146 gva_n, 0, flush, NULL); 147 } 148 goto check_status; 149 150 do_ex_hypercall: 151 status = hyperv_flush_tlb_others_ex(cpus, info); 152 153 check_status: 154 local_irq_restore(flags); 155 156 if (!(status & HV_HYPERCALL_RESULT_MASK)) 157 return; 158 do_native: 159 native_flush_tlb_others(cpus, info); 160 } 161 162 static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus, 163 const struct flush_tlb_info *info) 164 { 165 int nr_bank = 0, max_gvas, gva_n; 166 struct hv_tlb_flush_ex **flush_pcpu; 167 struct hv_tlb_flush_ex *flush; 168 u64 status; 169 170 if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 171 return U64_MAX; 172 173 flush_pcpu = (struct hv_tlb_flush_ex **) 174 this_cpu_ptr(hyperv_pcpu_input_arg); 175 176 flush = *flush_pcpu; 177 178 if (info->mm) { 179 /* 180 * AddressSpace argument must match the CR3 with PCID bits 181 * stripped out. 182 */ 183 flush->address_space = virt_to_phys(info->mm->pgd); 184 flush->address_space &= CR3_ADDR_MASK; 185 flush->flags = 0; 186 } else { 187 flush->address_space = 0; 188 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; 189 } 190 191 flush->hv_vp_set.valid_bank_mask = 0; 192 193 flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K; 194 nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus); 195 if (nr_bank < 0) 196 return U64_MAX; 197 198 /* 199 * We can flush not more than max_gvas with one hypercall. Flush the 200 * whole address space if we were asked to do more. 201 */ 202 max_gvas = 203 (PAGE_SIZE - sizeof(*flush) - nr_bank * 204 sizeof(flush->hv_vp_set.bank_contents[0])) / 205 sizeof(flush->gva_list[0]); 206 207 if (info->end == TLB_FLUSH_ALL) { 208 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; 209 status = hv_do_rep_hypercall( 210 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX, 211 0, nr_bank, flush, NULL); 212 } else if (info->end && 213 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) { 214 status = hv_do_rep_hypercall( 215 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX, 216 0, nr_bank, flush, NULL); 217 } else { 218 gva_n = fill_gva_list(flush->gva_list, nr_bank, 219 info->start, info->end); 220 status = hv_do_rep_hypercall( 221 HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX, 222 gva_n, nr_bank, flush, NULL); 223 } 224 225 return status; 226 } 227 228 void hyperv_setup_mmu_ops(void) 229 { 230 if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED)) 231 return; 232 233 pr_info("Using hypercall for remote TLB flush\n"); 234 pv_ops.mmu.flush_tlb_others = hyperv_flush_tlb_others; 235 pv_ops.mmu.tlb_remove_table = tlb_remove_table; 236 } 237