1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Support KVM gust page tracking 4 * 5 * This feature allows us to track page access in guest. Currently, only 6 * write access is tracked. 7 * 8 * Copyright(C) 2015 Intel Corporation. 9 * 10 * Author: 11 * Xiao Guangrong <guangrong.xiao@linux.intel.com> 12 */ 13 14 #include <linux/kvm_host.h> 15 #include <linux/rculist.h> 16 17 #include <asm/kvm_page_track.h> 18 19 #include "mmu.h" 20 #include "mmu_internal.h" 21 22 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) 23 { 24 int i; 25 26 for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { 27 kvfree(slot->arch.gfn_track[i]); 28 slot->arch.gfn_track[i] = NULL; 29 } 30 } 31 32 int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, 33 unsigned long npages) 34 { 35 int i; 36 37 for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { 38 slot->arch.gfn_track[i] = 39 kvcalloc(npages, sizeof(*slot->arch.gfn_track[i]), 40 GFP_KERNEL_ACCOUNT); 41 if (!slot->arch.gfn_track[i]) 42 goto track_free; 43 } 44 45 return 0; 46 47 track_free: 48 kvm_page_track_free_memslot(slot); 49 return -ENOMEM; 50 } 51 52 static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode) 53 { 54 if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX) 55 return false; 56 57 return true; 58 } 59 60 static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, 61 enum kvm_page_track_mode mode, short count) 62 { 63 int index, val; 64 65 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 66 67 val = slot->arch.gfn_track[mode][index]; 68 69 if (WARN_ON(val + count < 0 || val + count > USHRT_MAX)) 70 return; 71 72 slot->arch.gfn_track[mode][index] += count; 73 } 74 75 /* 76 * add guest page to the tracking pool so that corresponding access on that 77 * page will be intercepted. 78 * 79 * It should be called under the protection both of mmu-lock and kvm->srcu 80 * or kvm->slots_lock. 81 * 82 * @kvm: the guest instance we are interested in. 83 * @slot: the @gfn belongs to. 84 * @gfn: the guest page. 85 * @mode: tracking mode, currently only write track is supported. 86 */ 87 void kvm_slot_page_track_add_page(struct kvm *kvm, 88 struct kvm_memory_slot *slot, gfn_t gfn, 89 enum kvm_page_track_mode mode) 90 { 91 92 if (WARN_ON(!page_track_mode_is_valid(mode))) 93 return; 94 95 update_gfn_track(slot, gfn, mode, 1); 96 97 /* 98 * new track stops large page mapping for the 99 * tracked page. 100 */ 101 kvm_mmu_gfn_disallow_lpage(slot, gfn); 102 103 if (mode == KVM_PAGE_TRACK_WRITE) 104 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) 105 kvm_flush_remote_tlbs(kvm); 106 } 107 EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); 108 109 /* 110 * remove the guest page from the tracking pool which stops the interception 111 * of corresponding access on that page. It is the opposed operation of 112 * kvm_slot_page_track_add_page(). 113 * 114 * It should be called under the protection both of mmu-lock and kvm->srcu 115 * or kvm->slots_lock. 116 * 117 * @kvm: the guest instance we are interested in. 118 * @slot: the @gfn belongs to. 119 * @gfn: the guest page. 120 * @mode: tracking mode, currently only write track is supported. 121 */ 122 void kvm_slot_page_track_remove_page(struct kvm *kvm, 123 struct kvm_memory_slot *slot, gfn_t gfn, 124 enum kvm_page_track_mode mode) 125 { 126 if (WARN_ON(!page_track_mode_is_valid(mode))) 127 return; 128 129 update_gfn_track(slot, gfn, mode, -1); 130 131 /* 132 * allow large page mapping for the tracked page 133 * after the tracker is gone. 134 */ 135 kvm_mmu_gfn_allow_lpage(slot, gfn); 136 } 137 EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page); 138 139 /* 140 * check if the corresponding access on the specified guest page is tracked. 141 */ 142 bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, 143 enum kvm_page_track_mode mode) 144 { 145 struct kvm_memory_slot *slot; 146 int index; 147 148 if (WARN_ON(!page_track_mode_is_valid(mode))) 149 return false; 150 151 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 152 if (!slot) 153 return false; 154 155 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 156 return !!READ_ONCE(slot->arch.gfn_track[mode][index]); 157 } 158 159 void kvm_page_track_cleanup(struct kvm *kvm) 160 { 161 struct kvm_page_track_notifier_head *head; 162 163 head = &kvm->arch.track_notifier_head; 164 cleanup_srcu_struct(&head->track_srcu); 165 } 166 167 void kvm_page_track_init(struct kvm *kvm) 168 { 169 struct kvm_page_track_notifier_head *head; 170 171 head = &kvm->arch.track_notifier_head; 172 init_srcu_struct(&head->track_srcu); 173 INIT_HLIST_HEAD(&head->track_notifier_list); 174 } 175 176 /* 177 * register the notifier so that event interception for the tracked guest 178 * pages can be received. 179 */ 180 void 181 kvm_page_track_register_notifier(struct kvm *kvm, 182 struct kvm_page_track_notifier_node *n) 183 { 184 struct kvm_page_track_notifier_head *head; 185 186 head = &kvm->arch.track_notifier_head; 187 188 write_lock(&kvm->mmu_lock); 189 hlist_add_head_rcu(&n->node, &head->track_notifier_list); 190 write_unlock(&kvm->mmu_lock); 191 } 192 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); 193 194 /* 195 * stop receiving the event interception. It is the opposed operation of 196 * kvm_page_track_register_notifier(). 197 */ 198 void 199 kvm_page_track_unregister_notifier(struct kvm *kvm, 200 struct kvm_page_track_notifier_node *n) 201 { 202 struct kvm_page_track_notifier_head *head; 203 204 head = &kvm->arch.track_notifier_head; 205 206 write_lock(&kvm->mmu_lock); 207 hlist_del_rcu(&n->node); 208 write_unlock(&kvm->mmu_lock); 209 synchronize_srcu(&head->track_srcu); 210 } 211 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); 212 213 /* 214 * Notify the node that write access is intercepted and write emulation is 215 * finished at this time. 216 * 217 * The node should figure out if the written page is the one that node is 218 * interested in by itself. 219 */ 220 void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 221 int bytes) 222 { 223 struct kvm_page_track_notifier_head *head; 224 struct kvm_page_track_notifier_node *n; 225 int idx; 226 227 head = &vcpu->kvm->arch.track_notifier_head; 228 229 if (hlist_empty(&head->track_notifier_list)) 230 return; 231 232 idx = srcu_read_lock(&head->track_srcu); 233 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 234 srcu_read_lock_held(&head->track_srcu)) 235 if (n->track_write) 236 n->track_write(vcpu, gpa, new, bytes, n); 237 srcu_read_unlock(&head->track_srcu, idx); 238 } 239 240 /* 241 * Notify the node that memory slot is being removed or moved so that it can 242 * drop write-protection for the pages in the memory slot. 243 * 244 * The node should figure out it has any write-protected pages in this slot 245 * by itself. 246 */ 247 void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot) 248 { 249 struct kvm_page_track_notifier_head *head; 250 struct kvm_page_track_notifier_node *n; 251 int idx; 252 253 head = &kvm->arch.track_notifier_head; 254 255 if (hlist_empty(&head->track_notifier_list)) 256 return; 257 258 idx = srcu_read_lock(&head->track_srcu); 259 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 260 srcu_read_lock_held(&head->track_srcu)) 261 if (n->track_flush_slot) 262 n->track_flush_slot(kvm, slot, n); 263 srcu_read_unlock(&head->track_srcu, idx); 264 } 265