1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Support KVM gust page tracking 4 * 5 * This feature allows us to track page access in guest. Currently, only 6 * write access is tracked. 7 * 8 * Copyright(C) 2015 Intel Corporation. 9 * 10 * Author: 11 * Xiao Guangrong <guangrong.xiao@linux.intel.com> 12 */ 13 14 #include <linux/kvm_host.h> 15 #include <linux/rculist.h> 16 17 #include <asm/kvm_host.h> 18 #include <asm/kvm_page_track.h> 19 20 #include "mmu.h" 21 22 void kvm_page_track_free_memslot(struct kvm_memory_slot *free, 23 struct kvm_memory_slot *dont) 24 { 25 int i; 26 27 for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) 28 if (!dont || free->arch.gfn_track[i] != 29 dont->arch.gfn_track[i]) { 30 kvfree(free->arch.gfn_track[i]); 31 free->arch.gfn_track[i] = NULL; 32 } 33 } 34 35 int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, 36 unsigned long npages) 37 { 38 int i; 39 40 for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { 41 slot->arch.gfn_track[i] = 42 kvcalloc(npages, sizeof(*slot->arch.gfn_track[i]), 43 GFP_KERNEL_ACCOUNT); 44 if (!slot->arch.gfn_track[i]) 45 goto track_free; 46 } 47 48 return 0; 49 50 track_free: 51 kvm_page_track_free_memslot(slot, NULL); 52 return -ENOMEM; 53 } 54 55 static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode) 56 { 57 if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX) 58 return false; 59 60 return true; 61 } 62 63 static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, 64 enum kvm_page_track_mode mode, short count) 65 { 66 int index, val; 67 68 index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); 69 70 val = slot->arch.gfn_track[mode][index]; 71 72 if (WARN_ON(val + count < 0 || val + count > USHRT_MAX)) 73 return; 74 75 slot->arch.gfn_track[mode][index] += count; 76 } 77 78 /* 79 * add guest page to the tracking pool so that corresponding access on that 80 * page will be intercepted. 81 * 82 * It should be called under the protection both of mmu-lock and kvm->srcu 83 * or kvm->slots_lock. 84 * 85 * @kvm: the guest instance we are interested in. 86 * @slot: the @gfn belongs to. 87 * @gfn: the guest page. 88 * @mode: tracking mode, currently only write track is supported. 89 */ 90 void kvm_slot_page_track_add_page(struct kvm *kvm, 91 struct kvm_memory_slot *slot, gfn_t gfn, 92 enum kvm_page_track_mode mode) 93 { 94 95 if (WARN_ON(!page_track_mode_is_valid(mode))) 96 return; 97 98 update_gfn_track(slot, gfn, mode, 1); 99 100 /* 101 * new track stops large page mapping for the 102 * tracked page. 103 */ 104 kvm_mmu_gfn_disallow_lpage(slot, gfn); 105 106 if (mode == KVM_PAGE_TRACK_WRITE) 107 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn)) 108 kvm_flush_remote_tlbs(kvm); 109 } 110 EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); 111 112 /* 113 * remove the guest page from the tracking pool which stops the interception 114 * of corresponding access on that page. It is the opposed operation of 115 * kvm_slot_page_track_add_page(). 116 * 117 * It should be called under the protection both of mmu-lock and kvm->srcu 118 * or kvm->slots_lock. 119 * 120 * @kvm: the guest instance we are interested in. 121 * @slot: the @gfn belongs to. 122 * @gfn: the guest page. 123 * @mode: tracking mode, currently only write track is supported. 124 */ 125 void kvm_slot_page_track_remove_page(struct kvm *kvm, 126 struct kvm_memory_slot *slot, gfn_t gfn, 127 enum kvm_page_track_mode mode) 128 { 129 if (WARN_ON(!page_track_mode_is_valid(mode))) 130 return; 131 132 update_gfn_track(slot, gfn, mode, -1); 133 134 /* 135 * allow large page mapping for the tracked page 136 * after the tracker is gone. 137 */ 138 kvm_mmu_gfn_allow_lpage(slot, gfn); 139 } 140 EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page); 141 142 /* 143 * check if the corresponding access on the specified guest page is tracked. 144 */ 145 bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, 146 enum kvm_page_track_mode mode) 147 { 148 struct kvm_memory_slot *slot; 149 int index; 150 151 if (WARN_ON(!page_track_mode_is_valid(mode))) 152 return false; 153 154 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 155 if (!slot) 156 return false; 157 158 index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); 159 return !!READ_ONCE(slot->arch.gfn_track[mode][index]); 160 } 161 162 void kvm_page_track_cleanup(struct kvm *kvm) 163 { 164 struct kvm_page_track_notifier_head *head; 165 166 head = &kvm->arch.track_notifier_head; 167 cleanup_srcu_struct(&head->track_srcu); 168 } 169 170 void kvm_page_track_init(struct kvm *kvm) 171 { 172 struct kvm_page_track_notifier_head *head; 173 174 head = &kvm->arch.track_notifier_head; 175 init_srcu_struct(&head->track_srcu); 176 INIT_HLIST_HEAD(&head->track_notifier_list); 177 } 178 179 /* 180 * register the notifier so that event interception for the tracked guest 181 * pages can be received. 182 */ 183 void 184 kvm_page_track_register_notifier(struct kvm *kvm, 185 struct kvm_page_track_notifier_node *n) 186 { 187 struct kvm_page_track_notifier_head *head; 188 189 head = &kvm->arch.track_notifier_head; 190 191 spin_lock(&kvm->mmu_lock); 192 hlist_add_head_rcu(&n->node, &head->track_notifier_list); 193 spin_unlock(&kvm->mmu_lock); 194 } 195 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); 196 197 /* 198 * stop receiving the event interception. It is the opposed operation of 199 * kvm_page_track_register_notifier(). 200 */ 201 void 202 kvm_page_track_unregister_notifier(struct kvm *kvm, 203 struct kvm_page_track_notifier_node *n) 204 { 205 struct kvm_page_track_notifier_head *head; 206 207 head = &kvm->arch.track_notifier_head; 208 209 spin_lock(&kvm->mmu_lock); 210 hlist_del_rcu(&n->node); 211 spin_unlock(&kvm->mmu_lock); 212 synchronize_srcu(&head->track_srcu); 213 } 214 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); 215 216 /* 217 * Notify the node that write access is intercepted and write emulation is 218 * finished at this time. 219 * 220 * The node should figure out if the written page is the one that node is 221 * interested in by itself. 222 */ 223 void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 224 int bytes) 225 { 226 struct kvm_page_track_notifier_head *head; 227 struct kvm_page_track_notifier_node *n; 228 int idx; 229 230 head = &vcpu->kvm->arch.track_notifier_head; 231 232 if (hlist_empty(&head->track_notifier_list)) 233 return; 234 235 idx = srcu_read_lock(&head->track_srcu); 236 hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) 237 if (n->track_write) 238 n->track_write(vcpu, gpa, new, bytes, n); 239 srcu_read_unlock(&head->track_srcu, idx); 240 } 241 242 /* 243 * Notify the node that memory slot is being removed or moved so that it can 244 * drop write-protection for the pages in the memory slot. 245 * 246 * The node should figure out it has any write-protected pages in this slot 247 * by itself. 248 */ 249 void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot) 250 { 251 struct kvm_page_track_notifier_head *head; 252 struct kvm_page_track_notifier_node *n; 253 int idx; 254 255 head = &kvm->arch.track_notifier_head; 256 257 if (hlist_empty(&head->track_notifier_list)) 258 return; 259 260 idx = srcu_read_lock(&head->track_srcu); 261 hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) 262 if (n->track_flush_slot) 263 n->track_flush_slot(kvm, slot, n); 264 srcu_read_unlock(&head->track_srcu, idx); 265 } 266