1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Support KVM gust page tracking 4 * 5 * This feature allows us to track page access in guest. Currently, only 6 * write access is tracked. 7 * 8 * Copyright(C) 2015 Intel Corporation. 9 * 10 * Author: 11 * Xiao Guangrong <guangrong.xiao@linux.intel.com> 12 */ 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/kvm_host.h> 16 #include <linux/rculist.h> 17 18 #include "mmu.h" 19 #include "mmu_internal.h" 20 #include "page_track.h" 21 22 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) 23 { 24 return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || 25 !tdp_enabled || kvm_shadow_root_allocated(kvm); 26 } 27 28 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) 29 { 30 kvfree(slot->arch.gfn_write_track); 31 slot->arch.gfn_write_track = NULL; 32 } 33 34 static int __kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot, 35 unsigned long npages) 36 { 37 const size_t size = sizeof(*slot->arch.gfn_write_track); 38 39 if (!slot->arch.gfn_write_track) 40 slot->arch.gfn_write_track = __vcalloc(npages, size, 41 GFP_KERNEL_ACCOUNT); 42 43 return slot->arch.gfn_write_track ? 0 : -ENOMEM; 44 } 45 46 int kvm_page_track_create_memslot(struct kvm *kvm, 47 struct kvm_memory_slot *slot, 48 unsigned long npages) 49 { 50 if (!kvm_page_track_write_tracking_enabled(kvm)) 51 return 0; 52 53 return __kvm_page_track_write_tracking_alloc(slot, npages); 54 } 55 56 int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot) 57 { 58 return __kvm_page_track_write_tracking_alloc(slot, slot->npages); 59 } 60 61 static void update_gfn_write_track(struct kvm_memory_slot *slot, gfn_t gfn, 62 short count) 63 { 64 int index, val; 65 66 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 67 68 val = slot->arch.gfn_write_track[index]; 69 70 if (WARN_ON_ONCE(val + count < 0 || val + count > USHRT_MAX)) 71 return; 72 73 slot->arch.gfn_write_track[index] += count; 74 } 75 76 /* 77 * add guest page to the tracking pool so that corresponding access on that 78 * page will be intercepted. 79 * 80 * It should be called under the protection both of mmu-lock and kvm->srcu 81 * or kvm->slots_lock. 82 * 83 * @kvm: the guest instance we are interested in. 84 * @slot: the @gfn belongs to. 85 * @gfn: the guest page. 86 */ 87 void kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot, 88 gfn_t gfn) 89 { 90 if (WARN_ON_ONCE(!kvm_page_track_write_tracking_enabled(kvm))) 91 return; 92 93 update_gfn_write_track(slot, gfn, 1); 94 95 /* 96 * new track stops large page mapping for the 97 * tracked page. 98 */ 99 kvm_mmu_gfn_disallow_lpage(slot, gfn); 100 101 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) 102 kvm_flush_remote_tlbs(kvm); 103 } 104 EXPORT_SYMBOL_GPL(kvm_write_track_add_gfn); 105 106 /* 107 * remove the guest page from the tracking pool which stops the interception 108 * of corresponding access on that page. 109 * 110 * It should be called under the protection both of mmu-lock and kvm->srcu 111 * or kvm->slots_lock. 112 * 113 * @kvm: the guest instance we are interested in. 114 * @slot: the @gfn belongs to. 115 * @gfn: the guest page. 116 */ 117 void kvm_write_track_remove_gfn(struct kvm *kvm, 118 struct kvm_memory_slot *slot, gfn_t gfn) 119 { 120 if (WARN_ON_ONCE(!kvm_page_track_write_tracking_enabled(kvm))) 121 return; 122 123 update_gfn_write_track(slot, gfn, -1); 124 125 /* 126 * allow large page mapping for the tracked page 127 * after the tracker is gone. 128 */ 129 kvm_mmu_gfn_allow_lpage(slot, gfn); 130 } 131 EXPORT_SYMBOL_GPL(kvm_write_track_remove_gfn); 132 133 /* 134 * check if the corresponding access on the specified guest page is tracked. 135 */ 136 bool kvm_gfn_is_write_tracked(struct kvm *kvm, 137 const struct kvm_memory_slot *slot, gfn_t gfn) 138 { 139 int index; 140 141 if (!slot) 142 return false; 143 144 if (!kvm_page_track_write_tracking_enabled(kvm)) 145 return false; 146 147 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 148 return !!READ_ONCE(slot->arch.gfn_write_track[index]); 149 } 150 151 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 152 void kvm_page_track_cleanup(struct kvm *kvm) 153 { 154 struct kvm_page_track_notifier_head *head; 155 156 head = &kvm->arch.track_notifier_head; 157 cleanup_srcu_struct(&head->track_srcu); 158 } 159 160 int kvm_page_track_init(struct kvm *kvm) 161 { 162 struct kvm_page_track_notifier_head *head; 163 164 head = &kvm->arch.track_notifier_head; 165 INIT_HLIST_HEAD(&head->track_notifier_list); 166 return init_srcu_struct(&head->track_srcu); 167 } 168 169 /* 170 * register the notifier so that event interception for the tracked guest 171 * pages can be received. 172 */ 173 void 174 kvm_page_track_register_notifier(struct kvm *kvm, 175 struct kvm_page_track_notifier_node *n) 176 { 177 struct kvm_page_track_notifier_head *head; 178 179 head = &kvm->arch.track_notifier_head; 180 181 write_lock(&kvm->mmu_lock); 182 hlist_add_head_rcu(&n->node, &head->track_notifier_list); 183 write_unlock(&kvm->mmu_lock); 184 } 185 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); 186 187 /* 188 * stop receiving the event interception. It is the opposed operation of 189 * kvm_page_track_register_notifier(). 190 */ 191 void 192 kvm_page_track_unregister_notifier(struct kvm *kvm, 193 struct kvm_page_track_notifier_node *n) 194 { 195 struct kvm_page_track_notifier_head *head; 196 197 head = &kvm->arch.track_notifier_head; 198 199 write_lock(&kvm->mmu_lock); 200 hlist_del_rcu(&n->node); 201 write_unlock(&kvm->mmu_lock); 202 synchronize_srcu(&head->track_srcu); 203 } 204 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); 205 206 /* 207 * Notify the node that write access is intercepted and write emulation is 208 * finished at this time. 209 * 210 * The node should figure out if the written page is the one that node is 211 * interested in by itself. 212 */ 213 void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes) 214 { 215 struct kvm_page_track_notifier_head *head; 216 struct kvm_page_track_notifier_node *n; 217 int idx; 218 219 head = &kvm->arch.track_notifier_head; 220 221 if (hlist_empty(&head->track_notifier_list)) 222 return; 223 224 idx = srcu_read_lock(&head->track_srcu); 225 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 226 srcu_read_lock_held(&head->track_srcu)) 227 if (n->track_write) 228 n->track_write(gpa, new, bytes, n); 229 srcu_read_unlock(&head->track_srcu, idx); 230 } 231 232 /* 233 * Notify external page track nodes that a memory region is being removed from 234 * the VM, e.g. so that users can free any associated metadata. 235 */ 236 void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot) 237 { 238 struct kvm_page_track_notifier_head *head; 239 struct kvm_page_track_notifier_node *n; 240 int idx; 241 242 head = &kvm->arch.track_notifier_head; 243 244 if (hlist_empty(&head->track_notifier_list)) 245 return; 246 247 idx = srcu_read_lock(&head->track_srcu); 248 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 249 srcu_read_lock_held(&head->track_srcu)) 250 if (n->track_remove_region) 251 n->track_remove_region(slot->base_gfn, slot->npages, n); 252 srcu_read_unlock(&head->track_srcu, idx); 253 } 254 255 #endif 256