1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Support KVM gust page tracking 4 * 5 * This feature allows us to track page access in guest. Currently, only 6 * write access is tracked. 7 * 8 * Copyright(C) 2015 Intel Corporation. 9 * 10 * Author: 11 * Xiao Guangrong <guangrong.xiao@linux.intel.com> 12 */ 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/lockdep.h> 16 #include <linux/kvm_host.h> 17 #include <linux/rculist.h> 18 19 #include "mmu.h" 20 #include "mmu_internal.h" 21 #include "page_track.h" 22 23 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) 24 { 25 return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || 26 !tdp_enabled || kvm_shadow_root_allocated(kvm); 27 } 28 29 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) 30 { 31 kvfree(slot->arch.gfn_write_track); 32 slot->arch.gfn_write_track = NULL; 33 } 34 35 static int __kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot, 36 unsigned long npages) 37 { 38 const size_t size = sizeof(*slot->arch.gfn_write_track); 39 40 if (!slot->arch.gfn_write_track) 41 slot->arch.gfn_write_track = __vcalloc(npages, size, 42 GFP_KERNEL_ACCOUNT); 43 44 return slot->arch.gfn_write_track ? 0 : -ENOMEM; 45 } 46 47 int kvm_page_track_create_memslot(struct kvm *kvm, 48 struct kvm_memory_slot *slot, 49 unsigned long npages) 50 { 51 if (!kvm_page_track_write_tracking_enabled(kvm)) 52 return 0; 53 54 return __kvm_page_track_write_tracking_alloc(slot, npages); 55 } 56 57 int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot) 58 { 59 return __kvm_page_track_write_tracking_alloc(slot, slot->npages); 60 } 61 62 static void update_gfn_write_track(struct kvm_memory_slot *slot, gfn_t gfn, 63 short count) 64 { 65 int index, val; 66 67 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 68 69 val = slot->arch.gfn_write_track[index]; 70 71 if (WARN_ON_ONCE(val + count < 0 || val + count > USHRT_MAX)) 72 return; 73 74 slot->arch.gfn_write_track[index] += count; 75 } 76 77 /* 78 * add guest page to the tracking pool so that corresponding access on that 79 * page will be intercepted. 80 * 81 * @kvm: the guest instance we are interested in. 82 * @slot: the @gfn belongs to. 83 * @gfn: the guest page. 84 */ 85 void kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot, 86 gfn_t gfn) 87 { 88 lockdep_assert_held_write(&kvm->mmu_lock); 89 90 lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) || 91 srcu_read_lock_held(&kvm->srcu)); 92 93 if (WARN_ON_ONCE(!kvm_page_track_write_tracking_enabled(kvm))) 94 return; 95 96 update_gfn_write_track(slot, gfn, 1); 97 98 /* 99 * new track stops large page mapping for the 100 * tracked page. 101 */ 102 kvm_mmu_gfn_disallow_lpage(slot, gfn); 103 104 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) 105 kvm_flush_remote_tlbs(kvm); 106 } 107 EXPORT_SYMBOL_GPL(kvm_write_track_add_gfn); 108 109 /* 110 * remove the guest page from the tracking pool which stops the interception 111 * of corresponding access on that page. 112 * 113 * @kvm: the guest instance we are interested in. 114 * @slot: the @gfn belongs to. 115 * @gfn: the guest page. 116 */ 117 void kvm_write_track_remove_gfn(struct kvm *kvm, 118 struct kvm_memory_slot *slot, gfn_t gfn) 119 { 120 lockdep_assert_held_write(&kvm->mmu_lock); 121 122 lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) || 123 srcu_read_lock_held(&kvm->srcu)); 124 125 if (WARN_ON_ONCE(!kvm_page_track_write_tracking_enabled(kvm))) 126 return; 127 128 update_gfn_write_track(slot, gfn, -1); 129 130 /* 131 * allow large page mapping for the tracked page 132 * after the tracker is gone. 133 */ 134 kvm_mmu_gfn_allow_lpage(slot, gfn); 135 } 136 EXPORT_SYMBOL_GPL(kvm_write_track_remove_gfn); 137 138 /* 139 * check if the corresponding access on the specified guest page is tracked. 140 */ 141 bool kvm_gfn_is_write_tracked(struct kvm *kvm, 142 const struct kvm_memory_slot *slot, gfn_t gfn) 143 { 144 int index; 145 146 if (!slot) 147 return false; 148 149 if (!kvm_page_track_write_tracking_enabled(kvm)) 150 return false; 151 152 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 153 return !!READ_ONCE(slot->arch.gfn_write_track[index]); 154 } 155 156 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 157 void kvm_page_track_cleanup(struct kvm *kvm) 158 { 159 struct kvm_page_track_notifier_head *head; 160 161 head = &kvm->arch.track_notifier_head; 162 cleanup_srcu_struct(&head->track_srcu); 163 } 164 165 int kvm_page_track_init(struct kvm *kvm) 166 { 167 struct kvm_page_track_notifier_head *head; 168 169 head = &kvm->arch.track_notifier_head; 170 INIT_HLIST_HEAD(&head->track_notifier_list); 171 return init_srcu_struct(&head->track_srcu); 172 } 173 174 /* 175 * register the notifier so that event interception for the tracked guest 176 * pages can be received. 177 */ 178 void 179 kvm_page_track_register_notifier(struct kvm *kvm, 180 struct kvm_page_track_notifier_node *n) 181 { 182 struct kvm_page_track_notifier_head *head; 183 184 head = &kvm->arch.track_notifier_head; 185 186 write_lock(&kvm->mmu_lock); 187 hlist_add_head_rcu(&n->node, &head->track_notifier_list); 188 write_unlock(&kvm->mmu_lock); 189 } 190 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); 191 192 /* 193 * stop receiving the event interception. It is the opposed operation of 194 * kvm_page_track_register_notifier(). 195 */ 196 void 197 kvm_page_track_unregister_notifier(struct kvm *kvm, 198 struct kvm_page_track_notifier_node *n) 199 { 200 struct kvm_page_track_notifier_head *head; 201 202 head = &kvm->arch.track_notifier_head; 203 204 write_lock(&kvm->mmu_lock); 205 hlist_del_rcu(&n->node); 206 write_unlock(&kvm->mmu_lock); 207 synchronize_srcu(&head->track_srcu); 208 } 209 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); 210 211 /* 212 * Notify the node that write access is intercepted and write emulation is 213 * finished at this time. 214 * 215 * The node should figure out if the written page is the one that node is 216 * interested in by itself. 217 */ 218 void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes) 219 { 220 struct kvm_page_track_notifier_head *head; 221 struct kvm_page_track_notifier_node *n; 222 int idx; 223 224 head = &kvm->arch.track_notifier_head; 225 226 if (hlist_empty(&head->track_notifier_list)) 227 return; 228 229 idx = srcu_read_lock(&head->track_srcu); 230 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 231 srcu_read_lock_held(&head->track_srcu)) 232 if (n->track_write) 233 n->track_write(gpa, new, bytes, n); 234 srcu_read_unlock(&head->track_srcu, idx); 235 } 236 237 /* 238 * Notify external page track nodes that a memory region is being removed from 239 * the VM, e.g. so that users can free any associated metadata. 240 */ 241 void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot) 242 { 243 struct kvm_page_track_notifier_head *head; 244 struct kvm_page_track_notifier_node *n; 245 int idx; 246 247 head = &kvm->arch.track_notifier_head; 248 249 if (hlist_empty(&head->track_notifier_list)) 250 return; 251 252 idx = srcu_read_lock(&head->track_srcu); 253 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 254 srcu_read_lock_held(&head->track_srcu)) 255 if (n->track_remove_region) 256 n->track_remove_region(slot->base_gfn, slot->npages, n); 257 srcu_read_unlock(&head->track_srcu, idx); 258 } 259 260 #endif 261