1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Support KVM gust page tracking 4 * 5 * This feature allows us to track page access in guest. Currently, only 6 * write access is tracked. 7 * 8 * Copyright(C) 2015 Intel Corporation. 9 * 10 * Author: 11 * Xiao Guangrong <guangrong.xiao@linux.intel.com> 12 */ 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/kvm_host.h> 16 #include <linux/rculist.h> 17 18 #include "mmu.h" 19 #include "mmu_internal.h" 20 #include "page_track.h" 21 22 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) 23 { 24 return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || 25 !tdp_enabled || kvm_shadow_root_allocated(kvm); 26 } 27 28 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) 29 { 30 kvfree(slot->arch.gfn_write_track); 31 slot->arch.gfn_write_track = NULL; 32 } 33 34 static int __kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot, 35 unsigned long npages) 36 { 37 const size_t size = sizeof(*slot->arch.gfn_write_track); 38 39 if (!slot->arch.gfn_write_track) 40 slot->arch.gfn_write_track = __vcalloc(npages, size, 41 GFP_KERNEL_ACCOUNT); 42 43 return slot->arch.gfn_write_track ? 0 : -ENOMEM; 44 } 45 46 int kvm_page_track_create_memslot(struct kvm *kvm, 47 struct kvm_memory_slot *slot, 48 unsigned long npages) 49 { 50 if (!kvm_page_track_write_tracking_enabled(kvm)) 51 return 0; 52 53 return __kvm_page_track_write_tracking_alloc(slot, npages); 54 } 55 56 int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot) 57 { 58 return __kvm_page_track_write_tracking_alloc(slot, slot->npages); 59 } 60 61 static void update_gfn_write_track(struct kvm_memory_slot *slot, gfn_t gfn, 62 short count) 63 { 64 int index, val; 65 66 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 67 68 val = slot->arch.gfn_write_track[index]; 69 70 if (WARN_ON_ONCE(val + count < 0 || val + count > USHRT_MAX)) 71 return; 72 73 slot->arch.gfn_write_track[index] += count; 74 } 75 76 /* 77 * add guest page to the tracking pool so that corresponding access on that 78 * page will be intercepted. 79 * 80 * It should be called under the protection both of mmu-lock and kvm->srcu 81 * or kvm->slots_lock. 82 * 83 * @kvm: the guest instance we are interested in. 84 * @slot: the @gfn belongs to. 85 * @gfn: the guest page. 86 */ 87 void kvm_slot_page_track_add_page(struct kvm *kvm, 88 struct kvm_memory_slot *slot, gfn_t gfn) 89 { 90 if (WARN_ON_ONCE(!kvm_page_track_write_tracking_enabled(kvm))) 91 return; 92 93 update_gfn_write_track(slot, gfn, 1); 94 95 /* 96 * new track stops large page mapping for the 97 * tracked page. 98 */ 99 kvm_mmu_gfn_disallow_lpage(slot, gfn); 100 101 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) 102 kvm_flush_remote_tlbs(kvm); 103 } 104 EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); 105 106 /* 107 * remove the guest page from the tracking pool which stops the interception 108 * of corresponding access on that page. It is the opposed operation of 109 * kvm_slot_page_track_add_page(). 110 * 111 * It should be called under the protection both of mmu-lock and kvm->srcu 112 * or kvm->slots_lock. 113 * 114 * @kvm: the guest instance we are interested in. 115 * @slot: the @gfn belongs to. 116 * @gfn: the guest page. 117 */ 118 void kvm_slot_page_track_remove_page(struct kvm *kvm, 119 struct kvm_memory_slot *slot, gfn_t gfn) 120 { 121 if (WARN_ON_ONCE(!kvm_page_track_write_tracking_enabled(kvm))) 122 return; 123 124 update_gfn_write_track(slot, gfn, -1); 125 126 /* 127 * allow large page mapping for the tracked page 128 * after the tracker is gone. 129 */ 130 kvm_mmu_gfn_allow_lpage(slot, gfn); 131 } 132 EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page); 133 134 /* 135 * check if the corresponding access on the specified guest page is tracked. 136 */ 137 bool kvm_slot_page_track_is_active(struct kvm *kvm, 138 const struct kvm_memory_slot *slot, 139 gfn_t gfn) 140 { 141 int index; 142 143 if (!slot) 144 return false; 145 146 if (!kvm_page_track_write_tracking_enabled(kvm)) 147 return false; 148 149 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 150 return !!READ_ONCE(slot->arch.gfn_write_track[index]); 151 } 152 153 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING 154 void kvm_page_track_cleanup(struct kvm *kvm) 155 { 156 struct kvm_page_track_notifier_head *head; 157 158 head = &kvm->arch.track_notifier_head; 159 cleanup_srcu_struct(&head->track_srcu); 160 } 161 162 int kvm_page_track_init(struct kvm *kvm) 163 { 164 struct kvm_page_track_notifier_head *head; 165 166 head = &kvm->arch.track_notifier_head; 167 INIT_HLIST_HEAD(&head->track_notifier_list); 168 return init_srcu_struct(&head->track_srcu); 169 } 170 171 /* 172 * register the notifier so that event interception for the tracked guest 173 * pages can be received. 174 */ 175 void 176 kvm_page_track_register_notifier(struct kvm *kvm, 177 struct kvm_page_track_notifier_node *n) 178 { 179 struct kvm_page_track_notifier_head *head; 180 181 head = &kvm->arch.track_notifier_head; 182 183 write_lock(&kvm->mmu_lock); 184 hlist_add_head_rcu(&n->node, &head->track_notifier_list); 185 write_unlock(&kvm->mmu_lock); 186 } 187 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); 188 189 /* 190 * stop receiving the event interception. It is the opposed operation of 191 * kvm_page_track_register_notifier(). 192 */ 193 void 194 kvm_page_track_unregister_notifier(struct kvm *kvm, 195 struct kvm_page_track_notifier_node *n) 196 { 197 struct kvm_page_track_notifier_head *head; 198 199 head = &kvm->arch.track_notifier_head; 200 201 write_lock(&kvm->mmu_lock); 202 hlist_del_rcu(&n->node); 203 write_unlock(&kvm->mmu_lock); 204 synchronize_srcu(&head->track_srcu); 205 } 206 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); 207 208 /* 209 * Notify the node that write access is intercepted and write emulation is 210 * finished at this time. 211 * 212 * The node should figure out if the written page is the one that node is 213 * interested in by itself. 214 */ 215 void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes) 216 { 217 struct kvm_page_track_notifier_head *head; 218 struct kvm_page_track_notifier_node *n; 219 int idx; 220 221 head = &kvm->arch.track_notifier_head; 222 223 if (hlist_empty(&head->track_notifier_list)) 224 return; 225 226 idx = srcu_read_lock(&head->track_srcu); 227 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 228 srcu_read_lock_held(&head->track_srcu)) 229 if (n->track_write) 230 n->track_write(gpa, new, bytes, n); 231 srcu_read_unlock(&head->track_srcu, idx); 232 } 233 234 /* 235 * Notify external page track nodes that a memory region is being removed from 236 * the VM, e.g. so that users can free any associated metadata. 237 */ 238 void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot) 239 { 240 struct kvm_page_track_notifier_head *head; 241 struct kvm_page_track_notifier_node *n; 242 int idx; 243 244 head = &kvm->arch.track_notifier_head; 245 246 if (hlist_empty(&head->track_notifier_list)) 247 return; 248 249 idx = srcu_read_lock(&head->track_srcu); 250 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 251 srcu_read_lock_held(&head->track_srcu)) 252 if (n->track_remove_region) 253 n->track_remove_region(slot->base_gfn, slot->npages, n); 254 srcu_read_unlock(&head->track_srcu, idx); 255 } 256 257 #endif 258