1 // SPDX-License-Identifier: GPL-2.0-only
3  * Support KVM gust page tracking
23 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)  in kvm_page_track_write_tracking_enabled()  argument
26 	       !tdp_enabled || kvm_shadow_root_allocated(kvm);  in kvm_page_track_write_tracking_enabled()
31 	kvfree(slot->arch.gfn_write_track);  in kvm_page_track_free_memslot()
32 	slot->arch.gfn_write_track = NULL;  in kvm_page_track_free_memslot()
38 	const size_t size = sizeof(*slot->arch.gfn_write_track);  in __kvm_page_track_write_tracking_alloc()
40 	if (!slot->arch.gfn_write_track)  in __kvm_page_track_write_tracking_alloc()
41 		slot->arch.gfn_write_track = __vcalloc(npages, size,  in __kvm_page_track_write_tracking_alloc()
44 	return slot->arch.gfn_write_track ? 0 : -ENOMEM;  in __kvm_page_track_write_tracking_alloc()
47 int kvm_page_track_create_memslot(struct kvm *kvm,  in kvm_page_track_create_memslot()  argument
51 	if (!kvm_page_track_write_tracking_enabled(kvm))  in kvm_page_track_create_memslot()
59 	return __kvm_page_track_write_tracking_alloc(slot, slot->npages);  in kvm_page_track_write_tracking_alloc()
67 	index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);  in update_gfn_write_track()
69 	val = slot->arch.gfn_write_track[index];  in update_gfn_write_track()
74 	slot->arch.gfn_write_track[index] += count;  in update_gfn_write_track()
77 void __kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,  in __kvm_write_track_add_gfn()  argument
80 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_write_track_add_gfn()
82 	lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||  in __kvm_write_track_add_gfn()
83 			    srcu_read_lock_held(&kvm->srcu));  in __kvm_write_track_add_gfn()
85 	if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))  in __kvm_write_track_add_gfn()
96 	if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))  in __kvm_write_track_add_gfn()
97 		kvm_flush_remote_tlbs(kvm);  in __kvm_write_track_add_gfn()
100 void __kvm_write_track_remove_gfn(struct kvm *kvm,  in __kvm_write_track_remove_gfn()  argument
103 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_write_track_remove_gfn()
105 	lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||  in __kvm_write_track_remove_gfn()
106 			    srcu_read_lock_held(&kvm->srcu));  in __kvm_write_track_remove_gfn()
108 	if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))  in __kvm_write_track_remove_gfn()
111 	update_gfn_write_track(slot, gfn, -1);  in __kvm_write_track_remove_gfn()
123 bool kvm_gfn_is_write_tracked(struct kvm *kvm,  in kvm_gfn_is_write_tracked()  argument
131 	if (!kvm_page_track_write_tracking_enabled(kvm))  in kvm_gfn_is_write_tracked()
134 	index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);  in kvm_gfn_is_write_tracked()
135 	return !!READ_ONCE(slot->arch.gfn_write_track[index]);  in kvm_gfn_is_write_tracked()
139 void kvm_page_track_cleanup(struct kvm *kvm)  in kvm_page_track_cleanup()  argument
143 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_cleanup()
144 	cleanup_srcu_struct(&head->track_srcu);  in kvm_page_track_cleanup()
147 int kvm_page_track_init(struct kvm *kvm)  in kvm_page_track_init()  argument
151 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_init()
152 	INIT_HLIST_HEAD(&head->track_notifier_list);  in kvm_page_track_init()
153 	return init_srcu_struct(&head->track_srcu);  in kvm_page_track_init()
160 int kvm_page_track_register_notifier(struct kvm *kvm,  in kvm_page_track_register_notifier()  argument
165 	if (!kvm || kvm->mm != current->mm)  in kvm_page_track_register_notifier()
166 		return -ESRCH;  in kvm_page_track_register_notifier()
168 	kvm_get_kvm(kvm);  in kvm_page_track_register_notifier()
170 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_register_notifier()
172 	write_lock(&kvm->mmu_lock);  in kvm_page_track_register_notifier()
173 	hlist_add_head_rcu(&n->node, &head->track_notifier_list);  in kvm_page_track_register_notifier()
174 	write_unlock(&kvm->mmu_lock);  in kvm_page_track_register_notifier()
183 void kvm_page_track_unregister_notifier(struct kvm *kvm,  in kvm_page_track_unregister_notifier()  argument
188 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_unregister_notifier()
190 	write_lock(&kvm->mmu_lock);  in kvm_page_track_unregister_notifier()
191 	hlist_del_rcu(&n->node);  in kvm_page_track_unregister_notifier()
192 	write_unlock(&kvm->mmu_lock);  in kvm_page_track_unregister_notifier()
193 	synchronize_srcu(&head->track_srcu);  in kvm_page_track_unregister_notifier()
195 	kvm_put_kvm(kvm);  in kvm_page_track_unregister_notifier()
206 void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes)  in __kvm_page_track_write()  argument
212 	head = &kvm->arch.track_notifier_head;  in __kvm_page_track_write()
214 	if (hlist_empty(&head->track_notifier_list))  in __kvm_page_track_write()
217 	idx = srcu_read_lock(&head->track_srcu);  in __kvm_page_track_write()
218 	hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,  in __kvm_page_track_write()
219 				  srcu_read_lock_held(&head->track_srcu))  in __kvm_page_track_write()
220 		if (n->track_write)  in __kvm_page_track_write()
221 			n->track_write(gpa, new, bytes, n);  in __kvm_page_track_write()
222 	srcu_read_unlock(&head->track_srcu, idx);  in __kvm_page_track_write()
229 void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot)  in kvm_page_track_delete_slot()  argument
235 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_delete_slot()
237 	if (hlist_empty(&head->track_notifier_list))  in kvm_page_track_delete_slot()
240 	idx = srcu_read_lock(&head->track_srcu);  in kvm_page_track_delete_slot()
241 	hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,  in kvm_page_track_delete_slot()
242 				  srcu_read_lock_held(&head->track_srcu))  in kvm_page_track_delete_slot()
243 		if (n->track_remove_region)  in kvm_page_track_delete_slot()
244 			n->track_remove_region(slot->base_gfn, slot->npages, n);  in kvm_page_track_delete_slot()
245 	srcu_read_unlock(&head->track_srcu, idx);  in kvm_page_track_delete_slot()
252  * @kvm: the guest instance we are interested in.
255 int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_write_track_add_gfn()  argument
260 	idx = srcu_read_lock(&kvm->srcu);  in kvm_write_track_add_gfn()
262 	slot = gfn_to_memslot(kvm, gfn);  in kvm_write_track_add_gfn()
264 		srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_add_gfn()
265 		return -EINVAL;  in kvm_write_track_add_gfn()
268 	write_lock(&kvm->mmu_lock);  in kvm_write_track_add_gfn()
269 	__kvm_write_track_add_gfn(kvm, slot, gfn);  in kvm_write_track_add_gfn()
270 	write_unlock(&kvm->mmu_lock);  in kvm_write_track_add_gfn()
272 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_add_gfn()
282  * @kvm: the guest instance we are interested in.
285 int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_write_track_remove_gfn()  argument
290 	idx = srcu_read_lock(&kvm->srcu);  in kvm_write_track_remove_gfn()
292 	slot = gfn_to_memslot(kvm, gfn);  in kvm_write_track_remove_gfn()
294 		srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_remove_gfn()
295 		return -EINVAL;  in kvm_write_track_remove_gfn()
298 	write_lock(&kvm->mmu_lock);  in kvm_write_track_remove_gfn()
299 	__kvm_write_track_remove_gfn(kvm, slot, gfn);  in kvm_write_track_remove_gfn()
300 	write_unlock(&kvm->mmu_lock);  in kvm_write_track_remove_gfn()
302 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_remove_gfn()