1c50d8ae3SPaolo Bonzini // SPDX-License-Identifier: GPL-2.0-only
2c50d8ae3SPaolo Bonzini /*
3c50d8ae3SPaolo Bonzini * Support KVM gust page tracking
4c50d8ae3SPaolo Bonzini *
5c50d8ae3SPaolo Bonzini * This feature allows us to track page access in guest. Currently, only
6c50d8ae3SPaolo Bonzini * write access is tracked.
7c50d8ae3SPaolo Bonzini *
8c50d8ae3SPaolo Bonzini * Copyright(C) 2015 Intel Corporation.
9c50d8ae3SPaolo Bonzini *
10c50d8ae3SPaolo Bonzini * Author:
11c50d8ae3SPaolo Bonzini * Xiao Guangrong <guangrong.xiao@linux.intel.com>
12c50d8ae3SPaolo Bonzini */
138d20bd63SSean Christopherson #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14c50d8ae3SPaolo Bonzini
15e18c5429SSean Christopherson #include <linux/lockdep.h>
16c50d8ae3SPaolo Bonzini #include <linux/kvm_host.h>
17c50d8ae3SPaolo Bonzini #include <linux/rculist.h>
18c50d8ae3SPaolo Bonzini
194139b197SPeter Xu #include "mmu.h"
206ca9a6f3SSean Christopherson #include "mmu_internal.h"
2158ea7cf7SSean Christopherson #include "page_track.h"
22c50d8ae3SPaolo Bonzini
kvm_page_track_write_tracking_enabled(struct kvm * kvm)231e76a3ceSDavid Stevens bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
24deae4a10SDavid Stevens {
25deae4a10SDavid Stevens return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) ||
261e76a3ceSDavid Stevens !tdp_enabled || kvm_shadow_root_allocated(kvm);
27deae4a10SDavid Stevens }
28deae4a10SDavid Stevens
kvm_page_track_free_memslot(struct kvm_memory_slot * slot)29e96c81eeSSean Christopherson void kvm_page_track_free_memslot(struct kvm_memory_slot *slot)
30c50d8ae3SPaolo Bonzini {
31338068b5SSean Christopherson kvfree(slot->arch.gfn_write_track);
32338068b5SSean Christopherson slot->arch.gfn_write_track = NULL;
33c50d8ae3SPaolo Bonzini }
34338068b5SSean Christopherson
__kvm_page_track_write_tracking_alloc(struct kvm_memory_slot * slot,unsigned long npages)35338068b5SSean Christopherson static int __kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot,
36338068b5SSean Christopherson unsigned long npages)
37338068b5SSean Christopherson {
38338068b5SSean Christopherson const size_t size = sizeof(*slot->arch.gfn_write_track);
39338068b5SSean Christopherson
40338068b5SSean Christopherson if (!slot->arch.gfn_write_track)
41338068b5SSean Christopherson slot->arch.gfn_write_track = __vcalloc(npages, size,
42338068b5SSean Christopherson GFP_KERNEL_ACCOUNT);
43338068b5SSean Christopherson
44338068b5SSean Christopherson return slot->arch.gfn_write_track ? 0 : -ENOMEM;
45c50d8ae3SPaolo Bonzini }
46c50d8ae3SPaolo Bonzini
kvm_page_track_create_memslot(struct kvm * kvm,struct kvm_memory_slot * slot,unsigned long npages)47deae4a10SDavid Stevens int kvm_page_track_create_memslot(struct kvm *kvm,
48deae4a10SDavid Stevens struct kvm_memory_slot *slot,
49c50d8ae3SPaolo Bonzini unsigned long npages)
50c50d8ae3SPaolo Bonzini {
51338068b5SSean Christopherson if (!kvm_page_track_write_tracking_enabled(kvm))
52c50d8ae3SPaolo Bonzini return 0;
53c50d8ae3SPaolo Bonzini
54338068b5SSean Christopherson return __kvm_page_track_write_tracking_alloc(slot, npages);
55c50d8ae3SPaolo Bonzini }
56c50d8ae3SPaolo Bonzini
kvm_page_track_write_tracking_alloc(struct kvm_memory_slot * slot)571e76a3ceSDavid Stevens int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot)
58deae4a10SDavid Stevens {
59338068b5SSean Christopherson return __kvm_page_track_write_tracking_alloc(slot, slot->npages);
60deae4a10SDavid Stevens }
61deae4a10SDavid Stevens
update_gfn_write_track(struct kvm_memory_slot * slot,gfn_t gfn,short count)62338068b5SSean Christopherson static void update_gfn_write_track(struct kvm_memory_slot *slot, gfn_t gfn,
63338068b5SSean Christopherson short count)
64c50d8ae3SPaolo Bonzini {
65c50d8ae3SPaolo Bonzini int index, val;
66c50d8ae3SPaolo Bonzini
673bae0459SSean Christopherson index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);
68c50d8ae3SPaolo Bonzini
69338068b5SSean Christopherson val = slot->arch.gfn_write_track[index];
70c50d8ae3SPaolo Bonzini
7120ba462dSSean Christopherson if (WARN_ON_ONCE(val + count < 0 || val + count > USHRT_MAX))
72c50d8ae3SPaolo Bonzini return;
73c50d8ae3SPaolo Bonzini
74338068b5SSean Christopherson slot->arch.gfn_write_track[index] += count;
75c50d8ae3SPaolo Bonzini }
76c50d8ae3SPaolo Bonzini
__kvm_write_track_add_gfn(struct kvm * kvm,struct kvm_memory_slot * slot,gfn_t gfn)7796316a06SSean Christopherson void __kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,
787b574863SSean Christopherson gfn_t gfn)
79c50d8ae3SPaolo Bonzini {
80e18c5429SSean Christopherson lockdep_assert_held_write(&kvm->mmu_lock);
81e18c5429SSean Christopherson
82e18c5429SSean Christopherson lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||
83e18c5429SSean Christopherson srcu_read_lock_held(&kvm->srcu));
84e18c5429SSean Christopherson
85427c76aeSSean Christopherson if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))
86c50d8ae3SPaolo Bonzini return;
87c50d8ae3SPaolo Bonzini
88338068b5SSean Christopherson update_gfn_write_track(slot, gfn, 1);
89c50d8ae3SPaolo Bonzini
90c50d8ae3SPaolo Bonzini /*
91c50d8ae3SPaolo Bonzini * new track stops large page mapping for the
92c50d8ae3SPaolo Bonzini * tracked page.
93c50d8ae3SPaolo Bonzini */
94c50d8ae3SPaolo Bonzini kvm_mmu_gfn_disallow_lpage(slot, gfn);
95c50d8ae3SPaolo Bonzini
963ad93562SKeqian Zhu if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
97c50d8ae3SPaolo Bonzini kvm_flush_remote_tlbs(kvm);
98c50d8ae3SPaolo Bonzini }
99c50d8ae3SPaolo Bonzini
__kvm_write_track_remove_gfn(struct kvm * kvm,struct kvm_memory_slot * slot,gfn_t gfn)10096316a06SSean Christopherson void __kvm_write_track_remove_gfn(struct kvm *kvm,
101338068b5SSean Christopherson struct kvm_memory_slot *slot, gfn_t gfn)
102c50d8ae3SPaolo Bonzini {
103e18c5429SSean Christopherson lockdep_assert_held_write(&kvm->mmu_lock);
104e18c5429SSean Christopherson
105e18c5429SSean Christopherson lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||
106e18c5429SSean Christopherson srcu_read_lock_held(&kvm->srcu));
107e18c5429SSean Christopherson
108427c76aeSSean Christopherson if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))
109c50d8ae3SPaolo Bonzini return;
110c50d8ae3SPaolo Bonzini
111338068b5SSean Christopherson update_gfn_write_track(slot, gfn, -1);
112c50d8ae3SPaolo Bonzini
113c50d8ae3SPaolo Bonzini /*
114c50d8ae3SPaolo Bonzini * allow large page mapping for the tracked page
115c50d8ae3SPaolo Bonzini * after the tracker is gone.
116c50d8ae3SPaolo Bonzini */
117c50d8ae3SPaolo Bonzini kvm_mmu_gfn_allow_lpage(slot, gfn);
118c50d8ae3SPaolo Bonzini }
119c50d8ae3SPaolo Bonzini
12053597858SDavid Matlack /*
12153597858SDavid Matlack * check if the corresponding access on the specified guest page is tracked.
12253597858SDavid Matlack */
kvm_gfn_is_write_tracked(struct kvm * kvm,const struct kvm_memory_slot * slot,gfn_t gfn)1237b574863SSean Christopherson bool kvm_gfn_is_write_tracked(struct kvm *kvm,
1247b574863SSean Christopherson const struct kvm_memory_slot *slot, gfn_t gfn)
12588810413SDavid Matlack {
12688810413SDavid Matlack int index;
12788810413SDavid Matlack
12888810413SDavid Matlack if (!slot)
12988810413SDavid Matlack return false;
13088810413SDavid Matlack
131338068b5SSean Christopherson if (!kvm_page_track_write_tracking_enabled(kvm))
132deae4a10SDavid Stevens return false;
133deae4a10SDavid Stevens
13488810413SDavid Matlack index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);
135338068b5SSean Christopherson return !!READ_ONCE(slot->arch.gfn_write_track[index]);
13688810413SDavid Matlack }
13788810413SDavid Matlack
138e998fb1aSSean Christopherson #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING
kvm_page_track_cleanup(struct kvm * kvm)139c50d8ae3SPaolo Bonzini void kvm_page_track_cleanup(struct kvm *kvm)
140c50d8ae3SPaolo Bonzini {
141c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_head *head;
142c50d8ae3SPaolo Bonzini
143c50d8ae3SPaolo Bonzini head = &kvm->arch.track_notifier_head;
144c50d8ae3SPaolo Bonzini cleanup_srcu_struct(&head->track_srcu);
145c50d8ae3SPaolo Bonzini }
146c50d8ae3SPaolo Bonzini
kvm_page_track_init(struct kvm * kvm)147eb7511bfSHaimin Zhang int kvm_page_track_init(struct kvm *kvm)
148c50d8ae3SPaolo Bonzini {
149c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_head *head;
150c50d8ae3SPaolo Bonzini
151c50d8ae3SPaolo Bonzini head = &kvm->arch.track_notifier_head;
152c50d8ae3SPaolo Bonzini INIT_HLIST_HEAD(&head->track_notifier_list);
153eb7511bfSHaimin Zhang return init_srcu_struct(&head->track_srcu);
154c50d8ae3SPaolo Bonzini }
155c50d8ae3SPaolo Bonzini
156c50d8ae3SPaolo Bonzini /*
157c50d8ae3SPaolo Bonzini * register the notifier so that event interception for the tracked guest
158c50d8ae3SPaolo Bonzini * pages can be received.
159c50d8ae3SPaolo Bonzini */
kvm_page_track_register_notifier(struct kvm * kvm,struct kvm_page_track_notifier_node * n)160*f22b1e85SSean Christopherson int kvm_page_track_register_notifier(struct kvm *kvm,
161c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_node *n)
162c50d8ae3SPaolo Bonzini {
163c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_head *head;
164c50d8ae3SPaolo Bonzini
165*f22b1e85SSean Christopherson if (!kvm || kvm->mm != current->mm)
166*f22b1e85SSean Christopherson return -ESRCH;
167*f22b1e85SSean Christopherson
168*f22b1e85SSean Christopherson kvm_get_kvm(kvm);
169*f22b1e85SSean Christopherson
170c50d8ae3SPaolo Bonzini head = &kvm->arch.track_notifier_head;
171c50d8ae3SPaolo Bonzini
172531810caSBen Gardon write_lock(&kvm->mmu_lock);
173c50d8ae3SPaolo Bonzini hlist_add_head_rcu(&n->node, &head->track_notifier_list);
174531810caSBen Gardon write_unlock(&kvm->mmu_lock);
175*f22b1e85SSean Christopherson return 0;
176c50d8ae3SPaolo Bonzini }
177c50d8ae3SPaolo Bonzini EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
178c50d8ae3SPaolo Bonzini
179c50d8ae3SPaolo Bonzini /*
180c50d8ae3SPaolo Bonzini * stop receiving the event interception. It is the opposed operation of
181c50d8ae3SPaolo Bonzini * kvm_page_track_register_notifier().
182c50d8ae3SPaolo Bonzini */
kvm_page_track_unregister_notifier(struct kvm * kvm,struct kvm_page_track_notifier_node * n)183*f22b1e85SSean Christopherson void kvm_page_track_unregister_notifier(struct kvm *kvm,
184c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_node *n)
185c50d8ae3SPaolo Bonzini {
186c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_head *head;
187c50d8ae3SPaolo Bonzini
188c50d8ae3SPaolo Bonzini head = &kvm->arch.track_notifier_head;
189c50d8ae3SPaolo Bonzini
190531810caSBen Gardon write_lock(&kvm->mmu_lock);
191c50d8ae3SPaolo Bonzini hlist_del_rcu(&n->node);
192531810caSBen Gardon write_unlock(&kvm->mmu_lock);
193c50d8ae3SPaolo Bonzini synchronize_srcu(&head->track_srcu);
194*f22b1e85SSean Christopherson
195*f22b1e85SSean Christopherson kvm_put_kvm(kvm);
196c50d8ae3SPaolo Bonzini }
197c50d8ae3SPaolo Bonzini EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);
198c50d8ae3SPaolo Bonzini
199c50d8ae3SPaolo Bonzini /*
200c50d8ae3SPaolo Bonzini * Notify the node that write access is intercepted and write emulation is
201c50d8ae3SPaolo Bonzini * finished at this time.
202c50d8ae3SPaolo Bonzini *
203c50d8ae3SPaolo Bonzini * The node should figure out if the written page is the one that node is
204c50d8ae3SPaolo Bonzini * interested in by itself.
205c50d8ae3SPaolo Bonzini */
__kvm_page_track_write(struct kvm * kvm,gpa_t gpa,const u8 * new,int bytes)206e998fb1aSSean Christopherson void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes)
207c50d8ae3SPaolo Bonzini {
208c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_head *head;
209c50d8ae3SPaolo Bonzini struct kvm_page_track_notifier_node *n;
210c50d8ae3SPaolo Bonzini int idx;
211c50d8ae3SPaolo Bonzini
212e998fb1aSSean Christopherson head = &kvm->arch.track_notifier_head;
213c50d8ae3SPaolo Bonzini
214c50d8ae3SPaolo Bonzini if (hlist_empty(&head->track_notifier_list))
215c50d8ae3SPaolo Bonzini return;
216c50d8ae3SPaolo Bonzini
217c50d8ae3SPaolo Bonzini idx = srcu_read_lock(&head->track_srcu);
218df9a30fdSMadhuparna Bhowmik hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,
219df9a30fdSMadhuparna Bhowmik srcu_read_lock_held(&head->track_srcu))
220c50d8ae3SPaolo Bonzini if (n->track_write)
221b271e17dSSean Christopherson n->track_write(gpa, new, bytes, n);
222c50d8ae3SPaolo Bonzini srcu_read_unlock(&head->track_srcu, idx);
223c50d8ae3SPaolo Bonzini }
224c50d8ae3SPaolo Bonzini
225c50d8ae3SPaolo Bonzini /*
226b83ab124SYan Zhao * Notify external page track nodes that a memory region is being removed from
227b83ab124SYan Zhao * the VM, e.g. so that users can free any associated metadata.
228b83ab124SYan Zhao */
kvm_page_track_delete_slot(struct kvm * kvm,struct kvm_memory_slot * slot)229b83ab124SYan Zhao void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
230b83ab124SYan Zhao {
231b83ab124SYan Zhao struct kvm_page_track_notifier_head *head;
232b83ab124SYan Zhao struct kvm_page_track_notifier_node *n;
233b83ab124SYan Zhao int idx;
234b83ab124SYan Zhao
235b83ab124SYan Zhao head = &kvm->arch.track_notifier_head;
236b83ab124SYan Zhao
237b83ab124SYan Zhao if (hlist_empty(&head->track_notifier_list))
238b83ab124SYan Zhao return;
239b83ab124SYan Zhao
240b83ab124SYan Zhao idx = srcu_read_lock(&head->track_srcu);
241b83ab124SYan Zhao hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,
242b83ab124SYan Zhao srcu_read_lock_held(&head->track_srcu))
243b83ab124SYan Zhao if (n->track_remove_region)
244b83ab124SYan Zhao n->track_remove_region(slot->base_gfn, slot->npages, n);
245c50d8ae3SPaolo Bonzini srcu_read_unlock(&head->track_srcu, idx);
246c50d8ae3SPaolo Bonzini }
247e998fb1aSSean Christopherson
24896316a06SSean Christopherson /*
24996316a06SSean Christopherson * add guest page to the tracking pool so that corresponding access on that
25096316a06SSean Christopherson * page will be intercepted.
25196316a06SSean Christopherson *
25296316a06SSean Christopherson * @kvm: the guest instance we are interested in.
25396316a06SSean Christopherson * @gfn: the guest page.
25496316a06SSean Christopherson */
kvm_write_track_add_gfn(struct kvm * kvm,gfn_t gfn)25596316a06SSean Christopherson int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn)
25696316a06SSean Christopherson {
25796316a06SSean Christopherson struct kvm_memory_slot *slot;
25896316a06SSean Christopherson int idx;
25996316a06SSean Christopherson
26096316a06SSean Christopherson idx = srcu_read_lock(&kvm->srcu);
26196316a06SSean Christopherson
26296316a06SSean Christopherson slot = gfn_to_memslot(kvm, gfn);
26396316a06SSean Christopherson if (!slot) {
26496316a06SSean Christopherson srcu_read_unlock(&kvm->srcu, idx);
26596316a06SSean Christopherson return -EINVAL;
26696316a06SSean Christopherson }
26796316a06SSean Christopherson
26896316a06SSean Christopherson write_lock(&kvm->mmu_lock);
26996316a06SSean Christopherson __kvm_write_track_add_gfn(kvm, slot, gfn);
27096316a06SSean Christopherson write_unlock(&kvm->mmu_lock);
27196316a06SSean Christopherson
27296316a06SSean Christopherson srcu_read_unlock(&kvm->srcu, idx);
27396316a06SSean Christopherson
27496316a06SSean Christopherson return 0;
27596316a06SSean Christopherson }
27696316a06SSean Christopherson EXPORT_SYMBOL_GPL(kvm_write_track_add_gfn);
27796316a06SSean Christopherson
27896316a06SSean Christopherson /*
27996316a06SSean Christopherson * remove the guest page from the tracking pool which stops the interception
28096316a06SSean Christopherson * of corresponding access on that page.
28196316a06SSean Christopherson *
28296316a06SSean Christopherson * @kvm: the guest instance we are interested in.
28396316a06SSean Christopherson * @gfn: the guest page.
28496316a06SSean Christopherson */
kvm_write_track_remove_gfn(struct kvm * kvm,gfn_t gfn)28596316a06SSean Christopherson int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn)
28696316a06SSean Christopherson {
28796316a06SSean Christopherson struct kvm_memory_slot *slot;
28896316a06SSean Christopherson int idx;
28996316a06SSean Christopherson
29096316a06SSean Christopherson idx = srcu_read_lock(&kvm->srcu);
29196316a06SSean Christopherson
29296316a06SSean Christopherson slot = gfn_to_memslot(kvm, gfn);
29396316a06SSean Christopherson if (!slot) {
29496316a06SSean Christopherson srcu_read_unlock(&kvm->srcu, idx);
29596316a06SSean Christopherson return -EINVAL;
29696316a06SSean Christopherson }
29796316a06SSean Christopherson
29896316a06SSean Christopherson write_lock(&kvm->mmu_lock);
29996316a06SSean Christopherson __kvm_write_track_remove_gfn(kvm, slot, gfn);
30096316a06SSean Christopherson write_unlock(&kvm->mmu_lock);
30196316a06SSean Christopherson
30296316a06SSean Christopherson srcu_read_unlock(&kvm->srcu, idx);
30396316a06SSean Christopherson
30496316a06SSean Christopherson return 0;
30596316a06SSean Christopherson }
30696316a06SSean Christopherson EXPORT_SYMBOL_GPL(kvm_write_track_remove_gfn);
307e998fb1aSSean Christopherson #endif
308