1dfa78a20SBen Gardon // SPDX-License-Identifier: GPL-2.0
2dfa78a20SBen Gardon /*
3dfa78a20SBen Gardon * KVM dirty logging page splitting test
4dfa78a20SBen Gardon *
5dfa78a20SBen Gardon * Based on dirty_log_perf.c
6dfa78a20SBen Gardon *
7dfa78a20SBen Gardon * Copyright (C) 2018, Red Hat, Inc.
8dfa78a20SBen Gardon * Copyright (C) 2023, Google, Inc.
9dfa78a20SBen Gardon */
10dfa78a20SBen Gardon
11dfa78a20SBen Gardon #include <stdio.h>
12dfa78a20SBen Gardon #include <stdlib.h>
13dfa78a20SBen Gardon #include <pthread.h>
14dfa78a20SBen Gardon #include <linux/bitmap.h>
15dfa78a20SBen Gardon
16dfa78a20SBen Gardon #include "kvm_util.h"
17dfa78a20SBen Gardon #include "test_util.h"
18dfa78a20SBen Gardon #include "memstress.h"
19dfa78a20SBen Gardon #include "guest_modes.h"
20dfa78a20SBen Gardon
21dfa78a20SBen Gardon #define VCPUS 2
22dfa78a20SBen Gardon #define SLOTS 2
23dfa78a20SBen Gardon #define ITERATIONS 2
24dfa78a20SBen Gardon
25dfa78a20SBen Gardon static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
26dfa78a20SBen Gardon
27dfa78a20SBen Gardon static enum vm_mem_backing_src_type backing_src = VM_MEM_SRC_ANONYMOUS_HUGETLB;
28dfa78a20SBen Gardon
29dfa78a20SBen Gardon static u64 dirty_log_manual_caps;
30dfa78a20SBen Gardon static bool host_quit;
31dfa78a20SBen Gardon static int iteration;
32dfa78a20SBen Gardon static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
33dfa78a20SBen Gardon
34dfa78a20SBen Gardon struct kvm_page_stats {
35dfa78a20SBen Gardon uint64_t pages_4k;
36dfa78a20SBen Gardon uint64_t pages_2m;
37dfa78a20SBen Gardon uint64_t pages_1g;
38dfa78a20SBen Gardon uint64_t hugepages;
39dfa78a20SBen Gardon };
40dfa78a20SBen Gardon
get_page_stats(struct kvm_vm * vm,struct kvm_page_stats * stats,const char * stage)41dfa78a20SBen Gardon static void get_page_stats(struct kvm_vm *vm, struct kvm_page_stats *stats, const char *stage)
42dfa78a20SBen Gardon {
43dfa78a20SBen Gardon stats->pages_4k = vm_get_stat(vm, "pages_4k");
44dfa78a20SBen Gardon stats->pages_2m = vm_get_stat(vm, "pages_2m");
45dfa78a20SBen Gardon stats->pages_1g = vm_get_stat(vm, "pages_1g");
46dfa78a20SBen Gardon stats->hugepages = stats->pages_2m + stats->pages_1g;
47dfa78a20SBen Gardon
48dfa78a20SBen Gardon pr_debug("\nPage stats after %s: 4K: %ld 2M: %ld 1G: %ld huge: %ld\n",
49dfa78a20SBen Gardon stage, stats->pages_4k, stats->pages_2m, stats->pages_1g,
50dfa78a20SBen Gardon stats->hugepages);
51dfa78a20SBen Gardon }
52dfa78a20SBen Gardon
run_vcpu_iteration(struct kvm_vm * vm)53dfa78a20SBen Gardon static void run_vcpu_iteration(struct kvm_vm *vm)
54dfa78a20SBen Gardon {
55dfa78a20SBen Gardon int i;
56dfa78a20SBen Gardon
57dfa78a20SBen Gardon iteration++;
58dfa78a20SBen Gardon for (i = 0; i < VCPUS; i++) {
59dfa78a20SBen Gardon while (READ_ONCE(vcpu_last_completed_iteration[i]) !=
60dfa78a20SBen Gardon iteration)
61dfa78a20SBen Gardon ;
62dfa78a20SBen Gardon }
63dfa78a20SBen Gardon }
64dfa78a20SBen Gardon
vcpu_worker(struct memstress_vcpu_args * vcpu_args)65dfa78a20SBen Gardon static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
66dfa78a20SBen Gardon {
67dfa78a20SBen Gardon struct kvm_vcpu *vcpu = vcpu_args->vcpu;
68dfa78a20SBen Gardon int vcpu_idx = vcpu_args->vcpu_idx;
69dfa78a20SBen Gardon
70dfa78a20SBen Gardon while (!READ_ONCE(host_quit)) {
71dfa78a20SBen Gardon int current_iteration = READ_ONCE(iteration);
72dfa78a20SBen Gardon
73dfa78a20SBen Gardon vcpu_run(vcpu);
74dfa78a20SBen Gardon
75*6d85f51aSThomas Huth TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC);
76dfa78a20SBen Gardon
77dfa78a20SBen Gardon vcpu_last_completed_iteration[vcpu_idx] = current_iteration;
78dfa78a20SBen Gardon
79dfa78a20SBen Gardon /* Wait for the start of the next iteration to be signaled. */
80dfa78a20SBen Gardon while (current_iteration == READ_ONCE(iteration) &&
81dfa78a20SBen Gardon READ_ONCE(iteration) >= 0 &&
82dfa78a20SBen Gardon !READ_ONCE(host_quit))
83dfa78a20SBen Gardon ;
84dfa78a20SBen Gardon }
85dfa78a20SBen Gardon }
86dfa78a20SBen Gardon
run_test(enum vm_guest_mode mode,void * unused)87dfa78a20SBen Gardon static void run_test(enum vm_guest_mode mode, void *unused)
88dfa78a20SBen Gardon {
89dfa78a20SBen Gardon struct kvm_vm *vm;
90dfa78a20SBen Gardon unsigned long **bitmaps;
91dfa78a20SBen Gardon uint64_t guest_num_pages;
92dfa78a20SBen Gardon uint64_t host_num_pages;
93dfa78a20SBen Gardon uint64_t pages_per_slot;
94dfa78a20SBen Gardon int i;
95dfa78a20SBen Gardon uint64_t total_4k_pages;
96dfa78a20SBen Gardon struct kvm_page_stats stats_populated;
97dfa78a20SBen Gardon struct kvm_page_stats stats_dirty_logging_enabled;
98dfa78a20SBen Gardon struct kvm_page_stats stats_dirty_pass[ITERATIONS];
99dfa78a20SBen Gardon struct kvm_page_stats stats_clear_pass[ITERATIONS];
100dfa78a20SBen Gardon struct kvm_page_stats stats_dirty_logging_disabled;
101dfa78a20SBen Gardon struct kvm_page_stats stats_repopulated;
102dfa78a20SBen Gardon
103dfa78a20SBen Gardon vm = memstress_create_vm(mode, VCPUS, guest_percpu_mem_size,
104dfa78a20SBen Gardon SLOTS, backing_src, false);
105dfa78a20SBen Gardon
106dfa78a20SBen Gardon guest_num_pages = (VCPUS * guest_percpu_mem_size) >> vm->page_shift;
107dfa78a20SBen Gardon guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages);
108dfa78a20SBen Gardon host_num_pages = vm_num_host_pages(mode, guest_num_pages);
109dfa78a20SBen Gardon pages_per_slot = host_num_pages / SLOTS;
110dfa78a20SBen Gardon
111dfa78a20SBen Gardon bitmaps = memstress_alloc_bitmaps(SLOTS, pages_per_slot);
112dfa78a20SBen Gardon
113dfa78a20SBen Gardon if (dirty_log_manual_caps)
114dfa78a20SBen Gardon vm_enable_cap(vm, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2,
115dfa78a20SBen Gardon dirty_log_manual_caps);
116dfa78a20SBen Gardon
117dfa78a20SBen Gardon /* Start the iterations */
118dfa78a20SBen Gardon iteration = -1;
119dfa78a20SBen Gardon host_quit = false;
120dfa78a20SBen Gardon
121dfa78a20SBen Gardon for (i = 0; i < VCPUS; i++)
122dfa78a20SBen Gardon vcpu_last_completed_iteration[i] = -1;
123dfa78a20SBen Gardon
124dfa78a20SBen Gardon memstress_start_vcpu_threads(VCPUS, vcpu_worker);
125dfa78a20SBen Gardon
126dfa78a20SBen Gardon run_vcpu_iteration(vm);
127dfa78a20SBen Gardon get_page_stats(vm, &stats_populated, "populating memory");
128dfa78a20SBen Gardon
129dfa78a20SBen Gardon /* Enable dirty logging */
130dfa78a20SBen Gardon memstress_enable_dirty_logging(vm, SLOTS);
131dfa78a20SBen Gardon
132dfa78a20SBen Gardon get_page_stats(vm, &stats_dirty_logging_enabled, "enabling dirty logging");
133dfa78a20SBen Gardon
134dfa78a20SBen Gardon while (iteration < ITERATIONS) {
135dfa78a20SBen Gardon run_vcpu_iteration(vm);
136dfa78a20SBen Gardon get_page_stats(vm, &stats_dirty_pass[iteration - 1],
137dfa78a20SBen Gardon "dirtying memory");
138dfa78a20SBen Gardon
139dfa78a20SBen Gardon memstress_get_dirty_log(vm, bitmaps, SLOTS);
140dfa78a20SBen Gardon
141dfa78a20SBen Gardon if (dirty_log_manual_caps) {
142dfa78a20SBen Gardon memstress_clear_dirty_log(vm, bitmaps, SLOTS, pages_per_slot);
143dfa78a20SBen Gardon
144dfa78a20SBen Gardon get_page_stats(vm, &stats_clear_pass[iteration - 1], "clearing dirty log");
145dfa78a20SBen Gardon }
146dfa78a20SBen Gardon }
147dfa78a20SBen Gardon
148dfa78a20SBen Gardon /* Disable dirty logging */
149dfa78a20SBen Gardon memstress_disable_dirty_logging(vm, SLOTS);
150dfa78a20SBen Gardon
151dfa78a20SBen Gardon get_page_stats(vm, &stats_dirty_logging_disabled, "disabling dirty logging");
152dfa78a20SBen Gardon
153dfa78a20SBen Gardon /* Run vCPUs again to fault pages back in. */
154dfa78a20SBen Gardon run_vcpu_iteration(vm);
155dfa78a20SBen Gardon get_page_stats(vm, &stats_repopulated, "repopulating memory");
156dfa78a20SBen Gardon
157dfa78a20SBen Gardon /*
158dfa78a20SBen Gardon * Tell the vCPU threads to quit. No need to manually check that vCPUs
159dfa78a20SBen Gardon * have stopped running after disabling dirty logging, the join will
160dfa78a20SBen Gardon * wait for them to exit.
161dfa78a20SBen Gardon */
162dfa78a20SBen Gardon host_quit = true;
163dfa78a20SBen Gardon memstress_join_vcpu_threads(VCPUS);
164dfa78a20SBen Gardon
165dfa78a20SBen Gardon memstress_free_bitmaps(bitmaps, SLOTS);
166dfa78a20SBen Gardon memstress_destroy_vm(vm);
167dfa78a20SBen Gardon
168dfa78a20SBen Gardon /* Make assertions about the page counts. */
169dfa78a20SBen Gardon total_4k_pages = stats_populated.pages_4k;
170dfa78a20SBen Gardon total_4k_pages += stats_populated.pages_2m * 512;
171dfa78a20SBen Gardon total_4k_pages += stats_populated.pages_1g * 512 * 512;
172dfa78a20SBen Gardon
173dfa78a20SBen Gardon /*
174dfa78a20SBen Gardon * Check that all huge pages were split. Since large pages can only
175dfa78a20SBen Gardon * exist in the data slot, and the vCPUs should have dirtied all pages
176dfa78a20SBen Gardon * in the data slot, there should be no huge pages left after splitting.
177dfa78a20SBen Gardon * Splitting happens at dirty log enable time without
178dfa78a20SBen Gardon * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and after the first clear pass
179dfa78a20SBen Gardon * with that capability.
180dfa78a20SBen Gardon */
181dfa78a20SBen Gardon if (dirty_log_manual_caps) {
182*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_clear_pass[0].hugepages, 0);
183*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_clear_pass[0].pages_4k, total_4k_pages);
184*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_dirty_logging_enabled.hugepages, stats_populated.hugepages);
185dfa78a20SBen Gardon } else {
186*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_dirty_logging_enabled.hugepages, 0);
187*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_dirty_logging_enabled.pages_4k, total_4k_pages);
188dfa78a20SBen Gardon }
189dfa78a20SBen Gardon
190dfa78a20SBen Gardon /*
191dfa78a20SBen Gardon * Once dirty logging is disabled and the vCPUs have touched all their
192dfa78a20SBen Gardon * memory again, the page counts should be the same as they were
193dfa78a20SBen Gardon * right after initial population of memory.
194dfa78a20SBen Gardon */
195*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_populated.pages_4k, stats_repopulated.pages_4k);
196*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_populated.pages_2m, stats_repopulated.pages_2m);
197*6d85f51aSThomas Huth TEST_ASSERT_EQ(stats_populated.pages_1g, stats_repopulated.pages_1g);
198dfa78a20SBen Gardon }
199dfa78a20SBen Gardon
help(char * name)200dfa78a20SBen Gardon static void help(char *name)
201dfa78a20SBen Gardon {
202dfa78a20SBen Gardon puts("");
203dfa78a20SBen Gardon printf("usage: %s [-h] [-b vcpu bytes] [-s mem type]\n",
204dfa78a20SBen Gardon name);
205dfa78a20SBen Gardon puts("");
206dfa78a20SBen Gardon printf(" -b: specify the size of the memory region which should be\n"
207dfa78a20SBen Gardon " dirtied by each vCPU. e.g. 10M or 3G.\n"
208dfa78a20SBen Gardon " (default: 1G)\n");
209dfa78a20SBen Gardon backing_src_help("-s");
210dfa78a20SBen Gardon puts("");
211dfa78a20SBen Gardon }
212dfa78a20SBen Gardon
main(int argc,char * argv[])213dfa78a20SBen Gardon int main(int argc, char *argv[])
214dfa78a20SBen Gardon {
215dfa78a20SBen Gardon int opt;
216dfa78a20SBen Gardon
217dfa78a20SBen Gardon TEST_REQUIRE(get_kvm_param_bool("eager_page_split"));
218dfa78a20SBen Gardon TEST_REQUIRE(get_kvm_param_bool("tdp_mmu"));
219dfa78a20SBen Gardon
220dfa78a20SBen Gardon while ((opt = getopt(argc, argv, "b:hs:")) != -1) {
221dfa78a20SBen Gardon switch (opt) {
222dfa78a20SBen Gardon case 'b':
223dfa78a20SBen Gardon guest_percpu_mem_size = parse_size(optarg);
224dfa78a20SBen Gardon break;
225dfa78a20SBen Gardon case 'h':
226dfa78a20SBen Gardon help(argv[0]);
227dfa78a20SBen Gardon exit(0);
228dfa78a20SBen Gardon case 's':
229dfa78a20SBen Gardon backing_src = parse_backing_src_type(optarg);
230dfa78a20SBen Gardon break;
231dfa78a20SBen Gardon default:
232dfa78a20SBen Gardon help(argv[0]);
233dfa78a20SBen Gardon exit(1);
234dfa78a20SBen Gardon }
235dfa78a20SBen Gardon }
236dfa78a20SBen Gardon
237dfa78a20SBen Gardon if (!is_backing_src_hugetlb(backing_src)) {
238dfa78a20SBen Gardon pr_info("This test will only work reliably with HugeTLB memory. "
239dfa78a20SBen Gardon "It can work with THP, but that is best effort.\n");
240dfa78a20SBen Gardon }
241dfa78a20SBen Gardon
242dfa78a20SBen Gardon guest_modes_append_default();
243dfa78a20SBen Gardon
244dfa78a20SBen Gardon dirty_log_manual_caps = 0;
245dfa78a20SBen Gardon for_each_guest_mode(run_test, NULL);
246dfa78a20SBen Gardon
247dfa78a20SBen Gardon dirty_log_manual_caps =
248dfa78a20SBen Gardon kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
249dfa78a20SBen Gardon
250dfa78a20SBen Gardon if (dirty_log_manual_caps) {
251dfa78a20SBen Gardon dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE |
252dfa78a20SBen Gardon KVM_DIRTY_LOG_INITIALLY_SET);
253dfa78a20SBen Gardon for_each_guest_mode(run_test, NULL);
254dfa78a20SBen Gardon } else {
255dfa78a20SBen Gardon pr_info("Skipping testing with MANUAL_PROTECT as it is not supported");
256dfa78a20SBen Gardon }
257dfa78a20SBen Gardon
258dfa78a20SBen Gardon return 0;
259dfa78a20SBen Gardon }
260