1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KVM dirty logging page splitting test 4 * 5 * Based on dirty_log_perf.c 6 * 7 * Copyright (C) 2018, Red Hat, Inc. 8 * Copyright (C) 2023, Google, Inc. 9 */ 10 11 #include <stdio.h> 12 #include <stdlib.h> 13 #include <pthread.h> 14 #include <linux/bitmap.h> 15 16 #include "kvm_util.h" 17 #include "test_util.h" 18 #include "memstress.h" 19 #include "guest_modes.h" 20 21 #define VCPUS 2 22 #define SLOTS 2 23 #define ITERATIONS 2 24 25 static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE; 26 27 static enum vm_mem_backing_src_type backing_src = VM_MEM_SRC_ANONYMOUS_HUGETLB; 28 29 static u64 dirty_log_manual_caps; 30 static bool host_quit; 31 static int iteration; 32 static int vcpu_last_completed_iteration[KVM_MAX_VCPUS]; 33 34 struct kvm_page_stats { 35 uint64_t pages_4k; 36 uint64_t pages_2m; 37 uint64_t pages_1g; 38 uint64_t hugepages; 39 }; 40 41 static void get_page_stats(struct kvm_vm *vm, struct kvm_page_stats *stats, const char *stage) 42 { 43 stats->pages_4k = vm_get_stat(vm, "pages_4k"); 44 stats->pages_2m = vm_get_stat(vm, "pages_2m"); 45 stats->pages_1g = vm_get_stat(vm, "pages_1g"); 46 stats->hugepages = stats->pages_2m + stats->pages_1g; 47 48 pr_debug("\nPage stats after %s: 4K: %ld 2M: %ld 1G: %ld huge: %ld\n", 49 stage, stats->pages_4k, stats->pages_2m, stats->pages_1g, 50 stats->hugepages); 51 } 52 53 static void run_vcpu_iteration(struct kvm_vm *vm) 54 { 55 int i; 56 57 iteration++; 58 for (i = 0; i < VCPUS; i++) { 59 while (READ_ONCE(vcpu_last_completed_iteration[i]) != 60 iteration) 61 ; 62 } 63 } 64 65 static void vcpu_worker(struct memstress_vcpu_args *vcpu_args) 66 { 67 struct kvm_vcpu *vcpu = vcpu_args->vcpu; 68 int vcpu_idx = vcpu_args->vcpu_idx; 69 70 while (!READ_ONCE(host_quit)) { 71 int current_iteration = READ_ONCE(iteration); 72 73 vcpu_run(vcpu); 74 75 ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC); 76 77 vcpu_last_completed_iteration[vcpu_idx] = current_iteration; 78 79 /* Wait for the start of the next iteration to be signaled. */ 80 while (current_iteration == READ_ONCE(iteration) && 81 READ_ONCE(iteration) >= 0 && 82 !READ_ONCE(host_quit)) 83 ; 84 } 85 } 86 87 static void run_test(enum vm_guest_mode mode, void *unused) 88 { 89 struct kvm_vm *vm; 90 unsigned long **bitmaps; 91 uint64_t guest_num_pages; 92 uint64_t host_num_pages; 93 uint64_t pages_per_slot; 94 int i; 95 uint64_t total_4k_pages; 96 struct kvm_page_stats stats_populated; 97 struct kvm_page_stats stats_dirty_logging_enabled; 98 struct kvm_page_stats stats_dirty_pass[ITERATIONS]; 99 struct kvm_page_stats stats_clear_pass[ITERATIONS]; 100 struct kvm_page_stats stats_dirty_logging_disabled; 101 struct kvm_page_stats stats_repopulated; 102 103 vm = memstress_create_vm(mode, VCPUS, guest_percpu_mem_size, 104 SLOTS, backing_src, false); 105 106 guest_num_pages = (VCPUS * guest_percpu_mem_size) >> vm->page_shift; 107 guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages); 108 host_num_pages = vm_num_host_pages(mode, guest_num_pages); 109 pages_per_slot = host_num_pages / SLOTS; 110 111 bitmaps = memstress_alloc_bitmaps(SLOTS, pages_per_slot); 112 113 if (dirty_log_manual_caps) 114 vm_enable_cap(vm, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 115 dirty_log_manual_caps); 116 117 /* Start the iterations */ 118 iteration = -1; 119 host_quit = false; 120 121 for (i = 0; i < VCPUS; i++) 122 vcpu_last_completed_iteration[i] = -1; 123 124 memstress_start_vcpu_threads(VCPUS, vcpu_worker); 125 126 run_vcpu_iteration(vm); 127 get_page_stats(vm, &stats_populated, "populating memory"); 128 129 /* Enable dirty logging */ 130 memstress_enable_dirty_logging(vm, SLOTS); 131 132 get_page_stats(vm, &stats_dirty_logging_enabled, "enabling dirty logging"); 133 134 while (iteration < ITERATIONS) { 135 run_vcpu_iteration(vm); 136 get_page_stats(vm, &stats_dirty_pass[iteration - 1], 137 "dirtying memory"); 138 139 memstress_get_dirty_log(vm, bitmaps, SLOTS); 140 141 if (dirty_log_manual_caps) { 142 memstress_clear_dirty_log(vm, bitmaps, SLOTS, pages_per_slot); 143 144 get_page_stats(vm, &stats_clear_pass[iteration - 1], "clearing dirty log"); 145 } 146 } 147 148 /* Disable dirty logging */ 149 memstress_disable_dirty_logging(vm, SLOTS); 150 151 get_page_stats(vm, &stats_dirty_logging_disabled, "disabling dirty logging"); 152 153 /* Run vCPUs again to fault pages back in. */ 154 run_vcpu_iteration(vm); 155 get_page_stats(vm, &stats_repopulated, "repopulating memory"); 156 157 /* 158 * Tell the vCPU threads to quit. No need to manually check that vCPUs 159 * have stopped running after disabling dirty logging, the join will 160 * wait for them to exit. 161 */ 162 host_quit = true; 163 memstress_join_vcpu_threads(VCPUS); 164 165 memstress_free_bitmaps(bitmaps, SLOTS); 166 memstress_destroy_vm(vm); 167 168 /* Make assertions about the page counts. */ 169 total_4k_pages = stats_populated.pages_4k; 170 total_4k_pages += stats_populated.pages_2m * 512; 171 total_4k_pages += stats_populated.pages_1g * 512 * 512; 172 173 /* 174 * Check that all huge pages were split. Since large pages can only 175 * exist in the data slot, and the vCPUs should have dirtied all pages 176 * in the data slot, there should be no huge pages left after splitting. 177 * Splitting happens at dirty log enable time without 178 * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and after the first clear pass 179 * with that capability. 180 */ 181 if (dirty_log_manual_caps) { 182 ASSERT_EQ(stats_clear_pass[0].hugepages, 0); 183 ASSERT_EQ(stats_clear_pass[0].pages_4k, total_4k_pages); 184 ASSERT_EQ(stats_dirty_logging_enabled.hugepages, stats_populated.hugepages); 185 } else { 186 ASSERT_EQ(stats_dirty_logging_enabled.hugepages, 0); 187 ASSERT_EQ(stats_dirty_logging_enabled.pages_4k, total_4k_pages); 188 } 189 190 /* 191 * Once dirty logging is disabled and the vCPUs have touched all their 192 * memory again, the page counts should be the same as they were 193 * right after initial population of memory. 194 */ 195 ASSERT_EQ(stats_populated.pages_4k, stats_repopulated.pages_4k); 196 ASSERT_EQ(stats_populated.pages_2m, stats_repopulated.pages_2m); 197 ASSERT_EQ(stats_populated.pages_1g, stats_repopulated.pages_1g); 198 } 199 200 static void help(char *name) 201 { 202 puts(""); 203 printf("usage: %s [-h] [-b vcpu bytes] [-s mem type]\n", 204 name); 205 puts(""); 206 printf(" -b: specify the size of the memory region which should be\n" 207 " dirtied by each vCPU. e.g. 10M or 3G.\n" 208 " (default: 1G)\n"); 209 backing_src_help("-s"); 210 puts(""); 211 } 212 213 int main(int argc, char *argv[]) 214 { 215 int opt; 216 217 TEST_REQUIRE(get_kvm_param_bool("eager_page_split")); 218 TEST_REQUIRE(get_kvm_param_bool("tdp_mmu")); 219 220 while ((opt = getopt(argc, argv, "b:hs:")) != -1) { 221 switch (opt) { 222 case 'b': 223 guest_percpu_mem_size = parse_size(optarg); 224 break; 225 case 'h': 226 help(argv[0]); 227 exit(0); 228 case 's': 229 backing_src = parse_backing_src_type(optarg); 230 break; 231 default: 232 help(argv[0]); 233 exit(1); 234 } 235 } 236 237 if (!is_backing_src_hugetlb(backing_src)) { 238 pr_info("This test will only work reliably with HugeTLB memory. " 239 "It can work with THP, but that is best effort.\n"); 240 } 241 242 guest_modes_append_default(); 243 244 dirty_log_manual_caps = 0; 245 for_each_guest_mode(run_test, NULL); 246 247 dirty_log_manual_caps = 248 kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); 249 250 if (dirty_log_manual_caps) { 251 dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | 252 KVM_DIRTY_LOG_INITIALLY_SET); 253 for_each_guest_mode(run_test, NULL); 254 } else { 255 pr_info("Skipping testing with MANUAL_PROTECT as it is not supported"); 256 } 257 258 return 0; 259 } 260