1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <sys/mman.h>
4 #include <sys/prctl.h>
5 #include <sys/wait.h>
6 #include <stdbool.h>
7 #include <time.h>
8 #include <string.h>
9 #include <numa.h>
10 #include <unistd.h>
11 #include <fcntl.h>
12 #include <stdint.h>
13 #include <err.h>
14 
15 #include "../kselftest.h"
16 #include <include/vdso/time64.h>
17 #include "vm_util.h"
18 
19 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
20 #define KSM_FP(s) (KSM_SYSFS_PATH s)
21 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
22 #define KSM_PAGE_COUNT_DEFAULT 10l
23 #define KSM_PROT_STR_DEFAULT "rw"
24 #define KSM_USE_ZERO_PAGES_DEFAULT false
25 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
26 #define KSM_MERGE_TYPE_DEFAULT 0
27 #define MB (1ul << 20)
28 
29 struct ksm_sysfs {
30 	unsigned long max_page_sharing;
31 	unsigned long merge_across_nodes;
32 	unsigned long pages_to_scan;
33 	unsigned long run;
34 	unsigned long sleep_millisecs;
35 	unsigned long stable_node_chains_prune_millisecs;
36 	unsigned long use_zero_pages;
37 };
38 
39 enum ksm_merge_type {
40 	KSM_MERGE_MADVISE,
41 	KSM_MERGE_PRCTL,
42 	KSM_MERGE_LAST = KSM_MERGE_PRCTL
43 };
44 
45 enum ksm_test_name {
46 	CHECK_KSM_MERGE,
47 	CHECK_KSM_UNMERGE,
48 	CHECK_KSM_GET_MERGE_TYPE,
49 	CHECK_KSM_ZERO_PAGE_MERGE,
50 	CHECK_KSM_NUMA_MERGE,
51 	KSM_MERGE_TIME,
52 	KSM_MERGE_TIME_HUGE_PAGES,
53 	KSM_UNMERGE_TIME,
54 	KSM_COW_TIME
55 };
56 
57 int debug;
58 
ksm_write_sysfs(const char * file_path,unsigned long val)59 static int ksm_write_sysfs(const char *file_path, unsigned long val)
60 {
61 	FILE *f = fopen(file_path, "w");
62 
63 	if (!f) {
64 		fprintf(stderr, "f %s\n", file_path);
65 		perror("fopen");
66 		return 1;
67 	}
68 	if (fprintf(f, "%lu", val) < 0) {
69 		perror("fprintf");
70 		fclose(f);
71 		return 1;
72 	}
73 	fclose(f);
74 
75 	return 0;
76 }
77 
ksm_read_sysfs(const char * file_path,unsigned long * val)78 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
79 {
80 	FILE *f = fopen(file_path, "r");
81 
82 	if (!f) {
83 		fprintf(stderr, "f %s\n", file_path);
84 		perror("fopen");
85 		return 1;
86 	}
87 	if (fscanf(f, "%lu", val) != 1) {
88 		perror("fscanf");
89 		fclose(f);
90 		return 1;
91 	}
92 	fclose(f);
93 
94 	return 0;
95 }
96 
ksm_print_sysfs(void)97 static void ksm_print_sysfs(void)
98 {
99 	unsigned long max_page_sharing, pages_sharing, pages_shared;
100 	unsigned long full_scans, pages_unshared, pages_volatile;
101 	unsigned long stable_node_chains, stable_node_dups;
102 	long general_profit;
103 
104 	if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
105 	    ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
106 	    ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) ||
107 	    ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) ||
108 	    ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) ||
109 	    ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) ||
110 	    ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) ||
111 	    ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) ||
112 	    ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit))
113 		return;
114 
115 	printf("pages_shared      : %lu\n", pages_shared);
116 	printf("pages_sharing     : %lu\n", pages_sharing);
117 	printf("max_page_sharing  : %lu\n", max_page_sharing);
118 	printf("full_scans        : %lu\n", full_scans);
119 	printf("pages_unshared    : %lu\n", pages_unshared);
120 	printf("pages_volatile    : %lu\n", pages_volatile);
121 	printf("stable_node_chains: %lu\n", stable_node_chains);
122 	printf("stable_node_dups  : %lu\n", stable_node_dups);
123 	printf("general_profit    : %ld\n", general_profit);
124 }
125 
ksm_print_procfs(void)126 static void ksm_print_procfs(void)
127 {
128 	const char *file_name = "/proc/self/ksm_stat";
129 	char buffer[512];
130 	FILE *f = fopen(file_name, "r");
131 
132 	if (!f) {
133 		fprintf(stderr, "f %s\n", file_name);
134 		perror("fopen");
135 		return;
136 	}
137 
138 	while (fgets(buffer, sizeof(buffer), f))
139 		printf("%s", buffer);
140 
141 	fclose(f);
142 }
143 
str_to_prot(char * prot_str)144 static int str_to_prot(char *prot_str)
145 {
146 	int prot = 0;
147 
148 	if ((strchr(prot_str, 'r')) != NULL)
149 		prot |= PROT_READ;
150 	if ((strchr(prot_str, 'w')) != NULL)
151 		prot |= PROT_WRITE;
152 	if ((strchr(prot_str, 'x')) != NULL)
153 		prot |= PROT_EXEC;
154 
155 	return prot;
156 }
157 
print_help(void)158 static void print_help(void)
159 {
160 	printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
161 	       "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
162 
163 	printf("Supported <test type>:\n"
164 	       " -M (page merging)\n"
165 	       " -Z (zero pages merging)\n"
166 	       " -N (merging of pages in different NUMA nodes)\n"
167 	       " -U (page unmerging)\n"
168 	       " -P evaluate merging time and speed.\n"
169 	       "    For this test, the size of duplicated memory area (in MiB)\n"
170 	       "    must be provided using -s option\n"
171 	       " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
172 	       "    For this test, the size of duplicated memory area (in MiB)\n"
173 	       "    must be provided using -s option\n"
174 	       " -D evaluate unmerging time and speed when disabling KSM.\n"
175 	       "    For this test, the size of duplicated memory area (in MiB)\n"
176 	       "    must be provided using -s option\n"
177 	       " -C evaluate the time required to break COW of merged pages.\n\n");
178 
179 	printf(" -a: specify the access protections of pages.\n"
180 	       "     <prot> must be of the form [rwx].\n"
181 	       "     Default: %s\n", KSM_PROT_STR_DEFAULT);
182 	printf(" -p: specify the number of pages to test.\n"
183 	       "     Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
184 	printf(" -l: limit the maximum running time (in seconds) for a test.\n"
185 	       "     Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
186 	printf(" -z: change use_zero_pages tunable\n"
187 	       "     Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
188 	printf(" -m: change merge_across_nodes tunable\n"
189 	       "     Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
190 	printf(" -d: turn debugging output on\n");
191 	printf(" -s: the size of duplicated memory area (in MiB)\n");
192 	printf(" -t: KSM merge type\n"
193 	       "     Default: 0\n"
194 	       "     0: madvise merging\n"
195 	       "     1: prctl merging\n");
196 
197 	exit(0);
198 }
199 
allocate_memory(void * ptr,int prot,int mapping,char data,size_t map_size)200 static void  *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
201 {
202 	void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
203 
204 	if (!map_ptr) {
205 		perror("mmap");
206 		return NULL;
207 	}
208 	memset(map_ptr, data, map_size);
209 	if (mprotect(map_ptr, map_size, prot)) {
210 		perror("mprotect");
211 		munmap(map_ptr, map_size);
212 		return NULL;
213 	}
214 
215 	return map_ptr;
216 }
217 
ksm_do_scan(int scan_count,struct timespec start_time,int timeout)218 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
219 {
220 	struct timespec cur_time;
221 	unsigned long cur_scan, init_scan;
222 
223 	if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
224 		return 1;
225 	cur_scan = init_scan;
226 
227 	while (cur_scan < init_scan + scan_count) {
228 		if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
229 			return 1;
230 		if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
231 			perror("clock_gettime");
232 			return 1;
233 		}
234 		if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
235 			printf("Scan time limit exceeded\n");
236 			return 1;
237 		}
238 	}
239 
240 	return 0;
241 }
242 
ksm_merge_pages(int merge_type,void * addr,size_t size,struct timespec start_time,int timeout)243 static int ksm_merge_pages(int merge_type, void *addr, size_t size,
244 			struct timespec start_time, int timeout)
245 {
246 	if (merge_type == KSM_MERGE_MADVISE) {
247 		if (madvise(addr, size, MADV_MERGEABLE)) {
248 			perror("madvise");
249 			return 1;
250 		}
251 	} else if (merge_type == KSM_MERGE_PRCTL) {
252 		if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) {
253 			perror("prctl");
254 			return 1;
255 		}
256 	}
257 
258 	if (ksm_write_sysfs(KSM_FP("run"), 1))
259 		return 1;
260 
261 	/* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
262 	if (ksm_do_scan(2, start_time, timeout))
263 		return 1;
264 
265 	return 0;
266 }
267 
ksm_unmerge_pages(void * addr,size_t size,struct timespec start_time,int timeout)268 static int ksm_unmerge_pages(void *addr, size_t size,
269 			     struct timespec start_time, int timeout)
270 {
271 	if (madvise(addr, size, MADV_UNMERGEABLE)) {
272 		perror("madvise");
273 		return 1;
274 	}
275 	return 0;
276 }
277 
assert_ksm_pages_count(long dupl_page_count)278 static bool assert_ksm_pages_count(long dupl_page_count)
279 {
280 	unsigned long max_page_sharing, pages_sharing, pages_shared;
281 
282 	if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
283 	    ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
284 	    ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
285 		return false;
286 
287 	if (debug) {
288 		ksm_print_sysfs();
289 		ksm_print_procfs();
290 	}
291 
292 	/*
293 	 * Since there must be at least 2 pages for merging and 1 page can be
294 	 * shared with the limited number of pages (max_page_sharing), sometimes
295 	 * there are 'leftover' pages that cannot be merged. For example, if there
296 	 * are 11 pages and max_page_sharing = 10, then only 10 pages will be
297 	 * merged and the 11th page won't be affected. As a result, when the number
298 	 * of duplicate pages is divided by max_page_sharing and the remainder is 1,
299 	 * pages_shared and pages_sharing values will be equal between dupl_page_count
300 	 * and dupl_page_count - 1.
301 	 */
302 	if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
303 		if (pages_shared == dupl_page_count / max_page_sharing &&
304 		    pages_sharing == pages_shared * (max_page_sharing - 1))
305 			return true;
306 	} else {
307 		if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
308 		    pages_sharing == dupl_page_count - pages_shared)
309 			return true;
310 	}
311 
312 	return false;
313 }
314 
ksm_save_def(struct ksm_sysfs * ksm_sysfs)315 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
316 {
317 	if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
318 	    numa_available() ? 0 :
319 		ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
320 	    ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
321 	    ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
322 	    ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
323 	    ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
324 			   &ksm_sysfs->stable_node_chains_prune_millisecs) ||
325 	    ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
326 		return 1;
327 
328 	return 0;
329 }
330 
ksm_restore(struct ksm_sysfs * ksm_sysfs)331 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
332 {
333 	if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
334 	    numa_available() ? 0 :
335 		ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
336 	    ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
337 	    ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
338 	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
339 	    ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
340 			    ksm_sysfs->stable_node_chains_prune_millisecs) ||
341 	    ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
342 		return 1;
343 
344 	return 0;
345 }
346 
check_ksm_merge(int merge_type,int mapping,int prot,long page_count,int timeout,size_t page_size)347 static int check_ksm_merge(int merge_type, int mapping, int prot,
348 			long page_count, int timeout, size_t page_size)
349 {
350 	void *map_ptr;
351 	struct timespec start_time;
352 
353 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
354 		perror("clock_gettime");
355 		return KSFT_FAIL;
356 	}
357 
358 	/* fill pages with the same data and merge them */
359 	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
360 	if (!map_ptr)
361 		return KSFT_FAIL;
362 
363 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
364 		goto err_out;
365 
366 	/* verify that the right number of pages are merged */
367 	if (assert_ksm_pages_count(page_count)) {
368 		printf("OK\n");
369 		munmap(map_ptr, page_size * page_count);
370 		if (merge_type == KSM_MERGE_PRCTL)
371 			prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
372 		return KSFT_PASS;
373 	}
374 
375 err_out:
376 	printf("Not OK\n");
377 	munmap(map_ptr, page_size * page_count);
378 	return KSFT_FAIL;
379 }
380 
check_ksm_unmerge(int merge_type,int mapping,int prot,int timeout,size_t page_size)381 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size)
382 {
383 	void *map_ptr;
384 	struct timespec start_time;
385 	int page_count = 2;
386 
387 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
388 		perror("clock_gettime");
389 		return KSFT_FAIL;
390 	}
391 
392 	/* fill pages with the same data and merge them */
393 	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
394 	if (!map_ptr)
395 		return KSFT_FAIL;
396 
397 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
398 		goto err_out;
399 
400 	/* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
401 	memset(map_ptr, '-', 1);
402 	memset(map_ptr + page_size, '+', 1);
403 
404 	/* get at least 1 scan, so KSM can detect that the pages were modified */
405 	if (ksm_do_scan(1, start_time, timeout))
406 		goto err_out;
407 
408 	/* check that unmerging was successful and 0 pages are currently merged */
409 	if (assert_ksm_pages_count(0)) {
410 		printf("OK\n");
411 		munmap(map_ptr, page_size * page_count);
412 		return KSFT_PASS;
413 	}
414 
415 err_out:
416 	printf("Not OK\n");
417 	munmap(map_ptr, page_size * page_count);
418 	return KSFT_FAIL;
419 }
420 
check_ksm_zero_page_merge(int merge_type,int mapping,int prot,long page_count,int timeout,bool use_zero_pages,size_t page_size)421 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count,
422 				int timeout, bool use_zero_pages, size_t page_size)
423 {
424 	void *map_ptr;
425 	struct timespec start_time;
426 
427 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
428 		perror("clock_gettime");
429 		return KSFT_FAIL;
430 	}
431 
432 	if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
433 		return KSFT_FAIL;
434 
435 	/* fill pages with zero and try to merge them */
436 	map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
437 	if (!map_ptr)
438 		return KSFT_FAIL;
439 
440 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
441 		goto err_out;
442 
443        /*
444 	* verify that the right number of pages are merged:
445 	* 1) if use_zero_pages is set to 1, empty pages are merged
446 	*    with the kernel zero page instead of with each other;
447 	* 2) if use_zero_pages is set to 0, empty pages are not treated specially
448 	*    and merged as usual.
449 	*/
450 	if (use_zero_pages && !assert_ksm_pages_count(0))
451 		goto err_out;
452 	else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
453 		goto err_out;
454 
455 	printf("OK\n");
456 	munmap(map_ptr, page_size * page_count);
457 	return KSFT_PASS;
458 
459 err_out:
460 	printf("Not OK\n");
461 	munmap(map_ptr, page_size * page_count);
462 	return KSFT_FAIL;
463 }
464 
get_next_mem_node(int node)465 static int get_next_mem_node(int node)
466 {
467 
468 	long node_size;
469 	int mem_node = 0;
470 	int i, max_node = numa_max_node();
471 
472 	for (i = node + 1; i <= max_node + node; i++) {
473 		mem_node = i % (max_node + 1);
474 		node_size = numa_node_size(mem_node, NULL);
475 		if (node_size > 0)
476 			break;
477 	}
478 	return mem_node;
479 }
480 
get_first_mem_node(void)481 static int get_first_mem_node(void)
482 {
483 	return get_next_mem_node(numa_max_node());
484 }
485 
check_ksm_numa_merge(int merge_type,int mapping,int prot,int timeout,bool merge_across_nodes,size_t page_size)486 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout,
487 				bool merge_across_nodes, size_t page_size)
488 {
489 	void *numa1_map_ptr, *numa2_map_ptr;
490 	struct timespec start_time;
491 	int page_count = 2;
492 	int first_node;
493 
494 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
495 		perror("clock_gettime");
496 		return KSFT_FAIL;
497 	}
498 
499 	if (numa_available() < 0) {
500 		perror("NUMA support not enabled");
501 		return KSFT_SKIP;
502 	}
503 	if (numa_num_configured_nodes() <= 1) {
504 		printf("At least 2 NUMA nodes must be available\n");
505 		return KSFT_SKIP;
506 	}
507 	if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
508 		return KSFT_FAIL;
509 
510 	/* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
511 	first_node = get_first_mem_node();
512 	numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
513 	numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
514 	if (!numa1_map_ptr || !numa2_map_ptr) {
515 		perror("numa_alloc_onnode");
516 		return KSFT_FAIL;
517 	}
518 
519 	memset(numa1_map_ptr, '*', page_size);
520 	memset(numa2_map_ptr, '*', page_size);
521 
522 	/* try to merge the pages */
523 	if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) ||
524 	    ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout))
525 		goto err_out;
526 
527        /*
528 	* verify that the right number of pages are merged:
529 	* 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
530 	* 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
531 	*    only 1 unique page in each node and they can't be shared.
532 	*/
533 	if (merge_across_nodes && !assert_ksm_pages_count(page_count))
534 		goto err_out;
535 	else if (!merge_across_nodes && !assert_ksm_pages_count(0))
536 		goto err_out;
537 
538 	numa_free(numa1_map_ptr, page_size);
539 	numa_free(numa2_map_ptr, page_size);
540 	printf("OK\n");
541 	return KSFT_PASS;
542 
543 err_out:
544 	numa_free(numa1_map_ptr, page_size);
545 	numa_free(numa2_map_ptr, page_size);
546 	printf("Not OK\n");
547 	return KSFT_FAIL;
548 }
549 
ksm_merge_hugepages_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)550 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot,
551 				int timeout, size_t map_size)
552 {
553 	void *map_ptr, *map_ptr_orig;
554 	struct timespec start_time, end_time;
555 	unsigned long scan_time_ns;
556 	int pagemap_fd, n_normal_pages, n_huge_pages;
557 
558 	map_size *= MB;
559 	size_t len = map_size;
560 
561 	len -= len % HPAGE_SIZE;
562 	map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
563 			MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
564 	map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
565 
566 	if (map_ptr_orig == MAP_FAILED)
567 		err(2, "initial mmap");
568 
569 	if (madvise(map_ptr, len, MADV_HUGEPAGE))
570 		err(2, "MADV_HUGEPAGE");
571 
572 	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
573 	if (pagemap_fd < 0)
574 		err(2, "open pagemap");
575 
576 	n_normal_pages = 0;
577 	n_huge_pages = 0;
578 	for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
579 		if (allocate_transhuge(p, pagemap_fd) < 0)
580 			n_normal_pages++;
581 		else
582 			n_huge_pages++;
583 	}
584 	printf("Number of normal pages:    %d\n", n_normal_pages);
585 	printf("Number of huge pages:    %d\n", n_huge_pages);
586 
587 	memset(map_ptr, '*', len);
588 
589 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
590 		perror("clock_gettime");
591 		goto err_out;
592 	}
593 	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
594 		goto err_out;
595 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
596 		perror("clock_gettime");
597 		goto err_out;
598 	}
599 
600 	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
601 		       (end_time.tv_nsec - start_time.tv_nsec);
602 
603 	printf("Total size:    %lu MiB\n", map_size / MB);
604 	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
605 	       scan_time_ns % NSEC_PER_SEC);
606 	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
607 					       ((double)scan_time_ns / NSEC_PER_SEC));
608 
609 	munmap(map_ptr_orig, len + HPAGE_SIZE);
610 	return KSFT_PASS;
611 
612 err_out:
613 	printf("Not OK\n");
614 	munmap(map_ptr_orig, len + HPAGE_SIZE);
615 	return KSFT_FAIL;
616 }
617 
ksm_merge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)618 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
619 {
620 	void *map_ptr;
621 	struct timespec start_time, end_time;
622 	unsigned long scan_time_ns;
623 
624 	map_size *= MB;
625 
626 	map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
627 	if (!map_ptr)
628 		return KSFT_FAIL;
629 
630 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
631 		perror("clock_gettime");
632 		goto err_out;
633 	}
634 	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
635 		goto err_out;
636 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
637 		perror("clock_gettime");
638 		goto err_out;
639 	}
640 
641 	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
642 		       (end_time.tv_nsec - start_time.tv_nsec);
643 
644 	printf("Total size:    %lu MiB\n", map_size / MB);
645 	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
646 	       scan_time_ns % NSEC_PER_SEC);
647 	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
648 					       ((double)scan_time_ns / NSEC_PER_SEC));
649 
650 	munmap(map_ptr, map_size);
651 	return KSFT_PASS;
652 
653 err_out:
654 	printf("Not OK\n");
655 	munmap(map_ptr, map_size);
656 	return KSFT_FAIL;
657 }
658 
ksm_unmerge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)659 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
660 {
661 	void *map_ptr;
662 	struct timespec start_time, end_time;
663 	unsigned long scan_time_ns;
664 
665 	map_size *= MB;
666 
667 	map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
668 	if (!map_ptr)
669 		return KSFT_FAIL;
670 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
671 		perror("clock_gettime");
672 		goto err_out;
673 	}
674 	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
675 		goto err_out;
676 
677 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
678 		perror("clock_gettime");
679 		goto err_out;
680 	}
681 	if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout))
682 		goto err_out;
683 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
684 		perror("clock_gettime");
685 		goto err_out;
686 	}
687 
688 	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
689 		       (end_time.tv_nsec - start_time.tv_nsec);
690 
691 	printf("Total size:    %lu MiB\n", map_size / MB);
692 	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
693 	       scan_time_ns % NSEC_PER_SEC);
694 	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
695 					       ((double)scan_time_ns / NSEC_PER_SEC));
696 
697 	munmap(map_ptr, map_size);
698 	return KSFT_PASS;
699 
700 err_out:
701 	printf("Not OK\n");
702 	munmap(map_ptr, map_size);
703 	return KSFT_FAIL;
704 }
705 
ksm_cow_time(int merge_type,int mapping,int prot,int timeout,size_t page_size)706 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size)
707 {
708 	void *map_ptr;
709 	struct timespec start_time, end_time;
710 	unsigned long cow_time_ns;
711 
712 	/* page_count must be less than 2*page_size */
713 	size_t page_count = 4000;
714 
715 	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
716 	if (!map_ptr)
717 		return KSFT_FAIL;
718 
719 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
720 		perror("clock_gettime");
721 		return KSFT_FAIL;
722 	}
723 	for (size_t i = 0; i < page_count - 1; i = i + 2)
724 		memset(map_ptr + page_size * i, '-', 1);
725 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
726 		perror("clock_gettime");
727 		return KSFT_FAIL;
728 	}
729 
730 	cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
731 		       (end_time.tv_nsec - start_time.tv_nsec);
732 
733 	printf("Total size:    %lu MiB\n\n", (page_size * page_count) / MB);
734 	printf("Not merged pages:\n");
735 	printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
736 	       cow_time_ns % NSEC_PER_SEC);
737 	printf("Average speed:  %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
738 					       ((double)cow_time_ns / NSEC_PER_SEC));
739 
740 	/* Create 2000 pairs of duplicate pages */
741 	for (size_t i = 0; i < page_count - 1; i = i + 2) {
742 		memset(map_ptr + page_size * i, '+', i / 2 + 1);
743 		memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
744 	}
745 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
746 		goto err_out;
747 
748 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
749 		perror("clock_gettime");
750 		goto err_out;
751 	}
752 	for (size_t i = 0; i < page_count - 1; i = i + 2)
753 		memset(map_ptr + page_size * i, '-', 1);
754 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
755 		perror("clock_gettime");
756 		goto err_out;
757 	}
758 
759 	cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
760 		       (end_time.tv_nsec - start_time.tv_nsec);
761 
762 	printf("Merged pages:\n");
763 	printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
764 	       cow_time_ns % NSEC_PER_SEC);
765 	printf("Average speed:  %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
766 					       ((double)cow_time_ns / NSEC_PER_SEC));
767 
768 	munmap(map_ptr, page_size * page_count);
769 	return KSFT_PASS;
770 
771 err_out:
772 	printf("Not OK\n");
773 	munmap(map_ptr, page_size * page_count);
774 	return KSFT_FAIL;
775 }
776 
main(int argc,char * argv[])777 int main(int argc, char *argv[])
778 {
779 	int ret, opt;
780 	int prot = 0;
781 	int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
782 	int merge_type = KSM_MERGE_TYPE_DEFAULT;
783 	long page_count = KSM_PAGE_COUNT_DEFAULT;
784 	size_t page_size = sysconf(_SC_PAGESIZE);
785 	struct ksm_sysfs ksm_sysfs_old;
786 	int test_name = CHECK_KSM_MERGE;
787 	bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
788 	bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
789 	long size_MB = 0;
790 
791 	while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) {
792 		switch (opt) {
793 		case 'a':
794 			prot = str_to_prot(optarg);
795 			break;
796 		case 'p':
797 			page_count = atol(optarg);
798 			if (page_count <= 0) {
799 				printf("The number of pages must be greater than 0\n");
800 				return KSFT_FAIL;
801 			}
802 			break;
803 		case 'l':
804 			ksm_scan_limit_sec = atoi(optarg);
805 			if (ksm_scan_limit_sec <= 0) {
806 				printf("Timeout value must be greater than 0\n");
807 				return KSFT_FAIL;
808 			}
809 			break;
810 		case 'h':
811 			print_help();
812 			break;
813 		case 'z':
814 			if (strcmp(optarg, "0") == 0)
815 				use_zero_pages = 0;
816 			else
817 				use_zero_pages = 1;
818 			break;
819 		case 'm':
820 			if (strcmp(optarg, "0") == 0)
821 				merge_across_nodes = 0;
822 			else
823 				merge_across_nodes = 1;
824 			break;
825 		case 'd':
826 			debug = 1;
827 			break;
828 		case 's':
829 			size_MB = atoi(optarg);
830 			if (size_MB <= 0) {
831 				printf("Size must be greater than 0\n");
832 				return KSFT_FAIL;
833 			}
834 			break;
835 		case 't':
836 			{
837 				int tmp = atoi(optarg);
838 
839 				if (tmp < 0 || tmp > KSM_MERGE_LAST) {
840 					printf("Invalid merge type\n");
841 					return KSFT_FAIL;
842 				}
843 				merge_type = tmp;
844 			}
845 			break;
846 		case 'M':
847 			break;
848 		case 'U':
849 			test_name = CHECK_KSM_UNMERGE;
850 			break;
851 		case 'Z':
852 			test_name = CHECK_KSM_ZERO_PAGE_MERGE;
853 			break;
854 		case 'N':
855 			test_name = CHECK_KSM_NUMA_MERGE;
856 			break;
857 		case 'P':
858 			test_name = KSM_MERGE_TIME;
859 			break;
860 		case 'H':
861 			test_name = KSM_MERGE_TIME_HUGE_PAGES;
862 			break;
863 		case 'D':
864 			test_name = KSM_UNMERGE_TIME;
865 			break;
866 		case 'C':
867 			test_name = KSM_COW_TIME;
868 			break;
869 		default:
870 			return KSFT_FAIL;
871 		}
872 	}
873 
874 	if (prot == 0)
875 		prot = str_to_prot(KSM_PROT_STR_DEFAULT);
876 
877 	if (access(KSM_SYSFS_PATH, F_OK)) {
878 		printf("Config KSM not enabled\n");
879 		return KSFT_SKIP;
880 	}
881 
882 	if (ksm_save_def(&ksm_sysfs_old)) {
883 		printf("Cannot save default tunables\n");
884 		return KSFT_FAIL;
885 	}
886 
887 	if (ksm_write_sysfs(KSM_FP("run"), 2) ||
888 	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
889 	    numa_available() ? 0 :
890 		ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
891 	    ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
892 		return KSFT_FAIL;
893 
894 	switch (test_name) {
895 	case CHECK_KSM_MERGE:
896 		ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
897 				      ksm_scan_limit_sec, page_size);
898 		break;
899 	case CHECK_KSM_UNMERGE:
900 		ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
901 					ksm_scan_limit_sec, page_size);
902 		break;
903 	case CHECK_KSM_ZERO_PAGE_MERGE:
904 		ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
905 						page_count, ksm_scan_limit_sec, use_zero_pages,
906 						page_size);
907 		break;
908 	case CHECK_KSM_NUMA_MERGE:
909 		ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
910 					ksm_scan_limit_sec, merge_across_nodes, page_size);
911 		break;
912 	case KSM_MERGE_TIME:
913 		if (size_MB == 0) {
914 			printf("Option '-s' is required.\n");
915 			return KSFT_FAIL;
916 		}
917 		ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
918 				ksm_scan_limit_sec, size_MB);
919 		break;
920 	case KSM_MERGE_TIME_HUGE_PAGES:
921 		if (size_MB == 0) {
922 			printf("Option '-s' is required.\n");
923 			return KSFT_FAIL;
924 		}
925 		ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
926 				ksm_scan_limit_sec, size_MB);
927 		break;
928 	case KSM_UNMERGE_TIME:
929 		if (size_MB == 0) {
930 			printf("Option '-s' is required.\n");
931 			return KSFT_FAIL;
932 		}
933 		ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
934 				       ksm_scan_limit_sec, size_MB);
935 		break;
936 	case KSM_COW_TIME:
937 		ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
938 				ksm_scan_limit_sec, page_size);
939 		break;
940 	}
941 
942 	if (ksm_restore(&ksm_sysfs_old)) {
943 		printf("Cannot restore default tunables\n");
944 		return KSFT_FAIL;
945 	}
946 
947 	return ret;
948 }
949