1 // SPDX-License-Identifier: GPL-2.0
2 /* Test selecting other page sizes for mmap/shmget.
3 
4    Before running this huge pages for each huge page size must have been
5    reserved.
6    For large pages beyond MAX_ORDER (like 1GB on x86) boot options must be used.
7    Also shmmax must be increased.
8    And you need to run as root to work around some weird permissions in shm.
9    And nothing using huge pages should run in parallel.
10    When the program aborts you may need to clean up the shm segments with
11    ipcrm -m by hand, like this
12    sudo ipcs | awk '$1 == "0x00000000" {print $2}' | xargs -n1 sudo ipcrm -m
13    (warning this will remove all if someone else uses them) */
14 
15 #define _GNU_SOURCE 1
16 #include <sys/mman.h>
17 #include <stdlib.h>
18 #include <stdio.h>
19 #include <sys/ipc.h>
20 #include <sys/shm.h>
21 #include <sys/stat.h>
22 #include <glob.h>
23 #include <assert.h>
24 #include <unistd.h>
25 #include <stdarg.h>
26 #include <string.h>
27 #include "vm_util.h"
28 
29 #define err(x) perror(x), exit(1)
30 
31 #define MAP_HUGE_2MB    (21 << MAP_HUGE_SHIFT)
32 #define MAP_HUGE_1GB    (30 << MAP_HUGE_SHIFT)
33 #define MAP_HUGE_SHIFT  26
34 #define MAP_HUGE_MASK   0x3f
35 #if !defined(MAP_HUGETLB)
36 #define MAP_HUGETLB	0x40000
37 #endif
38 
39 #define SHM_HUGETLB     04000   /* segment will use huge TLB pages */
40 #define SHM_HUGE_SHIFT  26
41 #define SHM_HUGE_MASK   0x3f
42 #define SHM_HUGE_2MB    (21 << SHM_HUGE_SHIFT)
43 #define SHM_HUGE_1GB    (30 << SHM_HUGE_SHIFT)
44 
45 #define NUM_PAGESIZES   5
46 
47 #define NUM_PAGES 4
48 
49 #define Dprintf(fmt...) // printf(fmt)
50 
51 unsigned long page_sizes[NUM_PAGESIZES];
52 int num_page_sizes;
53 
ilog2(unsigned long v)54 int ilog2(unsigned long v)
55 {
56 	int l = 0;
57 	while ((1UL << l) < v)
58 		l++;
59 	return l;
60 }
61 
find_pagesizes(void)62 void find_pagesizes(void)
63 {
64 	glob_t g;
65 	int i;
66 	glob("/sys/kernel/mm/hugepages/hugepages-*kB", 0, NULL, &g);
67 	assert(g.gl_pathc <= NUM_PAGESIZES);
68 	for (i = 0; i < g.gl_pathc; i++) {
69 		sscanf(g.gl_pathv[i], "/sys/kernel/mm/hugepages/hugepages-%lukB",
70 				&page_sizes[i]);
71 		page_sizes[i] <<= 10;
72 		printf("Found %luMB\n", page_sizes[i] >> 20);
73 	}
74 	num_page_sizes = g.gl_pathc;
75 	globfree(&g);
76 }
77 
show(unsigned long ps)78 void show(unsigned long ps)
79 {
80 	char buf[100];
81 	if (ps == getpagesize())
82 		return;
83 	printf("%luMB: ", ps >> 20);
84 	fflush(stdout);
85 	snprintf(buf, sizeof buf,
86 		"cat /sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages",
87 		ps >> 10);
88 	system(buf);
89 }
90 
read_sysfs(int warn,char * fmt,...)91 unsigned long read_sysfs(int warn, char *fmt, ...)
92 {
93 	char *line = NULL;
94 	size_t linelen = 0;
95 	char buf[100];
96 	FILE *f;
97 	va_list ap;
98 	unsigned long val = 0;
99 
100 	va_start(ap, fmt);
101 	vsnprintf(buf, sizeof buf, fmt, ap);
102 	va_end(ap);
103 
104 	f = fopen(buf, "r");
105 	if (!f) {
106 		if (warn)
107 			printf("missing %s\n", buf);
108 		return 0;
109 	}
110 	if (getline(&line, &linelen, f) > 0) {
111 		sscanf(line, "%lu", &val);
112 	}
113 	fclose(f);
114 	free(line);
115 	return val;
116 }
117 
read_free(unsigned long ps)118 unsigned long read_free(unsigned long ps)
119 {
120 	return read_sysfs(ps != getpagesize(),
121 			"/sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages",
122 			ps >> 10);
123 }
124 
test_mmap(unsigned long size,unsigned flags)125 void test_mmap(unsigned long size, unsigned flags)
126 {
127 	char *map;
128 	unsigned long before, after;
129 	int err;
130 
131 	before = read_free(size);
132 	map = mmap(NULL, size*NUM_PAGES, PROT_READ|PROT_WRITE,
133 			MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|flags, -1, 0);
134 
135 	if (map == (char *)-1) err("mmap");
136 	memset(map, 0xff, size*NUM_PAGES);
137 	after = read_free(size);
138 	Dprintf("before %lu after %lu diff %ld size %lu\n",
139 		before, after, before - after, size);
140 	assert(size == getpagesize() || (before - after) == NUM_PAGES);
141 	show(size);
142 	err = munmap(map, size * NUM_PAGES);
143 	assert(!err);
144 }
145 
test_shmget(unsigned long size,unsigned flags)146 void test_shmget(unsigned long size, unsigned flags)
147 {
148 	int id;
149 	unsigned long before, after;
150 	int err;
151 
152 	before = read_free(size);
153 	id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags);
154 	if (id < 0) err("shmget");
155 
156 	struct shm_info i;
157 	if (shmctl(id, SHM_INFO, (void *)&i) < 0) err("shmctl");
158 	Dprintf("alloc %lu res %lu\n", i.shm_tot, i.shm_rss);
159 
160 
161 	Dprintf("id %d\n", id);
162 	char *map = shmat(id, NULL, 0600);
163 	if (map == (char*)-1) err("shmat");
164 
165 	shmctl(id, IPC_RMID, NULL);
166 
167 	memset(map, 0xff, size*NUM_PAGES);
168 	after = read_free(size);
169 
170 	Dprintf("before %lu after %lu diff %ld size %lu\n",
171 		before, after, before - after, size);
172 	assert(size == getpagesize() || (before - after) == NUM_PAGES);
173 	show(size);
174 	err = shmdt(map);
175 	assert(!err);
176 }
177 
sanity_checks(void)178 void sanity_checks(void)
179 {
180 	int i;
181 	unsigned long largest = getpagesize();
182 
183 	for (i = 0; i < num_page_sizes; i++) {
184 		if (page_sizes[i] > largest)
185 			largest = page_sizes[i];
186 
187 		if (read_free(page_sizes[i]) < NUM_PAGES) {
188 			printf("Not enough huge pages for page size %lu MB, need %u\n",
189 				page_sizes[i] >> 20,
190 				NUM_PAGES);
191 			exit(0);
192 		}
193 	}
194 
195 	if (read_sysfs(0, "/proc/sys/kernel/shmmax") < NUM_PAGES * largest) {
196 		printf("Please do echo %lu > /proc/sys/kernel/shmmax", largest * NUM_PAGES);
197 		exit(0);
198 	}
199 
200 #if defined(__x86_64__)
201 	if (largest != 1U<<30) {
202 		printf("No GB pages available on x86-64\n"
203 		       "Please boot with hugepagesz=1G hugepages=%d\n", NUM_PAGES);
204 		exit(0);
205 	}
206 #endif
207 }
208 
main(void)209 int main(void)
210 {
211 	int i;
212 	unsigned default_hps = default_huge_page_size();
213 
214 	find_pagesizes();
215 
216 	sanity_checks();
217 
218 	for (i = 0; i < num_page_sizes; i++) {
219 		unsigned long ps = page_sizes[i];
220 		int arg = ilog2(ps) << MAP_HUGE_SHIFT;
221 		printf("Testing %luMB mmap with shift %x\n", ps >> 20, arg);
222 		test_mmap(ps, MAP_HUGETLB | arg);
223 	}
224 	printf("Testing default huge mmap\n");
225 	test_mmap(default_hps, MAP_HUGETLB);
226 
227 	puts("Testing non-huge shmget");
228 	test_shmget(getpagesize(), 0);
229 
230 	for (i = 0; i < num_page_sizes; i++) {
231 		unsigned long ps = page_sizes[i];
232 		int arg = ilog2(ps) << SHM_HUGE_SHIFT;
233 		printf("Testing %luMB shmget with shift %x\n", ps >> 20, arg);
234 		test_shmget(ps, SHM_HUGETLB | arg);
235 	}
236 	puts("default huge shmget");
237 	test_shmget(default_hps, SHM_HUGETLB);
238 
239 	return 0;
240 }
241