1 #include <fcntl.h> 2 #include <errno.h> 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <dirent.h> 7 #include <sys/ioctl.h> 8 #include <sys/mman.h> 9 #include <sys/stat.h> 10 #include <sys/types.h> 11 #include <pthread.h> 12 #include <assert.h> 13 #include <mm/gup_test.h> 14 #include "../kselftest.h" 15 16 #include "util.h" 17 18 #define MB (1UL << 20) 19 20 /* Just the flags we need, copied from mm.h: */ 21 #define FOLL_WRITE 0x01 /* check pte is writable */ 22 #define FOLL_TOUCH 0x02 /* mark page accessed */ 23 24 #define GUP_TEST_FILE "/sys/kernel/debug/gup_test" 25 26 static unsigned long cmd = GUP_FAST_BENCHMARK; 27 static int gup_fd, repeats = 1; 28 static unsigned long size = 128 * MB; 29 /* Serialize prints */ 30 static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER; 31 32 static char *cmd_to_str(unsigned long cmd) 33 { 34 switch (cmd) { 35 case GUP_FAST_BENCHMARK: 36 return "GUP_FAST_BENCHMARK"; 37 case PIN_FAST_BENCHMARK: 38 return "PIN_FAST_BENCHMARK"; 39 case PIN_LONGTERM_BENCHMARK: 40 return "PIN_LONGTERM_BENCHMARK"; 41 case GUP_BASIC_TEST: 42 return "GUP_BASIC_TEST"; 43 case PIN_BASIC_TEST: 44 return "PIN_BASIC_TEST"; 45 case DUMP_USER_PAGES_TEST: 46 return "DUMP_USER_PAGES_TEST"; 47 } 48 return "Unknown command"; 49 } 50 51 void *gup_thread(void *data) 52 { 53 struct gup_test gup = *(struct gup_test *)data; 54 int i; 55 56 /* Only report timing information on the *_BENCHMARK commands: */ 57 if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) || 58 (cmd == PIN_LONGTERM_BENCHMARK)) { 59 for (i = 0; i < repeats; i++) { 60 gup.size = size; 61 if (ioctl(gup_fd, cmd, &gup)) 62 perror("ioctl"), exit(1); 63 64 pthread_mutex_lock(&print_mutex); 65 printf("%s: Time: get:%lld put:%lld us", 66 cmd_to_str(cmd), gup.get_delta_usec, 67 gup.put_delta_usec); 68 if (gup.size != size) 69 printf(", truncated (size: %lld)", gup.size); 70 printf("\n"); 71 pthread_mutex_unlock(&print_mutex); 72 } 73 } else { 74 gup.size = size; 75 if (ioctl(gup_fd, cmd, &gup)) { 76 perror("ioctl"); 77 exit(1); 78 } 79 80 pthread_mutex_lock(&print_mutex); 81 printf("%s: done\n", cmd_to_str(cmd)); 82 if (gup.size != size) 83 printf("Truncated (size: %lld)\n", gup.size); 84 pthread_mutex_unlock(&print_mutex); 85 } 86 87 return NULL; 88 } 89 90 int main(int argc, char **argv) 91 { 92 struct gup_test gup = { 0 }; 93 int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; 94 int flags = MAP_PRIVATE, touch = 0; 95 char *file = "/dev/zero"; 96 pthread_t *tid; 97 char *p; 98 99 while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) { 100 switch (opt) { 101 case 'a': 102 cmd = PIN_FAST_BENCHMARK; 103 break; 104 case 'b': 105 cmd = PIN_BASIC_TEST; 106 break; 107 case 'L': 108 cmd = PIN_LONGTERM_BENCHMARK; 109 break; 110 case 'c': 111 cmd = DUMP_USER_PAGES_TEST; 112 /* 113 * Dump page 0 (index 1). May be overridden later, by 114 * user's non-option arguments. 115 * 116 * .which_pages is zero-based, so that zero can mean "do 117 * nothing". 118 */ 119 gup.which_pages[0] = 1; 120 break; 121 case 'p': 122 /* works only with DUMP_USER_PAGES_TEST */ 123 gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN; 124 break; 125 case 'F': 126 /* strtol, so you can pass flags in hex form */ 127 gup.gup_flags = strtol(optarg, 0, 0); 128 break; 129 case 'j': 130 nthreads = atoi(optarg); 131 break; 132 case 'm': 133 size = atoi(optarg) * MB; 134 break; 135 case 'r': 136 repeats = atoi(optarg); 137 break; 138 case 'n': 139 nr_pages = atoi(optarg); 140 break; 141 case 't': 142 thp = 1; 143 break; 144 case 'T': 145 thp = 0; 146 break; 147 case 'U': 148 cmd = GUP_BASIC_TEST; 149 break; 150 case 'u': 151 cmd = GUP_FAST_BENCHMARK; 152 break; 153 case 'w': 154 write = 1; 155 break; 156 case 'W': 157 write = 0; 158 break; 159 case 'f': 160 file = optarg; 161 break; 162 case 'S': 163 flags &= ~MAP_PRIVATE; 164 flags |= MAP_SHARED; 165 break; 166 case 'H': 167 flags |= (MAP_HUGETLB | MAP_ANONYMOUS); 168 break; 169 case 'z': 170 /* fault pages in gup, do not fault in userland */ 171 touch = 1; 172 break; 173 default: 174 return -1; 175 } 176 } 177 178 if (optind < argc) { 179 int extra_arg_count = 0; 180 /* 181 * For example: 182 * 183 * ./gup_test -c 0 1 0x1001 184 * 185 * ...to dump pages 0, 1, and 4097 186 */ 187 188 while ((optind < argc) && 189 (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) { 190 /* 191 * Do the 1-based indexing here, so that the user can 192 * use normal 0-based indexing on the command line. 193 */ 194 long page_index = strtol(argv[optind], 0, 0) + 1; 195 196 gup.which_pages[extra_arg_count] = page_index; 197 extra_arg_count++; 198 optind++; 199 } 200 } 201 202 filed = open(file, O_RDWR|O_CREAT); 203 if (filed < 0) { 204 perror("open"); 205 exit(filed); 206 } 207 208 gup.nr_pages_per_call = nr_pages; 209 if (write) 210 gup.gup_flags |= FOLL_WRITE; 211 212 gup_fd = open(GUP_TEST_FILE, O_RDWR); 213 if (gup_fd == -1) { 214 switch (errno) { 215 case EACCES: 216 if (getuid()) 217 printf("Please run this test as root\n"); 218 break; 219 case ENOENT: 220 if (opendir("/sys/kernel/debug") == NULL) { 221 printf("mount debugfs at /sys/kernel/debug\n"); 222 break; 223 } 224 printf("check if CONFIG_GUP_TEST is enabled in kernel config\n"); 225 break; 226 default: 227 perror("failed to open " GUP_TEST_FILE); 228 break; 229 } 230 exit(KSFT_SKIP); 231 } 232 233 p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0); 234 if (p == MAP_FAILED) { 235 perror("mmap"); 236 exit(1); 237 } 238 gup.addr = (unsigned long)p; 239 240 if (thp == 1) 241 madvise(p, size, MADV_HUGEPAGE); 242 else if (thp == 0) 243 madvise(p, size, MADV_NOHUGEPAGE); 244 245 /* 246 * FOLL_TOUCH, in gup_test, is used as an either/or case: either 247 * fault pages in from the kernel via FOLL_TOUCH, or fault them 248 * in here, from user space. This allows comparison of performance 249 * between those two cases. 250 */ 251 if (touch) { 252 gup.gup_flags |= FOLL_TOUCH; 253 } else { 254 for (; (unsigned long)p < gup.addr + size; p += PAGE_SIZE) 255 p[0] = 0; 256 } 257 258 tid = malloc(sizeof(pthread_t) * nthreads); 259 assert(tid); 260 for (i = 0; i < nthreads; i++) { 261 ret = pthread_create(&tid[i], NULL, gup_thread, &gup); 262 assert(ret == 0); 263 } 264 for (i = 0; i < nthreads; i++) { 265 ret = pthread_join(tid[i], NULL); 266 assert(ret == 0); 267 } 268 free(tid); 269 270 return 0; 271 } 272