1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Test module for stress and analyze performance of vmalloc allocator. 5 * (C) 2018 Uladzislau Rezki (Sony) <urezki@gmail.com> 6 */ 7 #include <linux/init.h> 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/vmalloc.h> 11 #include <linux/random.h> 12 #include <linux/kthread.h> 13 #include <linux/moduleparam.h> 14 #include <linux/completion.h> 15 #include <linux/delay.h> 16 #include <linux/rwsem.h> 17 #include <linux/mm.h> 18 #include <linux/rcupdate.h> 19 #include <linux/slab.h> 20 21 #define __param(type, name, init, msg) \ 22 static type name = init; \ 23 module_param(name, type, 0444); \ 24 MODULE_PARM_DESC(name, msg) \ 25 26 __param(int, nr_threads, 0, 27 "Number of workers to perform tests(min: 1 max: USHRT_MAX)"); 28 29 __param(bool, sequential_test_order, false, 30 "Use sequential stress tests order"); 31 32 __param(int, test_repeat_count, 1, 33 "Set test repeat counter"); 34 35 __param(int, test_loop_count, 1000000, 36 "Set test loop counter"); 37 38 __param(int, nr_pages, 0, 39 "Set number of pages for fix_size_alloc_test(default: 1)"); 40 41 __param(int, run_test_mask, INT_MAX, 42 "Set tests specified in the mask.\n\n" 43 "\t\tid: 1, name: fix_size_alloc_test\n" 44 "\t\tid: 2, name: full_fit_alloc_test\n" 45 "\t\tid: 4, name: long_busy_list_alloc_test\n" 46 "\t\tid: 8, name: random_size_alloc_test\n" 47 "\t\tid: 16, name: fix_align_alloc_test\n" 48 "\t\tid: 32, name: random_size_align_alloc_test\n" 49 "\t\tid: 64, name: align_shift_alloc_test\n" 50 "\t\tid: 128, name: pcpu_alloc_test\n" 51 "\t\tid: 256, name: kvfree_rcu_1_arg_vmalloc_test\n" 52 "\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n" 53 /* Add a new test case description here. */ 54 ); 55 56 /* 57 * Read write semaphore for synchronization of setup 58 * phase that is done in main thread and workers. 59 */ 60 static DECLARE_RWSEM(prepare_for_test_rwsem); 61 62 /* 63 * Completion tracking for worker threads. 64 */ 65 static DECLARE_COMPLETION(test_all_done_comp); 66 static atomic_t test_n_undone = ATOMIC_INIT(0); 67 68 static inline void 69 test_report_one_done(void) 70 { 71 if (atomic_dec_and_test(&test_n_undone)) 72 complete(&test_all_done_comp); 73 } 74 75 static int random_size_align_alloc_test(void) 76 { 77 unsigned long size, align, rnd; 78 void *ptr; 79 int i; 80 81 for (i = 0; i < test_loop_count; i++) { 82 get_random_bytes(&rnd, sizeof(rnd)); 83 84 /* 85 * Maximum 1024 pages, if PAGE_SIZE is 4096. 86 */ 87 align = 1 << (rnd % 23); 88 89 /* 90 * Maximum 10 pages. 91 */ 92 size = ((rnd % 10) + 1) * PAGE_SIZE; 93 94 ptr = __vmalloc_node(size, align, GFP_KERNEL | __GFP_ZERO, 0, 95 __builtin_return_address(0)); 96 if (!ptr) 97 return -1; 98 99 vfree(ptr); 100 } 101 102 return 0; 103 } 104 105 /* 106 * This test case is supposed to be failed. 107 */ 108 static int align_shift_alloc_test(void) 109 { 110 unsigned long align; 111 void *ptr; 112 int i; 113 114 for (i = 0; i < BITS_PER_LONG; i++) { 115 align = ((unsigned long) 1) << i; 116 117 ptr = __vmalloc_node(PAGE_SIZE, align, GFP_KERNEL|__GFP_ZERO, 0, 118 __builtin_return_address(0)); 119 if (!ptr) 120 return -1; 121 122 vfree(ptr); 123 } 124 125 return 0; 126 } 127 128 static int fix_align_alloc_test(void) 129 { 130 void *ptr; 131 int i; 132 133 for (i = 0; i < test_loop_count; i++) { 134 ptr = __vmalloc_node(5 * PAGE_SIZE, THREAD_ALIGN << 1, 135 GFP_KERNEL | __GFP_ZERO, 0, 136 __builtin_return_address(0)); 137 if (!ptr) 138 return -1; 139 140 vfree(ptr); 141 } 142 143 return 0; 144 } 145 146 static int random_size_alloc_test(void) 147 { 148 unsigned int n; 149 void *p; 150 int i; 151 152 for (i = 0; i < test_loop_count; i++) { 153 get_random_bytes(&n, sizeof(i)); 154 n = (n % 100) + 1; 155 156 p = vmalloc(n * PAGE_SIZE); 157 158 if (!p) 159 return -1; 160 161 *((__u8 *)p) = 1; 162 vfree(p); 163 } 164 165 return 0; 166 } 167 168 static int long_busy_list_alloc_test(void) 169 { 170 void *ptr_1, *ptr_2; 171 void **ptr; 172 int rv = -1; 173 int i; 174 175 ptr = vmalloc(sizeof(void *) * 15000); 176 if (!ptr) 177 return rv; 178 179 for (i = 0; i < 15000; i++) 180 ptr[i] = vmalloc(1 * PAGE_SIZE); 181 182 for (i = 0; i < test_loop_count; i++) { 183 ptr_1 = vmalloc(100 * PAGE_SIZE); 184 if (!ptr_1) 185 goto leave; 186 187 ptr_2 = vmalloc(1 * PAGE_SIZE); 188 if (!ptr_2) { 189 vfree(ptr_1); 190 goto leave; 191 } 192 193 *((__u8 *)ptr_1) = 0; 194 *((__u8 *)ptr_2) = 1; 195 196 vfree(ptr_1); 197 vfree(ptr_2); 198 } 199 200 /* Success */ 201 rv = 0; 202 203 leave: 204 for (i = 0; i < 15000; i++) 205 vfree(ptr[i]); 206 207 vfree(ptr); 208 return rv; 209 } 210 211 static int full_fit_alloc_test(void) 212 { 213 void **ptr, **junk_ptr, *tmp; 214 int junk_length; 215 int rv = -1; 216 int i; 217 218 junk_length = fls(num_online_cpus()); 219 junk_length *= (32 * 1024 * 1024 / PAGE_SIZE); 220 221 ptr = vmalloc(sizeof(void *) * junk_length); 222 if (!ptr) 223 return rv; 224 225 junk_ptr = vmalloc(sizeof(void *) * junk_length); 226 if (!junk_ptr) { 227 vfree(ptr); 228 return rv; 229 } 230 231 for (i = 0; i < junk_length; i++) { 232 ptr[i] = vmalloc(1 * PAGE_SIZE); 233 junk_ptr[i] = vmalloc(1 * PAGE_SIZE); 234 } 235 236 for (i = 0; i < junk_length; i++) 237 vfree(junk_ptr[i]); 238 239 for (i = 0; i < test_loop_count; i++) { 240 tmp = vmalloc(1 * PAGE_SIZE); 241 242 if (!tmp) 243 goto error; 244 245 *((__u8 *)tmp) = 1; 246 vfree(tmp); 247 } 248 249 /* Success */ 250 rv = 0; 251 252 error: 253 for (i = 0; i < junk_length; i++) 254 vfree(ptr[i]); 255 256 vfree(ptr); 257 vfree(junk_ptr); 258 259 return rv; 260 } 261 262 static int fix_size_alloc_test(void) 263 { 264 void *ptr; 265 int i; 266 267 for (i = 0; i < test_loop_count; i++) { 268 ptr = vmalloc((nr_pages > 0 ? nr_pages:1) * PAGE_SIZE); 269 270 if (!ptr) 271 return -1; 272 273 *((__u8 *)ptr) = 0; 274 275 vfree(ptr); 276 } 277 278 return 0; 279 } 280 281 static int 282 pcpu_alloc_test(void) 283 { 284 int rv = 0; 285 #ifndef CONFIG_NEED_PER_CPU_KM 286 void __percpu **pcpu; 287 size_t size, align; 288 int i; 289 290 pcpu = vmalloc(sizeof(void __percpu *) * 35000); 291 if (!pcpu) 292 return -1; 293 294 for (i = 0; i < 35000; i++) { 295 unsigned int r; 296 297 get_random_bytes(&r, sizeof(i)); 298 size = (r % (PAGE_SIZE / 4)) + 1; 299 300 /* 301 * Maximum PAGE_SIZE 302 */ 303 get_random_bytes(&r, sizeof(i)); 304 align = 1 << ((i % 11) + 1); 305 306 pcpu[i] = __alloc_percpu(size, align); 307 if (!pcpu[i]) 308 rv = -1; 309 } 310 311 for (i = 0; i < 35000; i++) 312 free_percpu(pcpu[i]); 313 314 vfree(pcpu); 315 #endif 316 return rv; 317 } 318 319 struct test_kvfree_rcu { 320 struct rcu_head rcu; 321 unsigned char array[20]; 322 }; 323 324 static int 325 kvfree_rcu_1_arg_vmalloc_test(void) 326 { 327 struct test_kvfree_rcu *p; 328 int i; 329 330 for (i = 0; i < test_loop_count; i++) { 331 p = vmalloc(1 * PAGE_SIZE); 332 if (!p) 333 return -1; 334 335 p->array[0] = 'a'; 336 kvfree_rcu(p); 337 } 338 339 return 0; 340 } 341 342 static int 343 kvfree_rcu_2_arg_vmalloc_test(void) 344 { 345 struct test_kvfree_rcu *p; 346 int i; 347 348 for (i = 0; i < test_loop_count; i++) { 349 p = vmalloc(1 * PAGE_SIZE); 350 if (!p) 351 return -1; 352 353 p->array[0] = 'a'; 354 kvfree_rcu(p, rcu); 355 } 356 357 return 0; 358 } 359 360 struct test_case_desc { 361 const char *test_name; 362 int (*test_func)(void); 363 }; 364 365 static struct test_case_desc test_case_array[] = { 366 { "fix_size_alloc_test", fix_size_alloc_test }, 367 { "full_fit_alloc_test", full_fit_alloc_test }, 368 { "long_busy_list_alloc_test", long_busy_list_alloc_test }, 369 { "random_size_alloc_test", random_size_alloc_test }, 370 { "fix_align_alloc_test", fix_align_alloc_test }, 371 { "random_size_align_alloc_test", random_size_align_alloc_test }, 372 { "align_shift_alloc_test", align_shift_alloc_test }, 373 { "pcpu_alloc_test", pcpu_alloc_test }, 374 { "kvfree_rcu_1_arg_vmalloc_test", kvfree_rcu_1_arg_vmalloc_test }, 375 { "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test }, 376 /* Add a new test case here. */ 377 }; 378 379 struct test_case_data { 380 int test_failed; 381 int test_passed; 382 u64 time; 383 }; 384 385 static struct test_driver { 386 struct task_struct *task; 387 struct test_case_data data[ARRAY_SIZE(test_case_array)]; 388 389 unsigned long start; 390 unsigned long stop; 391 } *tdriver; 392 393 static void shuffle_array(int *arr, int n) 394 { 395 unsigned int rnd; 396 int i, j, x; 397 398 for (i = n - 1; i > 0; i--) { 399 get_random_bytes(&rnd, sizeof(rnd)); 400 401 /* Cut the range. */ 402 j = rnd % i; 403 404 /* Swap indexes. */ 405 x = arr[i]; 406 arr[i] = arr[j]; 407 arr[j] = x; 408 } 409 } 410 411 static int test_func(void *private) 412 { 413 struct test_driver *t = private; 414 int random_array[ARRAY_SIZE(test_case_array)]; 415 int index, i, j; 416 ktime_t kt; 417 u64 delta; 418 419 for (i = 0; i < ARRAY_SIZE(test_case_array); i++) 420 random_array[i] = i; 421 422 if (!sequential_test_order) 423 shuffle_array(random_array, ARRAY_SIZE(test_case_array)); 424 425 /* 426 * Block until initialization is done. 427 */ 428 down_read(&prepare_for_test_rwsem); 429 430 t->start = get_cycles(); 431 for (i = 0; i < ARRAY_SIZE(test_case_array); i++) { 432 index = random_array[i]; 433 434 /* 435 * Skip tests if run_test_mask has been specified. 436 */ 437 if (!((run_test_mask & (1 << index)) >> index)) 438 continue; 439 440 kt = ktime_get(); 441 for (j = 0; j < test_repeat_count; j++) { 442 if (!test_case_array[index].test_func()) 443 t->data[index].test_passed++; 444 else 445 t->data[index].test_failed++; 446 } 447 448 /* 449 * Take an average time that test took. 450 */ 451 delta = (u64) ktime_us_delta(ktime_get(), kt); 452 do_div(delta, (u32) test_repeat_count); 453 454 t->data[index].time = delta; 455 } 456 t->stop = get_cycles(); 457 458 up_read(&prepare_for_test_rwsem); 459 test_report_one_done(); 460 461 /* 462 * Wait for the kthread_stop() call. 463 */ 464 while (!kthread_should_stop()) 465 msleep(10); 466 467 return 0; 468 } 469 470 static int 471 init_test_configurtion(void) 472 { 473 /* 474 * A maximum number of workers is defined as hard-coded 475 * value and set to USHRT_MAX. We add such gap just in 476 * case and for potential heavy stressing. 477 */ 478 nr_threads = clamp(nr_threads, 1, (int) USHRT_MAX); 479 480 /* Allocate the space for test instances. */ 481 tdriver = kvcalloc(nr_threads, sizeof(*tdriver), GFP_KERNEL); 482 if (tdriver == NULL) 483 return -1; 484 485 if (test_repeat_count <= 0) 486 test_repeat_count = 1; 487 488 if (test_loop_count <= 0) 489 test_loop_count = 1; 490 491 return 0; 492 } 493 494 static void do_concurrent_test(void) 495 { 496 int i, ret; 497 498 /* 499 * Set some basic configurations plus sanity check. 500 */ 501 ret = init_test_configurtion(); 502 if (ret < 0) 503 return; 504 505 /* 506 * Put on hold all workers. 507 */ 508 down_write(&prepare_for_test_rwsem); 509 510 for (i = 0; i < nr_threads; i++) { 511 struct test_driver *t = &tdriver[i]; 512 513 t->task = kthread_run(test_func, t, "vmalloc_test/%d", i); 514 515 if (!IS_ERR(t->task)) 516 /* Success. */ 517 atomic_inc(&test_n_undone); 518 else 519 pr_err("Failed to start %d kthread\n", i); 520 } 521 522 /* 523 * Now let the workers do their job. 524 */ 525 up_write(&prepare_for_test_rwsem); 526 527 /* 528 * Sleep quiet until all workers are done with 1 second 529 * interval. Since the test can take a lot of time we 530 * can run into a stack trace of the hung task. That is 531 * why we go with completion_timeout and HZ value. 532 */ 533 do { 534 ret = wait_for_completion_timeout(&test_all_done_comp, HZ); 535 } while (!ret); 536 537 for (i = 0; i < nr_threads; i++) { 538 struct test_driver *t = &tdriver[i]; 539 int j; 540 541 if (!IS_ERR(t->task)) 542 kthread_stop(t->task); 543 544 for (j = 0; j < ARRAY_SIZE(test_case_array); j++) { 545 if (!((run_test_mask & (1 << j)) >> j)) 546 continue; 547 548 pr_info( 549 "Summary: %s passed: %d failed: %d repeat: %d loops: %d avg: %llu usec\n", 550 test_case_array[j].test_name, 551 t->data[j].test_passed, 552 t->data[j].test_failed, 553 test_repeat_count, test_loop_count, 554 t->data[j].time); 555 } 556 557 pr_info("All test took worker%d=%lu cycles\n", 558 i, t->stop - t->start); 559 } 560 561 kvfree(tdriver); 562 } 563 564 static int vmalloc_test_init(void) 565 { 566 do_concurrent_test(); 567 return -EAGAIN; /* Fail will directly unload the module */ 568 } 569 570 static void vmalloc_test_exit(void) 571 { 572 } 573 574 module_init(vmalloc_test_init) 575 module_exit(vmalloc_test_exit) 576 577 MODULE_LICENSE("GPL"); 578 MODULE_AUTHOR("Uladzislau Rezki"); 579 MODULE_DESCRIPTION("vmalloc test module"); 580