1 /* 2 * multiorder.c: Multi-order radix tree entry testing 3 * Copyright (c) 2016 Intel Corporation 4 * Author: Ross Zwisler <ross.zwisler@linux.intel.com> 5 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 */ 16 #include <linux/radix-tree.h> 17 #include <linux/slab.h> 18 #include <linux/errno.h> 19 #include <pthread.h> 20 21 #include "test.h" 22 23 #define for_each_index(i, base, order) \ 24 for (i = base; i < base + (1 << order); i++) 25 26 static void __multiorder_tag_test(int index, int order) 27 { 28 RADIX_TREE(tree, GFP_KERNEL); 29 int base, err, i; 30 31 /* our canonical entry */ 32 base = index & ~((1 << order) - 1); 33 34 printv(2, "Multiorder tag test with index %d, canonical entry %d\n", 35 index, base); 36 37 err = item_insert_order(&tree, index, order); 38 assert(!err); 39 40 /* 41 * Verify we get collisions for covered indices. We try and fail to 42 * insert a value entry so we don't leak memory via 43 * item_insert_order(). 44 */ 45 for_each_index(i, base, order) { 46 err = __radix_tree_insert(&tree, i, order, xa_mk_value(0xA0)); 47 assert(err == -EEXIST); 48 } 49 50 for_each_index(i, base, order) { 51 assert(!radix_tree_tag_get(&tree, i, 0)); 52 assert(!radix_tree_tag_get(&tree, i, 1)); 53 } 54 55 assert(radix_tree_tag_set(&tree, index, 0)); 56 57 for_each_index(i, base, order) { 58 assert(radix_tree_tag_get(&tree, i, 0)); 59 assert(!radix_tree_tag_get(&tree, i, 1)); 60 } 61 62 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1); 63 assert(radix_tree_tag_clear(&tree, index, 0)); 64 65 for_each_index(i, base, order) { 66 assert(!radix_tree_tag_get(&tree, i, 0)); 67 assert(radix_tree_tag_get(&tree, i, 1)); 68 } 69 70 assert(radix_tree_tag_clear(&tree, index, 1)); 71 72 assert(!radix_tree_tagged(&tree, 0)); 73 assert(!radix_tree_tagged(&tree, 1)); 74 75 item_kill_tree(&tree); 76 } 77 78 static void __multiorder_tag_test2(unsigned order, unsigned long index2) 79 { 80 RADIX_TREE(tree, GFP_KERNEL); 81 unsigned long index = (1 << order); 82 index2 += index; 83 84 assert(item_insert_order(&tree, 0, order) == 0); 85 assert(item_insert(&tree, index2) == 0); 86 87 assert(radix_tree_tag_set(&tree, 0, 0)); 88 assert(radix_tree_tag_set(&tree, index2, 0)); 89 90 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 2); 91 92 item_kill_tree(&tree); 93 } 94 95 static void multiorder_tag_tests(void) 96 { 97 int i, j; 98 99 /* test multi-order entry for indices 0-7 with no sibling pointers */ 100 __multiorder_tag_test(0, 3); 101 __multiorder_tag_test(5, 3); 102 103 /* test multi-order entry for indices 8-15 with no sibling pointers */ 104 __multiorder_tag_test(8, 3); 105 __multiorder_tag_test(15, 3); 106 107 /* 108 * Our order 5 entry covers indices 0-31 in a tree with height=2. 109 * This is broken up as follows: 110 * 0-7: canonical entry 111 * 8-15: sibling 1 112 * 16-23: sibling 2 113 * 24-31: sibling 3 114 */ 115 __multiorder_tag_test(0, 5); 116 __multiorder_tag_test(29, 5); 117 118 /* same test, but with indices 32-63 */ 119 __multiorder_tag_test(32, 5); 120 __multiorder_tag_test(44, 5); 121 122 /* 123 * Our order 8 entry covers indices 0-255 in a tree with height=3. 124 * This is broken up as follows: 125 * 0-63: canonical entry 126 * 64-127: sibling 1 127 * 128-191: sibling 2 128 * 192-255: sibling 3 129 */ 130 __multiorder_tag_test(0, 8); 131 __multiorder_tag_test(190, 8); 132 133 /* same test, but with indices 256-511 */ 134 __multiorder_tag_test(256, 8); 135 __multiorder_tag_test(300, 8); 136 137 __multiorder_tag_test(0x12345678UL, 8); 138 139 for (i = 1; i < 10; i++) 140 for (j = 0; j < (10 << i); j++) 141 __multiorder_tag_test2(i, j); 142 } 143 144 static void multiorder_check(unsigned long index, int order) 145 { 146 unsigned long i; 147 unsigned long min = index & ~((1UL << order) - 1); 148 unsigned long max = min + (1UL << order); 149 void **slot; 150 struct item *item2 = item_create(min, order); 151 RADIX_TREE(tree, GFP_KERNEL); 152 153 printv(2, "Multiorder index %ld, order %d\n", index, order); 154 155 assert(item_insert_order(&tree, index, order) == 0); 156 157 for (i = min; i < max; i++) { 158 struct item *item = item_lookup(&tree, i); 159 assert(item != 0); 160 assert(item->index == index); 161 } 162 for (i = 0; i < min; i++) 163 item_check_absent(&tree, i); 164 for (i = max; i < 2*max; i++) 165 item_check_absent(&tree, i); 166 for (i = min; i < max; i++) 167 assert(radix_tree_insert(&tree, i, item2) == -EEXIST); 168 169 slot = radix_tree_lookup_slot(&tree, index); 170 free(*slot); 171 radix_tree_replace_slot(&tree, slot, item2); 172 for (i = min; i < max; i++) { 173 struct item *item = item_lookup(&tree, i); 174 assert(item != 0); 175 assert(item->index == min); 176 } 177 178 assert(item_delete(&tree, min) != 0); 179 180 for (i = 0; i < 2*max; i++) 181 item_check_absent(&tree, i); 182 } 183 184 static void multiorder_shrink(unsigned long index, int order) 185 { 186 unsigned long i; 187 unsigned long max = 1 << order; 188 RADIX_TREE(tree, GFP_KERNEL); 189 struct radix_tree_node *node; 190 191 printv(2, "Multiorder shrink index %ld, order %d\n", index, order); 192 193 assert(item_insert_order(&tree, 0, order) == 0); 194 195 node = tree.rnode; 196 197 assert(item_insert(&tree, index) == 0); 198 assert(node != tree.rnode); 199 200 assert(item_delete(&tree, index) != 0); 201 assert(node == tree.rnode); 202 203 for (i = 0; i < max; i++) { 204 struct item *item = item_lookup(&tree, i); 205 assert(item != 0); 206 assert(item->index == 0); 207 } 208 for (i = max; i < 2*max; i++) 209 item_check_absent(&tree, i); 210 211 if (!item_delete(&tree, 0)) { 212 printv(2, "failed to delete index %ld (order %d)\n", index, order); 213 abort(); 214 } 215 216 for (i = 0; i < 2*max; i++) 217 item_check_absent(&tree, i); 218 } 219 220 static void multiorder_insert_bug(void) 221 { 222 RADIX_TREE(tree, GFP_KERNEL); 223 224 item_insert(&tree, 0); 225 radix_tree_tag_set(&tree, 0, 0); 226 item_insert_order(&tree, 3 << 6, 6); 227 228 item_kill_tree(&tree); 229 } 230 231 void multiorder_iteration(void) 232 { 233 RADIX_TREE(tree, GFP_KERNEL); 234 struct radix_tree_iter iter; 235 void **slot; 236 int i, j, err; 237 238 printv(1, "Multiorder iteration test\n"); 239 240 #define NUM_ENTRIES 11 241 int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128}; 242 int order[NUM_ENTRIES] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7}; 243 244 for (i = 0; i < NUM_ENTRIES; i++) { 245 err = item_insert_order(&tree, index[i], order[i]); 246 assert(!err); 247 } 248 249 for (j = 0; j < 256; j++) { 250 for (i = 0; i < NUM_ENTRIES; i++) 251 if (j <= (index[i] | ((1 << order[i]) - 1))) 252 break; 253 254 radix_tree_for_each_slot(slot, &tree, &iter, j) { 255 int height = order[i] / RADIX_TREE_MAP_SHIFT; 256 int shift = height * RADIX_TREE_MAP_SHIFT; 257 unsigned long mask = (1UL << order[i]) - 1; 258 struct item *item = *slot; 259 260 assert((iter.index | mask) == (index[i] | mask)); 261 assert(iter.shift == shift); 262 assert(!radix_tree_is_internal_node(item)); 263 assert((item->index | mask) == (index[i] | mask)); 264 assert(item->order == order[i]); 265 i++; 266 } 267 } 268 269 item_kill_tree(&tree); 270 } 271 272 void multiorder_tagged_iteration(void) 273 { 274 RADIX_TREE(tree, GFP_KERNEL); 275 struct radix_tree_iter iter; 276 void **slot; 277 int i, j; 278 279 printv(1, "Multiorder tagged iteration test\n"); 280 281 #define MT_NUM_ENTRIES 9 282 int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128}; 283 int order[MT_NUM_ENTRIES] = {1, 0, 2, 4, 3, 1, 3, 0, 7}; 284 285 #define TAG_ENTRIES 7 286 int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128}; 287 288 for (i = 0; i < MT_NUM_ENTRIES; i++) 289 assert(!item_insert_order(&tree, index[i], order[i])); 290 291 assert(!radix_tree_tagged(&tree, 1)); 292 293 for (i = 0; i < TAG_ENTRIES; i++) 294 assert(radix_tree_tag_set(&tree, tag_index[i], 1)); 295 296 for (j = 0; j < 256; j++) { 297 int k; 298 299 for (i = 0; i < TAG_ENTRIES; i++) { 300 for (k = i; index[k] < tag_index[i]; k++) 301 ; 302 if (j <= (index[k] | ((1 << order[k]) - 1))) 303 break; 304 } 305 306 radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) { 307 unsigned long mask; 308 struct item *item = *slot; 309 for (k = i; index[k] < tag_index[i]; k++) 310 ; 311 mask = (1UL << order[k]) - 1; 312 313 assert((iter.index | mask) == (tag_index[i] | mask)); 314 assert(!radix_tree_is_internal_node(item)); 315 assert((item->index | mask) == (tag_index[i] | mask)); 316 assert(item->order == order[k]); 317 i++; 318 } 319 } 320 321 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) == 322 TAG_ENTRIES); 323 324 for (j = 0; j < 256; j++) { 325 int mask, k; 326 327 for (i = 0; i < TAG_ENTRIES; i++) { 328 for (k = i; index[k] < tag_index[i]; k++) 329 ; 330 if (j <= (index[k] | ((1 << order[k]) - 1))) 331 break; 332 } 333 334 radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) { 335 struct item *item = *slot; 336 for (k = i; index[k] < tag_index[i]; k++) 337 ; 338 mask = (1 << order[k]) - 1; 339 340 assert((iter.index | mask) == (tag_index[i] | mask)); 341 assert(!radix_tree_is_internal_node(item)); 342 assert((item->index | mask) == (tag_index[i] | mask)); 343 assert(item->order == order[k]); 344 i++; 345 } 346 } 347 348 assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0) 349 == TAG_ENTRIES); 350 i = 0; 351 radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) { 352 assert(iter.index == tag_index[i]); 353 i++; 354 } 355 356 item_kill_tree(&tree); 357 } 358 359 /* 360 * Basic join checks: make sure we can't find an entry in the tree after 361 * a larger entry has replaced it 362 */ 363 static void multiorder_join1(unsigned long index, 364 unsigned order1, unsigned order2) 365 { 366 unsigned long loc; 367 void *item, *item2 = item_create(index + 1, order1); 368 RADIX_TREE(tree, GFP_KERNEL); 369 370 item_insert_order(&tree, index, order2); 371 item = radix_tree_lookup(&tree, index); 372 radix_tree_join(&tree, index + 1, order1, item2); 373 loc = find_item(&tree, item); 374 if (loc == -1) 375 free(item); 376 item = radix_tree_lookup(&tree, index + 1); 377 assert(item == item2); 378 item_kill_tree(&tree); 379 } 380 381 /* 382 * Check that the accounting of value entries is handled correctly 383 * by joining a value entry to a normal pointer. 384 */ 385 static void multiorder_join2(unsigned order1, unsigned order2) 386 { 387 RADIX_TREE(tree, GFP_KERNEL); 388 struct radix_tree_node *node; 389 void *item1 = item_create(0, order1); 390 void *item2; 391 392 item_insert_order(&tree, 0, order2); 393 radix_tree_insert(&tree, 1 << order2, xa_mk_value(5)); 394 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); 395 assert(item2 == xa_mk_value(5)); 396 assert(node->exceptional == 1); 397 398 item2 = radix_tree_lookup(&tree, 0); 399 free(item2); 400 401 radix_tree_join(&tree, 0, order1, item1); 402 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); 403 assert(item2 == item1); 404 assert(node->exceptional == 0); 405 item_kill_tree(&tree); 406 } 407 408 /* 409 * This test revealed an accounting bug for value entries at one point. 410 * Nodes were being freed back into the pool with an elevated exception count 411 * by radix_tree_join() and then radix_tree_split() was failing to zero the 412 * count of exceptional entries. 413 */ 414 static void multiorder_join3(unsigned int order) 415 { 416 RADIX_TREE(tree, GFP_KERNEL); 417 struct radix_tree_node *node; 418 void **slot; 419 struct radix_tree_iter iter; 420 unsigned long i; 421 422 for (i = 0; i < (1 << order); i++) { 423 radix_tree_insert(&tree, i, xa_mk_value(5)); 424 } 425 426 radix_tree_join(&tree, 0, order, xa_mk_value(7)); 427 rcu_barrier(); 428 429 radix_tree_split(&tree, 0, 0); 430 431 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 432 radix_tree_iter_replace(&tree, &iter, slot, xa_mk_value(5)); 433 } 434 435 __radix_tree_lookup(&tree, 0, &node, NULL); 436 assert(node->exceptional == node->count); 437 438 item_kill_tree(&tree); 439 } 440 441 static void multiorder_join(void) 442 { 443 int i, j, idx; 444 445 for (idx = 0; idx < 1024; idx = idx * 2 + 3) { 446 for (i = 1; i < 15; i++) { 447 for (j = 0; j < i; j++) { 448 multiorder_join1(idx, i, j); 449 } 450 } 451 } 452 453 for (i = 1; i < 15; i++) { 454 for (j = 0; j < i; j++) { 455 multiorder_join2(i, j); 456 } 457 } 458 459 for (i = 3; i < 10; i++) { 460 multiorder_join3(i); 461 } 462 } 463 464 static void check_mem(unsigned old_order, unsigned new_order, unsigned alloc) 465 { 466 struct radix_tree_preload *rtp = &radix_tree_preloads; 467 if (rtp->nr != 0) 468 printv(2, "split(%u %u) remaining %u\n", old_order, new_order, 469 rtp->nr); 470 /* 471 * Can't check for equality here as some nodes may have been 472 * RCU-freed while we ran. But we should never finish with more 473 * nodes allocated since they should have all been preloaded. 474 */ 475 if (nr_allocated > alloc) 476 printv(2, "split(%u %u) allocated %u %u\n", old_order, new_order, 477 alloc, nr_allocated); 478 } 479 480 static void __multiorder_split(int old_order, int new_order) 481 { 482 RADIX_TREE(tree, GFP_ATOMIC); 483 void **slot; 484 struct radix_tree_iter iter; 485 unsigned alloc; 486 struct item *item; 487 488 radix_tree_preload(GFP_KERNEL); 489 assert(item_insert_order(&tree, 0, old_order) == 0); 490 radix_tree_preload_end(); 491 492 /* Wipe out the preloaded cache or it'll confuse check_mem() */ 493 radix_tree_cpu_dead(0); 494 495 item = radix_tree_tag_set(&tree, 0, 2); 496 497 radix_tree_split_preload(old_order, new_order, GFP_KERNEL); 498 alloc = nr_allocated; 499 radix_tree_split(&tree, 0, new_order); 500 check_mem(old_order, new_order, alloc); 501 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 502 radix_tree_iter_replace(&tree, &iter, slot, 503 item_create(iter.index, new_order)); 504 } 505 radix_tree_preload_end(); 506 507 item_kill_tree(&tree); 508 free(item); 509 } 510 511 static void __multiorder_split2(int old_order, int new_order) 512 { 513 RADIX_TREE(tree, GFP_KERNEL); 514 void **slot; 515 struct radix_tree_iter iter; 516 struct radix_tree_node *node; 517 void *item; 518 519 __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); 520 521 item = __radix_tree_lookup(&tree, 0, &node, NULL); 522 assert(item == xa_mk_value(5)); 523 assert(node->exceptional > 0); 524 525 radix_tree_split(&tree, 0, new_order); 526 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 527 radix_tree_iter_replace(&tree, &iter, slot, 528 item_create(iter.index, new_order)); 529 } 530 531 item = __radix_tree_lookup(&tree, 0, &node, NULL); 532 assert(item != xa_mk_value(5)); 533 assert(node->exceptional == 0); 534 535 item_kill_tree(&tree); 536 } 537 538 static void __multiorder_split3(int old_order, int new_order) 539 { 540 RADIX_TREE(tree, GFP_KERNEL); 541 void **slot; 542 struct radix_tree_iter iter; 543 struct radix_tree_node *node; 544 void *item; 545 546 __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); 547 548 item = __radix_tree_lookup(&tree, 0, &node, NULL); 549 assert(item == xa_mk_value(5)); 550 assert(node->exceptional > 0); 551 552 radix_tree_split(&tree, 0, new_order); 553 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 554 radix_tree_iter_replace(&tree, &iter, slot, xa_mk_value(7)); 555 } 556 557 item = __radix_tree_lookup(&tree, 0, &node, NULL); 558 assert(item == xa_mk_value(7)); 559 assert(node->exceptional > 0); 560 561 item_kill_tree(&tree); 562 563 __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); 564 565 item = __radix_tree_lookup(&tree, 0, &node, NULL); 566 assert(item == xa_mk_value(5)); 567 assert(node->exceptional > 0); 568 569 radix_tree_split(&tree, 0, new_order); 570 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 571 if (iter.index == (1 << new_order)) 572 radix_tree_iter_replace(&tree, &iter, slot, 573 xa_mk_value(7)); 574 else 575 radix_tree_iter_replace(&tree, &iter, slot, NULL); 576 } 577 578 item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL); 579 assert(item == xa_mk_value(7)); 580 assert(node->count == node->exceptional); 581 do { 582 node = node->parent; 583 if (!node) 584 break; 585 assert(node->count == 1); 586 assert(node->exceptional == 0); 587 } while (1); 588 589 item_kill_tree(&tree); 590 } 591 592 static void multiorder_split(void) 593 { 594 int i, j; 595 596 for (i = 3; i < 11; i++) 597 for (j = 0; j < i; j++) { 598 __multiorder_split(i, j); 599 __multiorder_split2(i, j); 600 __multiorder_split3(i, j); 601 } 602 } 603 604 static void multiorder_account(void) 605 { 606 RADIX_TREE(tree, GFP_KERNEL); 607 struct radix_tree_node *node; 608 void **slot; 609 610 item_insert_order(&tree, 0, 5); 611 612 __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); 613 __radix_tree_lookup(&tree, 0, &node, NULL); 614 assert(node->count == node->exceptional * 2); 615 radix_tree_delete(&tree, 1 << 5); 616 assert(node->exceptional == 0); 617 618 __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); 619 __radix_tree_lookup(&tree, 1 << 5, &node, &slot); 620 assert(node->count == node->exceptional * 2); 621 __radix_tree_replace(&tree, node, slot, NULL, NULL); 622 assert(node->exceptional == 0); 623 624 item_kill_tree(&tree); 625 } 626 627 bool stop_iteration = false; 628 629 static void *creator_func(void *ptr) 630 { 631 /* 'order' is set up to ensure we have sibling entries */ 632 unsigned int order = RADIX_TREE_MAP_SHIFT - 1; 633 struct radix_tree_root *tree = ptr; 634 int i; 635 636 for (i = 0; i < 10000; i++) { 637 item_insert_order(tree, 0, order); 638 item_delete_rcu(tree, 0); 639 } 640 641 stop_iteration = true; 642 return NULL; 643 } 644 645 static void *iterator_func(void *ptr) 646 { 647 struct radix_tree_root *tree = ptr; 648 struct radix_tree_iter iter; 649 struct item *item; 650 void **slot; 651 652 while (!stop_iteration) { 653 rcu_read_lock(); 654 radix_tree_for_each_slot(slot, tree, &iter, 0) { 655 item = radix_tree_deref_slot(slot); 656 657 if (!item) 658 continue; 659 if (radix_tree_deref_retry(item)) { 660 slot = radix_tree_iter_retry(&iter); 661 continue; 662 } 663 664 item_sanity(item, iter.index); 665 } 666 rcu_read_unlock(); 667 } 668 return NULL; 669 } 670 671 static void multiorder_iteration_race(void) 672 { 673 const int num_threads = sysconf(_SC_NPROCESSORS_ONLN); 674 pthread_t worker_thread[num_threads]; 675 RADIX_TREE(tree, GFP_KERNEL); 676 int i; 677 678 pthread_create(&worker_thread[0], NULL, &creator_func, &tree); 679 for (i = 1; i < num_threads; i++) 680 pthread_create(&worker_thread[i], NULL, &iterator_func, &tree); 681 682 for (i = 0; i < num_threads; i++) 683 pthread_join(worker_thread[i], NULL); 684 685 item_kill_tree(&tree); 686 } 687 688 void multiorder_checks(void) 689 { 690 int i; 691 692 for (i = 0; i < 20; i++) { 693 multiorder_check(200, i); 694 multiorder_check(0, i); 695 multiorder_check((1UL << i) + 1, i); 696 } 697 698 for (i = 0; i < 15; i++) 699 multiorder_shrink((1UL << (i + RADIX_TREE_MAP_SHIFT)), i); 700 701 multiorder_insert_bug(); 702 multiorder_tag_tests(); 703 multiorder_iteration(); 704 multiorder_tagged_iteration(); 705 multiorder_join(); 706 multiorder_split(); 707 multiorder_account(); 708 multiorder_iteration_race(); 709 710 radix_tree_cpu_dead(0); 711 } 712 713 int __weak main(void) 714 { 715 radix_tree_init(); 716 multiorder_checks(); 717 return 0; 718 } 719