1 /* 2 * multiorder.c: Multi-order radix tree entry testing 3 * Copyright (c) 2016 Intel Corporation 4 * Author: Ross Zwisler <ross.zwisler@linux.intel.com> 5 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 */ 16 #include <linux/radix-tree.h> 17 #include <linux/slab.h> 18 #include <linux/errno.h> 19 20 #include "test.h" 21 22 #define for_each_index(i, base, order) \ 23 for (i = base; i < base + (1 << order); i++) 24 25 static void __multiorder_tag_test(int index, int order) 26 { 27 RADIX_TREE(tree, GFP_KERNEL); 28 int base, err, i; 29 30 /* our canonical entry */ 31 base = index & ~((1 << order) - 1); 32 33 printf("Multiorder tag test with index %d, canonical entry %d\n", 34 index, base); 35 36 err = item_insert_order(&tree, index, order); 37 assert(!err); 38 39 /* 40 * Verify we get collisions for covered indices. We try and fail to 41 * insert an exceptional entry so we don't leak memory via 42 * item_insert_order(). 43 */ 44 for_each_index(i, base, order) { 45 err = __radix_tree_insert(&tree, i, order, 46 (void *)(0xA0 | RADIX_TREE_EXCEPTIONAL_ENTRY)); 47 assert(err == -EEXIST); 48 } 49 50 for_each_index(i, base, order) { 51 assert(!radix_tree_tag_get(&tree, i, 0)); 52 assert(!radix_tree_tag_get(&tree, i, 1)); 53 } 54 55 assert(radix_tree_tag_set(&tree, index, 0)); 56 57 for_each_index(i, base, order) { 58 assert(radix_tree_tag_get(&tree, i, 0)); 59 assert(!radix_tree_tag_get(&tree, i, 1)); 60 } 61 62 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1); 63 assert(radix_tree_tag_clear(&tree, index, 0)); 64 65 for_each_index(i, base, order) { 66 assert(!radix_tree_tag_get(&tree, i, 0)); 67 assert(radix_tree_tag_get(&tree, i, 1)); 68 } 69 70 assert(radix_tree_tag_clear(&tree, index, 1)); 71 72 assert(!radix_tree_tagged(&tree, 0)); 73 assert(!radix_tree_tagged(&tree, 1)); 74 75 item_kill_tree(&tree); 76 } 77 78 static void __multiorder_tag_test2(unsigned order, unsigned long index2) 79 { 80 RADIX_TREE(tree, GFP_KERNEL); 81 unsigned long index = (1 << order); 82 index2 += index; 83 84 assert(item_insert_order(&tree, 0, order) == 0); 85 assert(item_insert(&tree, index2) == 0); 86 87 assert(radix_tree_tag_set(&tree, 0, 0)); 88 assert(radix_tree_tag_set(&tree, index2, 0)); 89 90 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 2); 91 92 item_kill_tree(&tree); 93 } 94 95 static void multiorder_tag_tests(void) 96 { 97 int i, j; 98 99 /* test multi-order entry for indices 0-7 with no sibling pointers */ 100 __multiorder_tag_test(0, 3); 101 __multiorder_tag_test(5, 3); 102 103 /* test multi-order entry for indices 8-15 with no sibling pointers */ 104 __multiorder_tag_test(8, 3); 105 __multiorder_tag_test(15, 3); 106 107 /* 108 * Our order 5 entry covers indices 0-31 in a tree with height=2. 109 * This is broken up as follows: 110 * 0-7: canonical entry 111 * 8-15: sibling 1 112 * 16-23: sibling 2 113 * 24-31: sibling 3 114 */ 115 __multiorder_tag_test(0, 5); 116 __multiorder_tag_test(29, 5); 117 118 /* same test, but with indices 32-63 */ 119 __multiorder_tag_test(32, 5); 120 __multiorder_tag_test(44, 5); 121 122 /* 123 * Our order 8 entry covers indices 0-255 in a tree with height=3. 124 * This is broken up as follows: 125 * 0-63: canonical entry 126 * 64-127: sibling 1 127 * 128-191: sibling 2 128 * 192-255: sibling 3 129 */ 130 __multiorder_tag_test(0, 8); 131 __multiorder_tag_test(190, 8); 132 133 /* same test, but with indices 256-511 */ 134 __multiorder_tag_test(256, 8); 135 __multiorder_tag_test(300, 8); 136 137 __multiorder_tag_test(0x12345678UL, 8); 138 139 for (i = 1; i < 10; i++) 140 for (j = 0; j < (10 << i); j++) 141 __multiorder_tag_test2(i, j); 142 } 143 144 static void multiorder_check(unsigned long index, int order) 145 { 146 unsigned long i; 147 unsigned long min = index & ~((1UL << order) - 1); 148 unsigned long max = min + (1UL << order); 149 void **slot; 150 struct item *item2 = item_create(min, order); 151 RADIX_TREE(tree, GFP_KERNEL); 152 153 printf("Multiorder index %ld, order %d\n", index, order); 154 155 assert(item_insert_order(&tree, index, order) == 0); 156 157 for (i = min; i < max; i++) { 158 struct item *item = item_lookup(&tree, i); 159 assert(item != 0); 160 assert(item->index == index); 161 } 162 for (i = 0; i < min; i++) 163 item_check_absent(&tree, i); 164 for (i = max; i < 2*max; i++) 165 item_check_absent(&tree, i); 166 for (i = min; i < max; i++) 167 assert(radix_tree_insert(&tree, i, item2) == -EEXIST); 168 169 slot = radix_tree_lookup_slot(&tree, index); 170 free(*slot); 171 radix_tree_replace_slot(&tree, slot, item2); 172 for (i = min; i < max; i++) { 173 struct item *item = item_lookup(&tree, i); 174 assert(item != 0); 175 assert(item->index == min); 176 } 177 178 assert(item_delete(&tree, min) != 0); 179 180 for (i = 0; i < 2*max; i++) 181 item_check_absent(&tree, i); 182 } 183 184 static void multiorder_shrink(unsigned long index, int order) 185 { 186 unsigned long i; 187 unsigned long max = 1 << order; 188 RADIX_TREE(tree, GFP_KERNEL); 189 struct radix_tree_node *node; 190 191 printf("Multiorder shrink index %ld, order %d\n", index, order); 192 193 assert(item_insert_order(&tree, 0, order) == 0); 194 195 node = tree.rnode; 196 197 assert(item_insert(&tree, index) == 0); 198 assert(node != tree.rnode); 199 200 assert(item_delete(&tree, index) != 0); 201 assert(node == tree.rnode); 202 203 for (i = 0; i < max; i++) { 204 struct item *item = item_lookup(&tree, i); 205 assert(item != 0); 206 assert(item->index == 0); 207 } 208 for (i = max; i < 2*max; i++) 209 item_check_absent(&tree, i); 210 211 if (!item_delete(&tree, 0)) { 212 printf("failed to delete index %ld (order %d)\n", index, order); abort(); 213 } 214 215 for (i = 0; i < 2*max; i++) 216 item_check_absent(&tree, i); 217 } 218 219 static void multiorder_insert_bug(void) 220 { 221 RADIX_TREE(tree, GFP_KERNEL); 222 223 item_insert(&tree, 0); 224 radix_tree_tag_set(&tree, 0, 0); 225 item_insert_order(&tree, 3 << 6, 6); 226 227 item_kill_tree(&tree); 228 } 229 230 void multiorder_iteration(void) 231 { 232 RADIX_TREE(tree, GFP_KERNEL); 233 struct radix_tree_iter iter; 234 void **slot; 235 int i, j, err; 236 237 printf("Multiorder iteration test\n"); 238 239 #define NUM_ENTRIES 11 240 int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128}; 241 int order[NUM_ENTRIES] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7}; 242 243 for (i = 0; i < NUM_ENTRIES; i++) { 244 err = item_insert_order(&tree, index[i], order[i]); 245 assert(!err); 246 } 247 248 for (j = 0; j < 256; j++) { 249 for (i = 0; i < NUM_ENTRIES; i++) 250 if (j <= (index[i] | ((1 << order[i]) - 1))) 251 break; 252 253 radix_tree_for_each_slot(slot, &tree, &iter, j) { 254 int height = order[i] / RADIX_TREE_MAP_SHIFT; 255 int shift = height * RADIX_TREE_MAP_SHIFT; 256 unsigned long mask = (1UL << order[i]) - 1; 257 struct item *item = *slot; 258 259 assert((iter.index | mask) == (index[i] | mask)); 260 assert(iter.shift == shift); 261 assert(!radix_tree_is_internal_node(item)); 262 assert((item->index | mask) == (index[i] | mask)); 263 assert(item->order == order[i]); 264 i++; 265 } 266 } 267 268 item_kill_tree(&tree); 269 } 270 271 void multiorder_tagged_iteration(void) 272 { 273 RADIX_TREE(tree, GFP_KERNEL); 274 struct radix_tree_iter iter; 275 void **slot; 276 int i, j; 277 278 printf("Multiorder tagged iteration test\n"); 279 280 #define MT_NUM_ENTRIES 9 281 int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128}; 282 int order[MT_NUM_ENTRIES] = {1, 0, 2, 4, 3, 1, 3, 0, 7}; 283 284 #define TAG_ENTRIES 7 285 int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128}; 286 287 for (i = 0; i < MT_NUM_ENTRIES; i++) 288 assert(!item_insert_order(&tree, index[i], order[i])); 289 290 assert(!radix_tree_tagged(&tree, 1)); 291 292 for (i = 0; i < TAG_ENTRIES; i++) 293 assert(radix_tree_tag_set(&tree, tag_index[i], 1)); 294 295 for (j = 0; j < 256; j++) { 296 int k; 297 298 for (i = 0; i < TAG_ENTRIES; i++) { 299 for (k = i; index[k] < tag_index[i]; k++) 300 ; 301 if (j <= (index[k] | ((1 << order[k]) - 1))) 302 break; 303 } 304 305 radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) { 306 unsigned long mask; 307 struct item *item = *slot; 308 for (k = i; index[k] < tag_index[i]; k++) 309 ; 310 mask = (1UL << order[k]) - 1; 311 312 assert((iter.index | mask) == (tag_index[i] | mask)); 313 assert(!radix_tree_is_internal_node(item)); 314 assert((item->index | mask) == (tag_index[i] | mask)); 315 assert(item->order == order[k]); 316 i++; 317 } 318 } 319 320 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) == 321 TAG_ENTRIES); 322 323 for (j = 0; j < 256; j++) { 324 int mask, k; 325 326 for (i = 0; i < TAG_ENTRIES; i++) { 327 for (k = i; index[k] < tag_index[i]; k++) 328 ; 329 if (j <= (index[k] | ((1 << order[k]) - 1))) 330 break; 331 } 332 333 radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) { 334 struct item *item = *slot; 335 for (k = i; index[k] < tag_index[i]; k++) 336 ; 337 mask = (1 << order[k]) - 1; 338 339 assert((iter.index | mask) == (tag_index[i] | mask)); 340 assert(!radix_tree_is_internal_node(item)); 341 assert((item->index | mask) == (tag_index[i] | mask)); 342 assert(item->order == order[k]); 343 i++; 344 } 345 } 346 347 assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0) 348 == TAG_ENTRIES); 349 i = 0; 350 radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) { 351 assert(iter.index == tag_index[i]); 352 i++; 353 } 354 355 item_kill_tree(&tree); 356 } 357 358 static void multiorder_join1(unsigned long index, 359 unsigned order1, unsigned order2) 360 { 361 unsigned long loc; 362 void *item, *item2 = item_create(index + 1, order1); 363 RADIX_TREE(tree, GFP_KERNEL); 364 365 item_insert_order(&tree, index, order2); 366 item = radix_tree_lookup(&tree, index); 367 radix_tree_join(&tree, index + 1, order1, item2); 368 loc = find_item(&tree, item); 369 if (loc == -1) 370 free(item); 371 item = radix_tree_lookup(&tree, index + 1); 372 assert(item == item2); 373 item_kill_tree(&tree); 374 } 375 376 static void multiorder_join2(unsigned order1, unsigned order2) 377 { 378 RADIX_TREE(tree, GFP_KERNEL); 379 struct radix_tree_node *node; 380 void *item1 = item_create(0, order1); 381 void *item2; 382 383 item_insert_order(&tree, 0, order2); 384 radix_tree_insert(&tree, 1 << order2, (void *)0x12UL); 385 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); 386 assert(item2 == (void *)0x12UL); 387 assert(node->exceptional == 1); 388 389 radix_tree_join(&tree, 0, order1, item1); 390 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); 391 assert(item2 == item1); 392 assert(node->exceptional == 0); 393 item_kill_tree(&tree); 394 } 395 396 /* 397 * This test revealed an accounting bug for exceptional entries at one point. 398 * Nodes were being freed back into the pool with an elevated exception count 399 * by radix_tree_join() and then radix_tree_split() was failing to zero the 400 * count of exceptional entries. 401 */ 402 static void multiorder_join3(unsigned int order) 403 { 404 RADIX_TREE(tree, GFP_KERNEL); 405 struct radix_tree_node *node; 406 void **slot; 407 struct radix_tree_iter iter; 408 unsigned long i; 409 410 for (i = 0; i < (1 << order); i++) { 411 radix_tree_insert(&tree, i, (void *)0x12UL); 412 } 413 414 radix_tree_join(&tree, 0, order, (void *)0x16UL); 415 rcu_barrier(); 416 417 radix_tree_split(&tree, 0, 0); 418 419 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 420 radix_tree_iter_replace(&tree, &iter, slot, (void *)0x12UL); 421 } 422 423 __radix_tree_lookup(&tree, 0, &node, NULL); 424 assert(node->exceptional == node->count); 425 426 item_kill_tree(&tree); 427 } 428 429 static void multiorder_join(void) 430 { 431 int i, j, idx; 432 433 for (idx = 0; idx < 1024; idx = idx * 2 + 3) { 434 for (i = 1; i < 15; i++) { 435 for (j = 0; j < i; j++) { 436 multiorder_join1(idx, i, j); 437 } 438 } 439 } 440 441 for (i = 1; i < 15; i++) { 442 for (j = 0; j < i; j++) { 443 multiorder_join2(i, j); 444 } 445 } 446 447 for (i = 3; i < 10; i++) { 448 multiorder_join3(i); 449 } 450 } 451 452 static void check_mem(unsigned old_order, unsigned new_order, unsigned alloc) 453 { 454 struct radix_tree_preload *rtp = &radix_tree_preloads; 455 if (rtp->nr != 0) 456 printf("split(%u %u) remaining %u\n", old_order, new_order, 457 rtp->nr); 458 /* 459 * Can't check for equality here as some nodes may have been 460 * RCU-freed while we ran. But we should never finish with more 461 * nodes allocated since they should have all been preloaded. 462 */ 463 if (nr_allocated > alloc) 464 printf("split(%u %u) allocated %u %u\n", old_order, new_order, 465 alloc, nr_allocated); 466 } 467 468 static void __multiorder_split(int old_order, int new_order) 469 { 470 RADIX_TREE(tree, GFP_ATOMIC); 471 void **slot; 472 struct radix_tree_iter iter; 473 unsigned alloc; 474 475 radix_tree_preload(GFP_KERNEL); 476 assert(item_insert_order(&tree, 0, old_order) == 0); 477 radix_tree_preload_end(); 478 479 /* Wipe out the preloaded cache or it'll confuse check_mem() */ 480 radix_tree_cpu_dead(0); 481 482 radix_tree_tag_set(&tree, 0, 2); 483 484 radix_tree_split_preload(old_order, new_order, GFP_KERNEL); 485 alloc = nr_allocated; 486 radix_tree_split(&tree, 0, new_order); 487 check_mem(old_order, new_order, alloc); 488 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 489 radix_tree_iter_replace(&tree, &iter, slot, 490 item_create(iter.index, new_order)); 491 } 492 radix_tree_preload_end(); 493 494 item_kill_tree(&tree); 495 } 496 497 static void __multiorder_split2(int old_order, int new_order) 498 { 499 RADIX_TREE(tree, GFP_KERNEL); 500 void **slot; 501 struct radix_tree_iter iter; 502 struct radix_tree_node *node; 503 void *item; 504 505 __radix_tree_insert(&tree, 0, old_order, (void *)0x12); 506 507 item = __radix_tree_lookup(&tree, 0, &node, NULL); 508 assert(item == (void *)0x12); 509 assert(node->exceptional > 0); 510 511 radix_tree_split(&tree, 0, new_order); 512 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 513 radix_tree_iter_replace(&tree, &iter, slot, 514 item_create(iter.index, new_order)); 515 } 516 517 item = __radix_tree_lookup(&tree, 0, &node, NULL); 518 assert(item != (void *)0x12); 519 assert(node->exceptional == 0); 520 521 item_kill_tree(&tree); 522 } 523 524 static void __multiorder_split3(int old_order, int new_order) 525 { 526 RADIX_TREE(tree, GFP_KERNEL); 527 void **slot; 528 struct radix_tree_iter iter; 529 struct radix_tree_node *node; 530 void *item; 531 532 __radix_tree_insert(&tree, 0, old_order, (void *)0x12); 533 534 item = __radix_tree_lookup(&tree, 0, &node, NULL); 535 assert(item == (void *)0x12); 536 assert(node->exceptional > 0); 537 538 radix_tree_split(&tree, 0, new_order); 539 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 540 radix_tree_iter_replace(&tree, &iter, slot, (void *)0x16); 541 } 542 543 item = __radix_tree_lookup(&tree, 0, &node, NULL); 544 assert(item == (void *)0x16); 545 assert(node->exceptional > 0); 546 547 item_kill_tree(&tree); 548 549 __radix_tree_insert(&tree, 0, old_order, (void *)0x12); 550 551 item = __radix_tree_lookup(&tree, 0, &node, NULL); 552 assert(item == (void *)0x12); 553 assert(node->exceptional > 0); 554 555 radix_tree_split(&tree, 0, new_order); 556 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 557 if (iter.index == (1 << new_order)) 558 radix_tree_iter_replace(&tree, &iter, slot, 559 (void *)0x16); 560 else 561 radix_tree_iter_replace(&tree, &iter, slot, NULL); 562 } 563 564 item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL); 565 assert(item == (void *)0x16); 566 assert(node->count == node->exceptional); 567 do { 568 node = node->parent; 569 if (!node) 570 break; 571 assert(node->count == 1); 572 assert(node->exceptional == 0); 573 } while (1); 574 575 item_kill_tree(&tree); 576 } 577 578 static void multiorder_split(void) 579 { 580 int i, j; 581 582 for (i = 3; i < 11; i++) 583 for (j = 0; j < i; j++) { 584 __multiorder_split(i, j); 585 __multiorder_split2(i, j); 586 __multiorder_split3(i, j); 587 } 588 } 589 590 static void multiorder_account(void) 591 { 592 RADIX_TREE(tree, GFP_KERNEL); 593 struct radix_tree_node *node; 594 void **slot; 595 596 item_insert_order(&tree, 0, 5); 597 598 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); 599 __radix_tree_lookup(&tree, 0, &node, NULL); 600 assert(node->count == node->exceptional * 2); 601 radix_tree_delete(&tree, 1 << 5); 602 assert(node->exceptional == 0); 603 604 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); 605 __radix_tree_lookup(&tree, 1 << 5, &node, &slot); 606 assert(node->count == node->exceptional * 2); 607 __radix_tree_replace(&tree, node, slot, NULL, NULL, NULL); 608 assert(node->exceptional == 0); 609 610 item_kill_tree(&tree); 611 } 612 613 void multiorder_checks(void) 614 { 615 int i; 616 617 for (i = 0; i < 20; i++) { 618 multiorder_check(200, i); 619 multiorder_check(0, i); 620 multiorder_check((1UL << i) + 1, i); 621 } 622 623 for (i = 0; i < 15; i++) 624 multiorder_shrink((1UL << (i + RADIX_TREE_MAP_SHIFT)), i); 625 626 multiorder_insert_bug(); 627 multiorder_tag_tests(); 628 multiorder_iteration(); 629 multiorder_tagged_iteration(); 630 multiorder_join(); 631 multiorder_split(); 632 multiorder_account(); 633 634 radix_tree_cpu_dead(0); 635 } 636