1 /* 2 * multiorder.c: Multi-order radix tree entry testing 3 * Copyright (c) 2016 Intel Corporation 4 * Author: Ross Zwisler <ross.zwisler@linux.intel.com> 5 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 */ 16 #include <linux/radix-tree.h> 17 #include <linux/slab.h> 18 #include <linux/errno.h> 19 20 #include "test.h" 21 22 #define for_each_index(i, base, order) \ 23 for (i = base; i < base + (1 << order); i++) 24 25 static void __multiorder_tag_test(int index, int order) 26 { 27 RADIX_TREE(tree, GFP_KERNEL); 28 int base, err, i; 29 30 /* our canonical entry */ 31 base = index & ~((1 << order) - 1); 32 33 printf("Multiorder tag test with index %d, canonical entry %d\n", 34 index, base); 35 36 err = item_insert_order(&tree, index, order); 37 assert(!err); 38 39 /* 40 * Verify we get collisions for covered indices. We try and fail to 41 * insert an exceptional entry so we don't leak memory via 42 * item_insert_order(). 43 */ 44 for_each_index(i, base, order) { 45 err = __radix_tree_insert(&tree, i, order, 46 (void *)(0xA0 | RADIX_TREE_EXCEPTIONAL_ENTRY)); 47 assert(err == -EEXIST); 48 } 49 50 for_each_index(i, base, order) { 51 assert(!radix_tree_tag_get(&tree, i, 0)); 52 assert(!radix_tree_tag_get(&tree, i, 1)); 53 } 54 55 assert(radix_tree_tag_set(&tree, index, 0)); 56 57 for_each_index(i, base, order) { 58 assert(radix_tree_tag_get(&tree, i, 0)); 59 assert(!radix_tree_tag_get(&tree, i, 1)); 60 } 61 62 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1); 63 assert(radix_tree_tag_clear(&tree, index, 0)); 64 65 for_each_index(i, base, order) { 66 assert(!radix_tree_tag_get(&tree, i, 0)); 67 assert(radix_tree_tag_get(&tree, i, 1)); 68 } 69 70 assert(radix_tree_tag_clear(&tree, index, 1)); 71 72 assert(!radix_tree_tagged(&tree, 0)); 73 assert(!radix_tree_tagged(&tree, 1)); 74 75 item_kill_tree(&tree); 76 } 77 78 static void multiorder_tag_tests(void) 79 { 80 /* test multi-order entry for indices 0-7 with no sibling pointers */ 81 __multiorder_tag_test(0, 3); 82 __multiorder_tag_test(5, 3); 83 84 /* test multi-order entry for indices 8-15 with no sibling pointers */ 85 __multiorder_tag_test(8, 3); 86 __multiorder_tag_test(15, 3); 87 88 /* 89 * Our order 5 entry covers indices 0-31 in a tree with height=2. 90 * This is broken up as follows: 91 * 0-7: canonical entry 92 * 8-15: sibling 1 93 * 16-23: sibling 2 94 * 24-31: sibling 3 95 */ 96 __multiorder_tag_test(0, 5); 97 __multiorder_tag_test(29, 5); 98 99 /* same test, but with indices 32-63 */ 100 __multiorder_tag_test(32, 5); 101 __multiorder_tag_test(44, 5); 102 103 /* 104 * Our order 8 entry covers indices 0-255 in a tree with height=3. 105 * This is broken up as follows: 106 * 0-63: canonical entry 107 * 64-127: sibling 1 108 * 128-191: sibling 2 109 * 192-255: sibling 3 110 */ 111 __multiorder_tag_test(0, 8); 112 __multiorder_tag_test(190, 8); 113 114 /* same test, but with indices 256-511 */ 115 __multiorder_tag_test(256, 8); 116 __multiorder_tag_test(300, 8); 117 118 __multiorder_tag_test(0x12345678UL, 8); 119 } 120 121 static void multiorder_check(unsigned long index, int order) 122 { 123 unsigned long i; 124 unsigned long min = index & ~((1UL << order) - 1); 125 unsigned long max = min + (1UL << order); 126 void **slot; 127 struct item *item2 = item_create(min, order); 128 RADIX_TREE(tree, GFP_KERNEL); 129 130 printf("Multiorder index %ld, order %d\n", index, order); 131 132 assert(item_insert_order(&tree, index, order) == 0); 133 134 for (i = min; i < max; i++) { 135 struct item *item = item_lookup(&tree, i); 136 assert(item != 0); 137 assert(item->index == index); 138 } 139 for (i = 0; i < min; i++) 140 item_check_absent(&tree, i); 141 for (i = max; i < 2*max; i++) 142 item_check_absent(&tree, i); 143 for (i = min; i < max; i++) 144 assert(radix_tree_insert(&tree, i, item2) == -EEXIST); 145 146 slot = radix_tree_lookup_slot(&tree, index); 147 free(*slot); 148 radix_tree_replace_slot(&tree, slot, item2); 149 for (i = min; i < max; i++) { 150 struct item *item = item_lookup(&tree, i); 151 assert(item != 0); 152 assert(item->index == min); 153 } 154 155 assert(item_delete(&tree, min) != 0); 156 157 for (i = 0; i < 2*max; i++) 158 item_check_absent(&tree, i); 159 } 160 161 static void multiorder_shrink(unsigned long index, int order) 162 { 163 unsigned long i; 164 unsigned long max = 1 << order; 165 RADIX_TREE(tree, GFP_KERNEL); 166 struct radix_tree_node *node; 167 168 printf("Multiorder shrink index %ld, order %d\n", index, order); 169 170 assert(item_insert_order(&tree, 0, order) == 0); 171 172 node = tree.rnode; 173 174 assert(item_insert(&tree, index) == 0); 175 assert(node != tree.rnode); 176 177 assert(item_delete(&tree, index) != 0); 178 assert(node == tree.rnode); 179 180 for (i = 0; i < max; i++) { 181 struct item *item = item_lookup(&tree, i); 182 assert(item != 0); 183 assert(item->index == 0); 184 } 185 for (i = max; i < 2*max; i++) 186 item_check_absent(&tree, i); 187 188 if (!item_delete(&tree, 0)) { 189 printf("failed to delete index %ld (order %d)\n", index, order); abort(); 190 } 191 192 for (i = 0; i < 2*max; i++) 193 item_check_absent(&tree, i); 194 } 195 196 static void multiorder_insert_bug(void) 197 { 198 RADIX_TREE(tree, GFP_KERNEL); 199 200 item_insert(&tree, 0); 201 radix_tree_tag_set(&tree, 0, 0); 202 item_insert_order(&tree, 3 << 6, 6); 203 204 item_kill_tree(&tree); 205 } 206 207 void multiorder_iteration(void) 208 { 209 RADIX_TREE(tree, GFP_KERNEL); 210 struct radix_tree_iter iter; 211 void **slot; 212 int i, j, err; 213 214 printf("Multiorder iteration test\n"); 215 216 #define NUM_ENTRIES 11 217 int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128}; 218 int order[NUM_ENTRIES] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7}; 219 220 for (i = 0; i < NUM_ENTRIES; i++) { 221 err = item_insert_order(&tree, index[i], order[i]); 222 assert(!err); 223 } 224 225 for (j = 0; j < 256; j++) { 226 for (i = 0; i < NUM_ENTRIES; i++) 227 if (j <= (index[i] | ((1 << order[i]) - 1))) 228 break; 229 230 radix_tree_for_each_slot(slot, &tree, &iter, j) { 231 int height = order[i] / RADIX_TREE_MAP_SHIFT; 232 int shift = height * RADIX_TREE_MAP_SHIFT; 233 unsigned long mask = (1UL << order[i]) - 1; 234 struct item *item = *slot; 235 236 assert((iter.index | mask) == (index[i] | mask)); 237 assert(iter.shift == shift); 238 assert(!radix_tree_is_internal_node(item)); 239 assert((item->index | mask) == (index[i] | mask)); 240 assert(item->order == order[i]); 241 i++; 242 } 243 } 244 245 item_kill_tree(&tree); 246 } 247 248 void multiorder_tagged_iteration(void) 249 { 250 RADIX_TREE(tree, GFP_KERNEL); 251 struct radix_tree_iter iter; 252 void **slot; 253 int i, j; 254 255 printf("Multiorder tagged iteration test\n"); 256 257 #define MT_NUM_ENTRIES 9 258 int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128}; 259 int order[MT_NUM_ENTRIES] = {1, 0, 2, 4, 3, 1, 3, 0, 7}; 260 261 #define TAG_ENTRIES 7 262 int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128}; 263 264 for (i = 0; i < MT_NUM_ENTRIES; i++) 265 assert(!item_insert_order(&tree, index[i], order[i])); 266 267 assert(!radix_tree_tagged(&tree, 1)); 268 269 for (i = 0; i < TAG_ENTRIES; i++) 270 assert(radix_tree_tag_set(&tree, tag_index[i], 1)); 271 272 for (j = 0; j < 256; j++) { 273 int k; 274 275 for (i = 0; i < TAG_ENTRIES; i++) { 276 for (k = i; index[k] < tag_index[i]; k++) 277 ; 278 if (j <= (index[k] | ((1 << order[k]) - 1))) 279 break; 280 } 281 282 radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) { 283 unsigned long mask; 284 struct item *item = *slot; 285 for (k = i; index[k] < tag_index[i]; k++) 286 ; 287 mask = (1UL << order[k]) - 1; 288 289 assert((iter.index | mask) == (tag_index[i] | mask)); 290 assert(!radix_tree_is_internal_node(item)); 291 assert((item->index | mask) == (tag_index[i] | mask)); 292 assert(item->order == order[k]); 293 i++; 294 } 295 } 296 297 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) == 298 TAG_ENTRIES); 299 300 for (j = 0; j < 256; j++) { 301 int mask, k; 302 303 for (i = 0; i < TAG_ENTRIES; i++) { 304 for (k = i; index[k] < tag_index[i]; k++) 305 ; 306 if (j <= (index[k] | ((1 << order[k]) - 1))) 307 break; 308 } 309 310 radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) { 311 struct item *item = *slot; 312 for (k = i; index[k] < tag_index[i]; k++) 313 ; 314 mask = (1 << order[k]) - 1; 315 316 assert((iter.index | mask) == (tag_index[i] | mask)); 317 assert(!radix_tree_is_internal_node(item)); 318 assert((item->index | mask) == (tag_index[i] | mask)); 319 assert(item->order == order[k]); 320 i++; 321 } 322 } 323 324 assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0) 325 == TAG_ENTRIES); 326 i = 0; 327 radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) { 328 assert(iter.index == tag_index[i]); 329 i++; 330 } 331 332 item_kill_tree(&tree); 333 } 334 335 static void __multiorder_join(unsigned long index, 336 unsigned order1, unsigned order2) 337 { 338 unsigned long loc; 339 void *item, *item2 = item_create(index + 1, order1); 340 RADIX_TREE(tree, GFP_KERNEL); 341 342 item_insert_order(&tree, index, order2); 343 item = radix_tree_lookup(&tree, index); 344 radix_tree_join(&tree, index + 1, order1, item2); 345 loc = find_item(&tree, item); 346 if (loc == -1) 347 free(item); 348 item = radix_tree_lookup(&tree, index + 1); 349 assert(item == item2); 350 item_kill_tree(&tree); 351 } 352 353 static void __multiorder_join2(unsigned order1, unsigned order2) 354 { 355 RADIX_TREE(tree, GFP_KERNEL); 356 struct radix_tree_node *node; 357 void *item1 = item_create(0, order1); 358 void *item2; 359 360 item_insert_order(&tree, 0, order2); 361 radix_tree_insert(&tree, 1 << order2, (void *)0x12UL); 362 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); 363 assert(item2 == (void *)0x12UL); 364 assert(node->exceptional == 1); 365 366 radix_tree_join(&tree, 0, order1, item1); 367 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); 368 assert(item2 == item1); 369 assert(node->exceptional == 0); 370 item_kill_tree(&tree); 371 } 372 373 static void multiorder_join(void) 374 { 375 int i, j, idx; 376 377 for (idx = 0; idx < 1024; idx = idx * 2 + 3) { 378 for (i = 1; i < 15; i++) { 379 for (j = 0; j < i; j++) { 380 __multiorder_join(idx, i, j); 381 } 382 } 383 } 384 385 for (i = 1; i < 15; i++) { 386 for (j = 0; j < i; j++) { 387 __multiorder_join2(i, j); 388 } 389 } 390 } 391 392 static void check_mem(unsigned old_order, unsigned new_order, unsigned alloc) 393 { 394 struct radix_tree_preload *rtp = &radix_tree_preloads; 395 if (rtp->nr != 0) 396 printf("split(%u %u) remaining %u\n", old_order, new_order, 397 rtp->nr); 398 /* 399 * Can't check for equality here as some nodes may have been 400 * RCU-freed while we ran. But we should never finish with more 401 * nodes allocated since they should have all been preloaded. 402 */ 403 if (nr_allocated > alloc) 404 printf("split(%u %u) allocated %u %u\n", old_order, new_order, 405 alloc, nr_allocated); 406 } 407 408 static void __multiorder_split(int old_order, int new_order) 409 { 410 RADIX_TREE(tree, GFP_ATOMIC); 411 void **slot; 412 struct radix_tree_iter iter; 413 unsigned alloc; 414 415 radix_tree_preload(GFP_KERNEL); 416 assert(item_insert_order(&tree, 0, old_order) == 0); 417 radix_tree_preload_end(); 418 419 /* Wipe out the preloaded cache or it'll confuse check_mem() */ 420 radix_tree_cpu_dead(0); 421 422 radix_tree_tag_set(&tree, 0, 2); 423 424 radix_tree_split_preload(old_order, new_order, GFP_KERNEL); 425 alloc = nr_allocated; 426 radix_tree_split(&tree, 0, new_order); 427 check_mem(old_order, new_order, alloc); 428 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 429 radix_tree_iter_replace(&tree, &iter, slot, 430 item_create(iter.index, new_order)); 431 } 432 radix_tree_preload_end(); 433 434 item_kill_tree(&tree); 435 } 436 437 static void __multiorder_split2(int old_order, int new_order) 438 { 439 RADIX_TREE(tree, GFP_KERNEL); 440 void **slot; 441 struct radix_tree_iter iter; 442 struct radix_tree_node *node; 443 void *item; 444 445 __radix_tree_insert(&tree, 0, old_order, (void *)0x12); 446 447 item = __radix_tree_lookup(&tree, 0, &node, NULL); 448 assert(item == (void *)0x12); 449 assert(node->exceptional > 0); 450 451 radix_tree_split(&tree, 0, new_order); 452 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 453 radix_tree_iter_replace(&tree, &iter, slot, 454 item_create(iter.index, new_order)); 455 } 456 457 item = __radix_tree_lookup(&tree, 0, &node, NULL); 458 assert(item != (void *)0x12); 459 assert(node->exceptional == 0); 460 461 item_kill_tree(&tree); 462 } 463 464 static void __multiorder_split3(int old_order, int new_order) 465 { 466 RADIX_TREE(tree, GFP_KERNEL); 467 void **slot; 468 struct radix_tree_iter iter; 469 struct radix_tree_node *node; 470 void *item; 471 472 __radix_tree_insert(&tree, 0, old_order, (void *)0x12); 473 474 item = __radix_tree_lookup(&tree, 0, &node, NULL); 475 assert(item == (void *)0x12); 476 assert(node->exceptional > 0); 477 478 radix_tree_split(&tree, 0, new_order); 479 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 480 radix_tree_iter_replace(&tree, &iter, slot, (void *)0x16); 481 } 482 483 item = __radix_tree_lookup(&tree, 0, &node, NULL); 484 assert(item == (void *)0x16); 485 assert(node->exceptional > 0); 486 487 item_kill_tree(&tree); 488 489 __radix_tree_insert(&tree, 0, old_order, (void *)0x12); 490 491 item = __radix_tree_lookup(&tree, 0, &node, NULL); 492 assert(item == (void *)0x12); 493 assert(node->exceptional > 0); 494 495 radix_tree_split(&tree, 0, new_order); 496 radix_tree_for_each_slot(slot, &tree, &iter, 0) { 497 if (iter.index == (1 << new_order)) 498 radix_tree_iter_replace(&tree, &iter, slot, 499 (void *)0x16); 500 else 501 radix_tree_iter_replace(&tree, &iter, slot, NULL); 502 } 503 504 item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL); 505 assert(item == (void *)0x16); 506 assert(node->count == node->exceptional); 507 do { 508 node = node->parent; 509 if (!node) 510 break; 511 assert(node->count == 1); 512 assert(node->exceptional == 0); 513 } while (1); 514 515 item_kill_tree(&tree); 516 } 517 518 static void multiorder_split(void) 519 { 520 int i, j; 521 522 for (i = 3; i < 11; i++) 523 for (j = 0; j < i; j++) { 524 __multiorder_split(i, j); 525 __multiorder_split2(i, j); 526 __multiorder_split3(i, j); 527 } 528 } 529 530 static void multiorder_account(void) 531 { 532 RADIX_TREE(tree, GFP_KERNEL); 533 struct radix_tree_node *node; 534 void **slot; 535 536 item_insert_order(&tree, 0, 5); 537 538 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); 539 __radix_tree_lookup(&tree, 0, &node, NULL); 540 assert(node->count == node->exceptional * 2); 541 radix_tree_delete(&tree, 1 << 5); 542 assert(node->exceptional == 0); 543 544 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); 545 __radix_tree_lookup(&tree, 1 << 5, &node, &slot); 546 assert(node->count == node->exceptional * 2); 547 __radix_tree_replace(&tree, node, slot, NULL, NULL, NULL); 548 assert(node->exceptional == 0); 549 550 item_kill_tree(&tree); 551 } 552 553 void multiorder_checks(void) 554 { 555 int i; 556 557 for (i = 0; i < 20; i++) { 558 multiorder_check(200, i); 559 multiorder_check(0, i); 560 multiorder_check((1UL << i) + 1, i); 561 } 562 563 for (i = 0; i < 15; i++) 564 multiorder_shrink((1UL << (i + RADIX_TREE_MAP_SHIFT)), i); 565 566 multiorder_insert_bug(); 567 multiorder_tag_tests(); 568 multiorder_iteration(); 569 multiorder_tagged_iteration(); 570 multiorder_join(); 571 multiorder_split(); 572 multiorder_account(); 573 574 radix_tree_cpu_dead(0); 575 } 576