1 // SPDX-License-Identifier: GPL-2.0-only
14 __param(int, nnodes, 100, "Number of nodes in the rb-tree");
15 __param(int, perf_loops, 1000, "Number of iterations modifying the rb-tree");
16 __param(int, check_loops, 100, "Number of iterations modifying and verifying the rb-tree");
27 static struct rb_root_cached root = RB_ROOT_CACHED;  variable
32 static void insert(struct test_node *node, struct rb_root_cached *root)  in insert()  argument
34 	struct rb_node **new = &root->rb_root.rb_node, *parent = NULL;  in insert()
35 	u32 key = node->key;  in insert()
39 		if (key < rb_entry(parent, struct test_node, rb)->key)  in insert()
40 			new = &parent->rb_left;  in insert()
42 			new = &parent->rb_right;  in insert()
45 	rb_link_node(&node->rb, parent, new);  in insert()
46 	rb_insert_color(&node->rb, &root->rb_root);  in insert()
49 static void insert_cached(struct test_node *node, struct rb_root_cached *root)  in insert_cached()  argument
51 	struct rb_node **new = &root->rb_root.rb_node, *parent = NULL;  in insert_cached()
52 	u32 key = node->key;  in insert_cached()
57 		if (key < rb_entry(parent, struct test_node, rb)->key)  in insert_cached()
58 			new = &parent->rb_left;  in insert_cached()
60 			new = &parent->rb_right;  in insert_cached()
65 	rb_link_node(&node->rb, parent, new);  in insert_cached()
66 	rb_insert_color_cached(&node->rb, root, leftmost);  in insert_cached()
69 static inline void erase(struct test_node *node, struct rb_root_cached *root)  in erase()  argument
71 	rb_erase(&node->rb, &root->rb_root);  in erase()
74 static inline void erase_cached(struct test_node *node, struct rb_root_cached *root)  in erase_cached()  argument
76 	rb_erase_cached(&node->rb, root);  in erase_cached()
80 #define NODE_VAL(node) ((node)->val)  argument
85 static void insert_augmented(struct test_node *node,  in RB_DECLARE_CALLBACKS_MAX()
86 			     struct rb_root_cached *root)  in RB_DECLARE_CALLBACKS_MAX()
88 	struct rb_node **new = &root->rb_root.rb_node, *rb_parent = NULL;  in RB_DECLARE_CALLBACKS_MAX()
89 	u32 key = node->key;  in RB_DECLARE_CALLBACKS_MAX()
90 	u32 val = node->val;  in RB_DECLARE_CALLBACKS_MAX()
96 		if (parent->augmented < val)  in RB_DECLARE_CALLBACKS_MAX()
97 			parent->augmented = val;  in RB_DECLARE_CALLBACKS_MAX()
98 		if (key < parent->key)  in RB_DECLARE_CALLBACKS_MAX()
99 			new = &parent->rb.rb_left;  in RB_DECLARE_CALLBACKS_MAX()
101 			new = &parent->rb.rb_right;  in RB_DECLARE_CALLBACKS_MAX()
104 	node->augmented = val;  in RB_DECLARE_CALLBACKS_MAX()
105 	rb_link_node(&node->rb, rb_parent, new);  in RB_DECLARE_CALLBACKS_MAX()
106 	rb_insert_augmented(&node->rb, &root->rb_root, &augment_callbacks);  in RB_DECLARE_CALLBACKS_MAX()
109 static void insert_augmented_cached(struct test_node *node,  in insert_augmented_cached()  argument
110 				    struct rb_root_cached *root)  in insert_augmented_cached()  argument
112 	struct rb_node **new = &root->rb_root.rb_node, *rb_parent = NULL;  in insert_augmented_cached()
113 	u32 key = node->key;  in insert_augmented_cached()
114 	u32 val = node->val;  in insert_augmented_cached()
121 		if (parent->augmented < val)  in insert_augmented_cached()
122 			parent->augmented = val;  in insert_augmented_cached()
123 		if (key < parent->key)  in insert_augmented_cached()
124 			new = &parent->rb.rb_left;  in insert_augmented_cached()
126 			new = &parent->rb.rb_right;  in insert_augmented_cached()
131 	node->augmented = val;  in insert_augmented_cached()
132 	rb_link_node(&node->rb, rb_parent, new);  in insert_augmented_cached()
133 	rb_insert_augmented_cached(&node->rb, root,  in insert_augmented_cached()
138 static void erase_augmented(struct test_node *node, struct rb_root_cached *root)  in erase_augmented()  argument
140 	rb_erase_augmented(&node->rb, &root->rb_root, &augment_callbacks);  in erase_augmented()
143 static void erase_augmented_cached(struct test_node *node,  in erase_augmented_cached()  argument
144 				   struct rb_root_cached *root)  in erase_augmented_cached()  argument
146 	rb_erase_augmented_cached(&node->rb, root, &augment_callbacks);  in erase_augmented_cached()
160 	return !(rb->__rb_parent_color & 1);  in is_red()
175 	rbtree_postorder_for_each_entry_safe(cur, n, &root.rb_root, rb)  in check_postorder_foreach()
185 	for (rb = rb_first_postorder(&root.rb_root); rb; rb = rb_next_postorder(rb))  in check_postorder()
197 	for (rb = rb_first(&root.rb_root); rb; rb = rb_next(rb)) {  in check()
198 		struct test_node *node = rb_entry(rb, struct test_node, rb);  in check()  local
199 		WARN_ON_ONCE(node->key < prev_key);  in check()
205 			WARN_ON_ONCE((!rb->rb_left || !rb->rb_right) &&  in check()
207 		prev_key = node->key;  in check()
212 	WARN_ON_ONCE(count < (1 << black_path_count(rb_last(&root.rb_root))) - 1);  in check()
223 	for (rb = rb_first(&root.rb_root); rb; rb = rb_next(rb)) {  in check_augmented()
224 		struct test_node *node = rb_entry(rb, struct test_node, rb);  in check_augmented()  local
225 		u32 subtree, max = node->val;  in check_augmented()
226 		if (node->rb.rb_left) {  in check_augmented()
227 			subtree = rb_entry(node->rb.rb_left, struct test_node,  in check_augmented()
228 					   rb)->augmented;  in check_augmented()
232 		if (node->rb.rb_right) {  in check_augmented()
233 			subtree = rb_entry(node->rb.rb_right, struct test_node,  in check_augmented()
234 					   rb)->augmented;  in check_augmented()
238 		WARN_ON_ONCE(node->augmented != max);  in check_augmented()
246 	struct rb_node *node;  in rbtree_test_init()  local
250 		return -ENOMEM;  in rbtree_test_init()
261 			insert(nodes + j, &root);  in rbtree_test_init()
263 			erase(nodes + j, &root);  in rbtree_test_init()
267 	time = time2 - time1;  in rbtree_test_init()
270 	printk(" -> test 1 (latency of nnodes insert+delete): %llu cycles\n",  in rbtree_test_init()
277 			insert_cached(nodes + j, &root);  in rbtree_test_init()
279 			erase_cached(nodes + j, &root);  in rbtree_test_init()
283 	time = time2 - time1;  in rbtree_test_init()
286 	printk(" -> test 2 (latency of nnodes cached insert+delete): %llu cycles\n",  in rbtree_test_init()
290 		insert(nodes + i, &root);  in rbtree_test_init()
295 		for (node = rb_first(&root.rb_root); node; node = rb_next(node))  in rbtree_test_init()
300 	time = time2 - time1;  in rbtree_test_init()
303 	printk(" -> test 3 (latency of inorder traversal): %llu cycles\n",  in rbtree_test_init()
309 		node = rb_first(&root.rb_root);  in rbtree_test_init()
312 	time = time2 - time1;  in rbtree_test_init()
315 	printk(" -> test 4 (latency to fetch first node)\n");  in rbtree_test_init()
316 	printk("        non-cached: %llu cycles\n", (unsigned long long)time);  in rbtree_test_init()
321 		node = rb_first_cached(&root);  in rbtree_test_init()
324 	time = time2 - time1;  in rbtree_test_init()
330 		erase(nodes + i, &root);  in rbtree_test_init()
337 			insert(nodes + j, &root);  in rbtree_test_init()
340 			check(nnodes - j);  in rbtree_test_init()
341 			erase(nodes + j, &root);  in rbtree_test_init()
354 			insert_augmented(nodes + j, &root);  in rbtree_test_init()
356 			erase_augmented(nodes + j, &root);  in rbtree_test_init()
360 	time = time2 - time1;  in rbtree_test_init()
363 	printk(" -> test 1 (latency of nnodes insert+delete): %llu cycles\n", (unsigned long long)time);  in rbtree_test_init()
369 			insert_augmented_cached(nodes + j, &root);  in rbtree_test_init()
371 			erase_augmented_cached(nodes + j, &root);  in rbtree_test_init()
375 	time = time2 - time1;  in rbtree_test_init()
378 …printk(" -> test 2 (latency of nnodes cached insert+delete): %llu cycles\n", (unsigned long long)t…  in rbtree_test_init()
384 			insert_augmented(nodes + j, &root);  in rbtree_test_init()
387 			check_augmented(nnodes - j);  in rbtree_test_init()
388 			erase_augmented(nodes + j, &root);  in rbtree_test_init()
395 	return -EAGAIN; /* Fail will directly unload the module */  in rbtree_test_init()