1 // SPDX-License-Identifier: GPL-2.0 2 #include <stdlib.h> 3 #include <assert.h> 4 #include <stdio.h> 5 #include <linux/types.h> 6 #include <linux/kernel.h> 7 #include <linux/bitops.h> 8 9 #include "test.h" 10 11 struct item * 12 item_tag_set(struct radix_tree_root *root, unsigned long index, int tag) 13 { 14 return radix_tree_tag_set(root, index, tag); 15 } 16 17 struct item * 18 item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag) 19 { 20 return radix_tree_tag_clear(root, index, tag); 21 } 22 23 int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag) 24 { 25 return radix_tree_tag_get(root, index, tag); 26 } 27 28 struct item *item_create(unsigned long index, unsigned int order) 29 { 30 struct item *ret = malloc(sizeof(*ret)); 31 32 ret->index = index; 33 ret->order = order; 34 return ret; 35 } 36 37 int item_insert_order(struct radix_tree_root *root, unsigned long index, 38 unsigned order) 39 { 40 struct item *item = item_create(index, order); 41 int err = __radix_tree_insert(root, item->index, item->order, item); 42 if (err) 43 free(item); 44 return err; 45 } 46 47 int item_insert(struct radix_tree_root *root, unsigned long index) 48 { 49 return item_insert_order(root, index, 0); 50 } 51 52 void item_sanity(struct item *item, unsigned long index) 53 { 54 unsigned long mask; 55 assert(!radix_tree_is_internal_node(item)); 56 assert(item->order < BITS_PER_LONG); 57 mask = (1UL << item->order) - 1; 58 assert((item->index | mask) == (index | mask)); 59 } 60 61 void item_free(struct item *item, unsigned long index) 62 { 63 item_sanity(item, index); 64 free(item); 65 } 66 67 int item_delete(struct radix_tree_root *root, unsigned long index) 68 { 69 struct item *item = radix_tree_delete(root, index); 70 71 if (!item) 72 return 0; 73 74 item_free(item, index); 75 return 1; 76 } 77 78 static void item_free_rcu(struct rcu_head *head) 79 { 80 struct item *item = container_of(head, struct item, rcu_head); 81 82 free(item); 83 } 84 85 int item_delete_rcu(struct radix_tree_root *root, unsigned long index) 86 { 87 struct item *item = radix_tree_delete(root, index); 88 89 if (item) { 90 item_sanity(item, index); 91 call_rcu(&item->rcu_head, item_free_rcu); 92 return 1; 93 } 94 return 0; 95 } 96 97 void item_check_present(struct radix_tree_root *root, unsigned long index) 98 { 99 struct item *item; 100 101 item = radix_tree_lookup(root, index); 102 assert(item != NULL); 103 item_sanity(item, index); 104 } 105 106 struct item *item_lookup(struct radix_tree_root *root, unsigned long index) 107 { 108 return radix_tree_lookup(root, index); 109 } 110 111 void item_check_absent(struct radix_tree_root *root, unsigned long index) 112 { 113 struct item *item; 114 115 item = radix_tree_lookup(root, index); 116 assert(item == NULL); 117 } 118 119 /* 120 * Scan only the passed (start, start+nr] for present items 121 */ 122 void item_gang_check_present(struct radix_tree_root *root, 123 unsigned long start, unsigned long nr, 124 int chunk, int hop) 125 { 126 struct item *items[chunk]; 127 unsigned long into; 128 129 for (into = 0; into < nr; ) { 130 int nfound; 131 int nr_to_find = chunk; 132 int i; 133 134 if (nr_to_find > (nr - into)) 135 nr_to_find = nr - into; 136 137 nfound = radix_tree_gang_lookup(root, (void **)items, 138 start + into, nr_to_find); 139 assert(nfound == nr_to_find); 140 for (i = 0; i < nfound; i++) 141 assert(items[i]->index == start + into + i); 142 into += hop; 143 } 144 } 145 146 /* 147 * Scan the entire tree, only expecting present items (start, start+nr] 148 */ 149 void item_full_scan(struct radix_tree_root *root, unsigned long start, 150 unsigned long nr, int chunk) 151 { 152 struct item *items[chunk]; 153 unsigned long into = 0; 154 unsigned long this_index = start; 155 int nfound; 156 int i; 157 158 // printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk); 159 160 while ((nfound = radix_tree_gang_lookup(root, (void **)items, into, 161 chunk))) { 162 // printf("At 0x%08lx, nfound=%d\n", into, nfound); 163 for (i = 0; i < nfound; i++) { 164 assert(items[i]->index == this_index); 165 this_index++; 166 } 167 // printf("Found 0x%08lx->0x%08lx\n", 168 // items[0]->index, items[nfound-1]->index); 169 into = this_index; 170 } 171 if (chunk) 172 assert(this_index == start + nr); 173 nfound = radix_tree_gang_lookup(root, (void **)items, 174 this_index, chunk); 175 assert(nfound == 0); 176 } 177 178 /* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */ 179 int tag_tagged_items(struct xarray *xa, unsigned long start, unsigned long end, 180 unsigned batch, xa_mark_t iftag, xa_mark_t thentag) 181 { 182 XA_STATE(xas, xa, start); 183 unsigned int tagged = 0; 184 struct item *item; 185 186 if (batch == 0) 187 batch = 1; 188 189 xas_lock_irq(&xas); 190 xas_for_each_marked(&xas, item, end, iftag) { 191 xas_set_mark(&xas, thentag); 192 if (++tagged % batch) 193 continue; 194 195 xas_pause(&xas); 196 xas_unlock_irq(&xas); 197 rcu_barrier(); 198 xas_lock_irq(&xas); 199 } 200 xas_unlock_irq(&xas); 201 202 return tagged; 203 } 204 205 static int verify_node(struct radix_tree_node *slot, unsigned int tag, 206 int tagged) 207 { 208 int anyset = 0; 209 int i; 210 int j; 211 212 slot = entry_to_node(slot); 213 214 /* Verify consistency at this level */ 215 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) { 216 if (slot->tags[tag][i]) { 217 anyset = 1; 218 break; 219 } 220 } 221 if (tagged != anyset) { 222 printf("tag: %u, shift %u, tagged: %d, anyset: %d\n", 223 tag, slot->shift, tagged, anyset); 224 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) { 225 printf("tag %d: ", j); 226 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) 227 printf("%016lx ", slot->tags[j][i]); 228 printf("\n"); 229 } 230 return 1; 231 } 232 assert(tagged == anyset); 233 234 /* Go for next level */ 235 if (slot->shift > 0) { 236 for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) 237 if (slot->slots[i]) 238 if (verify_node(slot->slots[i], tag, 239 !!test_bit(i, slot->tags[tag]))) { 240 printf("Failure at off %d\n", i); 241 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) { 242 printf("tag %d: ", j); 243 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) 244 printf("%016lx ", slot->tags[j][i]); 245 printf("\n"); 246 } 247 return 1; 248 } 249 } 250 return 0; 251 } 252 253 void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag) 254 { 255 struct radix_tree_node *node = root->xa_head; 256 if (!radix_tree_is_internal_node(node)) 257 return; 258 verify_node(node, tag, !!root_tag_get(root, tag)); 259 } 260 261 void item_kill_tree(struct radix_tree_root *root) 262 { 263 struct radix_tree_iter iter; 264 void **slot; 265 struct item *items[32]; 266 int nfound; 267 268 radix_tree_for_each_slot(slot, root, &iter, 0) { 269 if (xa_is_value(*slot)) 270 radix_tree_delete(root, iter.index); 271 } 272 273 while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) { 274 int i; 275 276 for (i = 0; i < nfound; i++) { 277 void *ret; 278 279 ret = radix_tree_delete(root, items[i]->index); 280 assert(ret == items[i]); 281 free(items[i]); 282 } 283 } 284 assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0); 285 assert(root->xa_head == NULL); 286 } 287 288 void tree_verify_min_height(struct radix_tree_root *root, int maxindex) 289 { 290 unsigned shift; 291 struct radix_tree_node *node = root->xa_head; 292 if (!radix_tree_is_internal_node(node)) { 293 assert(maxindex == 0); 294 return; 295 } 296 297 node = entry_to_node(node); 298 assert(maxindex <= node_maxindex(node)); 299 300 shift = node->shift; 301 if (shift > 0) 302 assert(maxindex > shift_maxindex(shift - RADIX_TREE_MAP_SHIFT)); 303 else 304 assert(maxindex > 0); 305 } 306