xref: /openbmc/linux/kernel/bpf/lpm_trie.c (revision 96de2506)
1 /*
2  * Longest prefix match list implementation
3  *
4  * Copyright (c) 2016,2017 Daniel Mack
5  * Copyright (c) 2016 David Herrmann
6  *
7  * This file is subject to the terms and conditions of version 2 of the GNU
8  * General Public License.  See the file COPYING in the main directory of the
9  * Linux distribution for more details.
10  */
11 
12 #include <linux/bpf.h>
13 #include <linux/btf.h>
14 #include <linux/err.h>
15 #include <linux/slab.h>
16 #include <linux/spinlock.h>
17 #include <linux/vmalloc.h>
18 #include <net/ipv6.h>
19 #include <uapi/linux/btf.h>
20 
21 /* Intermediate node */
22 #define LPM_TREE_NODE_FLAG_IM BIT(0)
23 
24 struct lpm_trie_node;
25 
26 struct lpm_trie_node {
27 	struct rcu_head rcu;
28 	struct lpm_trie_node __rcu	*child[2];
29 	u32				prefixlen;
30 	u32				flags;
31 	u8				data[0];
32 };
33 
34 struct lpm_trie {
35 	struct bpf_map			map;
36 	struct lpm_trie_node __rcu	*root;
37 	size_t				n_entries;
38 	size_t				max_prefixlen;
39 	size_t				data_size;
40 	raw_spinlock_t			lock;
41 };
42 
43 /* This trie implements a longest prefix match algorithm that can be used to
44  * match IP addresses to a stored set of ranges.
45  *
46  * Data stored in @data of struct bpf_lpm_key and struct lpm_trie_node is
47  * interpreted as big endian, so data[0] stores the most significant byte.
48  *
49  * Match ranges are internally stored in instances of struct lpm_trie_node
50  * which each contain their prefix length as well as two pointers that may
51  * lead to more nodes containing more specific matches. Each node also stores
52  * a value that is defined by and returned to userspace via the update_elem
53  * and lookup functions.
54  *
55  * For instance, let's start with a trie that was created with a prefix length
56  * of 32, so it can be used for IPv4 addresses, and one single element that
57  * matches 192.168.0.0/16. The data array would hence contain
58  * [0xc0, 0xa8, 0x00, 0x00] in big-endian notation. This documentation will
59  * stick to IP-address notation for readability though.
60  *
61  * As the trie is empty initially, the new node (1) will be places as root
62  * node, denoted as (R) in the example below. As there are no other node, both
63  * child pointers are %NULL.
64  *
65  *              +----------------+
66  *              |       (1)  (R) |
67  *              | 192.168.0.0/16 |
68  *              |    value: 1    |
69  *              |   [0]    [1]   |
70  *              +----------------+
71  *
72  * Next, let's add a new node (2) matching 192.168.0.0/24. As there is already
73  * a node with the same data and a smaller prefix (ie, a less specific one),
74  * node (2) will become a child of (1). In child index depends on the next bit
75  * that is outside of what (1) matches, and that bit is 0, so (2) will be
76  * child[0] of (1):
77  *
78  *              +----------------+
79  *              |       (1)  (R) |
80  *              | 192.168.0.0/16 |
81  *              |    value: 1    |
82  *              |   [0]    [1]   |
83  *              +----------------+
84  *                   |
85  *    +----------------+
86  *    |       (2)      |
87  *    | 192.168.0.0/24 |
88  *    |    value: 2    |
89  *    |   [0]    [1]   |
90  *    +----------------+
91  *
92  * The child[1] slot of (1) could be filled with another node which has bit #17
93  * (the next bit after the ones that (1) matches on) set to 1. For instance,
94  * 192.168.128.0/24:
95  *
96  *              +----------------+
97  *              |       (1)  (R) |
98  *              | 192.168.0.0/16 |
99  *              |    value: 1    |
100  *              |   [0]    [1]   |
101  *              +----------------+
102  *                   |      |
103  *    +----------------+  +------------------+
104  *    |       (2)      |  |        (3)       |
105  *    | 192.168.0.0/24 |  | 192.168.128.0/24 |
106  *    |    value: 2    |  |     value: 3     |
107  *    |   [0]    [1]   |  |    [0]    [1]    |
108  *    +----------------+  +------------------+
109  *
110  * Let's add another node (4) to the game for 192.168.1.0/24. In order to place
111  * it, node (1) is looked at first, and because (4) of the semantics laid out
112  * above (bit #17 is 0), it would normally be attached to (1) as child[0].
113  * However, that slot is already allocated, so a new node is needed in between.
114  * That node does not have a value attached to it and it will never be
115  * returned to users as result of a lookup. It is only there to differentiate
116  * the traversal further. It will get a prefix as wide as necessary to
117  * distinguish its two children:
118  *
119  *                      +----------------+
120  *                      |       (1)  (R) |
121  *                      | 192.168.0.0/16 |
122  *                      |    value: 1    |
123  *                      |   [0]    [1]   |
124  *                      +----------------+
125  *                           |      |
126  *            +----------------+  +------------------+
127  *            |       (4)  (I) |  |        (3)       |
128  *            | 192.168.0.0/23 |  | 192.168.128.0/24 |
129  *            |    value: ---  |  |     value: 3     |
130  *            |   [0]    [1]   |  |    [0]    [1]    |
131  *            +----------------+  +------------------+
132  *                 |      |
133  *  +----------------+  +----------------+
134  *  |       (2)      |  |       (5)      |
135  *  | 192.168.0.0/24 |  | 192.168.1.0/24 |
136  *  |    value: 2    |  |     value: 5   |
137  *  |   [0]    [1]   |  |   [0]    [1]   |
138  *  +----------------+  +----------------+
139  *
140  * 192.168.1.1/32 would be a child of (5) etc.
141  *
142  * An intermediate node will be turned into a 'real' node on demand. In the
143  * example above, (4) would be re-used if 192.168.0.0/23 is added to the trie.
144  *
145  * A fully populated trie would have a height of 32 nodes, as the trie was
146  * created with a prefix length of 32.
147  *
148  * The lookup starts at the root node. If the current node matches and if there
149  * is a child that can be used to become more specific, the trie is traversed
150  * downwards. The last node in the traversal that is a non-intermediate one is
151  * returned.
152  */
153 
154 static inline int extract_bit(const u8 *data, size_t index)
155 {
156 	return !!(data[index / 8] & (1 << (7 - (index % 8))));
157 }
158 
159 /**
160  * longest_prefix_match() - determine the longest prefix
161  * @trie:	The trie to get internal sizes from
162  * @node:	The node to operate on
163  * @key:	The key to compare to @node
164  *
165  * Determine the longest prefix of @node that matches the bits in @key.
166  */
167 static size_t longest_prefix_match(const struct lpm_trie *trie,
168 				   const struct lpm_trie_node *node,
169 				   const struct bpf_lpm_trie_key *key)
170 {
171 	size_t prefixlen = 0;
172 	size_t i;
173 
174 	for (i = 0; i < trie->data_size; i++) {
175 		size_t b;
176 
177 		b = 8 - fls(node->data[i] ^ key->data[i]);
178 		prefixlen += b;
179 
180 		if (prefixlen >= node->prefixlen || prefixlen >= key->prefixlen)
181 			return min(node->prefixlen, key->prefixlen);
182 
183 		if (b < 8)
184 			break;
185 	}
186 
187 	return prefixlen;
188 }
189 
190 /* Called from syscall or from eBPF program */
191 static void *trie_lookup_elem(struct bpf_map *map, void *_key)
192 {
193 	struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
194 	struct lpm_trie_node *node, *found = NULL;
195 	struct bpf_lpm_trie_key *key = _key;
196 
197 	/* Start walking the trie from the root node ... */
198 
199 	for (node = rcu_dereference(trie->root); node;) {
200 		unsigned int next_bit;
201 		size_t matchlen;
202 
203 		/* Determine the longest prefix of @node that matches @key.
204 		 * If it's the maximum possible prefix for this trie, we have
205 		 * an exact match and can return it directly.
206 		 */
207 		matchlen = longest_prefix_match(trie, node, key);
208 		if (matchlen == trie->max_prefixlen) {
209 			found = node;
210 			break;
211 		}
212 
213 		/* If the number of bits that match is smaller than the prefix
214 		 * length of @node, bail out and return the node we have seen
215 		 * last in the traversal (ie, the parent).
216 		 */
217 		if (matchlen < node->prefixlen)
218 			break;
219 
220 		/* Consider this node as return candidate unless it is an
221 		 * artificially added intermediate one.
222 		 */
223 		if (!(node->flags & LPM_TREE_NODE_FLAG_IM))
224 			found = node;
225 
226 		/* If the node match is fully satisfied, let's see if we can
227 		 * become more specific. Determine the next bit in the key and
228 		 * traverse down.
229 		 */
230 		next_bit = extract_bit(key->data, node->prefixlen);
231 		node = rcu_dereference(node->child[next_bit]);
232 	}
233 
234 	if (!found)
235 		return NULL;
236 
237 	return found->data + trie->data_size;
238 }
239 
240 static struct lpm_trie_node *lpm_trie_node_alloc(const struct lpm_trie *trie,
241 						 const void *value)
242 {
243 	struct lpm_trie_node *node;
244 	size_t size = sizeof(struct lpm_trie_node) + trie->data_size;
245 
246 	if (value)
247 		size += trie->map.value_size;
248 
249 	node = kmalloc_node(size, GFP_ATOMIC | __GFP_NOWARN,
250 			    trie->map.numa_node);
251 	if (!node)
252 		return NULL;
253 
254 	node->flags = 0;
255 
256 	if (value)
257 		memcpy(node->data + trie->data_size, value,
258 		       trie->map.value_size);
259 
260 	return node;
261 }
262 
263 /* Called from syscall or from eBPF program */
264 static int trie_update_elem(struct bpf_map *map,
265 			    void *_key, void *value, u64 flags)
266 {
267 	struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
268 	struct lpm_trie_node *node, *im_node = NULL, *new_node = NULL;
269 	struct lpm_trie_node __rcu **slot;
270 	struct bpf_lpm_trie_key *key = _key;
271 	unsigned long irq_flags;
272 	unsigned int next_bit;
273 	size_t matchlen = 0;
274 	int ret = 0;
275 
276 	if (unlikely(flags > BPF_EXIST))
277 		return -EINVAL;
278 
279 	if (key->prefixlen > trie->max_prefixlen)
280 		return -EINVAL;
281 
282 	raw_spin_lock_irqsave(&trie->lock, irq_flags);
283 
284 	/* Allocate and fill a new node */
285 
286 	if (trie->n_entries == trie->map.max_entries) {
287 		ret = -ENOSPC;
288 		goto out;
289 	}
290 
291 	new_node = lpm_trie_node_alloc(trie, value);
292 	if (!new_node) {
293 		ret = -ENOMEM;
294 		goto out;
295 	}
296 
297 	trie->n_entries++;
298 
299 	new_node->prefixlen = key->prefixlen;
300 	RCU_INIT_POINTER(new_node->child[0], NULL);
301 	RCU_INIT_POINTER(new_node->child[1], NULL);
302 	memcpy(new_node->data, key->data, trie->data_size);
303 
304 	/* Now find a slot to attach the new node. To do that, walk the tree
305 	 * from the root and match as many bits as possible for each node until
306 	 * we either find an empty slot or a slot that needs to be replaced by
307 	 * an intermediate node.
308 	 */
309 	slot = &trie->root;
310 
311 	while ((node = rcu_dereference_protected(*slot,
312 					lockdep_is_held(&trie->lock)))) {
313 		matchlen = longest_prefix_match(trie, node, key);
314 
315 		if (node->prefixlen != matchlen ||
316 		    node->prefixlen == key->prefixlen ||
317 		    node->prefixlen == trie->max_prefixlen)
318 			break;
319 
320 		next_bit = extract_bit(key->data, node->prefixlen);
321 		slot = &node->child[next_bit];
322 	}
323 
324 	/* If the slot is empty (a free child pointer or an empty root),
325 	 * simply assign the @new_node to that slot and be done.
326 	 */
327 	if (!node) {
328 		rcu_assign_pointer(*slot, new_node);
329 		goto out;
330 	}
331 
332 	/* If the slot we picked already exists, replace it with @new_node
333 	 * which already has the correct data array set.
334 	 */
335 	if (node->prefixlen == matchlen) {
336 		new_node->child[0] = node->child[0];
337 		new_node->child[1] = node->child[1];
338 
339 		if (!(node->flags & LPM_TREE_NODE_FLAG_IM))
340 			trie->n_entries--;
341 
342 		rcu_assign_pointer(*slot, new_node);
343 		kfree_rcu(node, rcu);
344 
345 		goto out;
346 	}
347 
348 	/* If the new node matches the prefix completely, it must be inserted
349 	 * as an ancestor. Simply insert it between @node and *@slot.
350 	 */
351 	if (matchlen == key->prefixlen) {
352 		next_bit = extract_bit(node->data, matchlen);
353 		rcu_assign_pointer(new_node->child[next_bit], node);
354 		rcu_assign_pointer(*slot, new_node);
355 		goto out;
356 	}
357 
358 	im_node = lpm_trie_node_alloc(trie, NULL);
359 	if (!im_node) {
360 		ret = -ENOMEM;
361 		goto out;
362 	}
363 
364 	im_node->prefixlen = matchlen;
365 	im_node->flags |= LPM_TREE_NODE_FLAG_IM;
366 	memcpy(im_node->data, node->data, trie->data_size);
367 
368 	/* Now determine which child to install in which slot */
369 	if (extract_bit(key->data, matchlen)) {
370 		rcu_assign_pointer(im_node->child[0], node);
371 		rcu_assign_pointer(im_node->child[1], new_node);
372 	} else {
373 		rcu_assign_pointer(im_node->child[0], new_node);
374 		rcu_assign_pointer(im_node->child[1], node);
375 	}
376 
377 	/* Finally, assign the intermediate node to the determined spot */
378 	rcu_assign_pointer(*slot, im_node);
379 
380 out:
381 	if (ret) {
382 		if (new_node)
383 			trie->n_entries--;
384 
385 		kfree(new_node);
386 		kfree(im_node);
387 	}
388 
389 	raw_spin_unlock_irqrestore(&trie->lock, irq_flags);
390 
391 	return ret;
392 }
393 
394 /* Called from syscall or from eBPF program */
395 static int trie_delete_elem(struct bpf_map *map, void *_key)
396 {
397 	struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
398 	struct bpf_lpm_trie_key *key = _key;
399 	struct lpm_trie_node __rcu **trim, **trim2;
400 	struct lpm_trie_node *node, *parent;
401 	unsigned long irq_flags;
402 	unsigned int next_bit;
403 	size_t matchlen = 0;
404 	int ret = 0;
405 
406 	if (key->prefixlen > trie->max_prefixlen)
407 		return -EINVAL;
408 
409 	raw_spin_lock_irqsave(&trie->lock, irq_flags);
410 
411 	/* Walk the tree looking for an exact key/length match and keeping
412 	 * track of the path we traverse.  We will need to know the node
413 	 * we wish to delete, and the slot that points to the node we want
414 	 * to delete.  We may also need to know the nodes parent and the
415 	 * slot that contains it.
416 	 */
417 	trim = &trie->root;
418 	trim2 = trim;
419 	parent = NULL;
420 	while ((node = rcu_dereference_protected(
421 		       *trim, lockdep_is_held(&trie->lock)))) {
422 		matchlen = longest_prefix_match(trie, node, key);
423 
424 		if (node->prefixlen != matchlen ||
425 		    node->prefixlen == key->prefixlen)
426 			break;
427 
428 		parent = node;
429 		trim2 = trim;
430 		next_bit = extract_bit(key->data, node->prefixlen);
431 		trim = &node->child[next_bit];
432 	}
433 
434 	if (!node || node->prefixlen != key->prefixlen ||
435 	    (node->flags & LPM_TREE_NODE_FLAG_IM)) {
436 		ret = -ENOENT;
437 		goto out;
438 	}
439 
440 	trie->n_entries--;
441 
442 	/* If the node we are removing has two children, simply mark it
443 	 * as intermediate and we are done.
444 	 */
445 	if (rcu_access_pointer(node->child[0]) &&
446 	    rcu_access_pointer(node->child[1])) {
447 		node->flags |= LPM_TREE_NODE_FLAG_IM;
448 		goto out;
449 	}
450 
451 	/* If the parent of the node we are about to delete is an intermediate
452 	 * node, and the deleted node doesn't have any children, we can delete
453 	 * the intermediate parent as well and promote its other child
454 	 * up the tree.  Doing this maintains the invariant that all
455 	 * intermediate nodes have exactly 2 children and that there are no
456 	 * unnecessary intermediate nodes in the tree.
457 	 */
458 	if (parent && (parent->flags & LPM_TREE_NODE_FLAG_IM) &&
459 	    !node->child[0] && !node->child[1]) {
460 		if (node == rcu_access_pointer(parent->child[0]))
461 			rcu_assign_pointer(
462 				*trim2, rcu_access_pointer(parent->child[1]));
463 		else
464 			rcu_assign_pointer(
465 				*trim2, rcu_access_pointer(parent->child[0]));
466 		kfree_rcu(parent, rcu);
467 		kfree_rcu(node, rcu);
468 		goto out;
469 	}
470 
471 	/* The node we are removing has either zero or one child. If there
472 	 * is a child, move it into the removed node's slot then delete
473 	 * the node.  Otherwise just clear the slot and delete the node.
474 	 */
475 	if (node->child[0])
476 		rcu_assign_pointer(*trim, rcu_access_pointer(node->child[0]));
477 	else if (node->child[1])
478 		rcu_assign_pointer(*trim, rcu_access_pointer(node->child[1]));
479 	else
480 		RCU_INIT_POINTER(*trim, NULL);
481 	kfree_rcu(node, rcu);
482 
483 out:
484 	raw_spin_unlock_irqrestore(&trie->lock, irq_flags);
485 
486 	return ret;
487 }
488 
489 #define LPM_DATA_SIZE_MAX	256
490 #define LPM_DATA_SIZE_MIN	1
491 
492 #define LPM_VAL_SIZE_MAX	(KMALLOC_MAX_SIZE - LPM_DATA_SIZE_MAX - \
493 				 sizeof(struct lpm_trie_node))
494 #define LPM_VAL_SIZE_MIN	1
495 
496 #define LPM_KEY_SIZE(X)		(sizeof(struct bpf_lpm_trie_key) + (X))
497 #define LPM_KEY_SIZE_MAX	LPM_KEY_SIZE(LPM_DATA_SIZE_MAX)
498 #define LPM_KEY_SIZE_MIN	LPM_KEY_SIZE(LPM_DATA_SIZE_MIN)
499 
500 #define LPM_CREATE_FLAG_MASK	(BPF_F_NO_PREALLOC | BPF_F_NUMA_NODE |	\
501 				 BPF_F_RDONLY | BPF_F_WRONLY)
502 
503 static struct bpf_map *trie_alloc(union bpf_attr *attr)
504 {
505 	struct lpm_trie *trie;
506 	u64 cost = sizeof(*trie), cost_per_node;
507 	int ret;
508 
509 	if (!capable(CAP_SYS_ADMIN))
510 		return ERR_PTR(-EPERM);
511 
512 	/* check sanity of attributes */
513 	if (attr->max_entries == 0 ||
514 	    !(attr->map_flags & BPF_F_NO_PREALLOC) ||
515 	    attr->map_flags & ~LPM_CREATE_FLAG_MASK ||
516 	    attr->key_size < LPM_KEY_SIZE_MIN ||
517 	    attr->key_size > LPM_KEY_SIZE_MAX ||
518 	    attr->value_size < LPM_VAL_SIZE_MIN ||
519 	    attr->value_size > LPM_VAL_SIZE_MAX)
520 		return ERR_PTR(-EINVAL);
521 
522 	trie = kzalloc(sizeof(*trie), GFP_USER | __GFP_NOWARN);
523 	if (!trie)
524 		return ERR_PTR(-ENOMEM);
525 
526 	/* copy mandatory map attributes */
527 	bpf_map_init_from_attr(&trie->map, attr);
528 	trie->data_size = attr->key_size -
529 			  offsetof(struct bpf_lpm_trie_key, data);
530 	trie->max_prefixlen = trie->data_size * 8;
531 
532 	cost_per_node = sizeof(struct lpm_trie_node) +
533 			attr->value_size + trie->data_size;
534 	cost += (u64) attr->max_entries * cost_per_node;
535 	if (cost >= U32_MAX - PAGE_SIZE) {
536 		ret = -E2BIG;
537 		goto out_err;
538 	}
539 
540 	trie->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
541 
542 	ret = bpf_map_precharge_memlock(trie->map.pages);
543 	if (ret)
544 		goto out_err;
545 
546 	raw_spin_lock_init(&trie->lock);
547 
548 	return &trie->map;
549 out_err:
550 	kfree(trie);
551 	return ERR_PTR(ret);
552 }
553 
554 static void trie_free(struct bpf_map *map)
555 {
556 	struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
557 	struct lpm_trie_node __rcu **slot;
558 	struct lpm_trie_node *node;
559 
560 	/* Wait for outstanding programs to complete
561 	 * update/lookup/delete/get_next_key and free the trie.
562 	 */
563 	synchronize_rcu();
564 
565 	/* Always start at the root and walk down to a node that has no
566 	 * children. Then free that node, nullify its reference in the parent
567 	 * and start over.
568 	 */
569 
570 	for (;;) {
571 		slot = &trie->root;
572 
573 		for (;;) {
574 			node = rcu_dereference_protected(*slot, 1);
575 			if (!node)
576 				goto out;
577 
578 			if (rcu_access_pointer(node->child[0])) {
579 				slot = &node->child[0];
580 				continue;
581 			}
582 
583 			if (rcu_access_pointer(node->child[1])) {
584 				slot = &node->child[1];
585 				continue;
586 			}
587 
588 			kfree(node);
589 			RCU_INIT_POINTER(*slot, NULL);
590 			break;
591 		}
592 	}
593 
594 out:
595 	kfree(trie);
596 }
597 
598 static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
599 {
600 	struct lpm_trie_node *node, *next_node = NULL, *parent, *search_root;
601 	struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
602 	struct bpf_lpm_trie_key *key = _key, *next_key = _next_key;
603 	struct lpm_trie_node **node_stack = NULL;
604 	int err = 0, stack_ptr = -1;
605 	unsigned int next_bit;
606 	size_t matchlen;
607 
608 	/* The get_next_key follows postorder. For the 4 node example in
609 	 * the top of this file, the trie_get_next_key() returns the following
610 	 * one after another:
611 	 *   192.168.0.0/24
612 	 *   192.168.1.0/24
613 	 *   192.168.128.0/24
614 	 *   192.168.0.0/16
615 	 *
616 	 * The idea is to return more specific keys before less specific ones.
617 	 */
618 
619 	/* Empty trie */
620 	search_root = rcu_dereference(trie->root);
621 	if (!search_root)
622 		return -ENOENT;
623 
624 	/* For invalid key, find the leftmost node in the trie */
625 	if (!key || key->prefixlen > trie->max_prefixlen)
626 		goto find_leftmost;
627 
628 	node_stack = kmalloc_array(trie->max_prefixlen,
629 				   sizeof(struct lpm_trie_node *),
630 				   GFP_ATOMIC | __GFP_NOWARN);
631 	if (!node_stack)
632 		return -ENOMEM;
633 
634 	/* Try to find the exact node for the given key */
635 	for (node = search_root; node;) {
636 		node_stack[++stack_ptr] = node;
637 		matchlen = longest_prefix_match(trie, node, key);
638 		if (node->prefixlen != matchlen ||
639 		    node->prefixlen == key->prefixlen)
640 			break;
641 
642 		next_bit = extract_bit(key->data, node->prefixlen);
643 		node = rcu_dereference(node->child[next_bit]);
644 	}
645 	if (!node || node->prefixlen != key->prefixlen ||
646 	    (node->flags & LPM_TREE_NODE_FLAG_IM))
647 		goto find_leftmost;
648 
649 	/* The node with the exactly-matching key has been found,
650 	 * find the first node in postorder after the matched node.
651 	 */
652 	node = node_stack[stack_ptr];
653 	while (stack_ptr > 0) {
654 		parent = node_stack[stack_ptr - 1];
655 		if (rcu_dereference(parent->child[0]) == node) {
656 			search_root = rcu_dereference(parent->child[1]);
657 			if (search_root)
658 				goto find_leftmost;
659 		}
660 		if (!(parent->flags & LPM_TREE_NODE_FLAG_IM)) {
661 			next_node = parent;
662 			goto do_copy;
663 		}
664 
665 		node = parent;
666 		stack_ptr--;
667 	}
668 
669 	/* did not find anything */
670 	err = -ENOENT;
671 	goto free_stack;
672 
673 find_leftmost:
674 	/* Find the leftmost non-intermediate node, all intermediate nodes
675 	 * have exact two children, so this function will never return NULL.
676 	 */
677 	for (node = search_root; node;) {
678 		if (!(node->flags & LPM_TREE_NODE_FLAG_IM))
679 			next_node = node;
680 		node = rcu_dereference(node->child[0]);
681 	}
682 do_copy:
683 	next_key->prefixlen = next_node->prefixlen;
684 	memcpy((void *)next_key + offsetof(struct bpf_lpm_trie_key, data),
685 	       next_node->data, trie->data_size);
686 free_stack:
687 	kfree(node_stack);
688 	return err;
689 }
690 
691 static int trie_check_btf(const struct bpf_map *map,
692 			  const struct btf_type *key_type,
693 			  const struct btf_type *value_type)
694 {
695 	/* Keys must have struct bpf_lpm_trie_key embedded. */
696 	return BTF_INFO_KIND(key_type->info) != BTF_KIND_STRUCT ?
697 	       -EINVAL : 0;
698 }
699 
700 const struct bpf_map_ops trie_map_ops = {
701 	.map_alloc = trie_alloc,
702 	.map_free = trie_free,
703 	.map_get_next_key = trie_get_next_key,
704 	.map_lookup_elem = trie_lookup_elem,
705 	.map_update_elem = trie_update_elem,
706 	.map_delete_elem = trie_delete_elem,
707 	.map_check_btf = trie_check_btf,
708 };
709