xref: /openbmc/linux/lib/rhashtable.c (revision c819e2cf)
1 /*
2  * Resizable, Scalable, Concurrent Hash Table
3  *
4  * Copyright (c) 2014 Thomas Graf <tgraf@suug.ch>
5  * Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
6  *
7  * Based on the following paper:
8  * https://www.usenix.org/legacy/event/atc11/tech/final_files/Triplett.pdf
9  *
10  * Code partially derived from nft_hash
11  *
12  * This program is free software; you can redistribute it and/or modify
13  * it under the terms of the GNU General Public License version 2 as
14  * published by the Free Software Foundation.
15  */
16 
17 #include <linux/kernel.h>
18 #include <linux/init.h>
19 #include <linux/log2.h>
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
22 #include <linux/mm.h>
23 #include <linux/jhash.h>
24 #include <linux/random.h>
25 #include <linux/rhashtable.h>
26 
27 #define HASH_DEFAULT_SIZE	64UL
28 #define HASH_MIN_SIZE		4UL
29 
30 #define ASSERT_RHT_MUTEX(HT) BUG_ON(!lockdep_rht_mutex_is_held(HT))
31 
32 #ifdef CONFIG_PROVE_LOCKING
33 int lockdep_rht_mutex_is_held(const struct rhashtable *ht)
34 {
35 	return ht->p.mutex_is_held(ht->p.parent);
36 }
37 EXPORT_SYMBOL_GPL(lockdep_rht_mutex_is_held);
38 #endif
39 
40 static void *rht_obj(const struct rhashtable *ht, const struct rhash_head *he)
41 {
42 	return (void *) he - ht->p.head_offset;
43 }
44 
45 static u32 __hashfn(const struct rhashtable *ht, const void *key,
46 		      u32 len, u32 hsize)
47 {
48 	u32 h;
49 
50 	h = ht->p.hashfn(key, len, ht->p.hash_rnd);
51 
52 	return h & (hsize - 1);
53 }
54 
55 /**
56  * rhashtable_hashfn - compute hash for key of given length
57  * @ht:		hash table to compute for
58  * @key:	pointer to key
59  * @len:	length of key
60  *
61  * Computes the hash value using the hash function provided in the 'hashfn'
62  * of struct rhashtable_params. The returned value is guaranteed to be
63  * smaller than the number of buckets in the hash table.
64  */
65 u32 rhashtable_hashfn(const struct rhashtable *ht, const void *key, u32 len)
66 {
67 	struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
68 
69 	return __hashfn(ht, key, len, tbl->size);
70 }
71 EXPORT_SYMBOL_GPL(rhashtable_hashfn);
72 
73 static u32 obj_hashfn(const struct rhashtable *ht, const void *ptr, u32 hsize)
74 {
75 	if (unlikely(!ht->p.key_len)) {
76 		u32 h;
77 
78 		h = ht->p.obj_hashfn(ptr, ht->p.hash_rnd);
79 
80 		return h & (hsize - 1);
81 	}
82 
83 	return __hashfn(ht, ptr + ht->p.key_offset, ht->p.key_len, hsize);
84 }
85 
86 /**
87  * rhashtable_obj_hashfn - compute hash for hashed object
88  * @ht:		hash table to compute for
89  * @ptr:	pointer to hashed object
90  *
91  * Computes the hash value using the hash function `hashfn` respectively
92  * 'obj_hashfn' depending on whether the hash table is set up to work with
93  * a fixed length key. The returned value is guaranteed to be smaller than
94  * the number of buckets in the hash table.
95  */
96 u32 rhashtable_obj_hashfn(const struct rhashtable *ht, void *ptr)
97 {
98 	struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
99 
100 	return obj_hashfn(ht, ptr, tbl->size);
101 }
102 EXPORT_SYMBOL_GPL(rhashtable_obj_hashfn);
103 
104 static u32 head_hashfn(const struct rhashtable *ht,
105 		       const struct rhash_head *he, u32 hsize)
106 {
107 	return obj_hashfn(ht, rht_obj(ht, he), hsize);
108 }
109 
110 static struct bucket_table *bucket_table_alloc(size_t nbuckets)
111 {
112 	struct bucket_table *tbl;
113 	size_t size;
114 
115 	size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]);
116 	tbl = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
117 	if (tbl == NULL)
118 		tbl = vzalloc(size);
119 
120 	if (tbl == NULL)
121 		return NULL;
122 
123 	tbl->size = nbuckets;
124 
125 	return tbl;
126 }
127 
128 static void bucket_table_free(const struct bucket_table *tbl)
129 {
130 	kvfree(tbl);
131 }
132 
133 /**
134  * rht_grow_above_75 - returns true if nelems > 0.75 * table-size
135  * @ht:		hash table
136  * @new_size:	new table size
137  */
138 bool rht_grow_above_75(const struct rhashtable *ht, size_t new_size)
139 {
140 	/* Expand table when exceeding 75% load */
141 	return ht->nelems > (new_size / 4 * 3);
142 }
143 EXPORT_SYMBOL_GPL(rht_grow_above_75);
144 
145 /**
146  * rht_shrink_below_30 - returns true if nelems < 0.3 * table-size
147  * @ht:		hash table
148  * @new_size:	new table size
149  */
150 bool rht_shrink_below_30(const struct rhashtable *ht, size_t new_size)
151 {
152 	/* Shrink table beneath 30% load */
153 	return ht->nelems < (new_size * 3 / 10);
154 }
155 EXPORT_SYMBOL_GPL(rht_shrink_below_30);
156 
157 static void hashtable_chain_unzip(const struct rhashtable *ht,
158 				  const struct bucket_table *new_tbl,
159 				  struct bucket_table *old_tbl, size_t n)
160 {
161 	struct rhash_head *he, *p, *next;
162 	unsigned int h;
163 
164 	/* Old bucket empty, no work needed. */
165 	p = rht_dereference(old_tbl->buckets[n], ht);
166 	if (!p)
167 		return;
168 
169 	/* Advance the old bucket pointer one or more times until it
170 	 * reaches a node that doesn't hash to the same bucket as the
171 	 * previous node p. Call the previous node p;
172 	 */
173 	h = head_hashfn(ht, p, new_tbl->size);
174 	rht_for_each(he, p->next, ht) {
175 		if (head_hashfn(ht, he, new_tbl->size) != h)
176 			break;
177 		p = he;
178 	}
179 	RCU_INIT_POINTER(old_tbl->buckets[n], p->next);
180 
181 	/* Find the subsequent node which does hash to the same
182 	 * bucket as node P, or NULL if no such node exists.
183 	 */
184 	next = NULL;
185 	if (he) {
186 		rht_for_each(he, he->next, ht) {
187 			if (head_hashfn(ht, he, new_tbl->size) == h) {
188 				next = he;
189 				break;
190 			}
191 		}
192 	}
193 
194 	/* Set p's next pointer to that subsequent node pointer,
195 	 * bypassing the nodes which do not hash to p's bucket
196 	 */
197 	RCU_INIT_POINTER(p->next, next);
198 }
199 
200 /**
201  * rhashtable_expand - Expand hash table while allowing concurrent lookups
202  * @ht:		the hash table to expand
203  *
204  * A secondary bucket array is allocated and the hash entries are migrated
205  * while keeping them on both lists until the end of the RCU grace period.
206  *
207  * This function may only be called in a context where it is safe to call
208  * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
209  *
210  * The caller must ensure that no concurrent table mutations take place.
211  * It is however valid to have concurrent lookups if they are RCU protected.
212  */
213 int rhashtable_expand(struct rhashtable *ht)
214 {
215 	struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);
216 	struct rhash_head *he;
217 	unsigned int i, h;
218 	bool complete;
219 
220 	ASSERT_RHT_MUTEX(ht);
221 
222 	if (ht->p.max_shift && ht->shift >= ht->p.max_shift)
223 		return 0;
224 
225 	new_tbl = bucket_table_alloc(old_tbl->size * 2);
226 	if (new_tbl == NULL)
227 		return -ENOMEM;
228 
229 	ht->shift++;
230 
231 	/* For each new bucket, search the corresponding old bucket
232 	 * for the first entry that hashes to the new bucket, and
233 	 * link the new bucket to that entry. Since all the entries
234 	 * which will end up in the new bucket appear in the same
235 	 * old bucket, this constructs an entirely valid new hash
236 	 * table, but with multiple buckets "zipped" together into a
237 	 * single imprecise chain.
238 	 */
239 	for (i = 0; i < new_tbl->size; i++) {
240 		h = i & (old_tbl->size - 1);
241 		rht_for_each(he, old_tbl->buckets[h], ht) {
242 			if (head_hashfn(ht, he, new_tbl->size) == i) {
243 				RCU_INIT_POINTER(new_tbl->buckets[i], he);
244 				break;
245 			}
246 		}
247 	}
248 
249 	/* Publish the new table pointer. Lookups may now traverse
250 	 * the new table, but they will not benefit from any
251 	 * additional efficiency until later steps unzip the buckets.
252 	 */
253 	rcu_assign_pointer(ht->tbl, new_tbl);
254 
255 	/* Unzip interleaved hash chains */
256 	do {
257 		/* Wait for readers. All new readers will see the new
258 		 * table, and thus no references to the old table will
259 		 * remain.
260 		 */
261 		synchronize_rcu();
262 
263 		/* For each bucket in the old table (each of which
264 		 * contains items from multiple buckets of the new
265 		 * table): ...
266 		 */
267 		complete = true;
268 		for (i = 0; i < old_tbl->size; i++) {
269 			hashtable_chain_unzip(ht, new_tbl, old_tbl, i);
270 			if (old_tbl->buckets[i] != NULL)
271 				complete = false;
272 		}
273 	} while (!complete);
274 
275 	bucket_table_free(old_tbl);
276 	return 0;
277 }
278 EXPORT_SYMBOL_GPL(rhashtable_expand);
279 
280 /**
281  * rhashtable_shrink - Shrink hash table while allowing concurrent lookups
282  * @ht:		the hash table to shrink
283  *
284  * This function may only be called in a context where it is safe to call
285  * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
286  *
287  * The caller must ensure that no concurrent table mutations take place.
288  * It is however valid to have concurrent lookups if they are RCU protected.
289  */
290 int rhashtable_shrink(struct rhashtable *ht)
291 {
292 	struct bucket_table *ntbl, *tbl = rht_dereference(ht->tbl, ht);
293 	struct rhash_head __rcu **pprev;
294 	unsigned int i;
295 
296 	ASSERT_RHT_MUTEX(ht);
297 
298 	if (ht->shift <= ht->p.min_shift)
299 		return 0;
300 
301 	ntbl = bucket_table_alloc(tbl->size / 2);
302 	if (ntbl == NULL)
303 		return -ENOMEM;
304 
305 	ht->shift--;
306 
307 	/* Link each bucket in the new table to the first bucket
308 	 * in the old table that contains entries which will hash
309 	 * to the new bucket.
310 	 */
311 	for (i = 0; i < ntbl->size; i++) {
312 		ntbl->buckets[i] = tbl->buckets[i];
313 
314 		/* Link each bucket in the new table to the first bucket
315 		 * in the old table that contains entries which will hash
316 		 * to the new bucket.
317 		 */
318 		for (pprev = &ntbl->buckets[i]; *pprev != NULL;
319 		     pprev = &rht_dereference(*pprev, ht)->next)
320 			;
321 		RCU_INIT_POINTER(*pprev, tbl->buckets[i + ntbl->size]);
322 	}
323 
324 	/* Publish the new, valid hash table */
325 	rcu_assign_pointer(ht->tbl, ntbl);
326 
327 	/* Wait for readers. No new readers will have references to the
328 	 * old hash table.
329 	 */
330 	synchronize_rcu();
331 
332 	bucket_table_free(tbl);
333 
334 	return 0;
335 }
336 EXPORT_SYMBOL_GPL(rhashtable_shrink);
337 
338 /**
339  * rhashtable_insert - insert object into hash hash table
340  * @ht:		hash table
341  * @obj:	pointer to hash head inside object
342  *
343  * Will automatically grow the table via rhashtable_expand() if the the
344  * grow_decision function specified at rhashtable_init() returns true.
345  *
346  * The caller must ensure that no concurrent table mutations occur. It is
347  * however valid to have concurrent lookups if they are RCU protected.
348  */
349 void rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj)
350 {
351 	struct bucket_table *tbl = rht_dereference(ht->tbl, ht);
352 	u32 hash;
353 
354 	ASSERT_RHT_MUTEX(ht);
355 
356 	hash = head_hashfn(ht, obj, tbl->size);
357 	RCU_INIT_POINTER(obj->next, tbl->buckets[hash]);
358 	rcu_assign_pointer(tbl->buckets[hash], obj);
359 	ht->nelems++;
360 
361 	if (ht->p.grow_decision && ht->p.grow_decision(ht, tbl->size))
362 		rhashtable_expand(ht);
363 }
364 EXPORT_SYMBOL_GPL(rhashtable_insert);
365 
366 /**
367  * rhashtable_remove_pprev - remove object from hash table given previous element
368  * @ht:		hash table
369  * @obj:	pointer to hash head inside object
370  * @pprev:	pointer to previous element
371  *
372  * Identical to rhashtable_remove() but caller is alreayd aware of the element
373  * in front of the element to be deleted. This is in particular useful for
374  * deletion when combined with walking or lookup.
375  */
376 void rhashtable_remove_pprev(struct rhashtable *ht, struct rhash_head *obj,
377 			     struct rhash_head __rcu **pprev)
378 {
379 	struct bucket_table *tbl = rht_dereference(ht->tbl, ht);
380 
381 	ASSERT_RHT_MUTEX(ht);
382 
383 	RCU_INIT_POINTER(*pprev, obj->next);
384 	ht->nelems--;
385 
386 	if (ht->p.shrink_decision &&
387 	    ht->p.shrink_decision(ht, tbl->size))
388 		rhashtable_shrink(ht);
389 }
390 EXPORT_SYMBOL_GPL(rhashtable_remove_pprev);
391 
392 /**
393  * rhashtable_remove - remove object from hash table
394  * @ht:		hash table
395  * @obj:	pointer to hash head inside object
396  *
397  * Since the hash chain is single linked, the removal operation needs to
398  * walk the bucket chain upon removal. The removal operation is thus
399  * considerable slow if the hash table is not correctly sized.
400  *
401  * Will automatically shrink the table via rhashtable_expand() if the the
402  * shrink_decision function specified at rhashtable_init() returns true.
403  *
404  * The caller must ensure that no concurrent table mutations occur. It is
405  * however valid to have concurrent lookups if they are RCU protected.
406  */
407 bool rhashtable_remove(struct rhashtable *ht, struct rhash_head *obj)
408 {
409 	struct bucket_table *tbl = rht_dereference(ht->tbl, ht);
410 	struct rhash_head __rcu **pprev;
411 	struct rhash_head *he;
412 	u32 h;
413 
414 	ASSERT_RHT_MUTEX(ht);
415 
416 	h = head_hashfn(ht, obj, tbl->size);
417 
418 	pprev = &tbl->buckets[h];
419 	rht_for_each(he, tbl->buckets[h], ht) {
420 		if (he != obj) {
421 			pprev = &he->next;
422 			continue;
423 		}
424 
425 		rhashtable_remove_pprev(ht, he, pprev);
426 		return true;
427 	}
428 
429 	return false;
430 }
431 EXPORT_SYMBOL_GPL(rhashtable_remove);
432 
433 /**
434  * rhashtable_lookup - lookup key in hash table
435  * @ht:		hash table
436  * @key:	pointer to key
437  *
438  * Computes the hash value for the key and traverses the bucket chain looking
439  * for a entry with an identical key. The first matching entry is returned.
440  *
441  * This lookup function may only be used for fixed key hash table (key_len
442  * paramter set). It will BUG() if used inappropriately.
443  *
444  * Lookups may occur in parallel with hash mutations as long as the lookup is
445  * guarded by rcu_read_lock(). The caller must take care of this.
446  */
447 void *rhashtable_lookup(const struct rhashtable *ht, const void *key)
448 {
449 	const struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
450 	struct rhash_head *he;
451 	u32 h;
452 
453 	BUG_ON(!ht->p.key_len);
454 
455 	h = __hashfn(ht, key, ht->p.key_len, tbl->size);
456 	rht_for_each_rcu(he, tbl->buckets[h], ht) {
457 		if (memcmp(rht_obj(ht, he) + ht->p.key_offset, key,
458 			   ht->p.key_len))
459 			continue;
460 		return (void *) he - ht->p.head_offset;
461 	}
462 
463 	return NULL;
464 }
465 EXPORT_SYMBOL_GPL(rhashtable_lookup);
466 
467 /**
468  * rhashtable_lookup_compare - search hash table with compare function
469  * @ht:		hash table
470  * @hash:	hash value of desired entry
471  * @compare:	compare function, must return true on match
472  * @arg:	argument passed on to compare function
473  *
474  * Traverses the bucket chain behind the provided hash value and calls the
475  * specified compare function for each entry.
476  *
477  * Lookups may occur in parallel with hash mutations as long as the lookup is
478  * guarded by rcu_read_lock(). The caller must take care of this.
479  *
480  * Returns the first entry on which the compare function returned true.
481  */
482 void *rhashtable_lookup_compare(const struct rhashtable *ht, u32 hash,
483 				bool (*compare)(void *, void *), void *arg)
484 {
485 	const struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
486 	struct rhash_head *he;
487 
488 	if (unlikely(hash >= tbl->size))
489 		return NULL;
490 
491 	rht_for_each_rcu(he, tbl->buckets[hash], ht) {
492 		if (!compare(rht_obj(ht, he), arg))
493 			continue;
494 		return (void *) he - ht->p.head_offset;
495 	}
496 
497 	return NULL;
498 }
499 EXPORT_SYMBOL_GPL(rhashtable_lookup_compare);
500 
501 static size_t rounded_hashtable_size(struct rhashtable_params *params)
502 {
503 	return max(roundup_pow_of_two(params->nelem_hint * 4 / 3),
504 		   1UL << params->min_shift);
505 }
506 
507 /**
508  * rhashtable_init - initialize a new hash table
509  * @ht:		hash table to be initialized
510  * @params:	configuration parameters
511  *
512  * Initializes a new hash table based on the provided configuration
513  * parameters. A table can be configured either with a variable or
514  * fixed length key:
515  *
516  * Configuration Example 1: Fixed length keys
517  * struct test_obj {
518  *	int			key;
519  *	void *			my_member;
520  *	struct rhash_head	node;
521  * };
522  *
523  * struct rhashtable_params params = {
524  *	.head_offset = offsetof(struct test_obj, node),
525  *	.key_offset = offsetof(struct test_obj, key),
526  *	.key_len = sizeof(int),
527  *	.hashfn = jhash,
528  * #ifdef CONFIG_PROVE_LOCKING
529  *	.mutex_is_held = &my_mutex_is_held,
530  * #endif
531  * };
532  *
533  * Configuration Example 2: Variable length keys
534  * struct test_obj {
535  *	[...]
536  *	struct rhash_head	node;
537  * };
538  *
539  * u32 my_hash_fn(const void *data, u32 seed)
540  * {
541  *	struct test_obj *obj = data;
542  *
543  *	return [... hash ...];
544  * }
545  *
546  * struct rhashtable_params params = {
547  *	.head_offset = offsetof(struct test_obj, node),
548  *	.hashfn = jhash,
549  *	.obj_hashfn = my_hash_fn,
550  * #ifdef CONFIG_PROVE_LOCKING
551  *	.mutex_is_held = &my_mutex_is_held,
552  * #endif
553  * };
554  */
555 int rhashtable_init(struct rhashtable *ht, struct rhashtable_params *params)
556 {
557 	struct bucket_table *tbl;
558 	size_t size;
559 
560 	size = HASH_DEFAULT_SIZE;
561 
562 	if ((params->key_len && !params->hashfn) ||
563 	    (!params->key_len && !params->obj_hashfn))
564 		return -EINVAL;
565 
566 	params->min_shift = max_t(size_t, params->min_shift,
567 				  ilog2(HASH_MIN_SIZE));
568 
569 	if (params->nelem_hint)
570 		size = rounded_hashtable_size(params);
571 
572 	tbl = bucket_table_alloc(size);
573 	if (tbl == NULL)
574 		return -ENOMEM;
575 
576 	memset(ht, 0, sizeof(*ht));
577 	ht->shift = ilog2(tbl->size);
578 	memcpy(&ht->p, params, sizeof(*params));
579 	RCU_INIT_POINTER(ht->tbl, tbl);
580 
581 	if (!ht->p.hash_rnd)
582 		get_random_bytes(&ht->p.hash_rnd, sizeof(ht->p.hash_rnd));
583 
584 	return 0;
585 }
586 EXPORT_SYMBOL_GPL(rhashtable_init);
587 
588 /**
589  * rhashtable_destroy - destroy hash table
590  * @ht:		the hash table to destroy
591  *
592  * Frees the bucket array. This function is not rcu safe, therefore the caller
593  * has to make sure that no resizing may happen by unpublishing the hashtable
594  * and waiting for the quiescent cycle before releasing the bucket array.
595  */
596 void rhashtable_destroy(const struct rhashtable *ht)
597 {
598 	bucket_table_free(ht->tbl);
599 }
600 EXPORT_SYMBOL_GPL(rhashtable_destroy);
601 
602 /**************************************************************************
603  * Self Test
604  **************************************************************************/
605 
606 #ifdef CONFIG_TEST_RHASHTABLE
607 
608 #define TEST_HT_SIZE	8
609 #define TEST_ENTRIES	2048
610 #define TEST_PTR	((void *) 0xdeadbeef)
611 #define TEST_NEXPANDS	4
612 
613 #ifdef CONFIG_PROVE_LOCKING
614 static int test_mutex_is_held(void *parent)
615 {
616 	return 1;
617 }
618 #endif
619 
620 struct test_obj {
621 	void			*ptr;
622 	int			value;
623 	struct rhash_head	node;
624 };
625 
626 static int __init test_rht_lookup(struct rhashtable *ht)
627 {
628 	unsigned int i;
629 
630 	for (i = 0; i < TEST_ENTRIES * 2; i++) {
631 		struct test_obj *obj;
632 		bool expected = !(i % 2);
633 		u32 key = i;
634 
635 		obj = rhashtable_lookup(ht, &key);
636 
637 		if (expected && !obj) {
638 			pr_warn("Test failed: Could not find key %u\n", key);
639 			return -ENOENT;
640 		} else if (!expected && obj) {
641 			pr_warn("Test failed: Unexpected entry found for key %u\n",
642 				key);
643 			return -EEXIST;
644 		} else if (expected && obj) {
645 			if (obj->ptr != TEST_PTR || obj->value != i) {
646 				pr_warn("Test failed: Lookup value mismatch %p!=%p, %u!=%u\n",
647 					obj->ptr, TEST_PTR, obj->value, i);
648 				return -EINVAL;
649 			}
650 		}
651 	}
652 
653 	return 0;
654 }
655 
656 static void test_bucket_stats(struct rhashtable *ht, bool quiet)
657 {
658 	unsigned int cnt, rcu_cnt, i, total = 0;
659 	struct test_obj *obj;
660 	struct bucket_table *tbl;
661 
662 	tbl = rht_dereference_rcu(ht->tbl, ht);
663 	for (i = 0; i < tbl->size; i++) {
664 		rcu_cnt = cnt = 0;
665 
666 		if (!quiet)
667 			pr_info(" [%#4x/%zu]", i, tbl->size);
668 
669 		rht_for_each_entry_rcu(obj, tbl->buckets[i], node) {
670 			cnt++;
671 			total++;
672 			if (!quiet)
673 				pr_cont(" [%p],", obj);
674 		}
675 
676 		rht_for_each_entry_rcu(obj, tbl->buckets[i], node)
677 			rcu_cnt++;
678 
679 		if (rcu_cnt != cnt)
680 			pr_warn("Test failed: Chain count mismach %d != %d",
681 				cnt, rcu_cnt);
682 
683 		if (!quiet)
684 			pr_cont("\n  [%#x] first element: %p, chain length: %u\n",
685 				i, tbl->buckets[i], cnt);
686 	}
687 
688 	pr_info("  Traversal complete: counted=%u, nelems=%zu, entries=%d\n",
689 		total, ht->nelems, TEST_ENTRIES);
690 
691 	if (total != ht->nelems || total != TEST_ENTRIES)
692 		pr_warn("Test failed: Total count mismatch ^^^");
693 }
694 
695 static int __init test_rhashtable(struct rhashtable *ht)
696 {
697 	struct bucket_table *tbl;
698 	struct test_obj *obj, *next;
699 	int err;
700 	unsigned int i;
701 
702 	/*
703 	 * Insertion Test:
704 	 * Insert TEST_ENTRIES into table with all keys even numbers
705 	 */
706 	pr_info("  Adding %d keys\n", TEST_ENTRIES);
707 	for (i = 0; i < TEST_ENTRIES; i++) {
708 		struct test_obj *obj;
709 
710 		obj = kzalloc(sizeof(*obj), GFP_KERNEL);
711 		if (!obj) {
712 			err = -ENOMEM;
713 			goto error;
714 		}
715 
716 		obj->ptr = TEST_PTR;
717 		obj->value = i * 2;
718 
719 		rhashtable_insert(ht, &obj->node);
720 	}
721 
722 	rcu_read_lock();
723 	test_bucket_stats(ht, true);
724 	test_rht_lookup(ht);
725 	rcu_read_unlock();
726 
727 	for (i = 0; i < TEST_NEXPANDS; i++) {
728 		pr_info("  Table expansion iteration %u...\n", i);
729 		rhashtable_expand(ht);
730 
731 		rcu_read_lock();
732 		pr_info("  Verifying lookups...\n");
733 		test_rht_lookup(ht);
734 		rcu_read_unlock();
735 	}
736 
737 	for (i = 0; i < TEST_NEXPANDS; i++) {
738 		pr_info("  Table shrinkage iteration %u...\n", i);
739 		rhashtable_shrink(ht);
740 
741 		rcu_read_lock();
742 		pr_info("  Verifying lookups...\n");
743 		test_rht_lookup(ht);
744 		rcu_read_unlock();
745 	}
746 
747 	rcu_read_lock();
748 	test_bucket_stats(ht, true);
749 	rcu_read_unlock();
750 
751 	pr_info("  Deleting %d keys\n", TEST_ENTRIES);
752 	for (i = 0; i < TEST_ENTRIES; i++) {
753 		u32 key = i * 2;
754 
755 		obj = rhashtable_lookup(ht, &key);
756 		BUG_ON(!obj);
757 
758 		rhashtable_remove(ht, &obj->node);
759 		kfree(obj);
760 	}
761 
762 	return 0;
763 
764 error:
765 	tbl = rht_dereference_rcu(ht->tbl, ht);
766 	for (i = 0; i < tbl->size; i++)
767 		rht_for_each_entry_safe(obj, next, tbl->buckets[i], ht, node)
768 			kfree(obj);
769 
770 	return err;
771 }
772 
773 static int __init test_rht_init(void)
774 {
775 	struct rhashtable ht;
776 	struct rhashtable_params params = {
777 		.nelem_hint = TEST_HT_SIZE,
778 		.head_offset = offsetof(struct test_obj, node),
779 		.key_offset = offsetof(struct test_obj, value),
780 		.key_len = sizeof(int),
781 		.hashfn = jhash,
782 #ifdef CONFIG_PROVE_LOCKING
783 		.mutex_is_held = &test_mutex_is_held,
784 #endif
785 		.grow_decision = rht_grow_above_75,
786 		.shrink_decision = rht_shrink_below_30,
787 	};
788 	int err;
789 
790 	pr_info("Running resizable hashtable tests...\n");
791 
792 	err = rhashtable_init(&ht, &params);
793 	if (err < 0) {
794 		pr_warn("Test failed: Unable to initialize hashtable: %d\n",
795 			err);
796 		return err;
797 	}
798 
799 	err = test_rhashtable(&ht);
800 
801 	rhashtable_destroy(&ht);
802 
803 	return err;
804 }
805 
806 subsys_initcall(test_rht_init);
807 
808 #endif /* CONFIG_TEST_RHASHTABLE */
809