1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2013 Fusion IO.  All rights reserved.
4  */
5 
6 #include <linux/slab.h>
7 #include "btrfs-tests.h"
8 #include "../ctree.h"
9 #include "../disk-io.h"
10 #include "../free-space-cache.h"
11 #include "../block-group.h"
12 
13 #define BITS_PER_BITMAP		(PAGE_SIZE * 8UL)
14 
15 /*
16  * This test just does basic sanity checking, making sure we can add an extent
17  * entry and remove space from either end and the middle, and make sure we can
18  * remove space that covers adjacent extent entries.
19  */
20 static int test_extents(struct btrfs_block_group *cache)
21 {
22 	int ret = 0;
23 
24 	test_msg("running extent only tests");
25 
26 	/* First just make sure we can remove an entire entry */
27 	ret = btrfs_add_free_space(cache, 0, SZ_4M);
28 	if (ret) {
29 		test_err("error adding initial extents %d", ret);
30 		return ret;
31 	}
32 
33 	ret = btrfs_remove_free_space(cache, 0, SZ_4M);
34 	if (ret) {
35 		test_err("error removing extent %d", ret);
36 		return ret;
37 	}
38 
39 	if (test_check_exists(cache, 0, SZ_4M)) {
40 		test_err("full remove left some lingering space");
41 		return -1;
42 	}
43 
44 	/* Ok edge and middle cases now */
45 	ret = btrfs_add_free_space(cache, 0, SZ_4M);
46 	if (ret) {
47 		test_err("error adding half extent %d", ret);
48 		return ret;
49 	}
50 
51 	ret = btrfs_remove_free_space(cache, 3 * SZ_1M, SZ_1M);
52 	if (ret) {
53 		test_err("error removing tail end %d", ret);
54 		return ret;
55 	}
56 
57 	ret = btrfs_remove_free_space(cache, 0, SZ_1M);
58 	if (ret) {
59 		test_err("error removing front end %d", ret);
60 		return ret;
61 	}
62 
63 	ret = btrfs_remove_free_space(cache, SZ_2M, 4096);
64 	if (ret) {
65 		test_err("error removing middle piece %d", ret);
66 		return ret;
67 	}
68 
69 	if (test_check_exists(cache, 0, SZ_1M)) {
70 		test_err("still have space at the front");
71 		return -1;
72 	}
73 
74 	if (test_check_exists(cache, SZ_2M, 4096)) {
75 		test_err("still have space in the middle");
76 		return -1;
77 	}
78 
79 	if (test_check_exists(cache, 3 * SZ_1M, SZ_1M)) {
80 		test_err("still have space at the end");
81 		return -1;
82 	}
83 
84 	/* Cleanup */
85 	btrfs_remove_free_space_cache(cache);
86 
87 	return 0;
88 }
89 
90 static int test_bitmaps(struct btrfs_block_group *cache, u32 sectorsize)
91 {
92 	u64 next_bitmap_offset;
93 	int ret;
94 
95 	test_msg("running bitmap only tests");
96 
97 	ret = test_add_free_space_entry(cache, 0, SZ_4M, 1);
98 	if (ret) {
99 		test_err("couldn't create a bitmap entry %d", ret);
100 		return ret;
101 	}
102 
103 	ret = btrfs_remove_free_space(cache, 0, SZ_4M);
104 	if (ret) {
105 		test_err("error removing bitmap full range %d", ret);
106 		return ret;
107 	}
108 
109 	if (test_check_exists(cache, 0, SZ_4M)) {
110 		test_err("left some space in bitmap");
111 		return -1;
112 	}
113 
114 	ret = test_add_free_space_entry(cache, 0, SZ_4M, 1);
115 	if (ret) {
116 		test_err("couldn't add to our bitmap entry %d", ret);
117 		return ret;
118 	}
119 
120 	ret = btrfs_remove_free_space(cache, SZ_1M, SZ_2M);
121 	if (ret) {
122 		test_err("couldn't remove middle chunk %d", ret);
123 		return ret;
124 	}
125 
126 	/*
127 	 * The first bitmap we have starts at offset 0 so the next one is just
128 	 * at the end of the first bitmap.
129 	 */
130 	next_bitmap_offset = (u64)(BITS_PER_BITMAP * sectorsize);
131 
132 	/* Test a bit straddling two bitmaps */
133 	ret = test_add_free_space_entry(cache, next_bitmap_offset - SZ_2M,
134 					SZ_4M, 1);
135 	if (ret) {
136 		test_err("couldn't add space that straddles two bitmaps %d",
137 				ret);
138 		return ret;
139 	}
140 
141 	ret = btrfs_remove_free_space(cache, next_bitmap_offset - SZ_1M, SZ_2M);
142 	if (ret) {
143 		test_err("couldn't remove overlapping space %d", ret);
144 		return ret;
145 	}
146 
147 	if (test_check_exists(cache, next_bitmap_offset - SZ_1M, SZ_2M)) {
148 		test_err("left some space when removing overlapping");
149 		return -1;
150 	}
151 
152 	btrfs_remove_free_space_cache(cache);
153 
154 	return 0;
155 }
156 
157 /* This is the high grade jackassery */
158 static int test_bitmaps_and_extents(struct btrfs_block_group *cache,
159 				    u32 sectorsize)
160 {
161 	u64 bitmap_offset = (u64)(BITS_PER_BITMAP * sectorsize);
162 	int ret;
163 
164 	test_msg("running bitmap and extent tests");
165 
166 	/*
167 	 * First let's do something simple, an extent at the same offset as the
168 	 * bitmap, but the free space completely in the extent and then
169 	 * completely in the bitmap.
170 	 */
171 	ret = test_add_free_space_entry(cache, SZ_4M, SZ_1M, 1);
172 	if (ret) {
173 		test_err("couldn't create bitmap entry %d", ret);
174 		return ret;
175 	}
176 
177 	ret = test_add_free_space_entry(cache, 0, SZ_1M, 0);
178 	if (ret) {
179 		test_err("couldn't add extent entry %d", ret);
180 		return ret;
181 	}
182 
183 	ret = btrfs_remove_free_space(cache, 0, SZ_1M);
184 	if (ret) {
185 		test_err("couldn't remove extent entry %d", ret);
186 		return ret;
187 	}
188 
189 	if (test_check_exists(cache, 0, SZ_1M)) {
190 		test_err("left remnants after our remove");
191 		return -1;
192 	}
193 
194 	/* Now to add back the extent entry and remove from the bitmap */
195 	ret = test_add_free_space_entry(cache, 0, SZ_1M, 0);
196 	if (ret) {
197 		test_err("couldn't re-add extent entry %d", ret);
198 		return ret;
199 	}
200 
201 	ret = btrfs_remove_free_space(cache, SZ_4M, SZ_1M);
202 	if (ret) {
203 		test_err("couldn't remove from bitmap %d", ret);
204 		return ret;
205 	}
206 
207 	if (test_check_exists(cache, SZ_4M, SZ_1M)) {
208 		test_err("left remnants in the bitmap");
209 		return -1;
210 	}
211 
212 	/*
213 	 * Ok so a little more evil, extent entry and bitmap at the same offset,
214 	 * removing an overlapping chunk.
215 	 */
216 	ret = test_add_free_space_entry(cache, SZ_1M, SZ_4M, 1);
217 	if (ret) {
218 		test_err("couldn't add to a bitmap %d", ret);
219 		return ret;
220 	}
221 
222 	ret = btrfs_remove_free_space(cache, SZ_512K, 3 * SZ_1M);
223 	if (ret) {
224 		test_err("couldn't remove overlapping space %d", ret);
225 		return ret;
226 	}
227 
228 	if (test_check_exists(cache, SZ_512K, 3 * SZ_1M)) {
229 		test_err("left over pieces after removing overlapping");
230 		return -1;
231 	}
232 
233 	btrfs_remove_free_space_cache(cache);
234 
235 	/* Now with the extent entry offset into the bitmap */
236 	ret = test_add_free_space_entry(cache, SZ_4M, SZ_4M, 1);
237 	if (ret) {
238 		test_err("couldn't add space to the bitmap %d", ret);
239 		return ret;
240 	}
241 
242 	ret = test_add_free_space_entry(cache, SZ_2M, SZ_2M, 0);
243 	if (ret) {
244 		test_err("couldn't add extent to the cache %d", ret);
245 		return ret;
246 	}
247 
248 	ret = btrfs_remove_free_space(cache, 3 * SZ_1M, SZ_4M);
249 	if (ret) {
250 		test_err("problem removing overlapping space %d", ret);
251 		return ret;
252 	}
253 
254 	if (test_check_exists(cache, 3 * SZ_1M, SZ_4M)) {
255 		test_err("left something behind when removing space");
256 		return -1;
257 	}
258 
259 	/*
260 	 * This has blown up in the past, the extent entry starts before the
261 	 * bitmap entry, but we're trying to remove an offset that falls
262 	 * completely within the bitmap range and is in both the extent entry
263 	 * and the bitmap entry, looks like this
264 	 *
265 	 *   [ extent ]
266 	 *      [ bitmap ]
267 	 *        [ del ]
268 	 */
269 	btrfs_remove_free_space_cache(cache);
270 	ret = test_add_free_space_entry(cache, bitmap_offset + SZ_4M, SZ_4M, 1);
271 	if (ret) {
272 		test_err("couldn't add bitmap %d", ret);
273 		return ret;
274 	}
275 
276 	ret = test_add_free_space_entry(cache, bitmap_offset - SZ_1M,
277 					5 * SZ_1M, 0);
278 	if (ret) {
279 		test_err("couldn't add extent entry %d", ret);
280 		return ret;
281 	}
282 
283 	ret = btrfs_remove_free_space(cache, bitmap_offset + SZ_1M, 5 * SZ_1M);
284 	if (ret) {
285 		test_err("failed to free our space %d", ret);
286 		return ret;
287 	}
288 
289 	if (test_check_exists(cache, bitmap_offset + SZ_1M, 5 * SZ_1M)) {
290 		test_err("left stuff over");
291 		return -1;
292 	}
293 
294 	btrfs_remove_free_space_cache(cache);
295 
296 	/*
297 	 * This blew up before, we have part of the free space in a bitmap and
298 	 * then the entirety of the rest of the space in an extent.  This used
299 	 * to return -EAGAIN back from btrfs_remove_extent, make sure this
300 	 * doesn't happen.
301 	 */
302 	ret = test_add_free_space_entry(cache, SZ_1M, SZ_2M, 1);
303 	if (ret) {
304 		test_err("couldn't add bitmap entry %d", ret);
305 		return ret;
306 	}
307 
308 	ret = test_add_free_space_entry(cache, 3 * SZ_1M, SZ_1M, 0);
309 	if (ret) {
310 		test_err("couldn't add extent entry %d", ret);
311 		return ret;
312 	}
313 
314 	ret = btrfs_remove_free_space(cache, SZ_1M, 3 * SZ_1M);
315 	if (ret) {
316 		test_err("error removing bitmap and extent overlapping %d", ret);
317 		return ret;
318 	}
319 
320 	btrfs_remove_free_space_cache(cache);
321 	return 0;
322 }
323 
324 /* Used by test_steal_space_from_bitmap_to_extent(). */
325 static bool test_use_bitmap(struct btrfs_free_space_ctl *ctl,
326 			    struct btrfs_free_space *info)
327 {
328 	return ctl->free_extents > 0;
329 }
330 
331 /* Used by test_steal_space_from_bitmap_to_extent(). */
332 static int
333 check_num_extents_and_bitmaps(const struct btrfs_block_group *cache,
334 			      const int num_extents,
335 			      const int num_bitmaps)
336 {
337 	if (cache->free_space_ctl->free_extents != num_extents) {
338 		test_err(
339 		"incorrect # of extent entries in the cache: %d, expected %d",
340 			 cache->free_space_ctl->free_extents, num_extents);
341 		return -EINVAL;
342 	}
343 	if (cache->free_space_ctl->total_bitmaps != num_bitmaps) {
344 		test_err(
345 		"incorrect # of extent entries in the cache: %d, expected %d",
346 			 cache->free_space_ctl->total_bitmaps, num_bitmaps);
347 		return -EINVAL;
348 	}
349 	return 0;
350 }
351 
352 /* Used by test_steal_space_from_bitmap_to_extent(). */
353 static int check_cache_empty(struct btrfs_block_group *cache)
354 {
355 	u64 offset;
356 	u64 max_extent_size;
357 
358 	/*
359 	 * Now lets confirm that there's absolutely no free space left to
360 	 * allocate.
361 	 */
362 	if (cache->free_space_ctl->free_space != 0) {
363 		test_err("cache free space is not 0");
364 		return -EINVAL;
365 	}
366 
367 	/* And any allocation request, no matter how small, should fail now. */
368 	offset = btrfs_find_space_for_alloc(cache, 0, 4096, 0,
369 					    &max_extent_size);
370 	if (offset != 0) {
371 		test_err("space allocation did not fail, returned offset: %llu",
372 			 offset);
373 		return -EINVAL;
374 	}
375 
376 	/* And no extent nor bitmap entries in the cache anymore. */
377 	return check_num_extents_and_bitmaps(cache, 0, 0);
378 }
379 
380 /*
381  * Before we were able to steal free space from a bitmap entry to an extent
382  * entry, we could end up with 2 entries representing a contiguous free space.
383  * One would be an extent entry and the other a bitmap entry. Since in order
384  * to allocate space to a caller we use only 1 entry, we couldn't return that
385  * whole range to the caller if it was requested. This forced the caller to
386  * either assume ENOSPC or perform several smaller space allocations, which
387  * wasn't optimal as they could be spread all over the block group while under
388  * concurrency (extra overhead and fragmentation).
389  *
390  * This stealing approach is beneficial, since we always prefer to allocate
391  * from extent entries, both for clustered and non-clustered allocation
392  * requests.
393  */
394 static int
395 test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache,
396 				       u32 sectorsize)
397 {
398 	int ret;
399 	u64 offset;
400 	u64 max_extent_size;
401 	const struct btrfs_free_space_op test_free_space_ops = {
402 		.use_bitmap = test_use_bitmap,
403 	};
404 	const struct btrfs_free_space_op *orig_free_space_ops;
405 
406 	test_msg("running space stealing from bitmap to extent tests");
407 
408 	/*
409 	 * For this test, we want to ensure we end up with an extent entry
410 	 * immediately adjacent to a bitmap entry, where the bitmap starts
411 	 * at an offset where the extent entry ends. We keep adding and
412 	 * removing free space to reach into this state, but to get there
413 	 * we need to reach a point where marking new free space doesn't
414 	 * result in adding new extent entries or merging the new space
415 	 * with existing extent entries - the space ends up being marked
416 	 * in an existing bitmap that covers the new free space range.
417 	 *
418 	 * To get there, we need to reach the threshold defined set at
419 	 * cache->free_space_ctl->extents_thresh, which currently is
420 	 * 256 extents on a x86_64 system at least, and a few other
421 	 * conditions (check free_space_cache.c). Instead of making the
422 	 * test much longer and complicated, use a "use_bitmap" operation
423 	 * that forces use of bitmaps as soon as we have at least 1
424 	 * extent entry.
425 	 */
426 	orig_free_space_ops = cache->free_space_ctl->op;
427 	cache->free_space_ctl->op = &test_free_space_ops;
428 
429 	/*
430 	 * Extent entry covering free space range [128Mb - 256Kb, 128Mb - 128Kb[
431 	 */
432 	ret = test_add_free_space_entry(cache, SZ_128M - SZ_256K, SZ_128K, 0);
433 	if (ret) {
434 		test_err("couldn't add extent entry %d", ret);
435 		return ret;
436 	}
437 
438 	/* Bitmap entry covering free space range [128Mb + 512Kb, 256Mb[ */
439 	ret = test_add_free_space_entry(cache, SZ_128M + SZ_512K,
440 					SZ_128M - SZ_512K, 1);
441 	if (ret) {
442 		test_err("couldn't add bitmap entry %d", ret);
443 		return ret;
444 	}
445 
446 	ret = check_num_extents_and_bitmaps(cache, 2, 1);
447 	if (ret)
448 		return ret;
449 
450 	/*
451 	 * Now make only the first 256Kb of the bitmap marked as free, so that
452 	 * we end up with only the following ranges marked as free space:
453 	 *
454 	 * [128Mb - 256Kb, 128Mb - 128Kb[
455 	 * [128Mb + 512Kb, 128Mb + 768Kb[
456 	 */
457 	ret = btrfs_remove_free_space(cache,
458 				      SZ_128M + 768 * SZ_1K,
459 				      SZ_128M - 768 * SZ_1K);
460 	if (ret) {
461 		test_err("failed to free part of bitmap space %d", ret);
462 		return ret;
463 	}
464 
465 	/* Confirm that only those 2 ranges are marked as free. */
466 	if (!test_check_exists(cache, SZ_128M - SZ_256K, SZ_128K)) {
467 		test_err("free space range missing");
468 		return -ENOENT;
469 	}
470 	if (!test_check_exists(cache, SZ_128M + SZ_512K, SZ_256K)) {
471 		test_err("free space range missing");
472 		return -ENOENT;
473 	}
474 
475 	/*
476 	 * Confirm that the bitmap range [128Mb + 768Kb, 256Mb[ isn't marked
477 	 * as free anymore.
478 	 */
479 	if (test_check_exists(cache, SZ_128M + 768 * SZ_1K,
480 			      SZ_128M - 768 * SZ_1K)) {
481 		test_err("bitmap region not removed from space cache");
482 		return -EINVAL;
483 	}
484 
485 	/*
486 	 * Confirm that the region [128Mb + 256Kb, 128Mb + 512Kb[, which is
487 	 * covered by the bitmap, isn't marked as free.
488 	 */
489 	if (test_check_exists(cache, SZ_128M + SZ_256K, SZ_256K)) {
490 		test_err("invalid bitmap region marked as free");
491 		return -EINVAL;
492 	}
493 
494 	/*
495 	 * Confirm that the region [128Mb, 128Mb + 256Kb[, which is covered
496 	 * by the bitmap too, isn't marked as free either.
497 	 */
498 	if (test_check_exists(cache, SZ_128M, SZ_256K)) {
499 		test_err("invalid bitmap region marked as free");
500 		return -EINVAL;
501 	}
502 
503 	/*
504 	 * Now lets mark the region [128Mb, 128Mb + 512Kb[ as free too. But,
505 	 * lets make sure the free space cache marks it as free in the bitmap,
506 	 * and doesn't insert a new extent entry to represent this region.
507 	 */
508 	ret = btrfs_add_free_space(cache, SZ_128M, SZ_512K);
509 	if (ret) {
510 		test_err("error adding free space: %d", ret);
511 		return ret;
512 	}
513 	/* Confirm the region is marked as free. */
514 	if (!test_check_exists(cache, SZ_128M, SZ_512K)) {
515 		test_err("bitmap region not marked as free");
516 		return -ENOENT;
517 	}
518 
519 	/*
520 	 * Confirm that no new extent entries or bitmap entries were added to
521 	 * the cache after adding that free space region.
522 	 */
523 	ret = check_num_extents_and_bitmaps(cache, 2, 1);
524 	if (ret)
525 		return ret;
526 
527 	/*
528 	 * Now lets add a small free space region to the right of the previous
529 	 * one, which is not contiguous with it and is part of the bitmap too.
530 	 * The goal is to test that the bitmap entry space stealing doesn't
531 	 * steal this space region.
532 	 */
533 	ret = btrfs_add_free_space(cache, SZ_128M + SZ_16M, sectorsize);
534 	if (ret) {
535 		test_err("error adding free space: %d", ret);
536 		return ret;
537 	}
538 
539 	/*
540 	 * Confirm that no new extent entries or bitmap entries were added to
541 	 * the cache after adding that free space region.
542 	 */
543 	ret = check_num_extents_and_bitmaps(cache, 2, 1);
544 	if (ret)
545 		return ret;
546 
547 	/*
548 	 * Now mark the region [128Mb - 128Kb, 128Mb[ as free too. This will
549 	 * expand the range covered by the existing extent entry that represents
550 	 * the free space [128Mb - 256Kb, 128Mb - 128Kb[.
551 	 */
552 	ret = btrfs_add_free_space(cache, SZ_128M - SZ_128K, SZ_128K);
553 	if (ret) {
554 		test_err("error adding free space: %d", ret);
555 		return ret;
556 	}
557 	/* Confirm the region is marked as free. */
558 	if (!test_check_exists(cache, SZ_128M - SZ_128K, SZ_128K)) {
559 		test_err("extent region not marked as free");
560 		return -ENOENT;
561 	}
562 
563 	/*
564 	 * Confirm that our extent entry didn't stole all free space from the
565 	 * bitmap, because of the small 4Kb free space region.
566 	 */
567 	ret = check_num_extents_and_bitmaps(cache, 2, 1);
568 	if (ret)
569 		return ret;
570 
571 	/*
572 	 * So now we have the range [128Mb - 256Kb, 128Mb + 768Kb[ as free
573 	 * space. Without stealing bitmap free space into extent entry space,
574 	 * we would have all this free space represented by 2 entries in the
575 	 * cache:
576 	 *
577 	 * extent entry covering range: [128Mb - 256Kb, 128Mb[
578 	 * bitmap entry covering range: [128Mb, 128Mb + 768Kb[
579 	 *
580 	 * Attempting to allocate the whole free space (1Mb) would fail, because
581 	 * we can't allocate from multiple entries.
582 	 * With the bitmap free space stealing, we get a single extent entry
583 	 * that represents the 1Mb free space, and therefore we're able to
584 	 * allocate the whole free space at once.
585 	 */
586 	if (!test_check_exists(cache, SZ_128M - SZ_256K, SZ_1M)) {
587 		test_err("expected region not marked as free");
588 		return -ENOENT;
589 	}
590 
591 	if (cache->free_space_ctl->free_space != (SZ_1M + sectorsize)) {
592 		test_err("cache free space is not 1Mb + %u", sectorsize);
593 		return -EINVAL;
594 	}
595 
596 	offset = btrfs_find_space_for_alloc(cache,
597 					    0, SZ_1M, 0,
598 					    &max_extent_size);
599 	if (offset != (SZ_128M - SZ_256K)) {
600 		test_err(
601 	"failed to allocate 1Mb from space cache, returned offset is: %llu",
602 			 offset);
603 		return -EINVAL;
604 	}
605 
606 	/*
607 	 * All that remains is a sectorsize free space region in a bitmap.
608 	 * Confirm.
609 	 */
610 	ret = check_num_extents_and_bitmaps(cache, 1, 1);
611 	if (ret)
612 		return ret;
613 
614 	if (cache->free_space_ctl->free_space != sectorsize) {
615 		test_err("cache free space is not %u", sectorsize);
616 		return -EINVAL;
617 	}
618 
619 	offset = btrfs_find_space_for_alloc(cache,
620 					    0, sectorsize, 0,
621 					    &max_extent_size);
622 	if (offset != (SZ_128M + SZ_16M)) {
623 		test_err("failed to allocate %u, returned offset : %llu",
624 			 sectorsize, offset);
625 		return -EINVAL;
626 	}
627 
628 	ret = check_cache_empty(cache);
629 	if (ret)
630 		return ret;
631 
632 	btrfs_remove_free_space_cache(cache);
633 
634 	/*
635 	 * Now test a similar scenario, but where our extent entry is located
636 	 * to the right of the bitmap entry, so that we can check that stealing
637 	 * space from a bitmap to the front of an extent entry works.
638 	 */
639 
640 	/*
641 	 * Extent entry covering free space range [128Mb + 128Kb, 128Mb + 256Kb[
642 	 */
643 	ret = test_add_free_space_entry(cache, SZ_128M + SZ_128K, SZ_128K, 0);
644 	if (ret) {
645 		test_err("couldn't add extent entry %d", ret);
646 		return ret;
647 	}
648 
649 	/* Bitmap entry covering free space range [0, 128Mb - 512Kb[ */
650 	ret = test_add_free_space_entry(cache, 0, SZ_128M - SZ_512K, 1);
651 	if (ret) {
652 		test_err("couldn't add bitmap entry %d", ret);
653 		return ret;
654 	}
655 
656 	ret = check_num_extents_and_bitmaps(cache, 2, 1);
657 	if (ret)
658 		return ret;
659 
660 	/*
661 	 * Now make only the last 256Kb of the bitmap marked as free, so that
662 	 * we end up with only the following ranges marked as free space:
663 	 *
664 	 * [128Mb + 128b, 128Mb + 256Kb[
665 	 * [128Mb - 768Kb, 128Mb - 512Kb[
666 	 */
667 	ret = btrfs_remove_free_space(cache, 0, SZ_128M - 768 * SZ_1K);
668 	if (ret) {
669 		test_err("failed to free part of bitmap space %d", ret);
670 		return ret;
671 	}
672 
673 	/* Confirm that only those 2 ranges are marked as free. */
674 	if (!test_check_exists(cache, SZ_128M + SZ_128K, SZ_128K)) {
675 		test_err("free space range missing");
676 		return -ENOENT;
677 	}
678 	if (!test_check_exists(cache, SZ_128M - 768 * SZ_1K, SZ_256K)) {
679 		test_err("free space range missing");
680 		return -ENOENT;
681 	}
682 
683 	/*
684 	 * Confirm that the bitmap range [0, 128Mb - 768Kb[ isn't marked
685 	 * as free anymore.
686 	 */
687 	if (test_check_exists(cache, 0, SZ_128M - 768 * SZ_1K)) {
688 		test_err("bitmap region not removed from space cache");
689 		return -EINVAL;
690 	}
691 
692 	/*
693 	 * Confirm that the region [128Mb - 512Kb, 128Mb[, which is
694 	 * covered by the bitmap, isn't marked as free.
695 	 */
696 	if (test_check_exists(cache, SZ_128M - SZ_512K, SZ_512K)) {
697 		test_err("invalid bitmap region marked as free");
698 		return -EINVAL;
699 	}
700 
701 	/*
702 	 * Now lets mark the region [128Mb - 512Kb, 128Mb[ as free too. But,
703 	 * lets make sure the free space cache marks it as free in the bitmap,
704 	 * and doesn't insert a new extent entry to represent this region.
705 	 */
706 	ret = btrfs_add_free_space(cache, SZ_128M - SZ_512K, SZ_512K);
707 	if (ret) {
708 		test_err("error adding free space: %d", ret);
709 		return ret;
710 	}
711 	/* Confirm the region is marked as free. */
712 	if (!test_check_exists(cache, SZ_128M - SZ_512K, SZ_512K)) {
713 		test_err("bitmap region not marked as free");
714 		return -ENOENT;
715 	}
716 
717 	/*
718 	 * Confirm that no new extent entries or bitmap entries were added to
719 	 * the cache after adding that free space region.
720 	 */
721 	ret = check_num_extents_and_bitmaps(cache, 2, 1);
722 	if (ret)
723 		return ret;
724 
725 	/*
726 	 * Now lets add a small free space region to the left of the previous
727 	 * one, which is not contiguous with it and is part of the bitmap too.
728 	 * The goal is to test that the bitmap entry space stealing doesn't
729 	 * steal this space region.
730 	 */
731 	ret = btrfs_add_free_space(cache, SZ_32M, 2 * sectorsize);
732 	if (ret) {
733 		test_err("error adding free space: %d", ret);
734 		return ret;
735 	}
736 
737 	/*
738 	 * Now mark the region [128Mb, 128Mb + 128Kb[ as free too. This will
739 	 * expand the range covered by the existing extent entry that represents
740 	 * the free space [128Mb + 128Kb, 128Mb + 256Kb[.
741 	 */
742 	ret = btrfs_add_free_space(cache, SZ_128M, SZ_128K);
743 	if (ret) {
744 		test_err("error adding free space: %d", ret);
745 		return ret;
746 	}
747 	/* Confirm the region is marked as free. */
748 	if (!test_check_exists(cache, SZ_128M, SZ_128K)) {
749 		test_err("extent region not marked as free");
750 		return -ENOENT;
751 	}
752 
753 	/*
754 	 * Confirm that our extent entry didn't stole all free space from the
755 	 * bitmap, because of the small 2 * sectorsize free space region.
756 	 */
757 	ret = check_num_extents_and_bitmaps(cache, 2, 1);
758 	if (ret)
759 		return ret;
760 
761 	/*
762 	 * So now we have the range [128Mb - 768Kb, 128Mb + 256Kb[ as free
763 	 * space. Without stealing bitmap free space into extent entry space,
764 	 * we would have all this free space represented by 2 entries in the
765 	 * cache:
766 	 *
767 	 * extent entry covering range: [128Mb, 128Mb + 256Kb[
768 	 * bitmap entry covering range: [128Mb - 768Kb, 128Mb[
769 	 *
770 	 * Attempting to allocate the whole free space (1Mb) would fail, because
771 	 * we can't allocate from multiple entries.
772 	 * With the bitmap free space stealing, we get a single extent entry
773 	 * that represents the 1Mb free space, and therefore we're able to
774 	 * allocate the whole free space at once.
775 	 */
776 	if (!test_check_exists(cache, SZ_128M - 768 * SZ_1K, SZ_1M)) {
777 		test_err("expected region not marked as free");
778 		return -ENOENT;
779 	}
780 
781 	if (cache->free_space_ctl->free_space != (SZ_1M + 2 * sectorsize)) {
782 		test_err("cache free space is not 1Mb + %u", 2 * sectorsize);
783 		return -EINVAL;
784 	}
785 
786 	offset = btrfs_find_space_for_alloc(cache, 0, SZ_1M, 0,
787 					    &max_extent_size);
788 	if (offset != (SZ_128M - 768 * SZ_1K)) {
789 		test_err(
790 	"failed to allocate 1Mb from space cache, returned offset is: %llu",
791 			 offset);
792 		return -EINVAL;
793 	}
794 
795 	/*
796 	 * All that remains is 2 * sectorsize free space region
797 	 * in a bitmap. Confirm.
798 	 */
799 	ret = check_num_extents_and_bitmaps(cache, 1, 1);
800 	if (ret)
801 		return ret;
802 
803 	if (cache->free_space_ctl->free_space != 2 * sectorsize) {
804 		test_err("cache free space is not %u", 2 * sectorsize);
805 		return -EINVAL;
806 	}
807 
808 	offset = btrfs_find_space_for_alloc(cache,
809 					    0, 2 * sectorsize, 0,
810 					    &max_extent_size);
811 	if (offset != SZ_32M) {
812 		test_err("failed to allocate %u, offset: %llu",
813 			 2 * sectorsize, offset);
814 		return -EINVAL;
815 	}
816 
817 	ret = check_cache_empty(cache);
818 	if (ret)
819 		return ret;
820 
821 	cache->free_space_ctl->op = orig_free_space_ops;
822 	btrfs_remove_free_space_cache(cache);
823 
824 	return 0;
825 }
826 
827 static bool bytes_index_use_bitmap(struct btrfs_free_space_ctl *ctl,
828 				   struct btrfs_free_space *info)
829 {
830 	return true;
831 }
832 
833 static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize)
834 {
835 	const struct btrfs_free_space_op test_free_space_ops = {
836 		.use_bitmap = bytes_index_use_bitmap,
837 	};
838 	const struct btrfs_free_space_op *orig_free_space_ops;
839 	struct btrfs_free_space_ctl *ctl = cache->free_space_ctl;
840 	struct btrfs_free_space *entry;
841 	struct rb_node *node;
842 	u64 offset, max_extent_size, bytes;
843 	int ret, i;
844 
845 	test_msg("running bytes index tests");
846 
847 	/* First just validate that it does everything in order. */
848 	offset = 0;
849 	for (i = 0; i < 10; i++) {
850 		bytes = (i + 1) * SZ_1M;
851 		ret = test_add_free_space_entry(cache, offset, bytes, 0);
852 		if (ret) {
853 			test_err("couldn't add extent entry %d\n", ret);
854 			return ret;
855 		}
856 		offset += bytes + sectorsize;
857 	}
858 
859 	for (node = rb_first_cached(&ctl->free_space_bytes), i = 9; node;
860 	     node = rb_next(node), i--) {
861 		entry = rb_entry(node, struct btrfs_free_space, bytes_index);
862 		bytes = (i + 1) * SZ_1M;
863 		if (entry->bytes != bytes) {
864 			test_err("invalid bytes index order, found %llu expected %llu",
865 				 entry->bytes, bytes);
866 			return -EINVAL;
867 		}
868 	}
869 
870 	/* Now validate bitmaps do the correct thing. */
871 	btrfs_remove_free_space_cache(cache);
872 	for (i = 0; i < 2; i++) {
873 		offset = i * BITS_PER_BITMAP * sectorsize;
874 		bytes = (i + 1) * SZ_1M;
875 		ret = test_add_free_space_entry(cache, offset, bytes, 1);
876 		if (ret) {
877 			test_err("couldn't add bitmap entry");
878 			return ret;
879 		}
880 	}
881 
882 	for (node = rb_first_cached(&ctl->free_space_bytes), i = 1; node;
883 	     node = rb_next(node), i--) {
884 		entry = rb_entry(node, struct btrfs_free_space, bytes_index);
885 		bytes = (i + 1) * SZ_1M;
886 		if (entry->bytes != bytes) {
887 			test_err("invalid bytes index order, found %llu expected %llu",
888 				 entry->bytes, bytes);
889 			return -EINVAL;
890 		}
891 	}
892 
893 	/* Now validate bitmaps with different ->max_extent_size. */
894 	btrfs_remove_free_space_cache(cache);
895 	orig_free_space_ops = cache->free_space_ctl->op;
896 	cache->free_space_ctl->op = &test_free_space_ops;
897 
898 	ret = test_add_free_space_entry(cache, 0, sectorsize, 1);
899 	if (ret) {
900 		test_err("couldn't add bitmap entry");
901 		return ret;
902 	}
903 
904 	offset = BITS_PER_BITMAP * sectorsize;
905 	ret = test_add_free_space_entry(cache, offset, sectorsize, 1);
906 	if (ret) {
907 		test_err("couldn't add bitmap_entry");
908 		return ret;
909 	}
910 
911 	/*
912 	 * Now set a bunch of sectorsize extents in the first entry so it's
913 	 * ->bytes is large.
914 	 */
915 	for (i = 2; i < 20; i += 2) {
916 		offset = sectorsize * i;
917 		ret = btrfs_add_free_space(cache, offset, sectorsize);
918 		if (ret) {
919 			test_err("error populating sparse bitmap %d", ret);
920 			return ret;
921 		}
922 	}
923 
924 	/*
925 	 * Now set a contiguous extent in the second bitmap so its
926 	 * ->max_extent_size is larger than the first bitmaps.
927 	 */
928 	offset = (BITS_PER_BITMAP * sectorsize) + sectorsize;
929 	ret = btrfs_add_free_space(cache, offset, sectorsize);
930 	if (ret) {
931 		test_err("error adding contiguous extent %d", ret);
932 		return ret;
933 	}
934 
935 	/*
936 	 * Since we don't set ->max_extent_size unless we search everything
937 	 * should be indexed on bytes.
938 	 */
939 	entry = rb_entry(rb_first_cached(&ctl->free_space_bytes),
940 			 struct btrfs_free_space, bytes_index);
941 	if (entry->bytes != (10 * sectorsize)) {
942 		test_err("error, wrong entry in the first slot in bytes_index");
943 		return -EINVAL;
944 	}
945 
946 	max_extent_size = 0;
947 	offset = btrfs_find_space_for_alloc(cache, cache->start, sectorsize * 3,
948 					    0, &max_extent_size);
949 	if (offset != 0) {
950 		test_err("found space to alloc even though we don't have enough space");
951 		return -EINVAL;
952 	}
953 
954 	if (max_extent_size != (2 * sectorsize)) {
955 		test_err("got the wrong max_extent size %llu expected %llu",
956 			 max_extent_size, (unsigned long long)(2 * sectorsize));
957 		return -EINVAL;
958 	}
959 
960 	/*
961 	 * The search should have re-arranged the bytes index to use the
962 	 * ->max_extent_size, validate it's now what we expect it to be.
963 	 */
964 	entry = rb_entry(rb_first_cached(&ctl->free_space_bytes),
965 			 struct btrfs_free_space, bytes_index);
966 	if (entry->bytes != (2 * sectorsize)) {
967 		test_err("error, the bytes index wasn't recalculated properly");
968 		return -EINVAL;
969 	}
970 
971 	/* Add another sectorsize to re-arrange the tree back to ->bytes. */
972 	offset = (BITS_PER_BITMAP * sectorsize) - sectorsize;
973 	ret = btrfs_add_free_space(cache, offset, sectorsize);
974 	if (ret) {
975 		test_err("error adding extent to the sparse entry %d", ret);
976 		return ret;
977 	}
978 
979 	entry = rb_entry(rb_first_cached(&ctl->free_space_bytes),
980 			 struct btrfs_free_space, bytes_index);
981 	if (entry->bytes != (11 * sectorsize)) {
982 		test_err("error, wrong entry in the first slot in bytes_index");
983 		return -EINVAL;
984 	}
985 
986 	/*
987 	 * Now make sure we find our correct entry after searching that will
988 	 * result in a re-arranging of the tree.
989 	 */
990 	max_extent_size = 0;
991 	offset = btrfs_find_space_for_alloc(cache, cache->start, sectorsize * 2,
992 					    0, &max_extent_size);
993 	if (offset != (BITS_PER_BITMAP * sectorsize)) {
994 		test_err("error, found %llu instead of %llu for our alloc",
995 			 offset,
996 			 (unsigned long long)(BITS_PER_BITMAP * sectorsize));
997 		return -EINVAL;
998 	}
999 
1000 	cache->free_space_ctl->op = orig_free_space_ops;
1001 	btrfs_remove_free_space_cache(cache);
1002 	return 0;
1003 }
1004 
1005 int btrfs_test_free_space_cache(u32 sectorsize, u32 nodesize)
1006 {
1007 	struct btrfs_fs_info *fs_info;
1008 	struct btrfs_block_group *cache;
1009 	struct btrfs_root *root = NULL;
1010 	int ret = -ENOMEM;
1011 
1012 	test_msg("running btrfs free space cache tests");
1013 	fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize);
1014 	if (!fs_info) {
1015 		test_std_err(TEST_ALLOC_FS_INFO);
1016 		return -ENOMEM;
1017 	}
1018 
1019 	/*
1020 	 * For ppc64 (with 64k page size), bytes per bitmap might be
1021 	 * larger than 1G.  To make bitmap test available in ppc64,
1022 	 * alloc dummy block group whose size cross bitmaps.
1023 	 */
1024 	cache = btrfs_alloc_dummy_block_group(fs_info,
1025 				      BITS_PER_BITMAP * sectorsize + PAGE_SIZE);
1026 	if (!cache) {
1027 		test_std_err(TEST_ALLOC_BLOCK_GROUP);
1028 		btrfs_free_dummy_fs_info(fs_info);
1029 		return 0;
1030 	}
1031 
1032 	root = btrfs_alloc_dummy_root(fs_info);
1033 	if (IS_ERR(root)) {
1034 		test_std_err(TEST_ALLOC_ROOT);
1035 		ret = PTR_ERR(root);
1036 		goto out;
1037 	}
1038 
1039 	root->root_key.objectid = BTRFS_EXTENT_TREE_OBJECTID;
1040 	root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1041 	root->root_key.offset = 0;
1042 	btrfs_global_root_insert(root);
1043 
1044 	ret = test_extents(cache);
1045 	if (ret)
1046 		goto out;
1047 	ret = test_bitmaps(cache, sectorsize);
1048 	if (ret)
1049 		goto out;
1050 	ret = test_bitmaps_and_extents(cache, sectorsize);
1051 	if (ret)
1052 		goto out;
1053 
1054 	ret = test_steal_space_from_bitmap_to_extent(cache, sectorsize);
1055 	if (ret)
1056 		goto out;
1057 	ret = test_bytes_index(cache, sectorsize);
1058 out:
1059 	btrfs_free_dummy_block_group(cache);
1060 	btrfs_free_dummy_root(root);
1061 	btrfs_free_dummy_fs_info(fs_info);
1062 	return ret;
1063 }
1064