xref: /openbmc/linux/drivers/block/brd.c (revision 4f205687)
1 /*
2  * Ram backed block device driver.
3  *
4  * Copyright (C) 2007 Nick Piggin
5  * Copyright (C) 2007 Novell Inc.
6  *
7  * Parts derived from drivers/block/rd.c, and drivers/block/loop.c, copyright
8  * of their respective owners.
9  */
10 
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/major.h>
15 #include <linux/blkdev.h>
16 #include <linux/bio.h>
17 #include <linux/highmem.h>
18 #include <linux/mutex.h>
19 #include <linux/radix-tree.h>
20 #include <linux/fs.h>
21 #include <linux/slab.h>
22 #ifdef CONFIG_BLK_DEV_RAM_DAX
23 #include <linux/pfn_t.h>
24 #endif
25 
26 #include <asm/uaccess.h>
27 
28 #define SECTOR_SHIFT		9
29 #define PAGE_SECTORS_SHIFT	(PAGE_SHIFT - SECTOR_SHIFT)
30 #define PAGE_SECTORS		(1 << PAGE_SECTORS_SHIFT)
31 
32 /*
33  * Each block ramdisk device has a radix_tree brd_pages of pages that stores
34  * the pages containing the block device's contents. A brd page's ->index is
35  * its offset in PAGE_SIZE units. This is similar to, but in no way connected
36  * with, the kernel's pagecache or buffer cache (which sit above our block
37  * device).
38  */
39 struct brd_device {
40 	int		brd_number;
41 
42 	struct request_queue	*brd_queue;
43 	struct gendisk		*brd_disk;
44 	struct list_head	brd_list;
45 
46 	/*
47 	 * Backing store of pages and lock to protect it. This is the contents
48 	 * of the block device.
49 	 */
50 	spinlock_t		brd_lock;
51 	struct radix_tree_root	brd_pages;
52 };
53 
54 /*
55  * Look up and return a brd's page for a given sector.
56  */
57 static DEFINE_MUTEX(brd_mutex);
58 static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector)
59 {
60 	pgoff_t idx;
61 	struct page *page;
62 
63 	/*
64 	 * The page lifetime is protected by the fact that we have opened the
65 	 * device node -- brd pages will never be deleted under us, so we
66 	 * don't need any further locking or refcounting.
67 	 *
68 	 * This is strictly true for the radix-tree nodes as well (ie. we
69 	 * don't actually need the rcu_read_lock()), however that is not a
70 	 * documented feature of the radix-tree API so it is better to be
71 	 * safe here (we don't have total exclusion from radix tree updates
72 	 * here, only deletes).
73 	 */
74 	rcu_read_lock();
75 	idx = sector >> PAGE_SECTORS_SHIFT; /* sector to page index */
76 	page = radix_tree_lookup(&brd->brd_pages, idx);
77 	rcu_read_unlock();
78 
79 	BUG_ON(page && page->index != idx);
80 
81 	return page;
82 }
83 
84 /*
85  * Look up and return a brd's page for a given sector.
86  * If one does not exist, allocate an empty page, and insert that. Then
87  * return it.
88  */
89 static struct page *brd_insert_page(struct brd_device *brd, sector_t sector)
90 {
91 	pgoff_t idx;
92 	struct page *page;
93 	gfp_t gfp_flags;
94 
95 	page = brd_lookup_page(brd, sector);
96 	if (page)
97 		return page;
98 
99 	/*
100 	 * Must use NOIO because we don't want to recurse back into the
101 	 * block or filesystem layers from page reclaim.
102 	 *
103 	 * Cannot support DAX and highmem, because our ->direct_access
104 	 * routine for DAX must return memory that is always addressable.
105 	 * If DAX was reworked to use pfns and kmap throughout, this
106 	 * restriction might be able to be lifted.
107 	 */
108 	gfp_flags = GFP_NOIO | __GFP_ZERO;
109 #ifndef CONFIG_BLK_DEV_RAM_DAX
110 	gfp_flags |= __GFP_HIGHMEM;
111 #endif
112 	page = alloc_page(gfp_flags);
113 	if (!page)
114 		return NULL;
115 
116 	if (radix_tree_preload(GFP_NOIO)) {
117 		__free_page(page);
118 		return NULL;
119 	}
120 
121 	spin_lock(&brd->brd_lock);
122 	idx = sector >> PAGE_SECTORS_SHIFT;
123 	page->index = idx;
124 	if (radix_tree_insert(&brd->brd_pages, idx, page)) {
125 		__free_page(page);
126 		page = radix_tree_lookup(&brd->brd_pages, idx);
127 		BUG_ON(!page);
128 		BUG_ON(page->index != idx);
129 	}
130 	spin_unlock(&brd->brd_lock);
131 
132 	radix_tree_preload_end();
133 
134 	return page;
135 }
136 
137 static void brd_free_page(struct brd_device *brd, sector_t sector)
138 {
139 	struct page *page;
140 	pgoff_t idx;
141 
142 	spin_lock(&brd->brd_lock);
143 	idx = sector >> PAGE_SECTORS_SHIFT;
144 	page = radix_tree_delete(&brd->brd_pages, idx);
145 	spin_unlock(&brd->brd_lock);
146 	if (page)
147 		__free_page(page);
148 }
149 
150 static void brd_zero_page(struct brd_device *brd, sector_t sector)
151 {
152 	struct page *page;
153 
154 	page = brd_lookup_page(brd, sector);
155 	if (page)
156 		clear_highpage(page);
157 }
158 
159 /*
160  * Free all backing store pages and radix tree. This must only be called when
161  * there are no other users of the device.
162  */
163 #define FREE_BATCH 16
164 static void brd_free_pages(struct brd_device *brd)
165 {
166 	unsigned long pos = 0;
167 	struct page *pages[FREE_BATCH];
168 	int nr_pages;
169 
170 	do {
171 		int i;
172 
173 		nr_pages = radix_tree_gang_lookup(&brd->brd_pages,
174 				(void **)pages, pos, FREE_BATCH);
175 
176 		for (i = 0; i < nr_pages; i++) {
177 			void *ret;
178 
179 			BUG_ON(pages[i]->index < pos);
180 			pos = pages[i]->index;
181 			ret = radix_tree_delete(&brd->brd_pages, pos);
182 			BUG_ON(!ret || ret != pages[i]);
183 			__free_page(pages[i]);
184 		}
185 
186 		pos++;
187 
188 		/*
189 		 * This assumes radix_tree_gang_lookup always returns as
190 		 * many pages as possible. If the radix-tree code changes,
191 		 * so will this have to.
192 		 */
193 	} while (nr_pages == FREE_BATCH);
194 }
195 
196 /*
197  * copy_to_brd_setup must be called before copy_to_brd. It may sleep.
198  */
199 static int copy_to_brd_setup(struct brd_device *brd, sector_t sector, size_t n)
200 {
201 	unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
202 	size_t copy;
203 
204 	copy = min_t(size_t, n, PAGE_SIZE - offset);
205 	if (!brd_insert_page(brd, sector))
206 		return -ENOSPC;
207 	if (copy < n) {
208 		sector += copy >> SECTOR_SHIFT;
209 		if (!brd_insert_page(brd, sector))
210 			return -ENOSPC;
211 	}
212 	return 0;
213 }
214 
215 static void discard_from_brd(struct brd_device *brd,
216 			sector_t sector, size_t n)
217 {
218 	while (n >= PAGE_SIZE) {
219 		/*
220 		 * Don't want to actually discard pages here because
221 		 * re-allocating the pages can result in writeback
222 		 * deadlocks under heavy load.
223 		 */
224 		if (0)
225 			brd_free_page(brd, sector);
226 		else
227 			brd_zero_page(brd, sector);
228 		sector += PAGE_SIZE >> SECTOR_SHIFT;
229 		n -= PAGE_SIZE;
230 	}
231 }
232 
233 /*
234  * Copy n bytes from src to the brd starting at sector. Does not sleep.
235  */
236 static void copy_to_brd(struct brd_device *brd, const void *src,
237 			sector_t sector, size_t n)
238 {
239 	struct page *page;
240 	void *dst;
241 	unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
242 	size_t copy;
243 
244 	copy = min_t(size_t, n, PAGE_SIZE - offset);
245 	page = brd_lookup_page(brd, sector);
246 	BUG_ON(!page);
247 
248 	dst = kmap_atomic(page);
249 	memcpy(dst + offset, src, copy);
250 	kunmap_atomic(dst);
251 
252 	if (copy < n) {
253 		src += copy;
254 		sector += copy >> SECTOR_SHIFT;
255 		copy = n - copy;
256 		page = brd_lookup_page(brd, sector);
257 		BUG_ON(!page);
258 
259 		dst = kmap_atomic(page);
260 		memcpy(dst, src, copy);
261 		kunmap_atomic(dst);
262 	}
263 }
264 
265 /*
266  * Copy n bytes to dst from the brd starting at sector. Does not sleep.
267  */
268 static void copy_from_brd(void *dst, struct brd_device *brd,
269 			sector_t sector, size_t n)
270 {
271 	struct page *page;
272 	void *src;
273 	unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
274 	size_t copy;
275 
276 	copy = min_t(size_t, n, PAGE_SIZE - offset);
277 	page = brd_lookup_page(brd, sector);
278 	if (page) {
279 		src = kmap_atomic(page);
280 		memcpy(dst, src + offset, copy);
281 		kunmap_atomic(src);
282 	} else
283 		memset(dst, 0, copy);
284 
285 	if (copy < n) {
286 		dst += copy;
287 		sector += copy >> SECTOR_SHIFT;
288 		copy = n - copy;
289 		page = brd_lookup_page(brd, sector);
290 		if (page) {
291 			src = kmap_atomic(page);
292 			memcpy(dst, src, copy);
293 			kunmap_atomic(src);
294 		} else
295 			memset(dst, 0, copy);
296 	}
297 }
298 
299 /*
300  * Process a single bvec of a bio.
301  */
302 static int brd_do_bvec(struct brd_device *brd, struct page *page,
303 			unsigned int len, unsigned int off, int rw,
304 			sector_t sector)
305 {
306 	void *mem;
307 	int err = 0;
308 
309 	if (rw != READ) {
310 		err = copy_to_brd_setup(brd, sector, len);
311 		if (err)
312 			goto out;
313 	}
314 
315 	mem = kmap_atomic(page);
316 	if (rw == READ) {
317 		copy_from_brd(mem + off, brd, sector, len);
318 		flush_dcache_page(page);
319 	} else {
320 		flush_dcache_page(page);
321 		copy_to_brd(brd, mem + off, sector, len);
322 	}
323 	kunmap_atomic(mem);
324 
325 out:
326 	return err;
327 }
328 
329 static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio)
330 {
331 	struct block_device *bdev = bio->bi_bdev;
332 	struct brd_device *brd = bdev->bd_disk->private_data;
333 	int rw;
334 	struct bio_vec bvec;
335 	sector_t sector;
336 	struct bvec_iter iter;
337 
338 	sector = bio->bi_iter.bi_sector;
339 	if (bio_end_sector(bio) > get_capacity(bdev->bd_disk))
340 		goto io_error;
341 
342 	if (unlikely(bio->bi_rw & REQ_DISCARD)) {
343 		if (sector & ((PAGE_SIZE >> SECTOR_SHIFT) - 1) ||
344 		    bio->bi_iter.bi_size & ~PAGE_MASK)
345 			goto io_error;
346 		discard_from_brd(brd, sector, bio->bi_iter.bi_size);
347 		goto out;
348 	}
349 
350 	rw = bio_rw(bio);
351 	if (rw == READA)
352 		rw = READ;
353 
354 	bio_for_each_segment(bvec, bio, iter) {
355 		unsigned int len = bvec.bv_len;
356 		int err;
357 
358 		err = brd_do_bvec(brd, bvec.bv_page, len,
359 					bvec.bv_offset, rw, sector);
360 		if (err)
361 			goto io_error;
362 		sector += len >> SECTOR_SHIFT;
363 	}
364 
365 out:
366 	bio_endio(bio);
367 	return BLK_QC_T_NONE;
368 io_error:
369 	bio_io_error(bio);
370 	return BLK_QC_T_NONE;
371 }
372 
373 static int brd_rw_page(struct block_device *bdev, sector_t sector,
374 		       struct page *page, int rw)
375 {
376 	struct brd_device *brd = bdev->bd_disk->private_data;
377 	int err = brd_do_bvec(brd, page, PAGE_SIZE, 0, rw, sector);
378 	page_endio(page, rw & WRITE, err);
379 	return err;
380 }
381 
382 #ifdef CONFIG_BLK_DEV_RAM_DAX
383 static long brd_direct_access(struct block_device *bdev, sector_t sector,
384 			void __pmem **kaddr, pfn_t *pfn, long size)
385 {
386 	struct brd_device *brd = bdev->bd_disk->private_data;
387 	struct page *page;
388 
389 	if (!brd)
390 		return -ENODEV;
391 	page = brd_insert_page(brd, sector);
392 	if (!page)
393 		return -ENOSPC;
394 	*kaddr = (void __pmem *)page_address(page);
395 	*pfn = page_to_pfn_t(page);
396 
397 	return PAGE_SIZE;
398 }
399 #else
400 #define brd_direct_access NULL
401 #endif
402 
403 static int brd_ioctl(struct block_device *bdev, fmode_t mode,
404 			unsigned int cmd, unsigned long arg)
405 {
406 	int error;
407 	struct brd_device *brd = bdev->bd_disk->private_data;
408 
409 	if (cmd != BLKFLSBUF)
410 		return -ENOTTY;
411 
412 	/*
413 	 * ram device BLKFLSBUF has special semantics, we want to actually
414 	 * release and destroy the ramdisk data.
415 	 */
416 	mutex_lock(&brd_mutex);
417 	mutex_lock(&bdev->bd_mutex);
418 	error = -EBUSY;
419 	if (bdev->bd_openers <= 1) {
420 		/*
421 		 * Kill the cache first, so it isn't written back to the
422 		 * device.
423 		 *
424 		 * Another thread might instantiate more buffercache here,
425 		 * but there is not much we can do to close that race.
426 		 */
427 		kill_bdev(bdev);
428 		brd_free_pages(brd);
429 		error = 0;
430 	}
431 	mutex_unlock(&bdev->bd_mutex);
432 	mutex_unlock(&brd_mutex);
433 
434 	return error;
435 }
436 
437 static const struct block_device_operations brd_fops = {
438 	.owner =		THIS_MODULE,
439 	.rw_page =		brd_rw_page,
440 	.ioctl =		brd_ioctl,
441 	.direct_access =	brd_direct_access,
442 };
443 
444 /*
445  * And now the modules code and kernel interface.
446  */
447 static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT;
448 module_param(rd_nr, int, S_IRUGO);
449 MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices");
450 
451 int rd_size = CONFIG_BLK_DEV_RAM_SIZE;
452 module_param(rd_size, int, S_IRUGO);
453 MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes.");
454 
455 static int max_part = 1;
456 module_param(max_part, int, S_IRUGO);
457 MODULE_PARM_DESC(max_part, "Num Minors to reserve between devices");
458 
459 MODULE_LICENSE("GPL");
460 MODULE_ALIAS_BLOCKDEV_MAJOR(RAMDISK_MAJOR);
461 MODULE_ALIAS("rd");
462 
463 #ifndef MODULE
464 /* Legacy boot options - nonmodular */
465 static int __init ramdisk_size(char *str)
466 {
467 	rd_size = simple_strtol(str, NULL, 0);
468 	return 1;
469 }
470 __setup("ramdisk_size=", ramdisk_size);
471 #endif
472 
473 /*
474  * The device scheme is derived from loop.c. Keep them in synch where possible
475  * (should share code eventually).
476  */
477 static LIST_HEAD(brd_devices);
478 static DEFINE_MUTEX(brd_devices_mutex);
479 
480 static struct brd_device *brd_alloc(int i)
481 {
482 	struct brd_device *brd;
483 	struct gendisk *disk;
484 
485 	brd = kzalloc(sizeof(*brd), GFP_KERNEL);
486 	if (!brd)
487 		goto out;
488 	brd->brd_number		= i;
489 	spin_lock_init(&brd->brd_lock);
490 	INIT_RADIX_TREE(&brd->brd_pages, GFP_ATOMIC);
491 
492 	brd->brd_queue = blk_alloc_queue(GFP_KERNEL);
493 	if (!brd->brd_queue)
494 		goto out_free_dev;
495 
496 	blk_queue_make_request(brd->brd_queue, brd_make_request);
497 	blk_queue_max_hw_sectors(brd->brd_queue, 1024);
498 	blk_queue_bounce_limit(brd->brd_queue, BLK_BOUNCE_ANY);
499 
500 	/* This is so fdisk will align partitions on 4k, because of
501 	 * direct_access API needing 4k alignment, returning a PFN
502 	 * (This is only a problem on very small devices <= 4M,
503 	 *  otherwise fdisk will align on 1M. Regardless this call
504 	 *  is harmless)
505 	 */
506 	blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE);
507 
508 	brd->brd_queue->limits.discard_granularity = PAGE_SIZE;
509 	blk_queue_max_discard_sectors(brd->brd_queue, UINT_MAX);
510 	brd->brd_queue->limits.discard_zeroes_data = 1;
511 	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue);
512 
513 	disk = brd->brd_disk = alloc_disk(max_part);
514 	if (!disk)
515 		goto out_free_queue;
516 	disk->major		= RAMDISK_MAJOR;
517 	disk->first_minor	= i * max_part;
518 	disk->fops		= &brd_fops;
519 	disk->private_data	= brd;
520 	disk->queue		= brd->brd_queue;
521 	disk->flags		= GENHD_FL_EXT_DEVT;
522 	sprintf(disk->disk_name, "ram%d", i);
523 	set_capacity(disk, rd_size * 2);
524 
525 	return brd;
526 
527 out_free_queue:
528 	blk_cleanup_queue(brd->brd_queue);
529 out_free_dev:
530 	kfree(brd);
531 out:
532 	return NULL;
533 }
534 
535 static void brd_free(struct brd_device *brd)
536 {
537 	put_disk(brd->brd_disk);
538 	blk_cleanup_queue(brd->brd_queue);
539 	brd_free_pages(brd);
540 	kfree(brd);
541 }
542 
543 static struct brd_device *brd_init_one(int i, bool *new)
544 {
545 	struct brd_device *brd;
546 
547 	*new = false;
548 	list_for_each_entry(brd, &brd_devices, brd_list) {
549 		if (brd->brd_number == i)
550 			goto out;
551 	}
552 
553 	brd = brd_alloc(i);
554 	if (brd) {
555 		add_disk(brd->brd_disk);
556 		list_add_tail(&brd->brd_list, &brd_devices);
557 	}
558 	*new = true;
559 out:
560 	return brd;
561 }
562 
563 static void brd_del_one(struct brd_device *brd)
564 {
565 	list_del(&brd->brd_list);
566 	del_gendisk(brd->brd_disk);
567 	brd_free(brd);
568 }
569 
570 static struct kobject *brd_probe(dev_t dev, int *part, void *data)
571 {
572 	struct brd_device *brd;
573 	struct kobject *kobj;
574 	bool new;
575 
576 	mutex_lock(&brd_devices_mutex);
577 	brd = brd_init_one(MINOR(dev) / max_part, &new);
578 	kobj = brd ? get_disk(brd->brd_disk) : NULL;
579 	mutex_unlock(&brd_devices_mutex);
580 
581 	if (new)
582 		*part = 0;
583 
584 	return kobj;
585 }
586 
587 static int __init brd_init(void)
588 {
589 	struct brd_device *brd, *next;
590 	int i;
591 
592 	/*
593 	 * brd module now has a feature to instantiate underlying device
594 	 * structure on-demand, provided that there is an access dev node.
595 	 *
596 	 * (1) if rd_nr is specified, create that many upfront. else
597 	 *     it defaults to CONFIG_BLK_DEV_RAM_COUNT
598 	 * (2) User can further extend brd devices by create dev node themselves
599 	 *     and have kernel automatically instantiate actual device
600 	 *     on-demand. Example:
601 	 *		mknod /path/devnod_name b 1 X	# 1 is the rd major
602 	 *		fdisk -l /path/devnod_name
603 	 *	If (X / max_part) was not already created it will be created
604 	 *	dynamically.
605 	 */
606 
607 	if (register_blkdev(RAMDISK_MAJOR, "ramdisk"))
608 		return -EIO;
609 
610 	if (unlikely(!max_part))
611 		max_part = 1;
612 
613 	for (i = 0; i < rd_nr; i++) {
614 		brd = brd_alloc(i);
615 		if (!brd)
616 			goto out_free;
617 		list_add_tail(&brd->brd_list, &brd_devices);
618 	}
619 
620 	/* point of no return */
621 
622 	list_for_each_entry(brd, &brd_devices, brd_list)
623 		add_disk(brd->brd_disk);
624 
625 	blk_register_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS,
626 				  THIS_MODULE, brd_probe, NULL, NULL);
627 
628 	pr_info("brd: module loaded\n");
629 	return 0;
630 
631 out_free:
632 	list_for_each_entry_safe(brd, next, &brd_devices, brd_list) {
633 		list_del(&brd->brd_list);
634 		brd_free(brd);
635 	}
636 	unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
637 
638 	pr_info("brd: module NOT loaded !!!\n");
639 	return -ENOMEM;
640 }
641 
642 static void __exit brd_exit(void)
643 {
644 	struct brd_device *brd, *next;
645 
646 	list_for_each_entry_safe(brd, next, &brd_devices, brd_list)
647 		brd_del_one(brd);
648 
649 	blk_unregister_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS);
650 	unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
651 
652 	pr_info("brd: module unloaded\n");
653 }
654 
655 module_init(brd_init);
656 module_exit(brd_exit);
657 
658