1 /* 2 * Ram backed block device driver. 3 * 4 * Copyright (C) 2007 Nick Piggin 5 * Copyright (C) 2007 Novell Inc. 6 * 7 * Parts derived from drivers/block/rd.c, and drivers/block/loop.c, copyright 8 * of their respective owners. 9 */ 10 11 #include <linux/init.h> 12 #include <linux/initrd.h> 13 #include <linux/module.h> 14 #include <linux/moduleparam.h> 15 #include <linux/major.h> 16 #include <linux/blkdev.h> 17 #include <linux/bio.h> 18 #include <linux/highmem.h> 19 #include <linux/mutex.h> 20 #include <linux/radix-tree.h> 21 #include <linux/fs.h> 22 #include <linux/slab.h> 23 #ifdef CONFIG_BLK_DEV_RAM_DAX 24 #include <linux/pfn_t.h> 25 #include <linux/dax.h> 26 #include <linux/uio.h> 27 #endif 28 29 #include <linux/uaccess.h> 30 31 #define SECTOR_SHIFT 9 32 #define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) 33 #define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT) 34 35 /* 36 * Each block ramdisk device has a radix_tree brd_pages of pages that stores 37 * the pages containing the block device's contents. A brd page's ->index is 38 * its offset in PAGE_SIZE units. This is similar to, but in no way connected 39 * with, the kernel's pagecache or buffer cache (which sit above our block 40 * device). 41 */ 42 struct brd_device { 43 int brd_number; 44 45 struct request_queue *brd_queue; 46 struct gendisk *brd_disk; 47 #ifdef CONFIG_BLK_DEV_RAM_DAX 48 struct dax_device *dax_dev; 49 #endif 50 struct list_head brd_list; 51 52 /* 53 * Backing store of pages and lock to protect it. This is the contents 54 * of the block device. 55 */ 56 spinlock_t brd_lock; 57 struct radix_tree_root brd_pages; 58 }; 59 60 /* 61 * Look up and return a brd's page for a given sector. 62 */ 63 static DEFINE_MUTEX(brd_mutex); 64 static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector) 65 { 66 pgoff_t idx; 67 struct page *page; 68 69 /* 70 * The page lifetime is protected by the fact that we have opened the 71 * device node -- brd pages will never be deleted under us, so we 72 * don't need any further locking or refcounting. 73 * 74 * This is strictly true for the radix-tree nodes as well (ie. we 75 * don't actually need the rcu_read_lock()), however that is not a 76 * documented feature of the radix-tree API so it is better to be 77 * safe here (we don't have total exclusion from radix tree updates 78 * here, only deletes). 79 */ 80 rcu_read_lock(); 81 idx = sector >> PAGE_SECTORS_SHIFT; /* sector to page index */ 82 page = radix_tree_lookup(&brd->brd_pages, idx); 83 rcu_read_unlock(); 84 85 BUG_ON(page && page->index != idx); 86 87 return page; 88 } 89 90 /* 91 * Look up and return a brd's page for a given sector. 92 * If one does not exist, allocate an empty page, and insert that. Then 93 * return it. 94 */ 95 static struct page *brd_insert_page(struct brd_device *brd, sector_t sector) 96 { 97 pgoff_t idx; 98 struct page *page; 99 gfp_t gfp_flags; 100 101 page = brd_lookup_page(brd, sector); 102 if (page) 103 return page; 104 105 /* 106 * Must use NOIO because we don't want to recurse back into the 107 * block or filesystem layers from page reclaim. 108 * 109 * Cannot support DAX and highmem, because our ->direct_access 110 * routine for DAX must return memory that is always addressable. 111 * If DAX was reworked to use pfns and kmap throughout, this 112 * restriction might be able to be lifted. 113 */ 114 gfp_flags = GFP_NOIO | __GFP_ZERO; 115 #ifndef CONFIG_BLK_DEV_RAM_DAX 116 gfp_flags |= __GFP_HIGHMEM; 117 #endif 118 page = alloc_page(gfp_flags); 119 if (!page) 120 return NULL; 121 122 if (radix_tree_preload(GFP_NOIO)) { 123 __free_page(page); 124 return NULL; 125 } 126 127 spin_lock(&brd->brd_lock); 128 idx = sector >> PAGE_SECTORS_SHIFT; 129 page->index = idx; 130 if (radix_tree_insert(&brd->brd_pages, idx, page)) { 131 __free_page(page); 132 page = radix_tree_lookup(&brd->brd_pages, idx); 133 BUG_ON(!page); 134 BUG_ON(page->index != idx); 135 } 136 spin_unlock(&brd->brd_lock); 137 138 radix_tree_preload_end(); 139 140 return page; 141 } 142 143 /* 144 * Free all backing store pages and radix tree. This must only be called when 145 * there are no other users of the device. 146 */ 147 #define FREE_BATCH 16 148 static void brd_free_pages(struct brd_device *brd) 149 { 150 unsigned long pos = 0; 151 struct page *pages[FREE_BATCH]; 152 int nr_pages; 153 154 do { 155 int i; 156 157 nr_pages = radix_tree_gang_lookup(&brd->brd_pages, 158 (void **)pages, pos, FREE_BATCH); 159 160 for (i = 0; i < nr_pages; i++) { 161 void *ret; 162 163 BUG_ON(pages[i]->index < pos); 164 pos = pages[i]->index; 165 ret = radix_tree_delete(&brd->brd_pages, pos); 166 BUG_ON(!ret || ret != pages[i]); 167 __free_page(pages[i]); 168 } 169 170 pos++; 171 172 /* 173 * This assumes radix_tree_gang_lookup always returns as 174 * many pages as possible. If the radix-tree code changes, 175 * so will this have to. 176 */ 177 } while (nr_pages == FREE_BATCH); 178 } 179 180 /* 181 * copy_to_brd_setup must be called before copy_to_brd. It may sleep. 182 */ 183 static int copy_to_brd_setup(struct brd_device *brd, sector_t sector, size_t n) 184 { 185 unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT; 186 size_t copy; 187 188 copy = min_t(size_t, n, PAGE_SIZE - offset); 189 if (!brd_insert_page(brd, sector)) 190 return -ENOSPC; 191 if (copy < n) { 192 sector += copy >> SECTOR_SHIFT; 193 if (!brd_insert_page(brd, sector)) 194 return -ENOSPC; 195 } 196 return 0; 197 } 198 199 /* 200 * Copy n bytes from src to the brd starting at sector. Does not sleep. 201 */ 202 static void copy_to_brd(struct brd_device *brd, const void *src, 203 sector_t sector, size_t n) 204 { 205 struct page *page; 206 void *dst; 207 unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT; 208 size_t copy; 209 210 copy = min_t(size_t, n, PAGE_SIZE - offset); 211 page = brd_lookup_page(brd, sector); 212 BUG_ON(!page); 213 214 dst = kmap_atomic(page); 215 memcpy(dst + offset, src, copy); 216 kunmap_atomic(dst); 217 218 if (copy < n) { 219 src += copy; 220 sector += copy >> SECTOR_SHIFT; 221 copy = n - copy; 222 page = brd_lookup_page(brd, sector); 223 BUG_ON(!page); 224 225 dst = kmap_atomic(page); 226 memcpy(dst, src, copy); 227 kunmap_atomic(dst); 228 } 229 } 230 231 /* 232 * Copy n bytes to dst from the brd starting at sector. Does not sleep. 233 */ 234 static void copy_from_brd(void *dst, struct brd_device *brd, 235 sector_t sector, size_t n) 236 { 237 struct page *page; 238 void *src; 239 unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT; 240 size_t copy; 241 242 copy = min_t(size_t, n, PAGE_SIZE - offset); 243 page = brd_lookup_page(brd, sector); 244 if (page) { 245 src = kmap_atomic(page); 246 memcpy(dst, src + offset, copy); 247 kunmap_atomic(src); 248 } else 249 memset(dst, 0, copy); 250 251 if (copy < n) { 252 dst += copy; 253 sector += copy >> SECTOR_SHIFT; 254 copy = n - copy; 255 page = brd_lookup_page(brd, sector); 256 if (page) { 257 src = kmap_atomic(page); 258 memcpy(dst, src, copy); 259 kunmap_atomic(src); 260 } else 261 memset(dst, 0, copy); 262 } 263 } 264 265 /* 266 * Process a single bvec of a bio. 267 */ 268 static int brd_do_bvec(struct brd_device *brd, struct page *page, 269 unsigned int len, unsigned int off, bool is_write, 270 sector_t sector) 271 { 272 void *mem; 273 int err = 0; 274 275 if (is_write) { 276 err = copy_to_brd_setup(brd, sector, len); 277 if (err) 278 goto out; 279 } 280 281 mem = kmap_atomic(page); 282 if (!is_write) { 283 copy_from_brd(mem + off, brd, sector, len); 284 flush_dcache_page(page); 285 } else { 286 flush_dcache_page(page); 287 copy_to_brd(brd, mem + off, sector, len); 288 } 289 kunmap_atomic(mem); 290 291 out: 292 return err; 293 } 294 295 static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio) 296 { 297 struct brd_device *brd = bio->bi_disk->private_data; 298 struct bio_vec bvec; 299 sector_t sector; 300 struct bvec_iter iter; 301 302 sector = bio->bi_iter.bi_sector; 303 if (bio_end_sector(bio) > get_capacity(bio->bi_disk)) 304 goto io_error; 305 306 bio_for_each_segment(bvec, bio, iter) { 307 unsigned int len = bvec.bv_len; 308 int err; 309 310 err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset, 311 op_is_write(bio_op(bio)), sector); 312 if (err) 313 goto io_error; 314 sector += len >> SECTOR_SHIFT; 315 } 316 317 bio_endio(bio); 318 return BLK_QC_T_NONE; 319 io_error: 320 bio_io_error(bio); 321 return BLK_QC_T_NONE; 322 } 323 324 static int brd_rw_page(struct block_device *bdev, sector_t sector, 325 struct page *page, bool is_write) 326 { 327 struct brd_device *brd = bdev->bd_disk->private_data; 328 int err; 329 330 if (PageTransHuge(page)) 331 return -ENOTSUPP; 332 err = brd_do_bvec(brd, page, PAGE_SIZE, 0, is_write, sector); 333 page_endio(page, is_write, err); 334 return err; 335 } 336 337 #ifdef CONFIG_BLK_DEV_RAM_DAX 338 static long __brd_direct_access(struct brd_device *brd, pgoff_t pgoff, 339 long nr_pages, void **kaddr, pfn_t *pfn) 340 { 341 struct page *page; 342 343 if (!brd) 344 return -ENODEV; 345 page = brd_insert_page(brd, (sector_t)pgoff << PAGE_SECTORS_SHIFT); 346 if (!page) 347 return -ENOSPC; 348 *kaddr = page_address(page); 349 *pfn = page_to_pfn_t(page); 350 351 return 1; 352 } 353 354 static long brd_dax_direct_access(struct dax_device *dax_dev, 355 pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn) 356 { 357 struct brd_device *brd = dax_get_private(dax_dev); 358 359 return __brd_direct_access(brd, pgoff, nr_pages, kaddr, pfn); 360 } 361 362 static size_t brd_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, 363 void *addr, size_t bytes, struct iov_iter *i) 364 { 365 return copy_from_iter(addr, bytes, i); 366 } 367 368 static const struct dax_operations brd_dax_ops = { 369 .direct_access = brd_dax_direct_access, 370 .copy_from_iter = brd_dax_copy_from_iter, 371 }; 372 #endif 373 374 static const struct block_device_operations brd_fops = { 375 .owner = THIS_MODULE, 376 .rw_page = brd_rw_page, 377 }; 378 379 /* 380 * And now the modules code and kernel interface. 381 */ 382 static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT; 383 module_param(rd_nr, int, S_IRUGO); 384 MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices"); 385 386 unsigned long rd_size = CONFIG_BLK_DEV_RAM_SIZE; 387 module_param(rd_size, ulong, S_IRUGO); 388 MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes."); 389 390 static int max_part = 1; 391 module_param(max_part, int, S_IRUGO); 392 MODULE_PARM_DESC(max_part, "Num Minors to reserve between devices"); 393 394 MODULE_LICENSE("GPL"); 395 MODULE_ALIAS_BLOCKDEV_MAJOR(RAMDISK_MAJOR); 396 MODULE_ALIAS("rd"); 397 398 #ifndef MODULE 399 /* Legacy boot options - nonmodular */ 400 static int __init ramdisk_size(char *str) 401 { 402 rd_size = simple_strtol(str, NULL, 0); 403 return 1; 404 } 405 __setup("ramdisk_size=", ramdisk_size); 406 #endif 407 408 /* 409 * The device scheme is derived from loop.c. Keep them in synch where possible 410 * (should share code eventually). 411 */ 412 static LIST_HEAD(brd_devices); 413 static DEFINE_MUTEX(brd_devices_mutex); 414 415 static struct brd_device *brd_alloc(int i) 416 { 417 struct brd_device *brd; 418 struct gendisk *disk; 419 420 brd = kzalloc(sizeof(*brd), GFP_KERNEL); 421 if (!brd) 422 goto out; 423 brd->brd_number = i; 424 spin_lock_init(&brd->brd_lock); 425 INIT_RADIX_TREE(&brd->brd_pages, GFP_ATOMIC); 426 427 brd->brd_queue = blk_alloc_queue(GFP_KERNEL); 428 if (!brd->brd_queue) 429 goto out_free_dev; 430 431 blk_queue_make_request(brd->brd_queue, brd_make_request); 432 blk_queue_max_hw_sectors(brd->brd_queue, 1024); 433 434 /* This is so fdisk will align partitions on 4k, because of 435 * direct_access API needing 4k alignment, returning a PFN 436 * (This is only a problem on very small devices <= 4M, 437 * otherwise fdisk will align on 1M. Regardless this call 438 * is harmless) 439 */ 440 blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE); 441 disk = brd->brd_disk = alloc_disk(max_part); 442 if (!disk) 443 goto out_free_queue; 444 disk->major = RAMDISK_MAJOR; 445 disk->first_minor = i * max_part; 446 disk->fops = &brd_fops; 447 disk->private_data = brd; 448 disk->queue = brd->brd_queue; 449 disk->flags = GENHD_FL_EXT_DEVT; 450 sprintf(disk->disk_name, "ram%d", i); 451 set_capacity(disk, rd_size * 2); 452 453 #ifdef CONFIG_BLK_DEV_RAM_DAX 454 queue_flag_set_unlocked(QUEUE_FLAG_DAX, brd->brd_queue); 455 brd->dax_dev = alloc_dax(brd, disk->disk_name, &brd_dax_ops); 456 if (!brd->dax_dev) 457 goto out_free_inode; 458 #endif 459 460 461 return brd; 462 463 #ifdef CONFIG_BLK_DEV_RAM_DAX 464 out_free_inode: 465 kill_dax(brd->dax_dev); 466 put_dax(brd->dax_dev); 467 #endif 468 out_free_queue: 469 blk_cleanup_queue(brd->brd_queue); 470 out_free_dev: 471 kfree(brd); 472 out: 473 return NULL; 474 } 475 476 static void brd_free(struct brd_device *brd) 477 { 478 put_disk(brd->brd_disk); 479 blk_cleanup_queue(brd->brd_queue); 480 brd_free_pages(brd); 481 kfree(brd); 482 } 483 484 static struct brd_device *brd_init_one(int i, bool *new) 485 { 486 struct brd_device *brd; 487 488 *new = false; 489 list_for_each_entry(brd, &brd_devices, brd_list) { 490 if (brd->brd_number == i) 491 goto out; 492 } 493 494 brd = brd_alloc(i); 495 if (brd) { 496 add_disk(brd->brd_disk); 497 list_add_tail(&brd->brd_list, &brd_devices); 498 } 499 *new = true; 500 out: 501 return brd; 502 } 503 504 static void brd_del_one(struct brd_device *brd) 505 { 506 list_del(&brd->brd_list); 507 #ifdef CONFIG_BLK_DEV_RAM_DAX 508 kill_dax(brd->dax_dev); 509 put_dax(brd->dax_dev); 510 #endif 511 del_gendisk(brd->brd_disk); 512 brd_free(brd); 513 } 514 515 static struct kobject *brd_probe(dev_t dev, int *part, void *data) 516 { 517 struct brd_device *brd; 518 struct kobject *kobj; 519 bool new; 520 521 mutex_lock(&brd_devices_mutex); 522 brd = brd_init_one(MINOR(dev) / max_part, &new); 523 kobj = brd ? get_disk(brd->brd_disk) : NULL; 524 mutex_unlock(&brd_devices_mutex); 525 526 if (new) 527 *part = 0; 528 529 return kobj; 530 } 531 532 static int __init brd_init(void) 533 { 534 struct brd_device *brd, *next; 535 int i; 536 537 /* 538 * brd module now has a feature to instantiate underlying device 539 * structure on-demand, provided that there is an access dev node. 540 * 541 * (1) if rd_nr is specified, create that many upfront. else 542 * it defaults to CONFIG_BLK_DEV_RAM_COUNT 543 * (2) User can further extend brd devices by create dev node themselves 544 * and have kernel automatically instantiate actual device 545 * on-demand. Example: 546 * mknod /path/devnod_name b 1 X # 1 is the rd major 547 * fdisk -l /path/devnod_name 548 * If (X / max_part) was not already created it will be created 549 * dynamically. 550 */ 551 552 if (register_blkdev(RAMDISK_MAJOR, "ramdisk")) 553 return -EIO; 554 555 if (unlikely(!max_part)) 556 max_part = 1; 557 558 for (i = 0; i < rd_nr; i++) { 559 brd = brd_alloc(i); 560 if (!brd) 561 goto out_free; 562 list_add_tail(&brd->brd_list, &brd_devices); 563 } 564 565 /* point of no return */ 566 567 list_for_each_entry(brd, &brd_devices, brd_list) 568 add_disk(brd->brd_disk); 569 570 blk_register_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS, 571 THIS_MODULE, brd_probe, NULL, NULL); 572 573 pr_info("brd: module loaded\n"); 574 return 0; 575 576 out_free: 577 list_for_each_entry_safe(brd, next, &brd_devices, brd_list) { 578 list_del(&brd->brd_list); 579 brd_free(brd); 580 } 581 unregister_blkdev(RAMDISK_MAJOR, "ramdisk"); 582 583 pr_info("brd: module NOT loaded !!!\n"); 584 return -ENOMEM; 585 } 586 587 static void __exit brd_exit(void) 588 { 589 struct brd_device *brd, *next; 590 591 list_for_each_entry_safe(brd, next, &brd_devices, brd_list) 592 brd_del_one(brd); 593 594 blk_unregister_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS); 595 unregister_blkdev(RAMDISK_MAJOR, "ramdisk"); 596 597 pr_info("brd: module unloaded\n"); 598 } 599 600 module_init(brd_init); 601 module_exit(brd_exit); 602 603