187418ef9SColy Li // SPDX-License-Identifier: GPL-2.0 2cafe5635SKent Overstreet /* 3cafe5635SKent Overstreet * bcache setup/teardown code, and some metadata io - read a superblock and 4cafe5635SKent Overstreet * figure out what to do with it. 5cafe5635SKent Overstreet * 6cafe5635SKent Overstreet * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> 7cafe5635SKent Overstreet * Copyright 2012 Google, Inc. 8cafe5635SKent Overstreet */ 9cafe5635SKent Overstreet 10cafe5635SKent Overstreet #include "bcache.h" 11cafe5635SKent Overstreet #include "btree.h" 12cafe5635SKent Overstreet #include "debug.h" 1365d45231SKent Overstreet #include "extents.h" 14cafe5635SKent Overstreet #include "request.h" 15279afbadSKent Overstreet #include "writeback.h" 16d721a43fSColy Li #include "features.h" 17cafe5635SKent Overstreet 18c37511b8SKent Overstreet #include <linux/blkdev.h> 19cafe5635SKent Overstreet #include <linux/debugfs.h> 20cafe5635SKent Overstreet #include <linux/genhd.h> 2128935ab5SKent Overstreet #include <linux/idr.h> 2279826c35SKent Overstreet #include <linux/kthread.h> 23ee4a36f4SColy Li #include <linux/workqueue.h> 24cafe5635SKent Overstreet #include <linux/module.h> 25cafe5635SKent Overstreet #include <linux/random.h> 26cafe5635SKent Overstreet #include <linux/reboot.h> 27cafe5635SKent Overstreet #include <linux/sysfs.h> 28cafe5635SKent Overstreet 299aaf5165SColy Li unsigned int bch_cutoff_writeback; 309aaf5165SColy Li unsigned int bch_cutoff_writeback_sync; 319aaf5165SColy Li 32cafe5635SKent Overstreet static const char bcache_magic[] = { 33cafe5635SKent Overstreet 0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca, 34cafe5635SKent Overstreet 0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81 35cafe5635SKent Overstreet }; 36cafe5635SKent Overstreet 37cafe5635SKent Overstreet static const char invalid_uuid[] = { 38cafe5635SKent Overstreet 0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78, 39cafe5635SKent Overstreet 0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99 40cafe5635SKent Overstreet }; 41cafe5635SKent Overstreet 42cafe5635SKent Overstreet static struct kobject *bcache_kobj; 43cafe5635SKent Overstreet struct mutex bch_register_lock; 44a59ff6ccSColy Li bool bcache_is_reboot; 45cafe5635SKent Overstreet LIST_HEAD(bch_cache_sets); 46cafe5635SKent Overstreet static LIST_HEAD(uncached_devices); 47cafe5635SKent Overstreet 4828935ab5SKent Overstreet static int bcache_major; 491dbe32adSColy Li static DEFINE_IDA(bcache_device_idx); 50cafe5635SKent Overstreet static wait_queue_head_t unregister_wait; 51cafe5635SKent Overstreet struct workqueue_struct *bcache_wq; 520f843e65SGuoju Fang struct workqueue_struct *bch_journal_wq; 53cafe5635SKent Overstreet 54a59ff6ccSColy Li 55cafe5635SKent Overstreet #define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE) 561dbe32adSColy Li /* limitation of partitions number on single bcache device */ 571dbe32adSColy Li #define BCACHE_MINORS 128 581dbe32adSColy Li /* limitation of bcache devices number on single system */ 591dbe32adSColy Li #define BCACHE_DEVICE_IDX_MAX ((1U << MINORBITS)/BCACHE_MINORS) 60cafe5635SKent Overstreet 61cafe5635SKent Overstreet /* Superblock */ 62cafe5635SKent Overstreet 635b21403cSColy Li static const char *read_super_common(struct cache_sb *sb, struct block_device *bdev, 645b21403cSColy Li struct cache_sb_disk *s) 655b21403cSColy Li { 665b21403cSColy Li const char *err; 675b21403cSColy Li unsigned int i; 685b21403cSColy Li 695b21403cSColy Li sb->nbuckets = le64_to_cpu(s->nbuckets); 705b21403cSColy Li sb->bucket_size = le16_to_cpu(s->bucket_size); 715b21403cSColy Li 725b21403cSColy Li sb->nr_in_set = le16_to_cpu(s->nr_in_set); 735b21403cSColy Li sb->nr_this_dev = le16_to_cpu(s->nr_this_dev); 745b21403cSColy Li 755b21403cSColy Li err = "Too many buckets"; 765b21403cSColy Li if (sb->nbuckets > LONG_MAX) 775b21403cSColy Li goto err; 785b21403cSColy Li 795b21403cSColy Li err = "Not enough buckets"; 805b21403cSColy Li if (sb->nbuckets < 1 << 7) 815b21403cSColy Li goto err; 825b21403cSColy Li 83c557a5f7SColy Li err = "Bad block size (not power of 2)"; 84c557a5f7SColy Li if (!is_power_of_2(sb->block_size)) 85c557a5f7SColy Li goto err; 86c557a5f7SColy Li 87c557a5f7SColy Li err = "Bad block size (larger than page size)"; 88c557a5f7SColy Li if (sb->block_size > PAGE_SECTORS) 89c557a5f7SColy Li goto err; 90c557a5f7SColy Li 91c557a5f7SColy Li err = "Bad bucket size (not power of 2)"; 92c557a5f7SColy Li if (!is_power_of_2(sb->bucket_size)) 93c557a5f7SColy Li goto err; 94c557a5f7SColy Li 95c557a5f7SColy Li err = "Bad bucket size (smaller than page size)"; 96c557a5f7SColy Li if (sb->bucket_size < PAGE_SECTORS) 975b21403cSColy Li goto err; 985b21403cSColy Li 995b21403cSColy Li err = "Invalid superblock: device too small"; 1005b21403cSColy Li if (get_capacity(bdev->bd_disk) < 1015b21403cSColy Li sb->bucket_size * sb->nbuckets) 1025b21403cSColy Li goto err; 1035b21403cSColy Li 1045b21403cSColy Li err = "Bad UUID"; 1055b21403cSColy Li if (bch_is_zero(sb->set_uuid, 16)) 1065b21403cSColy Li goto err; 1075b21403cSColy Li 1085b21403cSColy Li err = "Bad cache device number in set"; 1095b21403cSColy Li if (!sb->nr_in_set || 1105b21403cSColy Li sb->nr_in_set <= sb->nr_this_dev || 1115b21403cSColy Li sb->nr_in_set > MAX_CACHES_PER_SET) 1125b21403cSColy Li goto err; 1135b21403cSColy Li 1145b21403cSColy Li err = "Journal buckets not sequential"; 1155b21403cSColy Li for (i = 0; i < sb->keys; i++) 1165b21403cSColy Li if (sb->d[i] != sb->first_bucket + i) 1175b21403cSColy Li goto err; 1185b21403cSColy Li 1195b21403cSColy Li err = "Too many journal buckets"; 1205b21403cSColy Li if (sb->first_bucket + sb->keys > sb->nbuckets) 1215b21403cSColy Li goto err; 1225b21403cSColy Li 1235b21403cSColy Li err = "Invalid superblock: first bucket comes before end of super"; 1245b21403cSColy Li if (sb->first_bucket * sb->bucket_size < 16) 1255b21403cSColy Li goto err; 1265b21403cSColy Li 1275b21403cSColy Li err = NULL; 1285b21403cSColy Li err: 1295b21403cSColy Li return err; 1305b21403cSColy Li } 1315b21403cSColy Li 1325b21403cSColy Li 133cafe5635SKent Overstreet static const char *read_super(struct cache_sb *sb, struct block_device *bdev, 134cfa0c56dSChristoph Hellwig struct cache_sb_disk **res) 135cafe5635SKent Overstreet { 136cafe5635SKent Overstreet const char *err; 137a702a692SChristoph Hellwig struct cache_sb_disk *s; 1386321bef0SChristoph Hellwig struct page *page; 1396f10f7d1SColy Li unsigned int i; 140cafe5635SKent Overstreet 1416321bef0SChristoph Hellwig page = read_cache_page_gfp(bdev->bd_inode->i_mapping, 1426321bef0SChristoph Hellwig SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL); 1436321bef0SChristoph Hellwig if (IS_ERR(page)) 144cafe5635SKent Overstreet return "IO error"; 1456321bef0SChristoph Hellwig s = page_address(page) + offset_in_page(SB_OFFSET); 146cafe5635SKent Overstreet 147cafe5635SKent Overstreet sb->offset = le64_to_cpu(s->offset); 148cafe5635SKent Overstreet sb->version = le64_to_cpu(s->version); 149cafe5635SKent Overstreet 150cafe5635SKent Overstreet memcpy(sb->magic, s->magic, 16); 151cafe5635SKent Overstreet memcpy(sb->uuid, s->uuid, 16); 152cafe5635SKent Overstreet memcpy(sb->set_uuid, s->set_uuid, 16); 153cafe5635SKent Overstreet memcpy(sb->label, s->label, SB_LABEL_SIZE); 154cafe5635SKent Overstreet 155cafe5635SKent Overstreet sb->flags = le64_to_cpu(s->flags); 156cafe5635SKent Overstreet sb->seq = le64_to_cpu(s->seq); 157cafe5635SKent Overstreet sb->last_mount = le32_to_cpu(s->last_mount); 158cafe5635SKent Overstreet sb->first_bucket = le16_to_cpu(s->first_bucket); 159cafe5635SKent Overstreet sb->keys = le16_to_cpu(s->keys); 160cafe5635SKent Overstreet 161cafe5635SKent Overstreet for (i = 0; i < SB_JOURNAL_BUCKETS; i++) 162cafe5635SKent Overstreet sb->d[i] = le64_to_cpu(s->d[i]); 163cafe5635SKent Overstreet 16446f5aa88SJoe Perches pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u\n", 165cafe5635SKent Overstreet sb->version, sb->flags, sb->seq, sb->keys); 166cafe5635SKent Overstreet 167aaf8dbeaSColy Li err = "Not a bcache superblock (bad offset)"; 168cafe5635SKent Overstreet if (sb->offset != SB_SECTOR) 169cafe5635SKent Overstreet goto err; 170cafe5635SKent Overstreet 171aaf8dbeaSColy Li err = "Not a bcache superblock (bad magic)"; 172cafe5635SKent Overstreet if (memcmp(sb->magic, bcache_magic, 16)) 173cafe5635SKent Overstreet goto err; 174cafe5635SKent Overstreet 175cafe5635SKent Overstreet err = "Too many journal buckets"; 176cafe5635SKent Overstreet if (sb->keys > SB_JOURNAL_BUCKETS) 177cafe5635SKent Overstreet goto err; 178cafe5635SKent Overstreet 179cafe5635SKent Overstreet err = "Bad checksum"; 180cafe5635SKent Overstreet if (s->csum != csum_set(s)) 181cafe5635SKent Overstreet goto err; 182cafe5635SKent Overstreet 183cafe5635SKent Overstreet err = "Bad UUID"; 184169ef1cfSKent Overstreet if (bch_is_zero(sb->uuid, 16)) 185cafe5635SKent Overstreet goto err; 186cafe5635SKent Overstreet 1878abb2a5dSKent Overstreet sb->block_size = le16_to_cpu(s->block_size); 1888abb2a5dSKent Overstreet 1898abb2a5dSKent Overstreet err = "Superblock block size smaller than device block size"; 1908abb2a5dSKent Overstreet if (sb->block_size << 9 < bdev_logical_block_size(bdev)) 1918abb2a5dSKent Overstreet goto err; 1928abb2a5dSKent Overstreet 1932903381fSKent Overstreet switch (sb->version) { 1942903381fSKent Overstreet case BCACHE_SB_VERSION_BDEV: 1952903381fSKent Overstreet sb->data_offset = BDEV_DATA_START_DEFAULT; 1962903381fSKent Overstreet break; 1972903381fSKent Overstreet case BCACHE_SB_VERSION_BDEV_WITH_OFFSET: 198d721a43fSColy Li case BCACHE_SB_VERSION_BDEV_WITH_FEATURES: 1992903381fSKent Overstreet sb->data_offset = le64_to_cpu(s->data_offset); 2002903381fSKent Overstreet 2012903381fSKent Overstreet err = "Bad data offset"; 2022903381fSKent Overstreet if (sb->data_offset < BDEV_DATA_START_DEFAULT) 203cafe5635SKent Overstreet goto err; 204cafe5635SKent Overstreet 2052903381fSKent Overstreet break; 2062903381fSKent Overstreet case BCACHE_SB_VERSION_CDEV: 2072903381fSKent Overstreet case BCACHE_SB_VERSION_CDEV_WITH_UUID: 2085b21403cSColy Li err = read_super_common(sb, bdev, s); 2095b21403cSColy Li if (err) 210cafe5635SKent Overstreet goto err; 2112903381fSKent Overstreet break; 212d721a43fSColy Li case BCACHE_SB_VERSION_CDEV_WITH_FEATURES: 213d721a43fSColy Li err = read_super_common(sb, bdev, s); 214d721a43fSColy Li if (err) 215d721a43fSColy Li goto err; 216d721a43fSColy Li sb->feature_compat = le64_to_cpu(s->feature_compat); 217d721a43fSColy Li sb->feature_incompat = le64_to_cpu(s->feature_incompat); 218d721a43fSColy Li sb->feature_ro_compat = le64_to_cpu(s->feature_ro_compat); 219d721a43fSColy Li break; 2202903381fSKent Overstreet default: 2212903381fSKent Overstreet err = "Unsupported superblock version"; 2222903381fSKent Overstreet goto err; 2232903381fSKent Overstreet } 2242903381fSKent Overstreet 22575cbb3f1SArnd Bergmann sb->last_mount = (u32)ktime_get_real_seconds(); 226cfa0c56dSChristoph Hellwig *res = s; 2276321bef0SChristoph Hellwig return NULL; 228cafe5635SKent Overstreet err: 2296321bef0SChristoph Hellwig put_page(page); 230cafe5635SKent Overstreet return err; 231cafe5635SKent Overstreet } 232cafe5635SKent Overstreet 2334246a0b6SChristoph Hellwig static void write_bdev_super_endio(struct bio *bio) 234cafe5635SKent Overstreet { 235cafe5635SKent Overstreet struct cached_dev *dc = bio->bi_private; 23608ec1e62SColy Li 23708ec1e62SColy Li if (bio->bi_status) 23808ec1e62SColy Li bch_count_backing_io_errors(dc, bio); 239cafe5635SKent Overstreet 240cb7a583eSKent Overstreet closure_put(&dc->sb_write); 241cafe5635SKent Overstreet } 242cafe5635SKent Overstreet 243475389aeSChristoph Hellwig static void __write_super(struct cache_sb *sb, struct cache_sb_disk *out, 244475389aeSChristoph Hellwig struct bio *bio) 245cafe5635SKent Overstreet { 2466f10f7d1SColy Li unsigned int i; 247cafe5635SKent Overstreet 248475389aeSChristoph Hellwig bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_META; 2494f024f37SKent Overstreet bio->bi_iter.bi_sector = SB_SECTOR; 250475389aeSChristoph Hellwig __bio_add_page(bio, virt_to_page(out), SB_SIZE, 251475389aeSChristoph Hellwig offset_in_page(out)); 252cafe5635SKent Overstreet 253cafe5635SKent Overstreet out->offset = cpu_to_le64(sb->offset); 254cafe5635SKent Overstreet 255cafe5635SKent Overstreet memcpy(out->uuid, sb->uuid, 16); 256cafe5635SKent Overstreet memcpy(out->set_uuid, sb->set_uuid, 16); 257cafe5635SKent Overstreet memcpy(out->label, sb->label, SB_LABEL_SIZE); 258cafe5635SKent Overstreet 259cafe5635SKent Overstreet out->flags = cpu_to_le64(sb->flags); 260cafe5635SKent Overstreet out->seq = cpu_to_le64(sb->seq); 261cafe5635SKent Overstreet 262cafe5635SKent Overstreet out->last_mount = cpu_to_le32(sb->last_mount); 263cafe5635SKent Overstreet out->first_bucket = cpu_to_le16(sb->first_bucket); 264cafe5635SKent Overstreet out->keys = cpu_to_le16(sb->keys); 265cafe5635SKent Overstreet 266cafe5635SKent Overstreet for (i = 0; i < sb->keys; i++) 267cafe5635SKent Overstreet out->d[i] = cpu_to_le64(sb->d[i]); 268cafe5635SKent Overstreet 269d721a43fSColy Li if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) { 270d721a43fSColy Li out->feature_compat = cpu_to_le64(sb->feature_compat); 271d721a43fSColy Li out->feature_incompat = cpu_to_le64(sb->feature_incompat); 272d721a43fSColy Li out->feature_ro_compat = cpu_to_le64(sb->feature_ro_compat); 273d721a43fSColy Li } 274d721a43fSColy Li 275d721a43fSColy Li out->version = cpu_to_le64(sb->version); 276cafe5635SKent Overstreet out->csum = csum_set(out); 277cafe5635SKent Overstreet 27846f5aa88SJoe Perches pr_debug("ver %llu, flags %llu, seq %llu\n", 279cafe5635SKent Overstreet sb->version, sb->flags, sb->seq); 280cafe5635SKent Overstreet 2814e49ea4aSMike Christie submit_bio(bio); 282cafe5635SKent Overstreet } 283cafe5635SKent Overstreet 284cb7a583eSKent Overstreet static void bch_write_bdev_super_unlock(struct closure *cl) 285cb7a583eSKent Overstreet { 286cb7a583eSKent Overstreet struct cached_dev *dc = container_of(cl, struct cached_dev, sb_write); 287cb7a583eSKent Overstreet 288cb7a583eSKent Overstreet up(&dc->sb_write_mutex); 289cb7a583eSKent Overstreet } 290cb7a583eSKent Overstreet 291cafe5635SKent Overstreet void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent) 292cafe5635SKent Overstreet { 293cb7a583eSKent Overstreet struct closure *cl = &dc->sb_write; 294cafe5635SKent Overstreet struct bio *bio = &dc->sb_bio; 295cafe5635SKent Overstreet 296cb7a583eSKent Overstreet down(&dc->sb_write_mutex); 297cb7a583eSKent Overstreet closure_init(cl, parent); 298cafe5635SKent Overstreet 299475389aeSChristoph Hellwig bio_init(bio, dc->sb_bv, 1); 30074d46992SChristoph Hellwig bio_set_dev(bio, dc->bdev); 301cafe5635SKent Overstreet bio->bi_end_io = write_bdev_super_endio; 302cafe5635SKent Overstreet bio->bi_private = dc; 303cafe5635SKent Overstreet 304cafe5635SKent Overstreet closure_get(cl); 30527a40ab9SColy Li /* I/O request sent to backing device */ 306475389aeSChristoph Hellwig __write_super(&dc->sb, dc->sb_disk, bio); 307cafe5635SKent Overstreet 308cb7a583eSKent Overstreet closure_return_with_destructor(cl, bch_write_bdev_super_unlock); 309cafe5635SKent Overstreet } 310cafe5635SKent Overstreet 3114246a0b6SChristoph Hellwig static void write_super_endio(struct bio *bio) 312cafe5635SKent Overstreet { 313cafe5635SKent Overstreet struct cache *ca = bio->bi_private; 314cafe5635SKent Overstreet 3155138ac67SColy Li /* is_read = 0 */ 3165138ac67SColy Li bch_count_io_errors(ca, bio->bi_status, 0, 3175138ac67SColy Li "writing superblock"); 318cb7a583eSKent Overstreet closure_put(&ca->set->sb_write); 319cb7a583eSKent Overstreet } 320cb7a583eSKent Overstreet 321cb7a583eSKent Overstreet static void bcache_write_super_unlock(struct closure *cl) 322cb7a583eSKent Overstreet { 323cb7a583eSKent Overstreet struct cache_set *c = container_of(cl, struct cache_set, sb_write); 324cb7a583eSKent Overstreet 325cb7a583eSKent Overstreet up(&c->sb_write_mutex); 326cafe5635SKent Overstreet } 327cafe5635SKent Overstreet 328cafe5635SKent Overstreet void bcache_write_super(struct cache_set *c) 329cafe5635SKent Overstreet { 330cb7a583eSKent Overstreet struct closure *cl = &c->sb_write; 331cafe5635SKent Overstreet struct cache *ca; 332d721a43fSColy Li unsigned int i, version = BCACHE_SB_VERSION_CDEV_WITH_UUID; 333cafe5635SKent Overstreet 334cb7a583eSKent Overstreet down(&c->sb_write_mutex); 335cb7a583eSKent Overstreet closure_init(cl, &c->cl); 336cafe5635SKent Overstreet 337cafe5635SKent Overstreet c->sb.seq++; 338cafe5635SKent Overstreet 339d721a43fSColy Li if (c->sb.version > version) 340d721a43fSColy Li version = c->sb.version; 341d721a43fSColy Li 342cafe5635SKent Overstreet for_each_cache(ca, c, i) { 343cafe5635SKent Overstreet struct bio *bio = &ca->sb_bio; 344cafe5635SKent Overstreet 345d721a43fSColy Li ca->sb.version = version; 346cafe5635SKent Overstreet ca->sb.seq = c->sb.seq; 347cafe5635SKent Overstreet ca->sb.last_mount = c->sb.last_mount; 348cafe5635SKent Overstreet 349cafe5635SKent Overstreet SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb)); 350cafe5635SKent Overstreet 351475389aeSChristoph Hellwig bio_init(bio, ca->sb_bv, 1); 35274d46992SChristoph Hellwig bio_set_dev(bio, ca->bdev); 353cafe5635SKent Overstreet bio->bi_end_io = write_super_endio; 354cafe5635SKent Overstreet bio->bi_private = ca; 355cafe5635SKent Overstreet 356cafe5635SKent Overstreet closure_get(cl); 357475389aeSChristoph Hellwig __write_super(&ca->sb, ca->sb_disk, bio); 358cafe5635SKent Overstreet } 359cafe5635SKent Overstreet 360cb7a583eSKent Overstreet closure_return_with_destructor(cl, bcache_write_super_unlock); 361cafe5635SKent Overstreet } 362cafe5635SKent Overstreet 363cafe5635SKent Overstreet /* UUID io */ 364cafe5635SKent Overstreet 3654246a0b6SChristoph Hellwig static void uuid_endio(struct bio *bio) 366cafe5635SKent Overstreet { 367cafe5635SKent Overstreet struct closure *cl = bio->bi_private; 368cb7a583eSKent Overstreet struct cache_set *c = container_of(cl, struct cache_set, uuid_write); 369cafe5635SKent Overstreet 3704e4cbee9SChristoph Hellwig cache_set_err_on(bio->bi_status, c, "accessing uuids"); 371cafe5635SKent Overstreet bch_bbio_free(bio, c); 372cafe5635SKent Overstreet closure_put(cl); 373cafe5635SKent Overstreet } 374cafe5635SKent Overstreet 375cb7a583eSKent Overstreet static void uuid_io_unlock(struct closure *cl) 376cb7a583eSKent Overstreet { 377cb7a583eSKent Overstreet struct cache_set *c = container_of(cl, struct cache_set, uuid_write); 378cb7a583eSKent Overstreet 379cb7a583eSKent Overstreet up(&c->uuid_write_mutex); 380cb7a583eSKent Overstreet } 381cb7a583eSKent Overstreet 382ad0d9e76SMike Christie static void uuid_io(struct cache_set *c, int op, unsigned long op_flags, 383cafe5635SKent Overstreet struct bkey *k, struct closure *parent) 384cafe5635SKent Overstreet { 385cb7a583eSKent Overstreet struct closure *cl = &c->uuid_write; 386cafe5635SKent Overstreet struct uuid_entry *u; 3876f10f7d1SColy Li unsigned int i; 38885b1492eSKent Overstreet char buf[80]; 389cafe5635SKent Overstreet 390cafe5635SKent Overstreet BUG_ON(!parent); 391cb7a583eSKent Overstreet down(&c->uuid_write_mutex); 392cb7a583eSKent Overstreet closure_init(cl, parent); 393cafe5635SKent Overstreet 394cafe5635SKent Overstreet for (i = 0; i < KEY_PTRS(k); i++) { 395cafe5635SKent Overstreet struct bio *bio = bch_bbio_alloc(c); 396cafe5635SKent Overstreet 3971eff9d32SJens Axboe bio->bi_opf = REQ_SYNC | REQ_META | op_flags; 3984f024f37SKent Overstreet bio->bi_iter.bi_size = KEY_SIZE(k) << 9; 399cafe5635SKent Overstreet 400cafe5635SKent Overstreet bio->bi_end_io = uuid_endio; 401cafe5635SKent Overstreet bio->bi_private = cl; 402ad0d9e76SMike Christie bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags); 403169ef1cfSKent Overstreet bch_bio_map(bio, c->uuids); 404cafe5635SKent Overstreet 405cafe5635SKent Overstreet bch_submit_bbio(bio, c, k, i); 406cafe5635SKent Overstreet 407ad0d9e76SMike Christie if (op != REQ_OP_WRITE) 408cafe5635SKent Overstreet break; 409cafe5635SKent Overstreet } 410cafe5635SKent Overstreet 411dc9d98d6SKent Overstreet bch_extent_to_text(buf, sizeof(buf), k); 41246f5aa88SJoe Perches pr_debug("%s UUIDs at %s\n", op == REQ_OP_WRITE ? "wrote" : "read", buf); 413cafe5635SKent Overstreet 414cafe5635SKent Overstreet for (u = c->uuids; u < c->uuids + c->nr_uuids; u++) 415169ef1cfSKent Overstreet if (!bch_is_zero(u->uuid, 16)) 41646f5aa88SJoe Perches pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u\n", 417cafe5635SKent Overstreet u - c->uuids, u->uuid, u->label, 418cafe5635SKent Overstreet u->first_reg, u->last_reg, u->invalidated); 419cafe5635SKent Overstreet 420cb7a583eSKent Overstreet closure_return_with_destructor(cl, uuid_io_unlock); 421cafe5635SKent Overstreet } 422cafe5635SKent Overstreet 423cafe5635SKent Overstreet static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl) 424cafe5635SKent Overstreet { 425cafe5635SKent Overstreet struct bkey *k = &j->uuid_bucket; 426cafe5635SKent Overstreet 42765d45231SKent Overstreet if (__bch_btree_ptr_invalid(c, k)) 428cafe5635SKent Overstreet return "bad uuid pointer"; 429cafe5635SKent Overstreet 430cafe5635SKent Overstreet bkey_copy(&c->uuid_bucket, k); 43170fd7614SChristoph Hellwig uuid_io(c, REQ_OP_READ, 0, k, cl); 432cafe5635SKent Overstreet 433cafe5635SKent Overstreet if (j->version < BCACHE_JSET_VERSION_UUIDv1) { 434cafe5635SKent Overstreet struct uuid_entry_v0 *u0 = (void *) c->uuids; 435cafe5635SKent Overstreet struct uuid_entry *u1 = (void *) c->uuids; 436cafe5635SKent Overstreet int i; 437cafe5635SKent Overstreet 438cafe5635SKent Overstreet closure_sync(cl); 439cafe5635SKent Overstreet 440cafe5635SKent Overstreet /* 441cafe5635SKent Overstreet * Since the new uuid entry is bigger than the old, we have to 442cafe5635SKent Overstreet * convert starting at the highest memory address and work down 443cafe5635SKent Overstreet * in order to do it in place 444cafe5635SKent Overstreet */ 445cafe5635SKent Overstreet 446cafe5635SKent Overstreet for (i = c->nr_uuids - 1; 447cafe5635SKent Overstreet i >= 0; 448cafe5635SKent Overstreet --i) { 449cafe5635SKent Overstreet memcpy(u1[i].uuid, u0[i].uuid, 16); 450cafe5635SKent Overstreet memcpy(u1[i].label, u0[i].label, 32); 451cafe5635SKent Overstreet 452cafe5635SKent Overstreet u1[i].first_reg = u0[i].first_reg; 453cafe5635SKent Overstreet u1[i].last_reg = u0[i].last_reg; 454cafe5635SKent Overstreet u1[i].invalidated = u0[i].invalidated; 455cafe5635SKent Overstreet 456cafe5635SKent Overstreet u1[i].flags = 0; 457cafe5635SKent Overstreet u1[i].sectors = 0; 458cafe5635SKent Overstreet } 459cafe5635SKent Overstreet } 460cafe5635SKent Overstreet 461cafe5635SKent Overstreet return NULL; 462cafe5635SKent Overstreet } 463cafe5635SKent Overstreet 464cafe5635SKent Overstreet static int __uuid_write(struct cache_set *c) 465cafe5635SKent Overstreet { 466cafe5635SKent Overstreet BKEY_PADDED(key) k; 467cafe5635SKent Overstreet struct closure cl; 4687a55948dSShenghui Wang struct cache *ca; 469cafe5635SKent Overstreet 4701fae7cf0SColy Li closure_init_stack(&cl); 471cafe5635SKent Overstreet lockdep_assert_held(&bch_register_lock); 472cafe5635SKent Overstreet 47378365411SKent Overstreet if (bch_bucket_alloc_set(c, RESERVE_BTREE, &k.key, 1, true)) 474cafe5635SKent Overstreet return 1; 475cafe5635SKent Overstreet 476cafe5635SKent Overstreet SET_KEY_SIZE(&k.key, c->sb.bucket_size); 477ad0d9e76SMike Christie uuid_io(c, REQ_OP_WRITE, 0, &k.key, &cl); 478cafe5635SKent Overstreet closure_sync(&cl); 479cafe5635SKent Overstreet 4807a55948dSShenghui Wang /* Only one bucket used for uuid write */ 4817a55948dSShenghui Wang ca = PTR_CACHE(c, &k.key, 0); 4827a55948dSShenghui Wang atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written); 4837a55948dSShenghui Wang 484cafe5635SKent Overstreet bkey_copy(&c->uuid_bucket, &k.key); 4853a3b6a4eSKent Overstreet bkey_put(c, &k.key); 486cafe5635SKent Overstreet return 0; 487cafe5635SKent Overstreet } 488cafe5635SKent Overstreet 489cafe5635SKent Overstreet int bch_uuid_write(struct cache_set *c) 490cafe5635SKent Overstreet { 491cafe5635SKent Overstreet int ret = __uuid_write(c); 492cafe5635SKent Overstreet 493cafe5635SKent Overstreet if (!ret) 494cafe5635SKent Overstreet bch_journal_meta(c, NULL); 495cafe5635SKent Overstreet 496cafe5635SKent Overstreet return ret; 497cafe5635SKent Overstreet } 498cafe5635SKent Overstreet 499cafe5635SKent Overstreet static struct uuid_entry *uuid_find(struct cache_set *c, const char *uuid) 500cafe5635SKent Overstreet { 501cafe5635SKent Overstreet struct uuid_entry *u; 502cafe5635SKent Overstreet 503cafe5635SKent Overstreet for (u = c->uuids; 504cafe5635SKent Overstreet u < c->uuids + c->nr_uuids; u++) 505cafe5635SKent Overstreet if (!memcmp(u->uuid, uuid, 16)) 506cafe5635SKent Overstreet return u; 507cafe5635SKent Overstreet 508cafe5635SKent Overstreet return NULL; 509cafe5635SKent Overstreet } 510cafe5635SKent Overstreet 511cafe5635SKent Overstreet static struct uuid_entry *uuid_find_empty(struct cache_set *c) 512cafe5635SKent Overstreet { 513cafe5635SKent Overstreet static const char zero_uuid[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; 5141fae7cf0SColy Li 515cafe5635SKent Overstreet return uuid_find(c, zero_uuid); 516cafe5635SKent Overstreet } 517cafe5635SKent Overstreet 518cafe5635SKent Overstreet /* 519cafe5635SKent Overstreet * Bucket priorities/gens: 520cafe5635SKent Overstreet * 521cafe5635SKent Overstreet * For each bucket, we store on disk its 522cafe5635SKent Overstreet * 8 bit gen 523cafe5635SKent Overstreet * 16 bit priority 524cafe5635SKent Overstreet * 525cafe5635SKent Overstreet * See alloc.c for an explanation of the gen. The priority is used to implement 526cafe5635SKent Overstreet * lru (and in the future other) cache replacement policies; for most purposes 527cafe5635SKent Overstreet * it's just an opaque integer. 528cafe5635SKent Overstreet * 529cafe5635SKent Overstreet * The gens and the priorities don't have a whole lot to do with each other, and 530cafe5635SKent Overstreet * it's actually the gens that must be written out at specific times - it's no 531cafe5635SKent Overstreet * big deal if the priorities don't get written, if we lose them we just reuse 532cafe5635SKent Overstreet * buckets in suboptimal order. 533cafe5635SKent Overstreet * 534cafe5635SKent Overstreet * On disk they're stored in a packed array, and in as many buckets are required 535cafe5635SKent Overstreet * to fit them all. The buckets we use to store them form a list; the journal 536cafe5635SKent Overstreet * header points to the first bucket, the first bucket points to the second 537cafe5635SKent Overstreet * bucket, et cetera. 538cafe5635SKent Overstreet * 539cafe5635SKent Overstreet * This code is used by the allocation code; periodically (whenever it runs out 540cafe5635SKent Overstreet * of buckets to allocate from) the allocation code will invalidate some 541cafe5635SKent Overstreet * buckets, but it can't use those buckets until their new gens are safely on 542cafe5635SKent Overstreet * disk. 543cafe5635SKent Overstreet */ 544cafe5635SKent Overstreet 5454246a0b6SChristoph Hellwig static void prio_endio(struct bio *bio) 546cafe5635SKent Overstreet { 547cafe5635SKent Overstreet struct cache *ca = bio->bi_private; 548cafe5635SKent Overstreet 5494e4cbee9SChristoph Hellwig cache_set_err_on(bio->bi_status, ca->set, "accessing priorities"); 550cafe5635SKent Overstreet bch_bbio_free(bio, ca->set); 551cafe5635SKent Overstreet closure_put(&ca->prio); 552cafe5635SKent Overstreet } 553cafe5635SKent Overstreet 554ad0d9e76SMike Christie static void prio_io(struct cache *ca, uint64_t bucket, int op, 555ad0d9e76SMike Christie unsigned long op_flags) 556cafe5635SKent Overstreet { 557cafe5635SKent Overstreet struct closure *cl = &ca->prio; 558cafe5635SKent Overstreet struct bio *bio = bch_bbio_alloc(ca->set); 559cafe5635SKent Overstreet 560cafe5635SKent Overstreet closure_init_stack(cl); 561cafe5635SKent Overstreet 5624f024f37SKent Overstreet bio->bi_iter.bi_sector = bucket * ca->sb.bucket_size; 56374d46992SChristoph Hellwig bio_set_dev(bio, ca->bdev); 5644f024f37SKent Overstreet bio->bi_iter.bi_size = bucket_bytes(ca); 565cafe5635SKent Overstreet 566cafe5635SKent Overstreet bio->bi_end_io = prio_endio; 567cafe5635SKent Overstreet bio->bi_private = ca; 568ad0d9e76SMike Christie bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags); 569169ef1cfSKent Overstreet bch_bio_map(bio, ca->disk_buckets); 570cafe5635SKent Overstreet 571771f393eSColy Li closure_bio_submit(ca->set, bio, &ca->prio); 572cafe5635SKent Overstreet closure_sync(cl); 573cafe5635SKent Overstreet } 574cafe5635SKent Overstreet 57584c529aeSAndrea Righi int bch_prio_write(struct cache *ca, bool wait) 576cafe5635SKent Overstreet { 577cafe5635SKent Overstreet int i; 578cafe5635SKent Overstreet struct bucket *b; 579cafe5635SKent Overstreet struct closure cl; 580cafe5635SKent Overstreet 58146f5aa88SJoe Perches pr_debug("free_prio=%zu, free_none=%zu, free_inc=%zu\n", 58284c529aeSAndrea Righi fifo_used(&ca->free[RESERVE_PRIO]), 58384c529aeSAndrea Righi fifo_used(&ca->free[RESERVE_NONE]), 58484c529aeSAndrea Righi fifo_used(&ca->free_inc)); 58584c529aeSAndrea Righi 58684c529aeSAndrea Righi /* 58784c529aeSAndrea Righi * Pre-check if there are enough free buckets. In the non-blocking 58884c529aeSAndrea Righi * scenario it's better to fail early rather than starting to allocate 58984c529aeSAndrea Righi * buckets and do a cleanup later in case of failure. 59084c529aeSAndrea Righi */ 59184c529aeSAndrea Righi if (!wait) { 59284c529aeSAndrea Righi size_t avail = fifo_used(&ca->free[RESERVE_PRIO]) + 59384c529aeSAndrea Righi fifo_used(&ca->free[RESERVE_NONE]); 59484c529aeSAndrea Righi if (prio_buckets(ca) > avail) 59584c529aeSAndrea Righi return -ENOMEM; 59684c529aeSAndrea Righi } 59784c529aeSAndrea Righi 598cafe5635SKent Overstreet closure_init_stack(&cl); 599cafe5635SKent Overstreet 600cafe5635SKent Overstreet lockdep_assert_held(&ca->set->bucket_lock); 601cafe5635SKent Overstreet 602cafe5635SKent Overstreet ca->disk_buckets->seq++; 603cafe5635SKent Overstreet 604cafe5635SKent Overstreet atomic_long_add(ca->sb.bucket_size * prio_buckets(ca), 605cafe5635SKent Overstreet &ca->meta_sectors_written); 606cafe5635SKent Overstreet 607cafe5635SKent Overstreet for (i = prio_buckets(ca) - 1; i >= 0; --i) { 608cafe5635SKent Overstreet long bucket; 609cafe5635SKent Overstreet struct prio_set *p = ca->disk_buckets; 610b1a67b0fSKent Overstreet struct bucket_disk *d = p->data; 611b1a67b0fSKent Overstreet struct bucket_disk *end = d + prios_per_bucket(ca); 612cafe5635SKent Overstreet 613cafe5635SKent Overstreet for (b = ca->buckets + i * prios_per_bucket(ca); 614cafe5635SKent Overstreet b < ca->buckets + ca->sb.nbuckets && d < end; 615cafe5635SKent Overstreet b++, d++) { 616cafe5635SKent Overstreet d->prio = cpu_to_le16(b->prio); 617cafe5635SKent Overstreet d->gen = b->gen; 618cafe5635SKent Overstreet } 619cafe5635SKent Overstreet 620cafe5635SKent Overstreet p->next_bucket = ca->prio_buckets[i + 1]; 62181ab4190SKent Overstreet p->magic = pset_magic(&ca->sb); 622169ef1cfSKent Overstreet p->csum = bch_crc64(&p->magic, bucket_bytes(ca) - 8); 623cafe5635SKent Overstreet 62484c529aeSAndrea Righi bucket = bch_bucket_alloc(ca, RESERVE_PRIO, wait); 625cafe5635SKent Overstreet BUG_ON(bucket == -1); 626cafe5635SKent Overstreet 627cafe5635SKent Overstreet mutex_unlock(&ca->set->bucket_lock); 628ad0d9e76SMike Christie prio_io(ca, bucket, REQ_OP_WRITE, 0); 629cafe5635SKent Overstreet mutex_lock(&ca->set->bucket_lock); 630cafe5635SKent Overstreet 631cafe5635SKent Overstreet ca->prio_buckets[i] = bucket; 632cafe5635SKent Overstreet atomic_dec_bug(&ca->buckets[bucket].pin); 633cafe5635SKent Overstreet } 634cafe5635SKent Overstreet 635cafe5635SKent Overstreet mutex_unlock(&ca->set->bucket_lock); 636cafe5635SKent Overstreet 637cafe5635SKent Overstreet bch_journal_meta(ca->set, &cl); 638cafe5635SKent Overstreet closure_sync(&cl); 639cafe5635SKent Overstreet 640cafe5635SKent Overstreet mutex_lock(&ca->set->bucket_lock); 641cafe5635SKent Overstreet 642cafe5635SKent Overstreet /* 643cafe5635SKent Overstreet * Don't want the old priorities to get garbage collected until after we 644cafe5635SKent Overstreet * finish writing the new ones, and they're journalled 645cafe5635SKent Overstreet */ 6462531d9eeSKent Overstreet for (i = 0; i < prio_buckets(ca); i++) { 6472531d9eeSKent Overstreet if (ca->prio_last_buckets[i]) 6482531d9eeSKent Overstreet __bch_bucket_free(ca, 6492531d9eeSKent Overstreet &ca->buckets[ca->prio_last_buckets[i]]); 6502531d9eeSKent Overstreet 651cafe5635SKent Overstreet ca->prio_last_buckets[i] = ca->prio_buckets[i]; 652cafe5635SKent Overstreet } 65384c529aeSAndrea Righi return 0; 6542531d9eeSKent Overstreet } 655cafe5635SKent Overstreet 65649d08d59SColy Li static int prio_read(struct cache *ca, uint64_t bucket) 657cafe5635SKent Overstreet { 658cafe5635SKent Overstreet struct prio_set *p = ca->disk_buckets; 659cafe5635SKent Overstreet struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d; 660cafe5635SKent Overstreet struct bucket *b; 6616f10f7d1SColy Li unsigned int bucket_nr = 0; 66249d08d59SColy Li int ret = -EIO; 663cafe5635SKent Overstreet 664cafe5635SKent Overstreet for (b = ca->buckets; 665cafe5635SKent Overstreet b < ca->buckets + ca->sb.nbuckets; 666cafe5635SKent Overstreet b++, d++) { 667cafe5635SKent Overstreet if (d == end) { 668cafe5635SKent Overstreet ca->prio_buckets[bucket_nr] = bucket; 669cafe5635SKent Overstreet ca->prio_last_buckets[bucket_nr] = bucket; 670cafe5635SKent Overstreet bucket_nr++; 671cafe5635SKent Overstreet 67270fd7614SChristoph Hellwig prio_io(ca, bucket, REQ_OP_READ, 0); 673cafe5635SKent Overstreet 674b0d30981SColy Li if (p->csum != 67549d08d59SColy Li bch_crc64(&p->magic, bucket_bytes(ca) - 8)) { 67646f5aa88SJoe Perches pr_warn("bad csum reading priorities\n"); 67749d08d59SColy Li goto out; 67849d08d59SColy Li } 679cafe5635SKent Overstreet 68049d08d59SColy Li if (p->magic != pset_magic(&ca->sb)) { 68146f5aa88SJoe Perches pr_warn("bad magic reading priorities\n"); 68249d08d59SColy Li goto out; 68349d08d59SColy Li } 684cafe5635SKent Overstreet 685cafe5635SKent Overstreet bucket = p->next_bucket; 686cafe5635SKent Overstreet d = p->data; 687cafe5635SKent Overstreet } 688cafe5635SKent Overstreet 689cafe5635SKent Overstreet b->prio = le16_to_cpu(d->prio); 6903a2fd9d5SKent Overstreet b->gen = b->last_gc = d->gen; 691cafe5635SKent Overstreet } 69249d08d59SColy Li 69349d08d59SColy Li ret = 0; 69449d08d59SColy Li out: 69549d08d59SColy Li return ret; 696cafe5635SKent Overstreet } 697cafe5635SKent Overstreet 698cafe5635SKent Overstreet /* Bcache device */ 699cafe5635SKent Overstreet 700cafe5635SKent Overstreet static int open_dev(struct block_device *b, fmode_t mode) 701cafe5635SKent Overstreet { 702cafe5635SKent Overstreet struct bcache_device *d = b->bd_disk->private_data; 7031fae7cf0SColy Li 704c4d951ddSKent Overstreet if (test_bit(BCACHE_DEV_CLOSING, &d->flags)) 705cafe5635SKent Overstreet return -ENXIO; 706cafe5635SKent Overstreet 707cafe5635SKent Overstreet closure_get(&d->cl); 708cafe5635SKent Overstreet return 0; 709cafe5635SKent Overstreet } 710cafe5635SKent Overstreet 711867e1162SEmil Goode static void release_dev(struct gendisk *b, fmode_t mode) 712cafe5635SKent Overstreet { 713cafe5635SKent Overstreet struct bcache_device *d = b->private_data; 7141fae7cf0SColy Li 715cafe5635SKent Overstreet closure_put(&d->cl); 716cafe5635SKent Overstreet } 717cafe5635SKent Overstreet 718cafe5635SKent Overstreet static int ioctl_dev(struct block_device *b, fmode_t mode, 719cafe5635SKent Overstreet unsigned int cmd, unsigned long arg) 720cafe5635SKent Overstreet { 721cafe5635SKent Overstreet struct bcache_device *d = b->bd_disk->private_data; 7220f0709e6SColy Li 723cafe5635SKent Overstreet return d->ioctl(d, mode, cmd, arg); 724cafe5635SKent Overstreet } 725cafe5635SKent Overstreet 726c62b37d9SChristoph Hellwig static const struct block_device_operations bcache_cached_ops = { 727c62b37d9SChristoph Hellwig .submit_bio = cached_dev_submit_bio, 728c62b37d9SChristoph Hellwig .open = open_dev, 729c62b37d9SChristoph Hellwig .release = release_dev, 730c62b37d9SChristoph Hellwig .ioctl = ioctl_dev, 731c62b37d9SChristoph Hellwig .owner = THIS_MODULE, 732c62b37d9SChristoph Hellwig }; 733c62b37d9SChristoph Hellwig 734c62b37d9SChristoph Hellwig static const struct block_device_operations bcache_flash_ops = { 735c62b37d9SChristoph Hellwig .submit_bio = flash_dev_submit_bio, 736cafe5635SKent Overstreet .open = open_dev, 737cafe5635SKent Overstreet .release = release_dev, 738cafe5635SKent Overstreet .ioctl = ioctl_dev, 739cafe5635SKent Overstreet .owner = THIS_MODULE, 740cafe5635SKent Overstreet }; 741cafe5635SKent Overstreet 742cafe5635SKent Overstreet void bcache_device_stop(struct bcache_device *d) 743cafe5635SKent Overstreet { 744c4d951ddSKent Overstreet if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags)) 74563d63b51SColy Li /* 74663d63b51SColy Li * closure_fn set to 74763d63b51SColy Li * - cached device: cached_dev_flush() 74863d63b51SColy Li * - flash dev: flash_dev_flush() 74963d63b51SColy Li */ 750cafe5635SKent Overstreet closure_queue(&d->cl); 751cafe5635SKent Overstreet } 752cafe5635SKent Overstreet 753ee668506SKent Overstreet static void bcache_device_unlink(struct bcache_device *d) 754ee668506SKent Overstreet { 755c4d951ddSKent Overstreet lockdep_assert_held(&bch_register_lock); 756c4d951ddSKent Overstreet 757c4d951ddSKent Overstreet if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) { 7586f10f7d1SColy Li unsigned int i; 759ee668506SKent Overstreet struct cache *ca; 760ee668506SKent Overstreet 761ee668506SKent Overstreet sysfs_remove_link(&d->c->kobj, d->name); 762ee668506SKent Overstreet sysfs_remove_link(&d->kobj, "cache"); 763ee668506SKent Overstreet 764ee668506SKent Overstreet for_each_cache(ca, d->c, i) 765ee668506SKent Overstreet bd_unlink_disk_holder(ca->bdev, d->disk); 766ee668506SKent Overstreet } 767c4d951ddSKent Overstreet } 768ee668506SKent Overstreet 769ee668506SKent Overstreet static void bcache_device_link(struct bcache_device *d, struct cache_set *c, 770ee668506SKent Overstreet const char *name) 771ee668506SKent Overstreet { 7726f10f7d1SColy Li unsigned int i; 773ee668506SKent Overstreet struct cache *ca; 7744b6efb4bSColy Li int ret; 775ee668506SKent Overstreet 776ee668506SKent Overstreet for_each_cache(ca, d->c, i) 777ee668506SKent Overstreet bd_link_disk_holder(ca->bdev, d->disk); 778ee668506SKent Overstreet 779ee668506SKent Overstreet snprintf(d->name, BCACHEDEVNAME_SIZE, 780ee668506SKent Overstreet "%s%u", name, d->id); 781ee668506SKent Overstreet 7824b6efb4bSColy Li ret = sysfs_create_link(&d->kobj, &c->kobj, "cache"); 7834b6efb4bSColy Li if (ret < 0) 78446f5aa88SJoe Perches pr_err("Couldn't create device -> cache set symlink\n"); 7854b6efb4bSColy Li 7864b6efb4bSColy Li ret = sysfs_create_link(&c->kobj, &d->kobj, d->name); 7874b6efb4bSColy Li if (ret < 0) 78846f5aa88SJoe Perches pr_err("Couldn't create cache set -> device symlink\n"); 789fecaee6fSZheng Liu 790fecaee6fSZheng Liu clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags); 791ee668506SKent Overstreet } 792ee668506SKent Overstreet 793cafe5635SKent Overstreet static void bcache_device_detach(struct bcache_device *d) 794cafe5635SKent Overstreet { 795cafe5635SKent Overstreet lockdep_assert_held(&bch_register_lock); 796cafe5635SKent Overstreet 797ea8c5356SColy Li atomic_dec(&d->c->attached_dev_nr); 798ea8c5356SColy Li 799c4d951ddSKent Overstreet if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) { 800cafe5635SKent Overstreet struct uuid_entry *u = d->c->uuids + d->id; 801cafe5635SKent Overstreet 802cafe5635SKent Overstreet SET_UUID_FLASH_ONLY(u, 0); 803cafe5635SKent Overstreet memcpy(u->uuid, invalid_uuid, 16); 80475cbb3f1SArnd Bergmann u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds()); 805cafe5635SKent Overstreet bch_uuid_write(d->c); 806cafe5635SKent Overstreet } 807cafe5635SKent Overstreet 808ee668506SKent Overstreet bcache_device_unlink(d); 809ee668506SKent Overstreet 810cafe5635SKent Overstreet d->c->devices[d->id] = NULL; 811cafe5635SKent Overstreet closure_put(&d->c->caching); 812cafe5635SKent Overstreet d->c = NULL; 813cafe5635SKent Overstreet } 814cafe5635SKent Overstreet 815cafe5635SKent Overstreet static void bcache_device_attach(struct bcache_device *d, struct cache_set *c, 8166f10f7d1SColy Li unsigned int id) 817cafe5635SKent Overstreet { 818cafe5635SKent Overstreet d->id = id; 819cafe5635SKent Overstreet d->c = c; 820cafe5635SKent Overstreet c->devices[id] = d; 821cafe5635SKent Overstreet 8222831231dSColy Li if (id >= c->devices_max_used) 8232831231dSColy Li c->devices_max_used = id + 1; 8242831231dSColy Li 825cafe5635SKent Overstreet closure_get(&c->caching); 826cafe5635SKent Overstreet } 827cafe5635SKent Overstreet 8281dbe32adSColy Li static inline int first_minor_to_idx(int first_minor) 8291dbe32adSColy Li { 8301dbe32adSColy Li return (first_minor/BCACHE_MINORS); 8311dbe32adSColy Li } 8321dbe32adSColy Li 8331dbe32adSColy Li static inline int idx_to_first_minor(int idx) 8341dbe32adSColy Li { 8351dbe32adSColy Li return (idx * BCACHE_MINORS); 8361dbe32adSColy Li } 8371dbe32adSColy Li 838cafe5635SKent Overstreet static void bcache_device_free(struct bcache_device *d) 839cafe5635SKent Overstreet { 8402d886951SColy Li struct gendisk *disk = d->disk; 8412d886951SColy Li 842cafe5635SKent Overstreet lockdep_assert_held(&bch_register_lock); 843cafe5635SKent Overstreet 8442d886951SColy Li if (disk) 84546f5aa88SJoe Perches pr_info("%s stopped\n", disk->disk_name); 8462d886951SColy Li else 84746f5aa88SJoe Perches pr_err("bcache device (NULL gendisk) stopped\n"); 848cafe5635SKent Overstreet 849cafe5635SKent Overstreet if (d->c) 850cafe5635SKent Overstreet bcache_device_detach(d); 8512d886951SColy Li 8522d886951SColy Li if (disk) { 85386da9f73SColy Li bool disk_added = (disk->flags & GENHD_FL_UP) != 0; 85486da9f73SColy Li 85586da9f73SColy Li if (disk_added) 8562d886951SColy Li del_gendisk(disk); 8572d886951SColy Li 8582d886951SColy Li if (disk->queue) 8592d886951SColy Li blk_cleanup_queue(disk->queue); 8602d886951SColy Li 8611dbe32adSColy Li ida_simple_remove(&bcache_device_idx, 8622d886951SColy Li first_minor_to_idx(disk->first_minor)); 86386da9f73SColy Li if (disk_added) 8642d886951SColy Li put_disk(disk); 86528935ab5SKent Overstreet } 866cafe5635SKent Overstreet 867d19936a2SKent Overstreet bioset_exit(&d->bio_split); 868958b4338SPekka Enberg kvfree(d->full_dirty_stripes); 869958b4338SPekka Enberg kvfree(d->stripe_sectors_dirty); 870cafe5635SKent Overstreet 871cafe5635SKent Overstreet closure_debug_destroy(&d->cl); 872cafe5635SKent Overstreet } 873cafe5635SKent Overstreet 8746f10f7d1SColy Li static int bcache_device_init(struct bcache_device *d, unsigned int block_size, 875c62b37d9SChristoph Hellwig sector_t sectors, struct block_device *cached_bdev, 876c62b37d9SChristoph Hellwig const struct block_device_operations *ops) 877cafe5635SKent Overstreet { 878cafe5635SKent Overstreet struct request_queue *q; 8795f2b18ecSBart Van Assche const size_t max_stripes = min_t(size_t, INT_MAX, 8805f2b18ecSBart Van Assche SIZE_MAX / sizeof(atomic_t)); 88165f0f017SColy Li uint64_t n; 8821dbe32adSColy Li int idx; 883279afbadSKent Overstreet 8842d679fc7SKent Overstreet if (!d->stripe_size) 8852d679fc7SKent Overstreet d->stripe_size = 1 << 31; 886279afbadSKent Overstreet 88765f0f017SColy Li n = DIV_ROUND_UP_ULL(sectors, d->stripe_size); 88865f0f017SColy Li if (!n || n > max_stripes) { 88965f0f017SColy Li pr_err("nr_stripes too large or invalid: %llu (start sector beyond end of disk?)\n", 89065f0f017SColy Li n); 891279afbadSKent Overstreet return -ENOMEM; 89248a915a8SKent Overstreet } 89365f0f017SColy Li d->nr_stripes = n; 894279afbadSKent Overstreet 895279afbadSKent Overstreet n = d->nr_stripes * sizeof(atomic_t); 896bc4e54f6SMichal Hocko d->stripe_sectors_dirty = kvzalloc(n, GFP_KERNEL); 897279afbadSKent Overstreet if (!d->stripe_sectors_dirty) 898279afbadSKent Overstreet return -ENOMEM; 899cafe5635SKent Overstreet 90048a915a8SKent Overstreet n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long); 901bc4e54f6SMichal Hocko d->full_dirty_stripes = kvzalloc(n, GFP_KERNEL); 90248a915a8SKent Overstreet if (!d->full_dirty_stripes) 90348a915a8SKent Overstreet return -ENOMEM; 90448a915a8SKent Overstreet 9051dbe32adSColy Li idx = ida_simple_get(&bcache_device_idx, 0, 9061dbe32adSColy Li BCACHE_DEVICE_IDX_MAX, GFP_KERNEL); 9071dbe32adSColy Li if (idx < 0) 9081dbe32adSColy Li return idx; 909b8c0d911SEric Wheeler 910d19936a2SKent Overstreet if (bioset_init(&d->bio_split, 4, offsetof(struct bbio, bio), 9119b4e9f5aSFlorian Schmaus BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER)) 9129b4e9f5aSFlorian Schmaus goto err; 9139b4e9f5aSFlorian Schmaus 9149b4e9f5aSFlorian Schmaus d->disk = alloc_disk(BCACHE_MINORS); 9159b4e9f5aSFlorian Schmaus if (!d->disk) 9169b4e9f5aSFlorian Schmaus goto err; 917cafe5635SKent Overstreet 918279afbadSKent Overstreet set_capacity(d->disk, sectors); 9191dbe32adSColy Li snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", idx); 920cafe5635SKent Overstreet 921cafe5635SKent Overstreet d->disk->major = bcache_major; 9221dbe32adSColy Li d->disk->first_minor = idx_to_first_minor(idx); 923c62b37d9SChristoph Hellwig d->disk->fops = ops; 924cafe5635SKent Overstreet d->disk->private_data = d; 925cafe5635SKent Overstreet 926c62b37d9SChristoph Hellwig q = blk_alloc_queue(NUMA_NO_NODE); 92728935ab5SKent Overstreet if (!q) 92828935ab5SKent Overstreet return -ENOMEM; 92928935ab5SKent Overstreet 930cafe5635SKent Overstreet d->disk->queue = q; 931dc3b17ccSJan Kara q->backing_dev_info->congested_data = d; 932cafe5635SKent Overstreet q->limits.max_hw_sectors = UINT_MAX; 933cafe5635SKent Overstreet q->limits.max_sectors = UINT_MAX; 934cafe5635SKent Overstreet q->limits.max_segment_size = UINT_MAX; 935cafe5635SKent Overstreet q->limits.max_segments = BIO_MAX_PAGES; 9362bb4cd5cSJens Axboe blk_queue_max_discard_sectors(q, UINT_MAX); 93790db6919SKent Overstreet q->limits.discard_granularity = 512; 938cafe5635SKent Overstreet q->limits.io_min = block_size; 939cafe5635SKent Overstreet q->limits.logical_block_size = block_size; 940cafe5635SKent Overstreet q->limits.physical_block_size = block_size; 941dcacbc12SMauricio Faria de Oliveira 942dcacbc12SMauricio Faria de Oliveira if (q->limits.logical_block_size > PAGE_SIZE && cached_bdev) { 943dcacbc12SMauricio Faria de Oliveira /* 944dcacbc12SMauricio Faria de Oliveira * This should only happen with BCACHE_SB_VERSION_BDEV. 945dcacbc12SMauricio Faria de Oliveira * Block/page size is checked for BCACHE_SB_VERSION_CDEV. 946dcacbc12SMauricio Faria de Oliveira */ 9474b25bbf5SColy Li pr_info("%s: sb/logical block size (%u) greater than page size (%lu) falling back to device logical block size (%u)\n", 948dcacbc12SMauricio Faria de Oliveira d->disk->disk_name, q->limits.logical_block_size, 949dcacbc12SMauricio Faria de Oliveira PAGE_SIZE, bdev_logical_block_size(cached_bdev)); 950dcacbc12SMauricio Faria de Oliveira 951dcacbc12SMauricio Faria de Oliveira /* This also adjusts physical block size/min io size if needed */ 952dcacbc12SMauricio Faria de Oliveira blk_queue_logical_block_size(q, bdev_logical_block_size(cached_bdev)); 953dcacbc12SMauricio Faria de Oliveira } 954dcacbc12SMauricio Faria de Oliveira 95544e1ebe2SBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NONROT, d->disk->queue); 95644e1ebe2SBart Van Assche blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, d->disk->queue); 95744e1ebe2SBart Van Assche blk_queue_flag_set(QUEUE_FLAG_DISCARD, d->disk->queue); 958cafe5635SKent Overstreet 95984b4ff9eSJens Axboe blk_queue_write_cache(q, true, true); 96054d12f2bSKent Overstreet 961cafe5635SKent Overstreet return 0; 9629b4e9f5aSFlorian Schmaus 9639b4e9f5aSFlorian Schmaus err: 9649b4e9f5aSFlorian Schmaus ida_simple_remove(&bcache_device_idx, idx); 9659b4e9f5aSFlorian Schmaus return -ENOMEM; 9669b4e9f5aSFlorian Schmaus 967cafe5635SKent Overstreet } 968cafe5635SKent Overstreet 969cafe5635SKent Overstreet /* Cached device */ 970cafe5635SKent Overstreet 971cafe5635SKent Overstreet static void calc_cached_dev_sectors(struct cache_set *c) 972cafe5635SKent Overstreet { 973cafe5635SKent Overstreet uint64_t sectors = 0; 974cafe5635SKent Overstreet struct cached_dev *dc; 975cafe5635SKent Overstreet 976cafe5635SKent Overstreet list_for_each_entry(dc, &c->cached_devs, list) 977cafe5635SKent Overstreet sectors += bdev_sectors(dc->bdev); 978cafe5635SKent Overstreet 979cafe5635SKent Overstreet c->cached_dev_sectors = sectors; 980cafe5635SKent Overstreet } 981cafe5635SKent Overstreet 9820f0709e6SColy Li #define BACKING_DEV_OFFLINE_TIMEOUT 5 9830f0709e6SColy Li static int cached_dev_status_update(void *arg) 9840f0709e6SColy Li { 9850f0709e6SColy Li struct cached_dev *dc = arg; 9860f0709e6SColy Li struct request_queue *q; 9870f0709e6SColy Li 9880f0709e6SColy Li /* 9890f0709e6SColy Li * If this delayed worker is stopping outside, directly quit here. 9900f0709e6SColy Li * dc->io_disable might be set via sysfs interface, so check it 9910f0709e6SColy Li * here too. 9920f0709e6SColy Li */ 9930f0709e6SColy Li while (!kthread_should_stop() && !dc->io_disable) { 9940f0709e6SColy Li q = bdev_get_queue(dc->bdev); 9950f0709e6SColy Li if (blk_queue_dying(q)) 9960f0709e6SColy Li dc->offline_seconds++; 9970f0709e6SColy Li else 9980f0709e6SColy Li dc->offline_seconds = 0; 9990f0709e6SColy Li 10000f0709e6SColy Li if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) { 100146f5aa88SJoe Perches pr_err("%s: device offline for %d seconds\n", 10020f0709e6SColy Li dc->backing_dev_name, 10030f0709e6SColy Li BACKING_DEV_OFFLINE_TIMEOUT); 100446f5aa88SJoe Perches pr_err("%s: disable I/O request due to backing device offline\n", 100546f5aa88SJoe Perches dc->disk.name); 10060f0709e6SColy Li dc->io_disable = true; 10070f0709e6SColy Li /* let others know earlier that io_disable is true */ 10080f0709e6SColy Li smp_mb(); 10090f0709e6SColy Li bcache_device_stop(&dc->disk); 10100f0709e6SColy Li break; 10110f0709e6SColy Li } 10120f0709e6SColy Li schedule_timeout_interruptible(HZ); 10130f0709e6SColy Li } 10140f0709e6SColy Li 10150f0709e6SColy Li wait_for_kthread_stop(); 10160f0709e6SColy Li return 0; 10170f0709e6SColy Li } 10180f0709e6SColy Li 10190f0709e6SColy Li 10200b13efecSColy Li int bch_cached_dev_run(struct cached_dev *dc) 1021cafe5635SKent Overstreet { 1022cafe5635SKent Overstreet struct bcache_device *d = &dc->disk; 1023792732d9SGeliang Tang char *buf = kmemdup_nul(dc->sb.label, SB_LABEL_SIZE, GFP_KERNEL); 1024a25c32beSGabriel de Perthuis char *env[] = { 1025a25c32beSGabriel de Perthuis "DRIVER=bcache", 1026a25c32beSGabriel de Perthuis kasprintf(GFP_KERNEL, "CACHED_UUID=%pU", dc->sb.uuid), 1027792732d9SGeliang Tang kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf ? : ""), 1028ab9e1400SGabriel de Perthuis NULL, 1029a25c32beSGabriel de Perthuis }; 1030cafe5635SKent Overstreet 1031e0faa3d7SColy Li if (dc->io_disable) { 103246f5aa88SJoe Perches pr_err("I/O disabled on cached dev %s\n", 1033e0faa3d7SColy Li dc->backing_dev_name); 10345d9e06d6SWei Yongjun kfree(env[1]); 10355d9e06d6SWei Yongjun kfree(env[2]); 10365d9e06d6SWei Yongjun kfree(buf); 10370b13efecSColy Li return -EIO; 1038e0faa3d7SColy Li } 10390b13efecSColy Li 10404d4d8573SAl Viro if (atomic_xchg(&dc->running, 1)) { 10414d4d8573SAl Viro kfree(env[1]); 10424d4d8573SAl Viro kfree(env[2]); 1043792732d9SGeliang Tang kfree(buf); 104446f5aa88SJoe Perches pr_info("cached dev %s is running already\n", 1045e0faa3d7SColy Li dc->backing_dev_name); 10460b13efecSColy Li return -EBUSY; 10474d4d8573SAl Viro } 1048cafe5635SKent Overstreet 1049cafe5635SKent Overstreet if (!d->c && 1050cafe5635SKent Overstreet BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) { 1051cafe5635SKent Overstreet struct closure cl; 10521fae7cf0SColy Li 1053cafe5635SKent Overstreet closure_init_stack(&cl); 1054cafe5635SKent Overstreet 1055cafe5635SKent Overstreet SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE); 1056cafe5635SKent Overstreet bch_write_bdev_super(dc, &cl); 1057cafe5635SKent Overstreet closure_sync(&cl); 1058cafe5635SKent Overstreet } 1059cafe5635SKent Overstreet 1060cafe5635SKent Overstreet add_disk(d->disk); 1061ee668506SKent Overstreet bd_link_disk_holder(dc->bdev, dc->disk.disk); 10623be11dbaSColy Li /* 10633be11dbaSColy Li * won't show up in the uevent file, use udevadm monitor -e instead 10643be11dbaSColy Li * only class / kset properties are persistent 10653be11dbaSColy Li */ 1066cafe5635SKent Overstreet kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env); 1067a25c32beSGabriel de Perthuis kfree(env[1]); 1068ab9e1400SGabriel de Perthuis kfree(env[2]); 1069792732d9SGeliang Tang kfree(buf); 1070a25c32beSGabriel de Perthuis 1071cafe5635SKent Overstreet if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") || 10720b13efecSColy Li sysfs_create_link(&disk_to_dev(d->disk)->kobj, 10730b13efecSColy Li &d->kobj, "bcache")) { 107446f5aa88SJoe Perches pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n"); 10750b13efecSColy Li return -ENOMEM; 10760b13efecSColy Li } 10770f0709e6SColy Li 10780f0709e6SColy Li dc->status_update_thread = kthread_run(cached_dev_status_update, 10790f0709e6SColy Li dc, "bcache_status_update"); 10800f0709e6SColy Li if (IS_ERR(dc->status_update_thread)) { 108146f5aa88SJoe Perches pr_warn("failed to create bcache_status_update kthread, continue to run without monitoring backing device status\n"); 10820f0709e6SColy Li } 10830b13efecSColy Li 10840b13efecSColy Li return 0; 1085cafe5635SKent Overstreet } 1086cafe5635SKent Overstreet 10873fd47bfeSColy Li /* 10883fd47bfeSColy Li * If BCACHE_DEV_RATE_DW_RUNNING is set, it means routine of the delayed 10893fd47bfeSColy Li * work dc->writeback_rate_update is running. Wait until the routine 10903fd47bfeSColy Li * quits (BCACHE_DEV_RATE_DW_RUNNING is clear), then continue to 10913fd47bfeSColy Li * cancel it. If BCACHE_DEV_RATE_DW_RUNNING is not clear after time_out 10923fd47bfeSColy Li * seconds, give up waiting here and continue to cancel it too. 10933fd47bfeSColy Li */ 10943fd47bfeSColy Li static void cancel_writeback_rate_update_dwork(struct cached_dev *dc) 10953fd47bfeSColy Li { 10963fd47bfeSColy Li int time_out = WRITEBACK_RATE_UPDATE_SECS_MAX * HZ; 10973fd47bfeSColy Li 10983fd47bfeSColy Li do { 10993fd47bfeSColy Li if (!test_bit(BCACHE_DEV_RATE_DW_RUNNING, 11003fd47bfeSColy Li &dc->disk.flags)) 11013fd47bfeSColy Li break; 11023fd47bfeSColy Li time_out--; 11033fd47bfeSColy Li schedule_timeout_interruptible(1); 11043fd47bfeSColy Li } while (time_out > 0); 11053fd47bfeSColy Li 11063fd47bfeSColy Li if (time_out == 0) 110746f5aa88SJoe Perches pr_warn("give up waiting for dc->writeback_write_update to quit\n"); 11083fd47bfeSColy Li 11093fd47bfeSColy Li cancel_delayed_work_sync(&dc->writeback_rate_update); 11103fd47bfeSColy Li } 11113fd47bfeSColy Li 1112cafe5635SKent Overstreet static void cached_dev_detach_finish(struct work_struct *w) 1113cafe5635SKent Overstreet { 1114cafe5635SKent Overstreet struct cached_dev *dc = container_of(w, struct cached_dev, detach); 1115cafe5635SKent Overstreet struct closure cl; 11161fae7cf0SColy Li 1117cafe5635SKent Overstreet closure_init_stack(&cl); 1118cafe5635SKent Overstreet 1119c4d951ddSKent Overstreet BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)); 11203b304d24SElena Reshetova BUG_ON(refcount_read(&dc->count)); 1121cafe5635SKent Overstreet 1122cafe5635SKent Overstreet 11233fd47bfeSColy Li if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags)) 11243fd47bfeSColy Li cancel_writeback_rate_update_dwork(dc); 11253fd47bfeSColy Li 11268d29c442STang Junhui if (!IS_ERR_OR_NULL(dc->writeback_thread)) { 11278d29c442STang Junhui kthread_stop(dc->writeback_thread); 11288d29c442STang Junhui dc->writeback_thread = NULL; 11298d29c442STang Junhui } 11308d29c442STang Junhui 1131cafe5635SKent Overstreet memset(&dc->sb.set_uuid, 0, 16); 1132cafe5635SKent Overstreet SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE); 1133cafe5635SKent Overstreet 1134cafe5635SKent Overstreet bch_write_bdev_super(dc, &cl); 1135cafe5635SKent Overstreet closure_sync(&cl); 1136cafe5635SKent Overstreet 113797ba3b81SColy Li mutex_lock(&bch_register_lock); 113897ba3b81SColy Li 113946010141SShenghui Wang calc_cached_dev_sectors(dc->disk.c); 1140cafe5635SKent Overstreet bcache_device_detach(&dc->disk); 1141cafe5635SKent Overstreet list_move(&dc->list, &uncached_devices); 1142cafe5635SKent Overstreet 1143c4d951ddSKent Overstreet clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags); 11445b1016e6SKent Overstreet clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags); 1145c4d951ddSKent Overstreet 1146cafe5635SKent Overstreet mutex_unlock(&bch_register_lock); 1147cafe5635SKent Overstreet 114846f5aa88SJoe Perches pr_info("Caching disabled for %s\n", dc->backing_dev_name); 1149cafe5635SKent Overstreet 1150cafe5635SKent Overstreet /* Drop ref we took in cached_dev_detach() */ 1151cafe5635SKent Overstreet closure_put(&dc->disk.cl); 1152cafe5635SKent Overstreet } 1153cafe5635SKent Overstreet 1154cafe5635SKent Overstreet void bch_cached_dev_detach(struct cached_dev *dc) 1155cafe5635SKent Overstreet { 1156cafe5635SKent Overstreet lockdep_assert_held(&bch_register_lock); 1157cafe5635SKent Overstreet 1158c4d951ddSKent Overstreet if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags)) 1159cafe5635SKent Overstreet return; 1160cafe5635SKent Overstreet 1161c4d951ddSKent Overstreet if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) 1162cafe5635SKent Overstreet return; 1163cafe5635SKent Overstreet 1164cafe5635SKent Overstreet /* 1165cafe5635SKent Overstreet * Block the device from being closed and freed until we're finished 1166cafe5635SKent Overstreet * detaching 1167cafe5635SKent Overstreet */ 1168cafe5635SKent Overstreet closure_get(&dc->disk.cl); 1169cafe5635SKent Overstreet 1170cafe5635SKent Overstreet bch_writeback_queue(dc); 11713fd47bfeSColy Li 1172cafe5635SKent Overstreet cached_dev_put(dc); 1173cafe5635SKent Overstreet } 1174cafe5635SKent Overstreet 117573ac105bSTang Junhui int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, 117673ac105bSTang Junhui uint8_t *set_uuid) 1177cafe5635SKent Overstreet { 117875cbb3f1SArnd Bergmann uint32_t rtime = cpu_to_le32((u32)ktime_get_real_seconds()); 1179cafe5635SKent Overstreet struct uuid_entry *u; 118086755b7aSMichael Lyle struct cached_dev *exist_dc, *t; 11810b13efecSColy Li int ret = 0; 1182cafe5635SKent Overstreet 118373ac105bSTang Junhui if ((set_uuid && memcmp(set_uuid, c->sb.set_uuid, 16)) || 118473ac105bSTang Junhui (!set_uuid && memcmp(dc->sb.set_uuid, c->sb.set_uuid, 16))) 1185cafe5635SKent Overstreet return -ENOENT; 1186cafe5635SKent Overstreet 1187cafe5635SKent Overstreet if (dc->disk.c) { 118846f5aa88SJoe Perches pr_err("Can't attach %s: already attached\n", 11896e916a7eSColy Li dc->backing_dev_name); 1190cafe5635SKent Overstreet return -EINVAL; 1191cafe5635SKent Overstreet } 1192cafe5635SKent Overstreet 1193cafe5635SKent Overstreet if (test_bit(CACHE_SET_STOPPING, &c->flags)) { 119446f5aa88SJoe Perches pr_err("Can't attach %s: shutting down\n", 11956e916a7eSColy Li dc->backing_dev_name); 1196cafe5635SKent Overstreet return -EINVAL; 1197cafe5635SKent Overstreet } 1198cafe5635SKent Overstreet 1199cafe5635SKent Overstreet if (dc->sb.block_size < c->sb.block_size) { 1200cafe5635SKent Overstreet /* Will die */ 120146f5aa88SJoe Perches pr_err("Couldn't attach %s: block size less than set's block size\n", 12026e916a7eSColy Li dc->backing_dev_name); 1203cafe5635SKent Overstreet return -EINVAL; 1204cafe5635SKent Overstreet } 1205cafe5635SKent Overstreet 120686755b7aSMichael Lyle /* Check whether already attached */ 120786755b7aSMichael Lyle list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) { 120886755b7aSMichael Lyle if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) { 120946f5aa88SJoe Perches pr_err("Tried to attach %s but duplicate UUID already attached\n", 12106e916a7eSColy Li dc->backing_dev_name); 121186755b7aSMichael Lyle 121286755b7aSMichael Lyle return -EINVAL; 121386755b7aSMichael Lyle } 121486755b7aSMichael Lyle } 121586755b7aSMichael Lyle 1216cafe5635SKent Overstreet u = uuid_find(c, dc->sb.uuid); 1217cafe5635SKent Overstreet 1218cafe5635SKent Overstreet if (u && 1219cafe5635SKent Overstreet (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE || 1220cafe5635SKent Overstreet BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) { 1221cafe5635SKent Overstreet memcpy(u->uuid, invalid_uuid, 16); 122275cbb3f1SArnd Bergmann u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds()); 1223cafe5635SKent Overstreet u = NULL; 1224cafe5635SKent Overstreet } 1225cafe5635SKent Overstreet 1226cafe5635SKent Overstreet if (!u) { 1227cafe5635SKent Overstreet if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) { 122846f5aa88SJoe Perches pr_err("Couldn't find uuid for %s in set\n", 12296e916a7eSColy Li dc->backing_dev_name); 1230cafe5635SKent Overstreet return -ENOENT; 1231cafe5635SKent Overstreet } 1232cafe5635SKent Overstreet 1233cafe5635SKent Overstreet u = uuid_find_empty(c); 1234cafe5635SKent Overstreet if (!u) { 123546f5aa88SJoe Perches pr_err("Not caching %s, no room for UUID\n", 12366e916a7eSColy Li dc->backing_dev_name); 1237cafe5635SKent Overstreet return -EINVAL; 1238cafe5635SKent Overstreet } 1239cafe5635SKent Overstreet } 1240cafe5635SKent Overstreet 12413be11dbaSColy Li /* 12423be11dbaSColy Li * Deadlocks since we're called via sysfs... 12433be11dbaSColy Li * sysfs_remove_file(&dc->kobj, &sysfs_attach); 1244cafe5635SKent Overstreet */ 1245cafe5635SKent Overstreet 1246169ef1cfSKent Overstreet if (bch_is_zero(u->uuid, 16)) { 1247cafe5635SKent Overstreet struct closure cl; 12481fae7cf0SColy Li 1249cafe5635SKent Overstreet closure_init_stack(&cl); 1250cafe5635SKent Overstreet 1251cafe5635SKent Overstreet memcpy(u->uuid, dc->sb.uuid, 16); 1252cafe5635SKent Overstreet memcpy(u->label, dc->sb.label, SB_LABEL_SIZE); 1253cafe5635SKent Overstreet u->first_reg = u->last_reg = rtime; 1254cafe5635SKent Overstreet bch_uuid_write(c); 1255cafe5635SKent Overstreet 1256cafe5635SKent Overstreet memcpy(dc->sb.set_uuid, c->sb.set_uuid, 16); 1257cafe5635SKent Overstreet SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN); 1258cafe5635SKent Overstreet 1259cafe5635SKent Overstreet bch_write_bdev_super(dc, &cl); 1260cafe5635SKent Overstreet closure_sync(&cl); 1261cafe5635SKent Overstreet } else { 1262cafe5635SKent Overstreet u->last_reg = rtime; 1263cafe5635SKent Overstreet bch_uuid_write(c); 1264cafe5635SKent Overstreet } 1265cafe5635SKent Overstreet 1266cafe5635SKent Overstreet bcache_device_attach(&dc->disk, c, u - c->uuids); 1267cafe5635SKent Overstreet list_move(&dc->list, &c->cached_devs); 1268cafe5635SKent Overstreet calc_cached_dev_sectors(c); 1269cafe5635SKent Overstreet 1270cafe5635SKent Overstreet /* 1271cafe5635SKent Overstreet * dc->c must be set before dc->count != 0 - paired with the mb in 1272cafe5635SKent Overstreet * cached_dev_get() 1273cafe5635SKent Overstreet */ 1274eb2b3d03SColy Li smp_wmb(); 12753b304d24SElena Reshetova refcount_set(&dc->count, 1); 1276cafe5635SKent Overstreet 127707cc6ef8SEric Wheeler /* Block writeback thread, but spawn it */ 127807cc6ef8SEric Wheeler down_write(&dc->writeback_lock); 127907cc6ef8SEric Wheeler if (bch_cached_dev_writeback_start(dc)) { 128007cc6ef8SEric Wheeler up_write(&dc->writeback_lock); 128146f5aa88SJoe Perches pr_err("Couldn't start writeback facilities for %s\n", 1282633bb2ceSColy Li dc->disk.disk->disk_name); 12839e5c3535SSlava Pestov return -ENOMEM; 128407cc6ef8SEric Wheeler } 12859e5c3535SSlava Pestov 1286cafe5635SKent Overstreet if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) { 1287cafe5635SKent Overstreet atomic_set(&dc->has_dirty, 1); 1288cafe5635SKent Overstreet bch_writeback_queue(dc); 1289cafe5635SKent Overstreet } 1290cafe5635SKent Overstreet 12912e17a262STang Junhui bch_sectors_dirty_init(&dc->disk); 12922e17a262STang Junhui 12930b13efecSColy Li ret = bch_cached_dev_run(dc); 12940b13efecSColy Li if (ret && (ret != -EBUSY)) { 12950b13efecSColy Li up_write(&dc->writeback_lock); 12965c2a634cSColy Li /* 12975c2a634cSColy Li * bch_register_lock is held, bcache_device_stop() is not 12985c2a634cSColy Li * able to be directly called. The kthread and kworker 12995c2a634cSColy Li * created previously in bch_cached_dev_writeback_start() 13005c2a634cSColy Li * have to be stopped manually here. 13015c2a634cSColy Li */ 13025c2a634cSColy Li kthread_stop(dc->writeback_thread); 13035c2a634cSColy Li cancel_writeback_rate_update_dwork(dc); 130446f5aa88SJoe Perches pr_err("Couldn't run cached device %s\n", 1305633bb2ceSColy Li dc->backing_dev_name); 13060b13efecSColy Li return ret; 13070b13efecSColy Li } 13080b13efecSColy Li 1309ee668506SKent Overstreet bcache_device_link(&dc->disk, c, "bdev"); 1310ea8c5356SColy Li atomic_inc(&c->attached_dev_nr); 1311cafe5635SKent Overstreet 131207cc6ef8SEric Wheeler /* Allow the writeback thread to proceed */ 131307cc6ef8SEric Wheeler up_write(&dc->writeback_lock); 131407cc6ef8SEric Wheeler 131546f5aa88SJoe Perches pr_info("Caching %s as %s on set %pU\n", 13166e916a7eSColy Li dc->backing_dev_name, 13176e916a7eSColy Li dc->disk.disk->disk_name, 1318cafe5635SKent Overstreet dc->disk.c->sb.set_uuid); 1319cafe5635SKent Overstreet return 0; 1320cafe5635SKent Overstreet } 1321cafe5635SKent Overstreet 13222d17456eSColy Li /* when dc->disk.kobj released */ 1323cafe5635SKent Overstreet void bch_cached_dev_release(struct kobject *kobj) 1324cafe5635SKent Overstreet { 1325cafe5635SKent Overstreet struct cached_dev *dc = container_of(kobj, struct cached_dev, 1326cafe5635SKent Overstreet disk.kobj); 1327cafe5635SKent Overstreet kfree(dc); 1328cafe5635SKent Overstreet module_put(THIS_MODULE); 1329cafe5635SKent Overstreet } 1330cafe5635SKent Overstreet 1331cafe5635SKent Overstreet static void cached_dev_free(struct closure *cl) 1332cafe5635SKent Overstreet { 1333cafe5635SKent Overstreet struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl); 1334cafe5635SKent Overstreet 13353fd47bfeSColy Li if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags)) 13363fd47bfeSColy Li cancel_writeback_rate_update_dwork(dc); 13373fd47bfeSColy Li 1338a664d0f0SSlava Pestov if (!IS_ERR_OR_NULL(dc->writeback_thread)) 13395e6926daSKent Overstreet kthread_stop(dc->writeback_thread); 13400f0709e6SColy Li if (!IS_ERR_OR_NULL(dc->status_update_thread)) 13410f0709e6SColy Li kthread_stop(dc->status_update_thread); 1342cafe5635SKent Overstreet 134380265d8dSColy Li mutex_lock(&bch_register_lock); 134480265d8dSColy Li 1345f59fce84SKent Overstreet if (atomic_read(&dc->running)) 1346ee668506SKent Overstreet bd_unlink_disk_holder(dc->bdev, dc->disk.disk); 1347cafe5635SKent Overstreet bcache_device_free(&dc->disk); 1348cafe5635SKent Overstreet list_del(&dc->list); 1349cafe5635SKent Overstreet 1350cafe5635SKent Overstreet mutex_unlock(&bch_register_lock); 1351cafe5635SKent Overstreet 1352475389aeSChristoph Hellwig if (dc->sb_disk) 1353475389aeSChristoph Hellwig put_page(virt_to_page(dc->sb_disk)); 1354e8547d42SLiang Chen 13550781c874SKent Overstreet if (!IS_ERR_OR_NULL(dc->bdev)) 1356cafe5635SKent Overstreet blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); 1357cafe5635SKent Overstreet 1358cafe5635SKent Overstreet wake_up(&unregister_wait); 1359cafe5635SKent Overstreet 1360cafe5635SKent Overstreet kobject_put(&dc->disk.kobj); 1361cafe5635SKent Overstreet } 1362cafe5635SKent Overstreet 1363cafe5635SKent Overstreet static void cached_dev_flush(struct closure *cl) 1364cafe5635SKent Overstreet { 1365cafe5635SKent Overstreet struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl); 1366cafe5635SKent Overstreet struct bcache_device *d = &dc->disk; 1367cafe5635SKent Overstreet 1368c9502ea4SKent Overstreet mutex_lock(&bch_register_lock); 1369c9502ea4SKent Overstreet bcache_device_unlink(d); 1370c9502ea4SKent Overstreet mutex_unlock(&bch_register_lock); 1371c9502ea4SKent Overstreet 1372cafe5635SKent Overstreet bch_cache_accounting_destroy(&dc->accounting); 1373cafe5635SKent Overstreet kobject_del(&d->kobj); 1374cafe5635SKent Overstreet 1375cafe5635SKent Overstreet continue_at(cl, cached_dev_free, system_wq); 1376cafe5635SKent Overstreet } 1377cafe5635SKent Overstreet 13786f10f7d1SColy Li static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) 1379cafe5635SKent Overstreet { 1380f59fce84SKent Overstreet int ret; 1381cafe5635SKent Overstreet struct io *io; 1382f59fce84SKent Overstreet struct request_queue *q = bdev_get_queue(dc->bdev); 1383cafe5635SKent Overstreet 1384cafe5635SKent Overstreet __module_get(THIS_MODULE); 1385cafe5635SKent Overstreet INIT_LIST_HEAD(&dc->list); 1386f59fce84SKent Overstreet closure_init(&dc->disk.cl, NULL); 1387f59fce84SKent Overstreet set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq); 1388cafe5635SKent Overstreet kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype); 1389cafe5635SKent Overstreet INIT_WORK(&dc->detach, cached_dev_detach_finish); 1390cb7a583eSKent Overstreet sema_init(&dc->sb_write_mutex, 1); 1391f59fce84SKent Overstreet INIT_LIST_HEAD(&dc->io_lru); 1392f59fce84SKent Overstreet spin_lock_init(&dc->io_lock); 1393f59fce84SKent Overstreet bch_cache_accounting_init(&dc->accounting, &dc->disk.cl); 1394cafe5635SKent Overstreet 1395cafe5635SKent Overstreet dc->sequential_cutoff = 4 << 20; 1396cafe5635SKent Overstreet 1397cafe5635SKent Overstreet for (io = dc->io; io < dc->io + RECENT_IO; io++) { 1398cafe5635SKent Overstreet list_add(&io->lru, &dc->io_lru); 1399cafe5635SKent Overstreet hlist_add_head(&io->hash, dc->io_hash + RECENT_IO); 1400cafe5635SKent Overstreet } 1401cafe5635SKent Overstreet 1402c78afc62SKent Overstreet dc->disk.stripe_size = q->limits.io_opt >> 9; 1403c78afc62SKent Overstreet 1404c78afc62SKent Overstreet if (dc->disk.stripe_size) 1405c78afc62SKent Overstreet dc->partial_stripes_expensive = 1406c78afc62SKent Overstreet q->limits.raid_partial_stripes_expensive; 1407c78afc62SKent Overstreet 1408279afbadSKent Overstreet ret = bcache_device_init(&dc->disk, block_size, 1409ff27668cSChristoph Hellwig dc->bdev->bd_part->nr_sects - dc->sb.data_offset, 1410c62b37d9SChristoph Hellwig dc->bdev, &bcache_cached_ops); 1411f59fce84SKent Overstreet if (ret) 1412f59fce84SKent Overstreet return ret; 1413f59fce84SKent Overstreet 1414dc3b17ccSJan Kara dc->disk.disk->queue->backing_dev_info->ra_pages = 1415dc3b17ccSJan Kara max(dc->disk.disk->queue->backing_dev_info->ra_pages, 1416dc3b17ccSJan Kara q->backing_dev_info->ra_pages); 1417f59fce84SKent Overstreet 1418c7b7bd07SColy Li atomic_set(&dc->io_errors, 0); 1419c7b7bd07SColy Li dc->io_disable = false; 1420c7b7bd07SColy Li dc->error_limit = DEFAULT_CACHED_DEV_ERROR_LIMIT; 14217e027ca4SColy Li /* default to auto */ 14227e027ca4SColy Li dc->stop_when_cache_set_failed = BCH_CACHED_DEV_STOP_AUTO; 14237e027ca4SColy Li 1424f59fce84SKent Overstreet bch_cached_dev_request_init(dc); 1425f59fce84SKent Overstreet bch_cached_dev_writeback_init(dc); 1426cafe5635SKent Overstreet return 0; 1427cafe5635SKent Overstreet } 1428cafe5635SKent Overstreet 1429cafe5635SKent Overstreet /* Cached device - bcache superblock */ 1430cafe5635SKent Overstreet 1431cfa0c56dSChristoph Hellwig static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk, 1432cafe5635SKent Overstreet struct block_device *bdev, 1433cafe5635SKent Overstreet struct cached_dev *dc) 1434cafe5635SKent Overstreet { 1435cafe5635SKent Overstreet const char *err = "cannot allocate memory"; 1436cafe5635SKent Overstreet struct cache_set *c; 14370b13efecSColy Li int ret = -ENOMEM; 1438cafe5635SKent Overstreet 14396e916a7eSColy Li bdevname(bdev, dc->backing_dev_name); 1440cafe5635SKent Overstreet memcpy(&dc->sb, sb, sizeof(struct cache_sb)); 1441cafe5635SKent Overstreet dc->bdev = bdev; 1442cafe5635SKent Overstreet dc->bdev->bd_holder = dc; 1443475389aeSChristoph Hellwig dc->sb_disk = sb_disk; 14446e916a7eSColy Li 1445f59fce84SKent Overstreet if (cached_dev_init(dc, sb->block_size << 9)) 1446f59fce84SKent Overstreet goto err; 1447cafe5635SKent Overstreet 1448cafe5635SKent Overstreet err = "error creating kobject"; 1449cafe5635SKent Overstreet if (kobject_add(&dc->disk.kobj, &part_to_dev(bdev->bd_part)->kobj, 1450cafe5635SKent Overstreet "bcache")) 1451cafe5635SKent Overstreet goto err; 1452cafe5635SKent Overstreet if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj)) 1453cafe5635SKent Overstreet goto err; 1454cafe5635SKent Overstreet 145546f5aa88SJoe Perches pr_info("registered backing device %s\n", dc->backing_dev_name); 1456f59fce84SKent Overstreet 1457cafe5635SKent Overstreet list_add(&dc->list, &uncached_devices); 1458e57fd746SColy Li /* attach to a matched cache set if it exists */ 1459cafe5635SKent Overstreet list_for_each_entry(c, &bch_cache_sets, list) 146073ac105bSTang Junhui bch_cached_dev_attach(dc, c, NULL); 1461cafe5635SKent Overstreet 1462cafe5635SKent Overstreet if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE || 14630b13efecSColy Li BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) { 14640b13efecSColy Li err = "failed to run cached device"; 14650b13efecSColy Li ret = bch_cached_dev_run(dc); 14660b13efecSColy Li if (ret) 14670b13efecSColy Li goto err; 14680b13efecSColy Li } 1469cafe5635SKent Overstreet 147088c12d42SColy Li return 0; 1471cafe5635SKent Overstreet err: 147246f5aa88SJoe Perches pr_notice("error %s: %s\n", dc->backing_dev_name, err); 1473f59fce84SKent Overstreet bcache_device_stop(&dc->disk); 14740b13efecSColy Li return ret; 1475cafe5635SKent Overstreet } 1476cafe5635SKent Overstreet 1477cafe5635SKent Overstreet /* Flash only volumes */ 1478cafe5635SKent Overstreet 14792d17456eSColy Li /* When d->kobj released */ 1480cafe5635SKent Overstreet void bch_flash_dev_release(struct kobject *kobj) 1481cafe5635SKent Overstreet { 1482cafe5635SKent Overstreet struct bcache_device *d = container_of(kobj, struct bcache_device, 1483cafe5635SKent Overstreet kobj); 1484cafe5635SKent Overstreet kfree(d); 1485cafe5635SKent Overstreet } 1486cafe5635SKent Overstreet 1487cafe5635SKent Overstreet static void flash_dev_free(struct closure *cl) 1488cafe5635SKent Overstreet { 1489cafe5635SKent Overstreet struct bcache_device *d = container_of(cl, struct bcache_device, cl); 14901fae7cf0SColy Li 1491e5112201SSlava Pestov mutex_lock(&bch_register_lock); 149299a27d59STang Junhui atomic_long_sub(bcache_dev_sectors_dirty(d), 149399a27d59STang Junhui &d->c->flash_dev_dirty_sectors); 1494cafe5635SKent Overstreet bcache_device_free(d); 1495e5112201SSlava Pestov mutex_unlock(&bch_register_lock); 1496cafe5635SKent Overstreet kobject_put(&d->kobj); 1497cafe5635SKent Overstreet } 1498cafe5635SKent Overstreet 1499cafe5635SKent Overstreet static void flash_dev_flush(struct closure *cl) 1500cafe5635SKent Overstreet { 1501cafe5635SKent Overstreet struct bcache_device *d = container_of(cl, struct bcache_device, cl); 1502cafe5635SKent Overstreet 1503e5112201SSlava Pestov mutex_lock(&bch_register_lock); 1504ee668506SKent Overstreet bcache_device_unlink(d); 1505e5112201SSlava Pestov mutex_unlock(&bch_register_lock); 1506cafe5635SKent Overstreet kobject_del(&d->kobj); 1507cafe5635SKent Overstreet continue_at(cl, flash_dev_free, system_wq); 1508cafe5635SKent Overstreet } 1509cafe5635SKent Overstreet 1510cafe5635SKent Overstreet static int flash_dev_run(struct cache_set *c, struct uuid_entry *u) 1511cafe5635SKent Overstreet { 1512cafe5635SKent Overstreet struct bcache_device *d = kzalloc(sizeof(struct bcache_device), 1513cafe5635SKent Overstreet GFP_KERNEL); 1514cafe5635SKent Overstreet if (!d) 1515cafe5635SKent Overstreet return -ENOMEM; 1516cafe5635SKent Overstreet 1517cafe5635SKent Overstreet closure_init(&d->cl, NULL); 1518cafe5635SKent Overstreet set_closure_fn(&d->cl, flash_dev_flush, system_wq); 1519cafe5635SKent Overstreet 1520cafe5635SKent Overstreet kobject_init(&d->kobj, &bch_flash_dev_ktype); 1521cafe5635SKent Overstreet 1522ff27668cSChristoph Hellwig if (bcache_device_init(d, block_bytes(c), u->sectors, 1523c62b37d9SChristoph Hellwig NULL, &bcache_flash_ops)) 1524cafe5635SKent Overstreet goto err; 1525cafe5635SKent Overstreet 1526cafe5635SKent Overstreet bcache_device_attach(d, c, u - c->uuids); 1527175206cfSTang Junhui bch_sectors_dirty_init(d); 1528cafe5635SKent Overstreet bch_flash_dev_request_init(d); 1529cafe5635SKent Overstreet add_disk(d->disk); 1530cafe5635SKent Overstreet 1531cafe5635SKent Overstreet if (kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache")) 1532cafe5635SKent Overstreet goto err; 1533cafe5635SKent Overstreet 1534cafe5635SKent Overstreet bcache_device_link(d, c, "volume"); 1535cafe5635SKent Overstreet 1536cafe5635SKent Overstreet return 0; 1537cafe5635SKent Overstreet err: 1538cafe5635SKent Overstreet kobject_put(&d->kobj); 1539cafe5635SKent Overstreet return -ENOMEM; 1540cafe5635SKent Overstreet } 1541cafe5635SKent Overstreet 1542cafe5635SKent Overstreet static int flash_devs_run(struct cache_set *c) 1543cafe5635SKent Overstreet { 1544cafe5635SKent Overstreet int ret = 0; 1545cafe5635SKent Overstreet struct uuid_entry *u; 1546cafe5635SKent Overstreet 1547cafe5635SKent Overstreet for (u = c->uuids; 154802aa8a8bSColy Li u < c->uuids + c->nr_uuids && !ret; 1549cafe5635SKent Overstreet u++) 1550cafe5635SKent Overstreet if (UUID_FLASH_ONLY(u)) 1551cafe5635SKent Overstreet ret = flash_dev_run(c, u); 1552cafe5635SKent Overstreet 1553cafe5635SKent Overstreet return ret; 1554cafe5635SKent Overstreet } 1555cafe5635SKent Overstreet 1556cafe5635SKent Overstreet int bch_flash_dev_create(struct cache_set *c, uint64_t size) 1557cafe5635SKent Overstreet { 1558cafe5635SKent Overstreet struct uuid_entry *u; 1559cafe5635SKent Overstreet 1560cafe5635SKent Overstreet if (test_bit(CACHE_SET_STOPPING, &c->flags)) 1561cafe5635SKent Overstreet return -EINTR; 1562cafe5635SKent Overstreet 1563bf0c55c9SSlava Pestov if (!test_bit(CACHE_SET_RUNNING, &c->flags)) 1564bf0c55c9SSlava Pestov return -EPERM; 1565bf0c55c9SSlava Pestov 1566cafe5635SKent Overstreet u = uuid_find_empty(c); 1567cafe5635SKent Overstreet if (!u) { 156846f5aa88SJoe Perches pr_err("Can't create volume, no room for UUID\n"); 1569cafe5635SKent Overstreet return -EINVAL; 1570cafe5635SKent Overstreet } 1571cafe5635SKent Overstreet 1572cafe5635SKent Overstreet get_random_bytes(u->uuid, 16); 1573cafe5635SKent Overstreet memset(u->label, 0, 32); 157475cbb3f1SArnd Bergmann u->first_reg = u->last_reg = cpu_to_le32((u32)ktime_get_real_seconds()); 1575cafe5635SKent Overstreet 1576cafe5635SKent Overstreet SET_UUID_FLASH_ONLY(u, 1); 1577cafe5635SKent Overstreet u->sectors = size >> 9; 1578cafe5635SKent Overstreet 1579cafe5635SKent Overstreet bch_uuid_write(c); 1580cafe5635SKent Overstreet 1581cafe5635SKent Overstreet return flash_dev_run(c, u); 1582cafe5635SKent Overstreet } 1583cafe5635SKent Overstreet 1584c7b7bd07SColy Li bool bch_cached_dev_error(struct cached_dev *dc) 1585c7b7bd07SColy Li { 1586c7b7bd07SColy Li if (!dc || test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags)) 1587c7b7bd07SColy Li return false; 1588c7b7bd07SColy Li 1589c7b7bd07SColy Li dc->io_disable = true; 1590c7b7bd07SColy Li /* make others know io_disable is true earlier */ 1591c7b7bd07SColy Li smp_mb(); 1592c7b7bd07SColy Li 1593c7b7bd07SColy Li pr_err("stop %s: too many IO errors on backing device %s\n", 15946e916a7eSColy Li dc->disk.disk->disk_name, dc->backing_dev_name); 1595c7b7bd07SColy Li 1596c7b7bd07SColy Li bcache_device_stop(&dc->disk); 1597c7b7bd07SColy Li return true; 1598c7b7bd07SColy Li } 1599c7b7bd07SColy Li 1600cafe5635SKent Overstreet /* Cache set */ 1601cafe5635SKent Overstreet 1602cafe5635SKent Overstreet __printf(2, 3) 1603cafe5635SKent Overstreet bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...) 1604cafe5635SKent Overstreet { 160546f5aa88SJoe Perches struct va_format vaf; 1606cafe5635SKent Overstreet va_list args; 1607cafe5635SKent Overstreet 160877c320ebSKent Overstreet if (c->on_error != ON_ERROR_PANIC && 160977c320ebSKent Overstreet test_bit(CACHE_SET_STOPPING, &c->flags)) 1610cafe5635SKent Overstreet return false; 1611cafe5635SKent Overstreet 1612771f393eSColy Li if (test_and_set_bit(CACHE_SET_IO_DISABLE, &c->flags)) 161346f5aa88SJoe Perches pr_info("CACHE_SET_IO_DISABLE already set\n"); 1614771f393eSColy Li 16153be11dbaSColy Li /* 16163be11dbaSColy Li * XXX: we can be called from atomic context 16173be11dbaSColy Li * acquire_console_sem(); 1618cafe5635SKent Overstreet */ 1619cafe5635SKent Overstreet 1620cafe5635SKent Overstreet va_start(args, fmt); 1621cafe5635SKent Overstreet 162246f5aa88SJoe Perches vaf.fmt = fmt; 162346f5aa88SJoe Perches vaf.va = &args; 162446f5aa88SJoe Perches 162546f5aa88SJoe Perches pr_err("error on %pU: %pV, disabling caching\n", 162646f5aa88SJoe Perches c->sb.set_uuid, &vaf); 162746f5aa88SJoe Perches 162846f5aa88SJoe Perches va_end(args); 1629cafe5635SKent Overstreet 163077c320ebSKent Overstreet if (c->on_error == ON_ERROR_PANIC) 163177c320ebSKent Overstreet panic("panic forced after error\n"); 163277c320ebSKent Overstreet 1633cafe5635SKent Overstreet bch_cache_set_unregister(c); 1634cafe5635SKent Overstreet return true; 1635cafe5635SKent Overstreet } 1636cafe5635SKent Overstreet 16372d17456eSColy Li /* When c->kobj released */ 1638cafe5635SKent Overstreet void bch_cache_set_release(struct kobject *kobj) 1639cafe5635SKent Overstreet { 1640cafe5635SKent Overstreet struct cache_set *c = container_of(kobj, struct cache_set, kobj); 16411fae7cf0SColy Li 1642cafe5635SKent Overstreet kfree(c); 1643cafe5635SKent Overstreet module_put(THIS_MODULE); 1644cafe5635SKent Overstreet } 1645cafe5635SKent Overstreet 1646cafe5635SKent Overstreet static void cache_set_free(struct closure *cl) 1647cafe5635SKent Overstreet { 1648cafe5635SKent Overstreet struct cache_set *c = container_of(cl, struct cache_set, cl); 1649cafe5635SKent Overstreet struct cache *ca; 16506f10f7d1SColy Li unsigned int i; 1651cafe5635SKent Overstreet 1652cafe5635SKent Overstreet debugfs_remove(c->debug); 1653cafe5635SKent Overstreet 1654cafe5635SKent Overstreet bch_open_buckets_free(c); 1655cafe5635SKent Overstreet bch_btree_cache_free(c); 1656cafe5635SKent Overstreet bch_journal_free(c); 1657cafe5635SKent Overstreet 1658a4b732a2SLiang Chen mutex_lock(&bch_register_lock); 1659cafe5635SKent Overstreet for_each_cache(ca, c, i) 1660c9a78332SSlava Pestov if (ca) { 1661c9a78332SSlava Pestov ca->set = NULL; 1662c9a78332SSlava Pestov c->cache[ca->sb.nr_this_dev] = NULL; 1663cafe5635SKent Overstreet kobject_put(&ca->kobj); 1664c9a78332SSlava Pestov } 1665cafe5635SKent Overstreet 166667539e85SKent Overstreet bch_bset_sort_state_free(&c->sort); 1667cafe5635SKent Overstreet free_pages((unsigned long) c->uuids, ilog2(bucket_pages(c))); 1668cafe5635SKent Overstreet 1669da415a09SNicholas Swenson if (c->moving_gc_wq) 1670da415a09SNicholas Swenson destroy_workqueue(c->moving_gc_wq); 1671d19936a2SKent Overstreet bioset_exit(&c->bio_split); 1672d19936a2SKent Overstreet mempool_exit(&c->fill_iter); 1673d19936a2SKent Overstreet mempool_exit(&c->bio_meta); 1674d19936a2SKent Overstreet mempool_exit(&c->search); 1675cafe5635SKent Overstreet kfree(c->devices); 1676cafe5635SKent Overstreet 1677cafe5635SKent Overstreet list_del(&c->list); 1678cafe5635SKent Overstreet mutex_unlock(&bch_register_lock); 1679cafe5635SKent Overstreet 168046f5aa88SJoe Perches pr_info("Cache set %pU unregistered\n", c->sb.set_uuid); 1681cafe5635SKent Overstreet wake_up(&unregister_wait); 1682cafe5635SKent Overstreet 1683cafe5635SKent Overstreet closure_debug_destroy(&c->cl); 1684cafe5635SKent Overstreet kobject_put(&c->kobj); 1685cafe5635SKent Overstreet } 1686cafe5635SKent Overstreet 1687cafe5635SKent Overstreet static void cache_set_flush(struct closure *cl) 1688cafe5635SKent Overstreet { 1689cafe5635SKent Overstreet struct cache_set *c = container_of(cl, struct cache_set, caching); 169079826c35SKent Overstreet struct cache *ca; 1691cafe5635SKent Overstreet struct btree *b; 16926f10f7d1SColy Li unsigned int i; 1693cafe5635SKent Overstreet 1694cafe5635SKent Overstreet bch_cache_accounting_destroy(&c->accounting); 1695cafe5635SKent Overstreet 1696cafe5635SKent Overstreet kobject_put(&c->internal); 1697cafe5635SKent Overstreet kobject_del(&c->kobj); 1698cafe5635SKent Overstreet 1699b387e9b5SColy Li if (!IS_ERR_OR_NULL(c->gc_thread)) 170072a44517SKent Overstreet kthread_stop(c->gc_thread); 170172a44517SKent Overstreet 1702cafe5635SKent Overstreet if (!IS_ERR_OR_NULL(c->root)) 1703cafe5635SKent Overstreet list_add(&c->root->list, &c->btree_cache); 1704cafe5635SKent Overstreet 1705e6dcbd3eSColy Li /* 1706e6dcbd3eSColy Li * Avoid flushing cached nodes if cache set is retiring 1707e6dcbd3eSColy Li * due to too many I/O errors detected. 1708e6dcbd3eSColy Li */ 1709e6dcbd3eSColy Li if (!test_bit(CACHE_SET_IO_DISABLE, &c->flags)) 17102a285686SKent Overstreet list_for_each_entry(b, &c->btree_cache, list) { 17112a285686SKent Overstreet mutex_lock(&b->write_lock); 1712cafe5635SKent Overstreet if (btree_node_dirty(b)) 17132a285686SKent Overstreet __bch_btree_node_write(b, NULL); 17142a285686SKent Overstreet mutex_unlock(&b->write_lock); 17152a285686SKent Overstreet } 1716cafe5635SKent Overstreet 171779826c35SKent Overstreet for_each_cache(ca, c, i) 171879826c35SKent Overstreet if (ca->alloc_thread) 171979826c35SKent Overstreet kthread_stop(ca->alloc_thread); 172079826c35SKent Overstreet 17215b1016e6SKent Overstreet if (c->journal.cur) { 1722dabb4433SKent Overstreet cancel_delayed_work_sync(&c->journal.work); 1723dabb4433SKent Overstreet /* flush last journal entry if needed */ 1724dabb4433SKent Overstreet c->journal.work.work.func(&c->journal.work.work); 17255b1016e6SKent Overstreet } 1726dabb4433SKent Overstreet 1727cafe5635SKent Overstreet closure_return(cl); 1728cafe5635SKent Overstreet } 1729cafe5635SKent Overstreet 17307e027ca4SColy Li /* 17317e027ca4SColy Li * This function is only called when CACHE_SET_IO_DISABLE is set, which means 17327e027ca4SColy Li * cache set is unregistering due to too many I/O errors. In this condition, 17337e027ca4SColy Li * the bcache device might be stopped, it depends on stop_when_cache_set_failed 17347e027ca4SColy Li * value and whether the broken cache has dirty data: 17357e027ca4SColy Li * 17367e027ca4SColy Li * dc->stop_when_cache_set_failed dc->has_dirty stop bcache device 17377e027ca4SColy Li * BCH_CACHED_STOP_AUTO 0 NO 17387e027ca4SColy Li * BCH_CACHED_STOP_AUTO 1 YES 17397e027ca4SColy Li * BCH_CACHED_DEV_STOP_ALWAYS 0 YES 17407e027ca4SColy Li * BCH_CACHED_DEV_STOP_ALWAYS 1 YES 17417e027ca4SColy Li * 17427e027ca4SColy Li * The expected behavior is, if stop_when_cache_set_failed is configured to 17437e027ca4SColy Li * "auto" via sysfs interface, the bcache device will not be stopped if the 17447e027ca4SColy Li * backing device is clean on the broken cache device. 17457e027ca4SColy Li */ 17467e027ca4SColy Li static void conditional_stop_bcache_device(struct cache_set *c, 17477e027ca4SColy Li struct bcache_device *d, 17487e027ca4SColy Li struct cached_dev *dc) 17497e027ca4SColy Li { 17507e027ca4SColy Li if (dc->stop_when_cache_set_failed == BCH_CACHED_DEV_STOP_ALWAYS) { 175146f5aa88SJoe Perches pr_warn("stop_when_cache_set_failed of %s is \"always\", stop it for failed cache set %pU.\n", 17527e027ca4SColy Li d->disk->disk_name, c->sb.set_uuid); 17537e027ca4SColy Li bcache_device_stop(d); 17547e027ca4SColy Li } else if (atomic_read(&dc->has_dirty)) { 17557e027ca4SColy Li /* 17567e027ca4SColy Li * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO 17577e027ca4SColy Li * and dc->has_dirty == 1 17587e027ca4SColy Li */ 175946f5aa88SJoe Perches pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is dirty, stop it to avoid potential data corruption.\n", 17607e027ca4SColy Li d->disk->disk_name); 17614fd8e138SColy Li /* 17624fd8e138SColy Li * There might be a small time gap that cache set is 17634fd8e138SColy Li * released but bcache device is not. Inside this time 17644fd8e138SColy Li * gap, regular I/O requests will directly go into 17654fd8e138SColy Li * backing device as no cache set attached to. This 17664fd8e138SColy Li * behavior may also introduce potential inconsistence 17674fd8e138SColy Li * data in writeback mode while cache is dirty. 17684fd8e138SColy Li * Therefore before calling bcache_device_stop() due 17694fd8e138SColy Li * to a broken cache device, dc->io_disable should be 17704fd8e138SColy Li * explicitly set to true. 17714fd8e138SColy Li */ 17724fd8e138SColy Li dc->io_disable = true; 17734fd8e138SColy Li /* make others know io_disable is true earlier */ 17744fd8e138SColy Li smp_mb(); 17757e027ca4SColy Li bcache_device_stop(d); 17767e027ca4SColy Li } else { 17777e027ca4SColy Li /* 17787e027ca4SColy Li * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO 17797e027ca4SColy Li * and dc->has_dirty == 0 17807e027ca4SColy Li */ 178146f5aa88SJoe Perches pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is clean, keep it alive.\n", 17827e027ca4SColy Li d->disk->disk_name); 17837e027ca4SColy Li } 17847e027ca4SColy Li } 17857e027ca4SColy Li 1786cafe5635SKent Overstreet static void __cache_set_unregister(struct closure *cl) 1787cafe5635SKent Overstreet { 1788cafe5635SKent Overstreet struct cache_set *c = container_of(cl, struct cache_set, caching); 17895caa52afSKent Overstreet struct cached_dev *dc; 17907e027ca4SColy Li struct bcache_device *d; 1791cafe5635SKent Overstreet size_t i; 1792cafe5635SKent Overstreet 1793cafe5635SKent Overstreet mutex_lock(&bch_register_lock); 1794cafe5635SKent Overstreet 17957e027ca4SColy Li for (i = 0; i < c->devices_max_used; i++) { 17967e027ca4SColy Li d = c->devices[i]; 17977e027ca4SColy Li if (!d) 17987e027ca4SColy Li continue; 17997e027ca4SColy Li 18005caa52afSKent Overstreet if (!UUID_FLASH_ONLY(&c->uuids[i]) && 18015caa52afSKent Overstreet test_bit(CACHE_SET_UNREGISTERING, &c->flags)) { 18027e027ca4SColy Li dc = container_of(d, struct cached_dev, disk); 18035caa52afSKent Overstreet bch_cached_dev_detach(dc); 18047e027ca4SColy Li if (test_bit(CACHE_SET_IO_DISABLE, &c->flags)) 18057e027ca4SColy Li conditional_stop_bcache_device(c, d, dc); 18065caa52afSKent Overstreet } else { 18077e027ca4SColy Li bcache_device_stop(d); 18085caa52afSKent Overstreet } 18095caa52afSKent Overstreet } 1810cafe5635SKent Overstreet 1811cafe5635SKent Overstreet mutex_unlock(&bch_register_lock); 1812cafe5635SKent Overstreet 1813cafe5635SKent Overstreet continue_at(cl, cache_set_flush, system_wq); 1814cafe5635SKent Overstreet } 1815cafe5635SKent Overstreet 1816cafe5635SKent Overstreet void bch_cache_set_stop(struct cache_set *c) 1817cafe5635SKent Overstreet { 1818cafe5635SKent Overstreet if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags)) 181963d63b51SColy Li /* closure_fn set to __cache_set_unregister() */ 1820cafe5635SKent Overstreet closure_queue(&c->caching); 1821cafe5635SKent Overstreet } 1822cafe5635SKent Overstreet 1823cafe5635SKent Overstreet void bch_cache_set_unregister(struct cache_set *c) 1824cafe5635SKent Overstreet { 1825cafe5635SKent Overstreet set_bit(CACHE_SET_UNREGISTERING, &c->flags); 1826cafe5635SKent Overstreet bch_cache_set_stop(c); 1827cafe5635SKent Overstreet } 1828cafe5635SKent Overstreet 1829cafe5635SKent Overstreet #define alloc_bucket_pages(gfp, c) \ 18305fe48867SColy Li ((void *) __get_free_pages(__GFP_ZERO|__GFP_COMP|gfp, ilog2(bucket_pages(c)))) 1831cafe5635SKent Overstreet 1832cafe5635SKent Overstreet struct cache_set *bch_cache_set_alloc(struct cache_sb *sb) 1833cafe5635SKent Overstreet { 1834cafe5635SKent Overstreet int iter_size; 1835cafe5635SKent Overstreet struct cache_set *c = kzalloc(sizeof(struct cache_set), GFP_KERNEL); 18361fae7cf0SColy Li 1837cafe5635SKent Overstreet if (!c) 1838cafe5635SKent Overstreet return NULL; 1839cafe5635SKent Overstreet 1840cafe5635SKent Overstreet __module_get(THIS_MODULE); 1841cafe5635SKent Overstreet closure_init(&c->cl, NULL); 1842cafe5635SKent Overstreet set_closure_fn(&c->cl, cache_set_free, system_wq); 1843cafe5635SKent Overstreet 1844cafe5635SKent Overstreet closure_init(&c->caching, &c->cl); 1845cafe5635SKent Overstreet set_closure_fn(&c->caching, __cache_set_unregister, system_wq); 1846cafe5635SKent Overstreet 1847cafe5635SKent Overstreet /* Maybe create continue_at_noreturn() and use it here? */ 1848cafe5635SKent Overstreet closure_set_stopped(&c->cl); 1849cafe5635SKent Overstreet closure_put(&c->cl); 1850cafe5635SKent Overstreet 1851cafe5635SKent Overstreet kobject_init(&c->kobj, &bch_cache_set_ktype); 1852cafe5635SKent Overstreet kobject_init(&c->internal, &bch_cache_set_internal_ktype); 1853cafe5635SKent Overstreet 1854cafe5635SKent Overstreet bch_cache_accounting_init(&c->accounting, &c->cl); 1855cafe5635SKent Overstreet 1856cafe5635SKent Overstreet memcpy(c->sb.set_uuid, sb->set_uuid, 16); 1857cafe5635SKent Overstreet c->sb.block_size = sb->block_size; 1858cafe5635SKent Overstreet c->sb.bucket_size = sb->bucket_size; 1859cafe5635SKent Overstreet c->sb.nr_in_set = sb->nr_in_set; 1860cafe5635SKent Overstreet c->sb.last_mount = sb->last_mount; 1861d721a43fSColy Li c->sb.version = sb->version; 1862d721a43fSColy Li if (c->sb.version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) { 1863d721a43fSColy Li c->sb.feature_compat = sb->feature_compat; 1864d721a43fSColy Li c->sb.feature_ro_compat = sb->feature_ro_compat; 1865d721a43fSColy Li c->sb.feature_incompat = sb->feature_incompat; 1866d721a43fSColy Li } 1867d721a43fSColy Li 1868cafe5635SKent Overstreet c->bucket_bits = ilog2(sb->bucket_size); 1869cafe5635SKent Overstreet c->block_bits = ilog2(sb->block_size); 1870cafe5635SKent Overstreet c->nr_uuids = bucket_bytes(c) / sizeof(struct uuid_entry); 18712831231dSColy Li c->devices_max_used = 0; 1872ea8c5356SColy Li atomic_set(&c->attached_dev_nr, 0); 1873ee811287SKent Overstreet c->btree_pages = bucket_pages(c); 1874cafe5635SKent Overstreet if (c->btree_pages > BTREE_MAX_PAGES) 1875cafe5635SKent Overstreet c->btree_pages = max_t(int, c->btree_pages / 4, 1876cafe5635SKent Overstreet BTREE_MAX_PAGES); 1877cafe5635SKent Overstreet 1878cb7a583eSKent Overstreet sema_init(&c->sb_write_mutex, 1); 1879e8e1d468SKent Overstreet mutex_init(&c->bucket_lock); 18800a63b66dSKent Overstreet init_waitqueue_head(&c->btree_cache_wait); 188134cf78bfSGuoju Fang spin_lock_init(&c->btree_cannibalize_lock); 188235fcd848SKent Overstreet init_waitqueue_head(&c->bucket_wait); 1883be628be0SKent Overstreet init_waitqueue_head(&c->gc_wait); 1884cb7a583eSKent Overstreet sema_init(&c->uuid_write_mutex, 1); 188565d22e91SKent Overstreet 188665d22e91SKent Overstreet spin_lock_init(&c->btree_gc_time.lock); 188765d22e91SKent Overstreet spin_lock_init(&c->btree_split_time.lock); 188865d22e91SKent Overstreet spin_lock_init(&c->btree_read_time.lock); 1889e8e1d468SKent Overstreet 1890cafe5635SKent Overstreet bch_moving_init_cache_set(c); 1891cafe5635SKent Overstreet 1892cafe5635SKent Overstreet INIT_LIST_HEAD(&c->list); 1893cafe5635SKent Overstreet INIT_LIST_HEAD(&c->cached_devs); 1894cafe5635SKent Overstreet INIT_LIST_HEAD(&c->btree_cache); 1895cafe5635SKent Overstreet INIT_LIST_HEAD(&c->btree_cache_freeable); 1896cafe5635SKent Overstreet INIT_LIST_HEAD(&c->btree_cache_freed); 1897cafe5635SKent Overstreet INIT_LIST_HEAD(&c->data_buckets); 1898cafe5635SKent Overstreet 1899cafe5635SKent Overstreet iter_size = (sb->bucket_size / sb->block_size + 1) * 1900cafe5635SKent Overstreet sizeof(struct btree_iter_set); 1901cafe5635SKent Overstreet 1902a42d3c64SColy Li c->devices = kcalloc(c->nr_uuids, sizeof(void *), GFP_KERNEL); 1903a42d3c64SColy Li if (!c->devices) 1904a42d3c64SColy Li goto err; 1905a42d3c64SColy Li 1906a42d3c64SColy Li if (mempool_init_slab_pool(&c->search, 32, bch_search_cache)) 1907a42d3c64SColy Li goto err; 1908a42d3c64SColy Li 1909a42d3c64SColy Li if (mempool_init_kmalloc_pool(&c->bio_meta, 2, 1910a42d3c64SColy Li sizeof(struct bbio) + 1911a42d3c64SColy Li sizeof(struct bio_vec) * bucket_pages(c))) 1912a42d3c64SColy Li goto err; 1913a42d3c64SColy Li 1914a42d3c64SColy Li if (mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size)) 1915a42d3c64SColy Li goto err; 1916a42d3c64SColy Li 1917a42d3c64SColy Li if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio), 1918a42d3c64SColy Li BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER)) 1919a42d3c64SColy Li goto err; 1920a42d3c64SColy Li 1921a42d3c64SColy Li c->uuids = alloc_bucket_pages(GFP_KERNEL, c); 1922a42d3c64SColy Li if (!c->uuids) 1923a42d3c64SColy Li goto err; 1924a42d3c64SColy Li 1925a42d3c64SColy Li c->moving_gc_wq = alloc_workqueue("bcache_gc", WQ_MEM_RECLAIM, 0); 1926a42d3c64SColy Li if (!c->moving_gc_wq) 1927a42d3c64SColy Li goto err; 1928a42d3c64SColy Li 1929a42d3c64SColy Li if (bch_journal_alloc(c)) 1930a42d3c64SColy Li goto err; 1931a42d3c64SColy Li 1932a42d3c64SColy Li if (bch_btree_cache_alloc(c)) 1933a42d3c64SColy Li goto err; 1934a42d3c64SColy Li 1935a42d3c64SColy Li if (bch_open_buckets_alloc(c)) 1936a42d3c64SColy Li goto err; 1937a42d3c64SColy Li 1938a42d3c64SColy Li if (bch_bset_sort_state_init(&c->sort, ilog2(c->btree_pages))) 1939cafe5635SKent Overstreet goto err; 1940cafe5635SKent Overstreet 1941cafe5635SKent Overstreet c->congested_read_threshold_us = 2000; 1942cafe5635SKent Overstreet c->congested_write_threshold_us = 20000; 19437ba0d830SColy Li c->error_limit = DEFAULT_IO_ERROR_LIMIT; 1944c5fcdedcSColy Li c->idle_max_writeback_rate_enabled = 1; 1945771f393eSColy Li WARN_ON(test_and_clear_bit(CACHE_SET_IO_DISABLE, &c->flags)); 1946cafe5635SKent Overstreet 1947cafe5635SKent Overstreet return c; 1948cafe5635SKent Overstreet err: 1949cafe5635SKent Overstreet bch_cache_set_unregister(c); 1950cafe5635SKent Overstreet return NULL; 1951cafe5635SKent Overstreet } 1952cafe5635SKent Overstreet 1953ce3e4cfbSColy Li static int run_cache_set(struct cache_set *c) 1954cafe5635SKent Overstreet { 1955cafe5635SKent Overstreet const char *err = "cannot allocate memory"; 1956cafe5635SKent Overstreet struct cached_dev *dc, *t; 1957cafe5635SKent Overstreet struct cache *ca; 1958c18536a7SKent Overstreet struct closure cl; 19596f10f7d1SColy Li unsigned int i; 196095f18c9dSShenghui Wang LIST_HEAD(journal); 196195f18c9dSShenghui Wang struct journal_replay *l; 1962cafe5635SKent Overstreet 1963c18536a7SKent Overstreet closure_init_stack(&cl); 1964cafe5635SKent Overstreet 1965cafe5635SKent Overstreet for_each_cache(ca, c, i) 1966cafe5635SKent Overstreet c->nbuckets += ca->sb.nbuckets; 1967be628be0SKent Overstreet set_gc_sectors(c); 1968cafe5635SKent Overstreet 1969cafe5635SKent Overstreet if (CACHE_SYNC(&c->sb)) { 1970cafe5635SKent Overstreet struct bkey *k; 1971cafe5635SKent Overstreet struct jset *j; 1972cafe5635SKent Overstreet 1973cafe5635SKent Overstreet err = "cannot allocate memory for journal"; 1974c18536a7SKent Overstreet if (bch_journal_read(c, &journal)) 1975cafe5635SKent Overstreet goto err; 1976cafe5635SKent Overstreet 197746f5aa88SJoe Perches pr_debug("btree_journal_read() done\n"); 1978cafe5635SKent Overstreet 1979cafe5635SKent Overstreet err = "no journal entries found"; 1980cafe5635SKent Overstreet if (list_empty(&journal)) 1981cafe5635SKent Overstreet goto err; 1982cafe5635SKent Overstreet 1983cafe5635SKent Overstreet j = &list_entry(journal.prev, struct journal_replay, list)->j; 1984cafe5635SKent Overstreet 1985cafe5635SKent Overstreet err = "IO error reading priorities"; 198649d08d59SColy Li for_each_cache(ca, c, i) { 198749d08d59SColy Li if (prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev])) 198849d08d59SColy Li goto err; 198949d08d59SColy Li } 1990cafe5635SKent Overstreet 1991cafe5635SKent Overstreet /* 1992cafe5635SKent Overstreet * If prio_read() fails it'll call cache_set_error and we'll 1993cafe5635SKent Overstreet * tear everything down right away, but if we perhaps checked 1994cafe5635SKent Overstreet * sooner we could avoid journal replay. 1995cafe5635SKent Overstreet */ 1996cafe5635SKent Overstreet 1997cafe5635SKent Overstreet k = &j->btree_root; 1998cafe5635SKent Overstreet 1999cafe5635SKent Overstreet err = "bad btree root"; 200065d45231SKent Overstreet if (__bch_btree_ptr_invalid(c, k)) 2001cafe5635SKent Overstreet goto err; 2002cafe5635SKent Overstreet 2003cafe5635SKent Overstreet err = "error reading btree root"; 2004b0d30981SColy Li c->root = bch_btree_node_get(c, NULL, k, 2005b0d30981SColy Li j->btree_level, 2006b0d30981SColy Li true, NULL); 2007cafe5635SKent Overstreet if (IS_ERR_OR_NULL(c->root)) 2008cafe5635SKent Overstreet goto err; 2009cafe5635SKent Overstreet 2010cafe5635SKent Overstreet list_del_init(&c->root->list); 2011cafe5635SKent Overstreet rw_unlock(true, c->root); 2012cafe5635SKent Overstreet 2013c18536a7SKent Overstreet err = uuid_read(c, j, &cl); 2014cafe5635SKent Overstreet if (err) 2015cafe5635SKent Overstreet goto err; 2016cafe5635SKent Overstreet 2017cafe5635SKent Overstreet err = "error in recovery"; 2018c18536a7SKent Overstreet if (bch_btree_check(c)) 2019cafe5635SKent Overstreet goto err; 2020cafe5635SKent Overstreet 2021cafe5635SKent Overstreet bch_journal_mark(c, &journal); 20222531d9eeSKent Overstreet bch_initial_gc_finish(c); 202346f5aa88SJoe Perches pr_debug("btree_check() done\n"); 2024cafe5635SKent Overstreet 2025cafe5635SKent Overstreet /* 2026cafe5635SKent Overstreet * bcache_journal_next() can't happen sooner, or 2027cafe5635SKent Overstreet * btree_gc_finish() will give spurious errors about last_gc > 2028cafe5635SKent Overstreet * gc_gen - this is a hack but oh well. 2029cafe5635SKent Overstreet */ 2030cafe5635SKent Overstreet bch_journal_next(&c->journal); 2031cafe5635SKent Overstreet 2032119ba0f8SKent Overstreet err = "error starting allocator thread"; 2033cafe5635SKent Overstreet for_each_cache(ca, c, i) 2034119ba0f8SKent Overstreet if (bch_cache_allocator_start(ca)) 2035119ba0f8SKent Overstreet goto err; 2036cafe5635SKent Overstreet 2037cafe5635SKent Overstreet /* 2038cafe5635SKent Overstreet * First place it's safe to allocate: btree_check() and 2039cafe5635SKent Overstreet * btree_gc_finish() have to run before we have buckets to 2040cafe5635SKent Overstreet * allocate, and bch_bucket_alloc_set() might cause a journal 2041cafe5635SKent Overstreet * entry to be written so bcache_journal_next() has to be called 2042cafe5635SKent Overstreet * first. 2043cafe5635SKent Overstreet * 2044cafe5635SKent Overstreet * If the uuids were in the old format we have to rewrite them 2045cafe5635SKent Overstreet * before the next journal entry is written: 2046cafe5635SKent Overstreet */ 2047cafe5635SKent Overstreet if (j->version < BCACHE_JSET_VERSION_UUID) 2048cafe5635SKent Overstreet __uuid_write(c); 2049cafe5635SKent Overstreet 2050ce3e4cfbSColy Li err = "bcache: replay journal failed"; 2051ce3e4cfbSColy Li if (bch_journal_replay(c, &journal)) 2052ce3e4cfbSColy Li goto err; 2053cafe5635SKent Overstreet } else { 205446f5aa88SJoe Perches pr_notice("invalidating existing data\n"); 2055cafe5635SKent Overstreet 2056cafe5635SKent Overstreet for_each_cache(ca, c, i) { 20576f10f7d1SColy Li unsigned int j; 2058cafe5635SKent Overstreet 2059cafe5635SKent Overstreet ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7, 2060cafe5635SKent Overstreet 2, SB_JOURNAL_BUCKETS); 2061cafe5635SKent Overstreet 2062cafe5635SKent Overstreet for (j = 0; j < ca->sb.keys; j++) 2063cafe5635SKent Overstreet ca->sb.d[j] = ca->sb.first_bucket + j; 2064cafe5635SKent Overstreet } 2065cafe5635SKent Overstreet 20662531d9eeSKent Overstreet bch_initial_gc_finish(c); 2067cafe5635SKent Overstreet 2068119ba0f8SKent Overstreet err = "error starting allocator thread"; 2069cafe5635SKent Overstreet for_each_cache(ca, c, i) 2070119ba0f8SKent Overstreet if (bch_cache_allocator_start(ca)) 2071119ba0f8SKent Overstreet goto err; 2072cafe5635SKent Overstreet 2073cafe5635SKent Overstreet mutex_lock(&c->bucket_lock); 2074cafe5635SKent Overstreet for_each_cache(ca, c, i) 207584c529aeSAndrea Righi bch_prio_write(ca, true); 2076cafe5635SKent Overstreet mutex_unlock(&c->bucket_lock); 2077cafe5635SKent Overstreet 2078cafe5635SKent Overstreet err = "cannot allocate new UUID bucket"; 2079cafe5635SKent Overstreet if (__uuid_write(c)) 208072a44517SKent Overstreet goto err; 2081cafe5635SKent Overstreet 2082cafe5635SKent Overstreet err = "cannot allocate new btree root"; 20832452cc89SSlava Pestov c->root = __bch_btree_node_alloc(c, NULL, 0, true, NULL); 2084cafe5635SKent Overstreet if (IS_ERR_OR_NULL(c->root)) 208572a44517SKent Overstreet goto err; 2086cafe5635SKent Overstreet 20872a285686SKent Overstreet mutex_lock(&c->root->write_lock); 2088cafe5635SKent Overstreet bkey_copy_key(&c->root->key, &MAX_KEY); 2089c18536a7SKent Overstreet bch_btree_node_write(c->root, &cl); 20902a285686SKent Overstreet mutex_unlock(&c->root->write_lock); 2091cafe5635SKent Overstreet 2092cafe5635SKent Overstreet bch_btree_set_root(c->root); 2093cafe5635SKent Overstreet rw_unlock(true, c->root); 2094cafe5635SKent Overstreet 2095cafe5635SKent Overstreet /* 2096cafe5635SKent Overstreet * We don't want to write the first journal entry until 2097cafe5635SKent Overstreet * everything is set up - fortunately journal entries won't be 2098cafe5635SKent Overstreet * written until the SET_CACHE_SYNC() here: 2099cafe5635SKent Overstreet */ 2100cafe5635SKent Overstreet SET_CACHE_SYNC(&c->sb, true); 2101cafe5635SKent Overstreet 2102cafe5635SKent Overstreet bch_journal_next(&c->journal); 2103c18536a7SKent Overstreet bch_journal_meta(c, &cl); 2104cafe5635SKent Overstreet } 2105cafe5635SKent Overstreet 210672a44517SKent Overstreet err = "error starting gc thread"; 210772a44517SKent Overstreet if (bch_gc_thread_start(c)) 210872a44517SKent Overstreet goto err; 210972a44517SKent Overstreet 2110c18536a7SKent Overstreet closure_sync(&cl); 211175cbb3f1SArnd Bergmann c->sb.last_mount = (u32)ktime_get_real_seconds(); 2112cafe5635SKent Overstreet bcache_write_super(c); 2113cafe5635SKent Overstreet 2114cafe5635SKent Overstreet list_for_each_entry_safe(dc, t, &uncached_devices, list) 211573ac105bSTang Junhui bch_cached_dev_attach(dc, c, NULL); 2116cafe5635SKent Overstreet 2117cafe5635SKent Overstreet flash_devs_run(c); 2118cafe5635SKent Overstreet 2119bf0c55c9SSlava Pestov set_bit(CACHE_SET_RUNNING, &c->flags); 2120ce3e4cfbSColy Li return 0; 2121cafe5635SKent Overstreet err: 212295f18c9dSShenghui Wang while (!list_empty(&journal)) { 212395f18c9dSShenghui Wang l = list_first_entry(&journal, struct journal_replay, list); 212495f18c9dSShenghui Wang list_del(&l->list); 212595f18c9dSShenghui Wang kfree(l); 212695f18c9dSShenghui Wang } 212795f18c9dSShenghui Wang 2128c18536a7SKent Overstreet closure_sync(&cl); 212968a53c95SColy Li 2130c8694948SKees Cook bch_cache_set_error(c, "%s", err); 2131ce3e4cfbSColy Li 2132ce3e4cfbSColy Li return -EIO; 2133cafe5635SKent Overstreet } 2134cafe5635SKent Overstreet 2135cafe5635SKent Overstreet static bool can_attach_cache(struct cache *ca, struct cache_set *c) 2136cafe5635SKent Overstreet { 2137cafe5635SKent Overstreet return ca->sb.block_size == c->sb.block_size && 21389eb8ebebSNicholas Swenson ca->sb.bucket_size == c->sb.bucket_size && 2139cafe5635SKent Overstreet ca->sb.nr_in_set == c->sb.nr_in_set; 2140cafe5635SKent Overstreet } 2141cafe5635SKent Overstreet 2142cafe5635SKent Overstreet static const char *register_cache_set(struct cache *ca) 2143cafe5635SKent Overstreet { 2144cafe5635SKent Overstreet char buf[12]; 2145cafe5635SKent Overstreet const char *err = "cannot allocate memory"; 2146cafe5635SKent Overstreet struct cache_set *c; 2147cafe5635SKent Overstreet 2148cafe5635SKent Overstreet list_for_each_entry(c, &bch_cache_sets, list) 2149cafe5635SKent Overstreet if (!memcmp(c->sb.set_uuid, ca->sb.set_uuid, 16)) { 2150cafe5635SKent Overstreet if (c->cache[ca->sb.nr_this_dev]) 2151cafe5635SKent Overstreet return "duplicate cache set member"; 2152cafe5635SKent Overstreet 2153cafe5635SKent Overstreet if (!can_attach_cache(ca, c)) 2154cafe5635SKent Overstreet return "cache sb does not match set"; 2155cafe5635SKent Overstreet 2156cafe5635SKent Overstreet if (!CACHE_SYNC(&ca->sb)) 2157cafe5635SKent Overstreet SET_CACHE_SYNC(&c->sb, false); 2158cafe5635SKent Overstreet 2159cafe5635SKent Overstreet goto found; 2160cafe5635SKent Overstreet } 2161cafe5635SKent Overstreet 2162cafe5635SKent Overstreet c = bch_cache_set_alloc(&ca->sb); 2163cafe5635SKent Overstreet if (!c) 2164cafe5635SKent Overstreet return err; 2165cafe5635SKent Overstreet 2166cafe5635SKent Overstreet err = "error creating kobject"; 2167cafe5635SKent Overstreet if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->sb.set_uuid) || 2168cafe5635SKent Overstreet kobject_add(&c->internal, &c->kobj, "internal")) 2169cafe5635SKent Overstreet goto err; 2170cafe5635SKent Overstreet 2171cafe5635SKent Overstreet if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj)) 2172cafe5635SKent Overstreet goto err; 2173cafe5635SKent Overstreet 2174cafe5635SKent Overstreet bch_debug_init_cache_set(c); 2175cafe5635SKent Overstreet 2176cafe5635SKent Overstreet list_add(&c->list, &bch_cache_sets); 2177cafe5635SKent Overstreet found: 2178cafe5635SKent Overstreet sprintf(buf, "cache%i", ca->sb.nr_this_dev); 2179cafe5635SKent Overstreet if (sysfs_create_link(&ca->kobj, &c->kobj, "set") || 2180cafe5635SKent Overstreet sysfs_create_link(&c->kobj, &ca->kobj, buf)) 2181cafe5635SKent Overstreet goto err; 2182cafe5635SKent Overstreet 2183117f636eSColy Li /* 2184117f636eSColy Li * A special case is both ca->sb.seq and c->sb.seq are 0, 2185117f636eSColy Li * such condition happens on a new created cache device whose 2186117f636eSColy Li * super block is never flushed yet. In this case c->sb.version 2187117f636eSColy Li * and other members should be updated too, otherwise we will 2188117f636eSColy Li * have a mistaken super block version in cache set. 2189117f636eSColy Li */ 2190117f636eSColy Li if (ca->sb.seq > c->sb.seq || c->sb.seq == 0) { 2191cafe5635SKent Overstreet c->sb.version = ca->sb.version; 2192cafe5635SKent Overstreet memcpy(c->sb.set_uuid, ca->sb.set_uuid, 16); 2193cafe5635SKent Overstreet c->sb.flags = ca->sb.flags; 2194cafe5635SKent Overstreet c->sb.seq = ca->sb.seq; 219546f5aa88SJoe Perches pr_debug("set version = %llu\n", c->sb.version); 2196cafe5635SKent Overstreet } 2197cafe5635SKent Overstreet 2198d83353b3SKent Overstreet kobject_get(&ca->kobj); 2199cafe5635SKent Overstreet ca->set = c; 2200cafe5635SKent Overstreet ca->set->cache[ca->sb.nr_this_dev] = ca; 2201cafe5635SKent Overstreet c->cache_by_alloc[c->caches_loaded++] = ca; 2202cafe5635SKent Overstreet 2203ce3e4cfbSColy Li if (c->caches_loaded == c->sb.nr_in_set) { 2204ce3e4cfbSColy Li err = "failed to run cache set"; 2205ce3e4cfbSColy Li if (run_cache_set(c) < 0) 2206ce3e4cfbSColy Li goto err; 2207ce3e4cfbSColy Li } 2208cafe5635SKent Overstreet 2209cafe5635SKent Overstreet return NULL; 2210cafe5635SKent Overstreet err: 2211cafe5635SKent Overstreet bch_cache_set_unregister(c); 2212cafe5635SKent Overstreet return err; 2213cafe5635SKent Overstreet } 2214cafe5635SKent Overstreet 2215cafe5635SKent Overstreet /* Cache device */ 2216cafe5635SKent Overstreet 22172d17456eSColy Li /* When ca->kobj released */ 2218cafe5635SKent Overstreet void bch_cache_release(struct kobject *kobj) 2219cafe5635SKent Overstreet { 2220cafe5635SKent Overstreet struct cache *ca = container_of(kobj, struct cache, kobj); 22216f10f7d1SColy Li unsigned int i; 2222cafe5635SKent Overstreet 2223c9a78332SSlava Pestov if (ca->set) { 2224c9a78332SSlava Pestov BUG_ON(ca->set->cache[ca->sb.nr_this_dev] != ca); 2225cafe5635SKent Overstreet ca->set->cache[ca->sb.nr_this_dev] = NULL; 2226c9a78332SSlava Pestov } 2227cafe5635SKent Overstreet 2228cafe5635SKent Overstreet free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca))); 2229cafe5635SKent Overstreet kfree(ca->prio_buckets); 2230cafe5635SKent Overstreet vfree(ca->buckets); 2231cafe5635SKent Overstreet 2232cafe5635SKent Overstreet free_heap(&ca->heap); 2233cafe5635SKent Overstreet free_fifo(&ca->free_inc); 223478365411SKent Overstreet 223578365411SKent Overstreet for (i = 0; i < RESERVE_NR; i++) 223678365411SKent Overstreet free_fifo(&ca->free[i]); 2237cafe5635SKent Overstreet 2238475389aeSChristoph Hellwig if (ca->sb_disk) 2239475389aeSChristoph Hellwig put_page(virt_to_page(ca->sb_disk)); 2240cafe5635SKent Overstreet 22410781c874SKent Overstreet if (!IS_ERR_OR_NULL(ca->bdev)) 2242cafe5635SKent Overstreet blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); 2243cafe5635SKent Overstreet 2244cafe5635SKent Overstreet kfree(ca); 2245cafe5635SKent Overstreet module_put(THIS_MODULE); 2246cafe5635SKent Overstreet } 2247cafe5635SKent Overstreet 2248c50d4d5dSYijing Wang static int cache_alloc(struct cache *ca) 2249cafe5635SKent Overstreet { 2250cafe5635SKent Overstreet size_t free; 2251682811b3STang Junhui size_t btree_buckets; 2252cafe5635SKent Overstreet struct bucket *b; 2253f6027bcaSDongbo Cao int ret = -ENOMEM; 2254f6027bcaSDongbo Cao const char *err = NULL; 2255cafe5635SKent Overstreet 2256cafe5635SKent Overstreet __module_get(THIS_MODULE); 2257cafe5635SKent Overstreet kobject_init(&ca->kobj, &bch_cache_ktype); 2258cafe5635SKent Overstreet 22593a83f467SMing Lei bio_init(&ca->journal.bio, ca->journal.bio.bi_inline_vecs, 8); 2260cafe5635SKent Overstreet 2261682811b3STang Junhui /* 2262682811b3STang Junhui * when ca->sb.njournal_buckets is not zero, journal exists, 2263682811b3STang Junhui * and in bch_journal_replay(), tree node may split, 2264682811b3STang Junhui * so bucket of RESERVE_BTREE type is needed, 2265682811b3STang Junhui * the worst situation is all journal buckets are valid journal, 2266682811b3STang Junhui * and all the keys need to replay, 2267682811b3STang Junhui * so the number of RESERVE_BTREE type buckets should be as much 2268682811b3STang Junhui * as journal buckets 2269682811b3STang Junhui */ 2270682811b3STang Junhui btree_buckets = ca->sb.njournal_buckets ?: 8; 227178365411SKent Overstreet free = roundup_pow_of_two(ca->sb.nbuckets) >> 10; 22723a646fd7SDongbo Cao if (!free) { 22733a646fd7SDongbo Cao ret = -EPERM; 22743a646fd7SDongbo Cao err = "ca->sb.nbuckets is too small"; 22753a646fd7SDongbo Cao goto err_free; 22763a646fd7SDongbo Cao } 2277cafe5635SKent Overstreet 2278f6027bcaSDongbo Cao if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets, 2279f6027bcaSDongbo Cao GFP_KERNEL)) { 2280f6027bcaSDongbo Cao err = "ca->free[RESERVE_BTREE] alloc failed"; 2281f6027bcaSDongbo Cao goto err_btree_alloc; 2282f6027bcaSDongbo Cao } 2283f6027bcaSDongbo Cao 2284f6027bcaSDongbo Cao if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), 2285f6027bcaSDongbo Cao GFP_KERNEL)) { 2286f6027bcaSDongbo Cao err = "ca->free[RESERVE_PRIO] alloc failed"; 2287f6027bcaSDongbo Cao goto err_prio_alloc; 2288f6027bcaSDongbo Cao } 2289f6027bcaSDongbo Cao 2290f6027bcaSDongbo Cao if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) { 2291f6027bcaSDongbo Cao err = "ca->free[RESERVE_MOVINGGC] alloc failed"; 2292f6027bcaSDongbo Cao goto err_movinggc_alloc; 2293f6027bcaSDongbo Cao } 2294f6027bcaSDongbo Cao 2295f6027bcaSDongbo Cao if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) { 2296f6027bcaSDongbo Cao err = "ca->free[RESERVE_NONE] alloc failed"; 2297f6027bcaSDongbo Cao goto err_none_alloc; 2298f6027bcaSDongbo Cao } 2299f6027bcaSDongbo Cao 2300f6027bcaSDongbo Cao if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) { 2301f6027bcaSDongbo Cao err = "ca->free_inc alloc failed"; 2302f6027bcaSDongbo Cao goto err_free_inc_alloc; 2303f6027bcaSDongbo Cao } 2304f6027bcaSDongbo Cao 2305f6027bcaSDongbo Cao if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) { 2306f6027bcaSDongbo Cao err = "ca->heap alloc failed"; 2307f6027bcaSDongbo Cao goto err_heap_alloc; 2308f6027bcaSDongbo Cao } 2309f6027bcaSDongbo Cao 2310f6027bcaSDongbo Cao ca->buckets = vzalloc(array_size(sizeof(struct bucket), 2311f6027bcaSDongbo Cao ca->sb.nbuckets)); 2312f6027bcaSDongbo Cao if (!ca->buckets) { 2313f6027bcaSDongbo Cao err = "ca->buckets alloc failed"; 2314f6027bcaSDongbo Cao goto err_buckets_alloc; 2315f6027bcaSDongbo Cao } 2316f6027bcaSDongbo Cao 2317f6027bcaSDongbo Cao ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t), 23186396bb22SKees Cook prio_buckets(ca), 2), 2319f6027bcaSDongbo Cao GFP_KERNEL); 2320f6027bcaSDongbo Cao if (!ca->prio_buckets) { 2321f6027bcaSDongbo Cao err = "ca->prio_buckets alloc failed"; 2322f6027bcaSDongbo Cao goto err_prio_buckets_alloc; 2323f6027bcaSDongbo Cao } 2324f6027bcaSDongbo Cao 2325f6027bcaSDongbo Cao ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca); 2326f6027bcaSDongbo Cao if (!ca->disk_buckets) { 2327f6027bcaSDongbo Cao err = "ca->disk_buckets alloc failed"; 2328f6027bcaSDongbo Cao goto err_disk_buckets_alloc; 2329f6027bcaSDongbo Cao } 2330cafe5635SKent Overstreet 2331cafe5635SKent Overstreet ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca); 2332cafe5635SKent Overstreet 2333cafe5635SKent Overstreet for_each_bucket(b, ca) 2334cafe5635SKent Overstreet atomic_set(&b->pin, 0); 2335cafe5635SKent Overstreet return 0; 2336f6027bcaSDongbo Cao 2337f6027bcaSDongbo Cao err_disk_buckets_alloc: 2338f6027bcaSDongbo Cao kfree(ca->prio_buckets); 2339f6027bcaSDongbo Cao err_prio_buckets_alloc: 2340f6027bcaSDongbo Cao vfree(ca->buckets); 2341f6027bcaSDongbo Cao err_buckets_alloc: 2342f6027bcaSDongbo Cao free_heap(&ca->heap); 2343f6027bcaSDongbo Cao err_heap_alloc: 2344f6027bcaSDongbo Cao free_fifo(&ca->free_inc); 2345f6027bcaSDongbo Cao err_free_inc_alloc: 2346f6027bcaSDongbo Cao free_fifo(&ca->free[RESERVE_NONE]); 2347f6027bcaSDongbo Cao err_none_alloc: 2348f6027bcaSDongbo Cao free_fifo(&ca->free[RESERVE_MOVINGGC]); 2349f6027bcaSDongbo Cao err_movinggc_alloc: 2350f6027bcaSDongbo Cao free_fifo(&ca->free[RESERVE_PRIO]); 2351f6027bcaSDongbo Cao err_prio_alloc: 2352f6027bcaSDongbo Cao free_fifo(&ca->free[RESERVE_BTREE]); 2353f6027bcaSDongbo Cao err_btree_alloc: 23543a646fd7SDongbo Cao err_free: 2355f6027bcaSDongbo Cao module_put(THIS_MODULE); 2356f6027bcaSDongbo Cao if (err) 235746f5aa88SJoe Perches pr_notice("error %s: %s\n", ca->cache_dev_name, err); 2358f6027bcaSDongbo Cao return ret; 2359cafe5635SKent Overstreet } 2360cafe5635SKent Overstreet 2361cfa0c56dSChristoph Hellwig static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk, 2362cafe5635SKent Overstreet struct block_device *bdev, struct cache *ca) 2363cafe5635SKent Overstreet { 2364d9dc1702SEric Wheeler const char *err = NULL; /* must be set for any error case */ 23659b299728SEric Wheeler int ret = 0; 2366cafe5635SKent Overstreet 23676e916a7eSColy Li bdevname(bdev, ca->cache_dev_name); 2368f59fce84SKent Overstreet memcpy(&ca->sb, sb, sizeof(struct cache_sb)); 2369cafe5635SKent Overstreet ca->bdev = bdev; 2370cafe5635SKent Overstreet ca->bdev->bd_holder = ca; 2371475389aeSChristoph Hellwig ca->sb_disk = sb_disk; 2372f59fce84SKent Overstreet 2373cc40daf9STang Junhui if (blk_queue_discard(bdev_get_queue(bdev))) 2374cafe5635SKent Overstreet ca->discard = CACHE_DISCARD(&ca->sb); 2375cafe5635SKent Overstreet 2376c50d4d5dSYijing Wang ret = cache_alloc(ca); 2377d9dc1702SEric Wheeler if (ret != 0) { 2378bb6d355cSColy Li /* 2379bb6d355cSColy Li * If we failed here, it means ca->kobj is not initialized yet, 2380bb6d355cSColy Li * kobject_put() won't be called and there is no chance to 2381bb6d355cSColy Li * call blkdev_put() to bdev in bch_cache_release(). So we 2382bb6d355cSColy Li * explicitly call blkdev_put() here. 2383bb6d355cSColy Li */ 2384cc40daf9STang Junhui blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); 2385d9dc1702SEric Wheeler if (ret == -ENOMEM) 2386d9dc1702SEric Wheeler err = "cache_alloc(): -ENOMEM"; 23873a646fd7SDongbo Cao else if (ret == -EPERM) 23883a646fd7SDongbo Cao err = "cache_alloc(): cache device is too small"; 2389d9dc1702SEric Wheeler else 2390d9dc1702SEric Wheeler err = "cache_alloc(): unknown error"; 2391f59fce84SKent Overstreet goto err; 2392d9dc1702SEric Wheeler } 2393f59fce84SKent Overstreet 2394b0d30981SColy Li if (kobject_add(&ca->kobj, 2395b0d30981SColy Li &part_to_dev(bdev->bd_part)->kobj, 2396b0d30981SColy Li "bcache")) { 23979b299728SEric Wheeler err = "error calling kobject_add"; 23989b299728SEric Wheeler ret = -ENOMEM; 23999b299728SEric Wheeler goto out; 24009b299728SEric Wheeler } 2401cafe5635SKent Overstreet 24024fa03402SKent Overstreet mutex_lock(&bch_register_lock); 2403cafe5635SKent Overstreet err = register_cache_set(ca); 24044fa03402SKent Overstreet mutex_unlock(&bch_register_lock); 24054fa03402SKent Overstreet 24069b299728SEric Wheeler if (err) { 24079b299728SEric Wheeler ret = -ENODEV; 24089b299728SEric Wheeler goto out; 24099b299728SEric Wheeler } 2410cafe5635SKent Overstreet 241146f5aa88SJoe Perches pr_info("registered cache device %s\n", ca->cache_dev_name); 24129b299728SEric Wheeler 2413d83353b3SKent Overstreet out: 2414d83353b3SKent Overstreet kobject_put(&ca->kobj); 24159b299728SEric Wheeler 2416cafe5635SKent Overstreet err: 24179b299728SEric Wheeler if (err) 241846f5aa88SJoe Perches pr_notice("error %s: %s\n", ca->cache_dev_name, err); 24199b299728SEric Wheeler 24209b299728SEric Wheeler return ret; 2421cafe5635SKent Overstreet } 2422cafe5635SKent Overstreet 2423cafe5635SKent Overstreet /* Global interfaces/init */ 2424cafe5635SKent Overstreet 2425fc2d5988SColy Li static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, 2426fc2d5988SColy Li const char *buffer, size_t size); 24270c277e21SColy Li static ssize_t bch_pending_bdevs_cleanup(struct kobject *k, 24280c277e21SColy Li struct kobj_attribute *attr, 24290c277e21SColy Li const char *buffer, size_t size); 2430cafe5635SKent Overstreet 2431cafe5635SKent Overstreet kobj_attribute_write(register, register_bcache); 2432cafe5635SKent Overstreet kobj_attribute_write(register_quiet, register_bcache); 24339e23ccf8SColy Li kobj_attribute_write(register_async, register_bcache); 24340c277e21SColy Li kobj_attribute_write(pendings_cleanup, bch_pending_bdevs_cleanup); 2435cafe5635SKent Overstreet 2436b3cf37bfSColy Li static bool bch_is_open_backing(struct block_device *bdev) 2437b3cf37bfSColy Li { 2438a9dd53adSGabriel de Perthuis struct cache_set *c, *tc; 2439a9dd53adSGabriel de Perthuis struct cached_dev *dc, *t; 2440a9dd53adSGabriel de Perthuis 2441a9dd53adSGabriel de Perthuis list_for_each_entry_safe(c, tc, &bch_cache_sets, list) 2442a9dd53adSGabriel de Perthuis list_for_each_entry_safe(dc, t, &c->cached_devs, list) 2443a9dd53adSGabriel de Perthuis if (dc->bdev == bdev) 2444a9dd53adSGabriel de Perthuis return true; 2445a9dd53adSGabriel de Perthuis list_for_each_entry_safe(dc, t, &uncached_devices, list) 2446a9dd53adSGabriel de Perthuis if (dc->bdev == bdev) 2447a9dd53adSGabriel de Perthuis return true; 2448a9dd53adSGabriel de Perthuis return false; 2449a9dd53adSGabriel de Perthuis } 2450a9dd53adSGabriel de Perthuis 2451b3cf37bfSColy Li static bool bch_is_open_cache(struct block_device *bdev) 2452b3cf37bfSColy Li { 2453a9dd53adSGabriel de Perthuis struct cache_set *c, *tc; 2454a9dd53adSGabriel de Perthuis struct cache *ca; 24556f10f7d1SColy Li unsigned int i; 2456a9dd53adSGabriel de Perthuis 2457a9dd53adSGabriel de Perthuis list_for_each_entry_safe(c, tc, &bch_cache_sets, list) 2458a9dd53adSGabriel de Perthuis for_each_cache(ca, c, i) 2459a9dd53adSGabriel de Perthuis if (ca->bdev == bdev) 2460a9dd53adSGabriel de Perthuis return true; 2461a9dd53adSGabriel de Perthuis return false; 2462a9dd53adSGabriel de Perthuis } 2463a9dd53adSGabriel de Perthuis 2464b3cf37bfSColy Li static bool bch_is_open(struct block_device *bdev) 2465b3cf37bfSColy Li { 2466a9dd53adSGabriel de Perthuis return bch_is_open_cache(bdev) || bch_is_open_backing(bdev); 2467a9dd53adSGabriel de Perthuis } 2468a9dd53adSGabriel de Perthuis 24699e23ccf8SColy Li struct async_reg_args { 2470ee4a36f4SColy Li struct delayed_work reg_work; 24719e23ccf8SColy Li char *path; 24729e23ccf8SColy Li struct cache_sb *sb; 24739e23ccf8SColy Li struct cache_sb_disk *sb_disk; 24749e23ccf8SColy Li struct block_device *bdev; 24759e23ccf8SColy Li }; 24769e23ccf8SColy Li 24779e23ccf8SColy Li static void register_bdev_worker(struct work_struct *work) 24789e23ccf8SColy Li { 24799e23ccf8SColy Li int fail = false; 24809e23ccf8SColy Li struct async_reg_args *args = 2481ee4a36f4SColy Li container_of(work, struct async_reg_args, reg_work.work); 24829e23ccf8SColy Li struct cached_dev *dc; 24839e23ccf8SColy Li 24849e23ccf8SColy Li dc = kzalloc(sizeof(*dc), GFP_KERNEL); 24859e23ccf8SColy Li if (!dc) { 24869e23ccf8SColy Li fail = true; 24879e23ccf8SColy Li put_page(virt_to_page(args->sb_disk)); 24889e23ccf8SColy Li blkdev_put(args->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 24899e23ccf8SColy Li goto out; 24909e23ccf8SColy Li } 24919e23ccf8SColy Li 24929e23ccf8SColy Li mutex_lock(&bch_register_lock); 24939e23ccf8SColy Li if (register_bdev(args->sb, args->sb_disk, args->bdev, dc) < 0) 24949e23ccf8SColy Li fail = true; 24959e23ccf8SColy Li mutex_unlock(&bch_register_lock); 24969e23ccf8SColy Li 24979e23ccf8SColy Li out: 24989e23ccf8SColy Li if (fail) 24999e23ccf8SColy Li pr_info("error %s: fail to register backing device\n", 25009e23ccf8SColy Li args->path); 25019e23ccf8SColy Li kfree(args->sb); 25029e23ccf8SColy Li kfree(args->path); 25039e23ccf8SColy Li kfree(args); 25049e23ccf8SColy Li module_put(THIS_MODULE); 25059e23ccf8SColy Li } 25069e23ccf8SColy Li 25079e23ccf8SColy Li static void register_cache_worker(struct work_struct *work) 25089e23ccf8SColy Li { 25099e23ccf8SColy Li int fail = false; 25109e23ccf8SColy Li struct async_reg_args *args = 2511ee4a36f4SColy Li container_of(work, struct async_reg_args, reg_work.work); 25129e23ccf8SColy Li struct cache *ca; 25139e23ccf8SColy Li 25149e23ccf8SColy Li ca = kzalloc(sizeof(*ca), GFP_KERNEL); 25159e23ccf8SColy Li if (!ca) { 25169e23ccf8SColy Li fail = true; 25179e23ccf8SColy Li put_page(virt_to_page(args->sb_disk)); 25189e23ccf8SColy Li blkdev_put(args->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 25199e23ccf8SColy Li goto out; 25209e23ccf8SColy Li } 25219e23ccf8SColy Li 25229e23ccf8SColy Li /* blkdev_put() will be called in bch_cache_release() */ 25239e23ccf8SColy Li if (register_cache(args->sb, args->sb_disk, args->bdev, ca) != 0) 25249e23ccf8SColy Li fail = true; 25259e23ccf8SColy Li 25269e23ccf8SColy Li out: 25279e23ccf8SColy Li if (fail) 25289e23ccf8SColy Li pr_info("error %s: fail to register cache device\n", 25299e23ccf8SColy Li args->path); 25309e23ccf8SColy Li kfree(args->sb); 25319e23ccf8SColy Li kfree(args->path); 25329e23ccf8SColy Li kfree(args); 25339e23ccf8SColy Li module_put(THIS_MODULE); 25349e23ccf8SColy Li } 25359e23ccf8SColy Li 25369e23ccf8SColy Li static void register_device_aync(struct async_reg_args *args) 25379e23ccf8SColy Li { 25389e23ccf8SColy Li if (SB_IS_BDEV(args->sb)) 2539ee4a36f4SColy Li INIT_DELAYED_WORK(&args->reg_work, register_bdev_worker); 25409e23ccf8SColy Li else 2541ee4a36f4SColy Li INIT_DELAYED_WORK(&args->reg_work, register_cache_worker); 25429e23ccf8SColy Li 2543ee4a36f4SColy Li /* 10 jiffies is enough for a delay */ 2544ee4a36f4SColy Li queue_delayed_work(system_wq, &args->reg_work, 10); 25459e23ccf8SColy Li } 25469e23ccf8SColy Li 2547cafe5635SKent Overstreet static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, 2548cafe5635SKent Overstreet const char *buffer, size_t size) 2549cafe5635SKent Overstreet { 255050246693SChristoph Hellwig const char *err; 255129cda393SColy Li char *path = NULL; 255250246693SChristoph Hellwig struct cache_sb *sb; 2553cfa0c56dSChristoph Hellwig struct cache_sb_disk *sb_disk; 2554fc8f19ccSChristoph Hellwig struct block_device *bdev; 255550246693SChristoph Hellwig ssize_t ret; 2556cafe5635SKent Overstreet 255750246693SChristoph Hellwig ret = -EBUSY; 255829cda393SColy Li err = "failed to reference bcache module"; 2559cafe5635SKent Overstreet if (!try_module_get(THIS_MODULE)) 256050246693SChristoph Hellwig goto out; 2561cafe5635SKent Overstreet 2562a59ff6ccSColy Li /* For latest state of bcache_is_reboot */ 2563a59ff6ccSColy Li smp_mb(); 256429cda393SColy Li err = "bcache is in reboot"; 2565a59ff6ccSColy Li if (bcache_is_reboot) 256650246693SChristoph Hellwig goto out_module_put; 2567a59ff6ccSColy Li 256850246693SChristoph Hellwig ret = -ENOMEM; 256950246693SChristoph Hellwig err = "cannot allocate memory"; 2570a56489d4SFlorian Schmaus path = kstrndup(buffer, size, GFP_KERNEL); 2571a56489d4SFlorian Schmaus if (!path) 257250246693SChristoph Hellwig goto out_module_put; 2573a56489d4SFlorian Schmaus 2574a56489d4SFlorian Schmaus sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL); 2575a56489d4SFlorian Schmaus if (!sb) 257650246693SChristoph Hellwig goto out_free_path; 2577cafe5635SKent Overstreet 257850246693SChristoph Hellwig ret = -EINVAL; 2579cafe5635SKent Overstreet err = "failed to open device"; 2580cafe5635SKent Overstreet bdev = blkdev_get_by_path(strim(path), 2581cafe5635SKent Overstreet FMODE_READ|FMODE_WRITE|FMODE_EXCL, 2582cafe5635SKent Overstreet sb); 2583f59fce84SKent Overstreet if (IS_ERR(bdev)) { 2584a9dd53adSGabriel de Perthuis if (bdev == ERR_PTR(-EBUSY)) { 2585a9dd53adSGabriel de Perthuis bdev = lookup_bdev(strim(path)); 2586789d21dbSJianjian Huo mutex_lock(&bch_register_lock); 2587a9dd53adSGabriel de Perthuis if (!IS_ERR(bdev) && bch_is_open(bdev)) 2588a9dd53adSGabriel de Perthuis err = "device already registered"; 2589a9dd53adSGabriel de Perthuis else 2590cafe5635SKent Overstreet err = "device busy"; 2591789d21dbSJianjian Huo mutex_unlock(&bch_register_lock); 25924b758df2SJan Kara if (!IS_ERR(bdev)) 25934b758df2SJan Kara bdput(bdev); 2594d7076f21SGabriel de Perthuis if (attr == &ksysfs_register_quiet) 259550246693SChristoph Hellwig goto done; 2596a9dd53adSGabriel de Perthuis } 259750246693SChristoph Hellwig goto out_free_sb; 2598f59fce84SKent Overstreet } 2599f59fce84SKent Overstreet 2600f59fce84SKent Overstreet err = "failed to set blocksize"; 2601f59fce84SKent Overstreet if (set_blocksize(bdev, 4096)) 260250246693SChristoph Hellwig goto out_blkdev_put; 2603cafe5635SKent Overstreet 2604cfa0c56dSChristoph Hellwig err = read_super(sb, bdev, &sb_disk); 2605cafe5635SKent Overstreet if (err) 260650246693SChristoph Hellwig goto out_blkdev_put; 2607cafe5635SKent Overstreet 2608cc40daf9STang Junhui err = "failed to register device"; 26099e23ccf8SColy Li if (attr == &ksysfs_register_async) { 26109e23ccf8SColy Li /* register in asynchronous way */ 26119e23ccf8SColy Li struct async_reg_args *args = 26129e23ccf8SColy Li kzalloc(sizeof(struct async_reg_args), GFP_KERNEL); 26139e23ccf8SColy Li 26149e23ccf8SColy Li if (!args) { 26159e23ccf8SColy Li ret = -ENOMEM; 26169e23ccf8SColy Li err = "cannot allocate memory"; 26179e23ccf8SColy Li goto out_put_sb_page; 26189e23ccf8SColy Li } 26199e23ccf8SColy Li 26209e23ccf8SColy Li args->path = path; 26219e23ccf8SColy Li args->sb = sb; 26229e23ccf8SColy Li args->sb_disk = sb_disk; 26239e23ccf8SColy Li args->bdev = bdev; 26249e23ccf8SColy Li register_device_aync(args); 26259e23ccf8SColy Li /* No wait and returns to user space */ 26269e23ccf8SColy Li goto async_done; 26279e23ccf8SColy Li } 26289e23ccf8SColy Li 26292903381fSKent Overstreet if (SB_IS_BDEV(sb)) { 2630cafe5635SKent Overstreet struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL); 26311fae7cf0SColy Li 2632f59fce84SKent Overstreet if (!dc) 263350246693SChristoph Hellwig goto out_put_sb_page; 2634cafe5635SKent Overstreet 26354fa03402SKent Overstreet mutex_lock(&bch_register_lock); 2636cfa0c56dSChristoph Hellwig ret = register_bdev(sb, sb_disk, bdev, dc); 26374fa03402SKent Overstreet mutex_unlock(&bch_register_lock); 2638bb6d355cSColy Li /* blkdev_put() will be called in cached_dev_free() */ 2639fc8f19ccSChristoph Hellwig if (ret < 0) 2640fc8f19ccSChristoph Hellwig goto out_free_sb; 2641cafe5635SKent Overstreet } else { 2642cafe5635SKent Overstreet struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL); 26431fae7cf0SColy Li 2644f59fce84SKent Overstreet if (!ca) 264550246693SChristoph Hellwig goto out_put_sb_page; 2646cafe5635SKent Overstreet 2647bb6d355cSColy Li /* blkdev_put() will be called in bch_cache_release() */ 2648cfa0c56dSChristoph Hellwig if (register_cache(sb, sb_disk, bdev, ca) != 0) 2649fc8f19ccSChristoph Hellwig goto out_free_sb; 265050246693SChristoph Hellwig } 265150246693SChristoph Hellwig 265250246693SChristoph Hellwig done: 2653f59fce84SKent Overstreet kfree(sb); 2654f59fce84SKent Overstreet kfree(path); 2655f59fce84SKent Overstreet module_put(THIS_MODULE); 26569e23ccf8SColy Li async_done: 265750246693SChristoph Hellwig return size; 2658f59fce84SKent Overstreet 265950246693SChristoph Hellwig out_put_sb_page: 2660cfa0c56dSChristoph Hellwig put_page(virt_to_page(sb_disk)); 266150246693SChristoph Hellwig out_blkdev_put: 2662cafe5635SKent Overstreet blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); 266350246693SChristoph Hellwig out_free_sb: 266450246693SChristoph Hellwig kfree(sb); 266550246693SChristoph Hellwig out_free_path: 266650246693SChristoph Hellwig kfree(path); 2667ae3cd299SColy Li path = NULL; 266850246693SChristoph Hellwig out_module_put: 266950246693SChristoph Hellwig module_put(THIS_MODULE); 267050246693SChristoph Hellwig out: 267146f5aa88SJoe Perches pr_info("error %s: %s\n", path?path:"", err); 267250246693SChristoph Hellwig return ret; 2673cafe5635SKent Overstreet } 2674cafe5635SKent Overstreet 26750c277e21SColy Li 26760c277e21SColy Li struct pdev { 26770c277e21SColy Li struct list_head list; 26780c277e21SColy Li struct cached_dev *dc; 26790c277e21SColy Li }; 26800c277e21SColy Li 26810c277e21SColy Li static ssize_t bch_pending_bdevs_cleanup(struct kobject *k, 26820c277e21SColy Li struct kobj_attribute *attr, 26830c277e21SColy Li const char *buffer, 26840c277e21SColy Li size_t size) 26850c277e21SColy Li { 26860c277e21SColy Li LIST_HEAD(pending_devs); 26870c277e21SColy Li ssize_t ret = size; 26880c277e21SColy Li struct cached_dev *dc, *tdc; 26890c277e21SColy Li struct pdev *pdev, *tpdev; 26900c277e21SColy Li struct cache_set *c, *tc; 26910c277e21SColy Li 26920c277e21SColy Li mutex_lock(&bch_register_lock); 26930c277e21SColy Li list_for_each_entry_safe(dc, tdc, &uncached_devices, list) { 26940c277e21SColy Li pdev = kmalloc(sizeof(struct pdev), GFP_KERNEL); 26950c277e21SColy Li if (!pdev) 26960c277e21SColy Li break; 26970c277e21SColy Li pdev->dc = dc; 26980c277e21SColy Li list_add(&pdev->list, &pending_devs); 26990c277e21SColy Li } 27000c277e21SColy Li 27010c277e21SColy Li list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) { 27020c277e21SColy Li list_for_each_entry_safe(c, tc, &bch_cache_sets, list) { 27030c277e21SColy Li char *pdev_set_uuid = pdev->dc->sb.set_uuid; 27040c277e21SColy Li char *set_uuid = c->sb.uuid; 27050c277e21SColy Li 27060c277e21SColy Li if (!memcmp(pdev_set_uuid, set_uuid, 16)) { 27070c277e21SColy Li list_del(&pdev->list); 27080c277e21SColy Li kfree(pdev); 27090c277e21SColy Li break; 27100c277e21SColy Li } 27110c277e21SColy Li } 27120c277e21SColy Li } 27130c277e21SColy Li mutex_unlock(&bch_register_lock); 27140c277e21SColy Li 27150c277e21SColy Li list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) { 271646f5aa88SJoe Perches pr_info("delete pdev %p\n", pdev); 27170c277e21SColy Li list_del(&pdev->list); 27180c277e21SColy Li bcache_device_stop(&pdev->dc->disk); 27190c277e21SColy Li kfree(pdev); 27200c277e21SColy Li } 27210c277e21SColy Li 27220c277e21SColy Li return ret; 27230c277e21SColy Li } 27240c277e21SColy Li 2725cafe5635SKent Overstreet static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x) 2726cafe5635SKent Overstreet { 2727a59ff6ccSColy Li if (bcache_is_reboot) 2728a59ff6ccSColy Li return NOTIFY_DONE; 2729a59ff6ccSColy Li 2730cafe5635SKent Overstreet if (code == SYS_DOWN || 2731cafe5635SKent Overstreet code == SYS_HALT || 2732cafe5635SKent Overstreet code == SYS_POWER_OFF) { 2733cafe5635SKent Overstreet DEFINE_WAIT(wait); 2734cafe5635SKent Overstreet unsigned long start = jiffies; 2735cafe5635SKent Overstreet bool stopped = false; 2736cafe5635SKent Overstreet 2737cafe5635SKent Overstreet struct cache_set *c, *tc; 2738cafe5635SKent Overstreet struct cached_dev *dc, *tdc; 2739cafe5635SKent Overstreet 2740cafe5635SKent Overstreet mutex_lock(&bch_register_lock); 2741cafe5635SKent Overstreet 2742a59ff6ccSColy Li if (bcache_is_reboot) 2743a59ff6ccSColy Li goto out; 2744a59ff6ccSColy Li 2745a59ff6ccSColy Li /* New registration is rejected since now */ 2746a59ff6ccSColy Li bcache_is_reboot = true; 2747a59ff6ccSColy Li /* 2748a59ff6ccSColy Li * Make registering caller (if there is) on other CPU 2749a59ff6ccSColy Li * core know bcache_is_reboot set to true earlier 2750a59ff6ccSColy Li */ 2751a59ff6ccSColy Li smp_mb(); 2752a59ff6ccSColy Li 2753cafe5635SKent Overstreet if (list_empty(&bch_cache_sets) && 2754cafe5635SKent Overstreet list_empty(&uncached_devices)) 2755cafe5635SKent Overstreet goto out; 2756cafe5635SKent Overstreet 2757a59ff6ccSColy Li mutex_unlock(&bch_register_lock); 2758a59ff6ccSColy Li 275946f5aa88SJoe Perches pr_info("Stopping all devices:\n"); 2760cafe5635SKent Overstreet 2761a59ff6ccSColy Li /* 2762a59ff6ccSColy Li * The reason bch_register_lock is not held to call 2763a59ff6ccSColy Li * bch_cache_set_stop() and bcache_device_stop() is to 2764a59ff6ccSColy Li * avoid potential deadlock during reboot, because cache 2765a59ff6ccSColy Li * set or bcache device stopping process will acqurie 2766a59ff6ccSColy Li * bch_register_lock too. 2767a59ff6ccSColy Li * 2768a59ff6ccSColy Li * We are safe here because bcache_is_reboot sets to 2769a59ff6ccSColy Li * true already, register_bcache() will reject new 2770a59ff6ccSColy Li * registration now. bcache_is_reboot also makes sure 2771a59ff6ccSColy Li * bcache_reboot() won't be re-entered on by other thread, 2772a59ff6ccSColy Li * so there is no race in following list iteration by 2773a59ff6ccSColy Li * list_for_each_entry_safe(). 2774a59ff6ccSColy Li */ 2775cafe5635SKent Overstreet list_for_each_entry_safe(c, tc, &bch_cache_sets, list) 2776cafe5635SKent Overstreet bch_cache_set_stop(c); 2777cafe5635SKent Overstreet 2778cafe5635SKent Overstreet list_for_each_entry_safe(dc, tdc, &uncached_devices, list) 2779cafe5635SKent Overstreet bcache_device_stop(&dc->disk); 2780cafe5635SKent Overstreet 2781eb8cbb6dSColy Li 2782eb8cbb6dSColy Li /* 2783eb8cbb6dSColy Li * Give an early chance for other kthreads and 2784eb8cbb6dSColy Li * kworkers to stop themselves 2785eb8cbb6dSColy Li */ 2786eb8cbb6dSColy Li schedule(); 2787eb8cbb6dSColy Li 2788cafe5635SKent Overstreet /* What's a condition variable? */ 2789cafe5635SKent Overstreet while (1) { 2790eb8cbb6dSColy Li long timeout = start + 10 * HZ - jiffies; 2791cafe5635SKent Overstreet 2792eb8cbb6dSColy Li mutex_lock(&bch_register_lock); 2793cafe5635SKent Overstreet stopped = list_empty(&bch_cache_sets) && 2794cafe5635SKent Overstreet list_empty(&uncached_devices); 2795cafe5635SKent Overstreet 2796cafe5635SKent Overstreet if (timeout < 0 || stopped) 2797cafe5635SKent Overstreet break; 2798cafe5635SKent Overstreet 2799cafe5635SKent Overstreet prepare_to_wait(&unregister_wait, &wait, 2800cafe5635SKent Overstreet TASK_UNINTERRUPTIBLE); 2801cafe5635SKent Overstreet 2802cafe5635SKent Overstreet mutex_unlock(&bch_register_lock); 2803cafe5635SKent Overstreet schedule_timeout(timeout); 2804cafe5635SKent Overstreet } 2805cafe5635SKent Overstreet 2806cafe5635SKent Overstreet finish_wait(&unregister_wait, &wait); 2807cafe5635SKent Overstreet 2808cafe5635SKent Overstreet if (stopped) 280946f5aa88SJoe Perches pr_info("All devices stopped\n"); 2810cafe5635SKent Overstreet else 281146f5aa88SJoe Perches pr_notice("Timeout waiting for devices to be closed\n"); 2812cafe5635SKent Overstreet out: 2813cafe5635SKent Overstreet mutex_unlock(&bch_register_lock); 2814cafe5635SKent Overstreet } 2815cafe5635SKent Overstreet 2816cafe5635SKent Overstreet return NOTIFY_DONE; 2817cafe5635SKent Overstreet } 2818cafe5635SKent Overstreet 2819cafe5635SKent Overstreet static struct notifier_block reboot = { 2820cafe5635SKent Overstreet .notifier_call = bcache_reboot, 2821cafe5635SKent Overstreet .priority = INT_MAX, /* before any real devices */ 2822cafe5635SKent Overstreet }; 2823cafe5635SKent Overstreet 2824cafe5635SKent Overstreet static void bcache_exit(void) 2825cafe5635SKent Overstreet { 2826cafe5635SKent Overstreet bch_debug_exit(); 2827cafe5635SKent Overstreet bch_request_exit(); 2828cafe5635SKent Overstreet if (bcache_kobj) 2829cafe5635SKent Overstreet kobject_put(bcache_kobj); 2830cafe5635SKent Overstreet if (bcache_wq) 2831cafe5635SKent Overstreet destroy_workqueue(bcache_wq); 28320f843e65SGuoju Fang if (bch_journal_wq) 28330f843e65SGuoju Fang destroy_workqueue(bch_journal_wq); 28340f843e65SGuoju Fang 28355c41c8a7SKent Overstreet if (bcache_major) 2836cafe5635SKent Overstreet unregister_blkdev(bcache_major, "bcache"); 2837cafe5635SKent Overstreet unregister_reboot_notifier(&reboot); 2838330a4db8SLiang Chen mutex_destroy(&bch_register_lock); 2839cafe5635SKent Overstreet } 2840cafe5635SKent Overstreet 28419aaf5165SColy Li /* Check and fixup module parameters */ 28429aaf5165SColy Li static void check_module_parameters(void) 28439aaf5165SColy Li { 28449aaf5165SColy Li if (bch_cutoff_writeback_sync == 0) 28459aaf5165SColy Li bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC; 28469aaf5165SColy Li else if (bch_cutoff_writeback_sync > CUTOFF_WRITEBACK_SYNC_MAX) { 284746f5aa88SJoe Perches pr_warn("set bch_cutoff_writeback_sync (%u) to max value %u\n", 28489aaf5165SColy Li bch_cutoff_writeback_sync, CUTOFF_WRITEBACK_SYNC_MAX); 28499aaf5165SColy Li bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC_MAX; 28509aaf5165SColy Li } 28519aaf5165SColy Li 28529aaf5165SColy Li if (bch_cutoff_writeback == 0) 28539aaf5165SColy Li bch_cutoff_writeback = CUTOFF_WRITEBACK; 28549aaf5165SColy Li else if (bch_cutoff_writeback > CUTOFF_WRITEBACK_MAX) { 285546f5aa88SJoe Perches pr_warn("set bch_cutoff_writeback (%u) to max value %u\n", 28569aaf5165SColy Li bch_cutoff_writeback, CUTOFF_WRITEBACK_MAX); 28579aaf5165SColy Li bch_cutoff_writeback = CUTOFF_WRITEBACK_MAX; 28589aaf5165SColy Li } 28599aaf5165SColy Li 28609aaf5165SColy Li if (bch_cutoff_writeback > bch_cutoff_writeback_sync) { 286146f5aa88SJoe Perches pr_warn("set bch_cutoff_writeback (%u) to %u\n", 28629aaf5165SColy Li bch_cutoff_writeback, bch_cutoff_writeback_sync); 28639aaf5165SColy Li bch_cutoff_writeback = bch_cutoff_writeback_sync; 28649aaf5165SColy Li } 28659aaf5165SColy Li } 28669aaf5165SColy Li 2867cafe5635SKent Overstreet static int __init bcache_init(void) 2868cafe5635SKent Overstreet { 2869cafe5635SKent Overstreet static const struct attribute *files[] = { 2870cafe5635SKent Overstreet &ksysfs_register.attr, 2871cafe5635SKent Overstreet &ksysfs_register_quiet.attr, 28726acd193bSJean Delvare #ifdef CONFIG_BCACHE_ASYNC_REGISTRATION 28739e23ccf8SColy Li &ksysfs_register_async.attr, 28740c8d3fceSColy Li #endif 28750c277e21SColy Li &ksysfs_pendings_cleanup.attr, 2876cafe5635SKent Overstreet NULL 2877cafe5635SKent Overstreet }; 2878cafe5635SKent Overstreet 28799aaf5165SColy Li check_module_parameters(); 28809aaf5165SColy Li 2881cafe5635SKent Overstreet mutex_init(&bch_register_lock); 2882cafe5635SKent Overstreet init_waitqueue_head(&unregister_wait); 2883cafe5635SKent Overstreet register_reboot_notifier(&reboot); 2884cafe5635SKent Overstreet 2885cafe5635SKent Overstreet bcache_major = register_blkdev(0, "bcache"); 28862ecf0cdbSZheng Liu if (bcache_major < 0) { 28872ecf0cdbSZheng Liu unregister_reboot_notifier(&reboot); 2888330a4db8SLiang Chen mutex_destroy(&bch_register_lock); 2889cafe5635SKent Overstreet return bcache_major; 28902ecf0cdbSZheng Liu } 2891cafe5635SKent Overstreet 289216c1fdf4SFlorian Schmaus bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM, 0); 289316c1fdf4SFlorian Schmaus if (!bcache_wq) 289416c1fdf4SFlorian Schmaus goto err; 289516c1fdf4SFlorian Schmaus 28960f843e65SGuoju Fang bch_journal_wq = alloc_workqueue("bch_journal", WQ_MEM_RECLAIM, 0); 28970f843e65SGuoju Fang if (!bch_journal_wq) 28980f843e65SGuoju Fang goto err; 28990f843e65SGuoju Fang 290016c1fdf4SFlorian Schmaus bcache_kobj = kobject_create_and_add("bcache", fs_kobj); 290116c1fdf4SFlorian Schmaus if (!bcache_kobj) 290216c1fdf4SFlorian Schmaus goto err; 290316c1fdf4SFlorian Schmaus 290416c1fdf4SFlorian Schmaus if (bch_request_init() || 2905330a4db8SLiang Chen sysfs_create_files(bcache_kobj, files)) 2906cafe5635SKent Overstreet goto err; 2907cafe5635SKent Overstreet 290891bafdf0SDongbo Cao bch_debug_init(); 290978ac2107SColy Li closure_debug_init(); 291078ac2107SColy Li 2911a59ff6ccSColy Li bcache_is_reboot = false; 2912a59ff6ccSColy Li 2913cafe5635SKent Overstreet return 0; 2914cafe5635SKent Overstreet err: 2915cafe5635SKent Overstreet bcache_exit(); 2916cafe5635SKent Overstreet return -ENOMEM; 2917cafe5635SKent Overstreet } 2918cafe5635SKent Overstreet 29199aaf5165SColy Li /* 29209aaf5165SColy Li * Module hooks 29219aaf5165SColy Li */ 2922cafe5635SKent Overstreet module_exit(bcache_exit); 2923cafe5635SKent Overstreet module_init(bcache_init); 2924009673d0SColy Li 29259aaf5165SColy Li module_param(bch_cutoff_writeback, uint, 0); 29269aaf5165SColy Li MODULE_PARM_DESC(bch_cutoff_writeback, "threshold to cutoff writeback"); 29279aaf5165SColy Li 29289aaf5165SColy Li module_param(bch_cutoff_writeback_sync, uint, 0); 29299aaf5165SColy Li MODULE_PARM_DESC(bch_cutoff_writeback_sync, "hard threshold to cutoff writeback"); 29309aaf5165SColy Li 2931009673d0SColy Li MODULE_DESCRIPTION("Bcache: a Linux block layer cache"); 2932009673d0SColy Li MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>"); 2933009673d0SColy Li MODULE_LICENSE("GPL"); 2934