xref: /openbmc/linux/drivers/md/bcache/super.c (revision 6ded34d1a54c046a45db071d3cb7b37bd0a4a31f)
1cafe5635SKent Overstreet /*
2cafe5635SKent Overstreet  * bcache setup/teardown code, and some metadata io - read a superblock and
3cafe5635SKent Overstreet  * figure out what to do with it.
4cafe5635SKent Overstreet  *
5cafe5635SKent Overstreet  * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6cafe5635SKent Overstreet  * Copyright 2012 Google, Inc.
7cafe5635SKent Overstreet  */
8cafe5635SKent Overstreet 
9cafe5635SKent Overstreet #include "bcache.h"
10cafe5635SKent Overstreet #include "btree.h"
11cafe5635SKent Overstreet #include "debug.h"
12cafe5635SKent Overstreet #include "request.h"
13cafe5635SKent Overstreet 
14c37511b8SKent Overstreet #include <linux/blkdev.h>
15cafe5635SKent Overstreet #include <linux/buffer_head.h>
16cafe5635SKent Overstreet #include <linux/debugfs.h>
17cafe5635SKent Overstreet #include <linux/genhd.h>
18cafe5635SKent Overstreet #include <linux/module.h>
19cafe5635SKent Overstreet #include <linux/random.h>
20cafe5635SKent Overstreet #include <linux/reboot.h>
21cafe5635SKent Overstreet #include <linux/sysfs.h>
22cafe5635SKent Overstreet 
23cafe5635SKent Overstreet MODULE_LICENSE("GPL");
24cafe5635SKent Overstreet MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
25cafe5635SKent Overstreet 
26cafe5635SKent Overstreet static const char bcache_magic[] = {
27cafe5635SKent Overstreet 	0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,
28cafe5635SKent Overstreet 	0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81
29cafe5635SKent Overstreet };
30cafe5635SKent Overstreet 
31cafe5635SKent Overstreet static const char invalid_uuid[] = {
32cafe5635SKent Overstreet 	0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,
33cafe5635SKent Overstreet 	0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
34cafe5635SKent Overstreet };
35cafe5635SKent Overstreet 
36cafe5635SKent Overstreet /* Default is -1; we skip past it for struct cached_dev's cache mode */
37cafe5635SKent Overstreet const char * const bch_cache_modes[] = {
38cafe5635SKent Overstreet 	"default",
39cafe5635SKent Overstreet 	"writethrough",
40cafe5635SKent Overstreet 	"writeback",
41cafe5635SKent Overstreet 	"writearound",
42cafe5635SKent Overstreet 	"none",
43cafe5635SKent Overstreet 	NULL
44cafe5635SKent Overstreet };
45cafe5635SKent Overstreet 
46cafe5635SKent Overstreet struct uuid_entry_v0 {
47cafe5635SKent Overstreet 	uint8_t		uuid[16];
48cafe5635SKent Overstreet 	uint8_t		label[32];
49cafe5635SKent Overstreet 	uint32_t	first_reg;
50cafe5635SKent Overstreet 	uint32_t	last_reg;
51cafe5635SKent Overstreet 	uint32_t	invalidated;
52cafe5635SKent Overstreet 	uint32_t	pad;
53cafe5635SKent Overstreet };
54cafe5635SKent Overstreet 
55cafe5635SKent Overstreet static struct kobject *bcache_kobj;
56cafe5635SKent Overstreet struct mutex bch_register_lock;
57cafe5635SKent Overstreet LIST_HEAD(bch_cache_sets);
58cafe5635SKent Overstreet static LIST_HEAD(uncached_devices);
59cafe5635SKent Overstreet 
60cafe5635SKent Overstreet static int bcache_major, bcache_minor;
61cafe5635SKent Overstreet static wait_queue_head_t unregister_wait;
62cafe5635SKent Overstreet struct workqueue_struct *bcache_wq;
63cafe5635SKent Overstreet 
64cafe5635SKent Overstreet #define BTREE_MAX_PAGES		(256 * 1024 / PAGE_SIZE)
65cafe5635SKent Overstreet 
66cafe5635SKent Overstreet static void bio_split_pool_free(struct bio_split_pool *p)
67cafe5635SKent Overstreet {
688ef74790SKent Overstreet 	if (p->bio_split_hook)
698ef74790SKent Overstreet 		mempool_destroy(p->bio_split_hook);
708ef74790SKent Overstreet 
71cafe5635SKent Overstreet 	if (p->bio_split)
72cafe5635SKent Overstreet 		bioset_free(p->bio_split);
73cafe5635SKent Overstreet }
74cafe5635SKent Overstreet 
75cafe5635SKent Overstreet static int bio_split_pool_init(struct bio_split_pool *p)
76cafe5635SKent Overstreet {
77cafe5635SKent Overstreet 	p->bio_split = bioset_create(4, 0);
78cafe5635SKent Overstreet 	if (!p->bio_split)
79cafe5635SKent Overstreet 		return -ENOMEM;
80cafe5635SKent Overstreet 
81cafe5635SKent Overstreet 	p->bio_split_hook = mempool_create_kmalloc_pool(4,
82cafe5635SKent Overstreet 				sizeof(struct bio_split_hook));
83cafe5635SKent Overstreet 	if (!p->bio_split_hook)
84cafe5635SKent Overstreet 		return -ENOMEM;
85cafe5635SKent Overstreet 
86cafe5635SKent Overstreet 	return 0;
87cafe5635SKent Overstreet }
88cafe5635SKent Overstreet 
89cafe5635SKent Overstreet /* Superblock */
90cafe5635SKent Overstreet 
91cafe5635SKent Overstreet static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
92cafe5635SKent Overstreet 			      struct page **res)
93cafe5635SKent Overstreet {
94cafe5635SKent Overstreet 	const char *err;
95cafe5635SKent Overstreet 	struct cache_sb *s;
96cafe5635SKent Overstreet 	struct buffer_head *bh = __bread(bdev, 1, SB_SIZE);
97cafe5635SKent Overstreet 	unsigned i;
98cafe5635SKent Overstreet 
99cafe5635SKent Overstreet 	if (!bh)
100cafe5635SKent Overstreet 		return "IO error";
101cafe5635SKent Overstreet 
102cafe5635SKent Overstreet 	s = (struct cache_sb *) bh->b_data;
103cafe5635SKent Overstreet 
104cafe5635SKent Overstreet 	sb->offset		= le64_to_cpu(s->offset);
105cafe5635SKent Overstreet 	sb->version		= le64_to_cpu(s->version);
106cafe5635SKent Overstreet 
107cafe5635SKent Overstreet 	memcpy(sb->magic,	s->magic, 16);
108cafe5635SKent Overstreet 	memcpy(sb->uuid,	s->uuid, 16);
109cafe5635SKent Overstreet 	memcpy(sb->set_uuid,	s->set_uuid, 16);
110cafe5635SKent Overstreet 	memcpy(sb->label,	s->label, SB_LABEL_SIZE);
111cafe5635SKent Overstreet 
112cafe5635SKent Overstreet 	sb->flags		= le64_to_cpu(s->flags);
113cafe5635SKent Overstreet 	sb->seq			= le64_to_cpu(s->seq);
114cafe5635SKent Overstreet 	sb->last_mount		= le32_to_cpu(s->last_mount);
115cafe5635SKent Overstreet 	sb->first_bucket	= le16_to_cpu(s->first_bucket);
116cafe5635SKent Overstreet 	sb->keys		= le16_to_cpu(s->keys);
117cafe5635SKent Overstreet 
118cafe5635SKent Overstreet 	for (i = 0; i < SB_JOURNAL_BUCKETS; i++)
119cafe5635SKent Overstreet 		sb->d[i] = le64_to_cpu(s->d[i]);
120cafe5635SKent Overstreet 
121cafe5635SKent Overstreet 	pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
122cafe5635SKent Overstreet 		 sb->version, sb->flags, sb->seq, sb->keys);
123cafe5635SKent Overstreet 
124cafe5635SKent Overstreet 	err = "Not a bcache superblock";
125cafe5635SKent Overstreet 	if (sb->offset != SB_SECTOR)
126cafe5635SKent Overstreet 		goto err;
127cafe5635SKent Overstreet 
128cafe5635SKent Overstreet 	if (memcmp(sb->magic, bcache_magic, 16))
129cafe5635SKent Overstreet 		goto err;
130cafe5635SKent Overstreet 
131cafe5635SKent Overstreet 	err = "Too many journal buckets";
132cafe5635SKent Overstreet 	if (sb->keys > SB_JOURNAL_BUCKETS)
133cafe5635SKent Overstreet 		goto err;
134cafe5635SKent Overstreet 
135cafe5635SKent Overstreet 	err = "Bad checksum";
136cafe5635SKent Overstreet 	if (s->csum != csum_set(s))
137cafe5635SKent Overstreet 		goto err;
138cafe5635SKent Overstreet 
139cafe5635SKent Overstreet 	err = "Bad UUID";
140169ef1cfSKent Overstreet 	if (bch_is_zero(sb->uuid, 16))
141cafe5635SKent Overstreet 		goto err;
142cafe5635SKent Overstreet 
1438abb2a5dSKent Overstreet 	sb->block_size	= le16_to_cpu(s->block_size);
1448abb2a5dSKent Overstreet 
1458abb2a5dSKent Overstreet 	err = "Superblock block size smaller than device block size";
1468abb2a5dSKent Overstreet 	if (sb->block_size << 9 < bdev_logical_block_size(bdev))
1478abb2a5dSKent Overstreet 		goto err;
1488abb2a5dSKent Overstreet 
1492903381fSKent Overstreet 	switch (sb->version) {
1502903381fSKent Overstreet 	case BCACHE_SB_VERSION_BDEV:
1512903381fSKent Overstreet 		sb->data_offset	= BDEV_DATA_START_DEFAULT;
1522903381fSKent Overstreet 		break;
1532903381fSKent Overstreet 	case BCACHE_SB_VERSION_BDEV_WITH_OFFSET:
1542903381fSKent Overstreet 		sb->data_offset	= le64_to_cpu(s->data_offset);
1552903381fSKent Overstreet 
1562903381fSKent Overstreet 		err = "Bad data offset";
1572903381fSKent Overstreet 		if (sb->data_offset < BDEV_DATA_START_DEFAULT)
158cafe5635SKent Overstreet 			goto err;
159cafe5635SKent Overstreet 
1602903381fSKent Overstreet 		break;
1612903381fSKent Overstreet 	case BCACHE_SB_VERSION_CDEV:
1622903381fSKent Overstreet 	case BCACHE_SB_VERSION_CDEV_WITH_UUID:
1632903381fSKent Overstreet 		sb->nbuckets	= le64_to_cpu(s->nbuckets);
1642903381fSKent Overstreet 		sb->block_size	= le16_to_cpu(s->block_size);
1652903381fSKent Overstreet 		sb->bucket_size	= le16_to_cpu(s->bucket_size);
1662903381fSKent Overstreet 
1672903381fSKent Overstreet 		sb->nr_in_set	= le16_to_cpu(s->nr_in_set);
1682903381fSKent Overstreet 		sb->nr_this_dev	= le16_to_cpu(s->nr_this_dev);
169cafe5635SKent Overstreet 
170cafe5635SKent Overstreet 		err = "Too many buckets";
171cafe5635SKent Overstreet 		if (sb->nbuckets > LONG_MAX)
172cafe5635SKent Overstreet 			goto err;
173cafe5635SKent Overstreet 
174cafe5635SKent Overstreet 		err = "Not enough buckets";
175cafe5635SKent Overstreet 		if (sb->nbuckets < 1 << 7)
176cafe5635SKent Overstreet 			goto err;
177cafe5635SKent Overstreet 
1782903381fSKent Overstreet 		err = "Bad block/bucket size";
1792903381fSKent Overstreet 		if (!is_power_of_2(sb->block_size) ||
1802903381fSKent Overstreet 		    sb->block_size > PAGE_SECTORS ||
1812903381fSKent Overstreet 		    !is_power_of_2(sb->bucket_size) ||
1822903381fSKent Overstreet 		    sb->bucket_size < PAGE_SECTORS)
1832903381fSKent Overstreet 			goto err;
1842903381fSKent Overstreet 
185cafe5635SKent Overstreet 		err = "Invalid superblock: device too small";
186cafe5635SKent Overstreet 		if (get_capacity(bdev->bd_disk) < sb->bucket_size * sb->nbuckets)
187cafe5635SKent Overstreet 			goto err;
188cafe5635SKent Overstreet 
189cafe5635SKent Overstreet 		err = "Bad UUID";
190169ef1cfSKent Overstreet 		if (bch_is_zero(sb->set_uuid, 16))
191cafe5635SKent Overstreet 			goto err;
192cafe5635SKent Overstreet 
193cafe5635SKent Overstreet 		err = "Bad cache device number in set";
194cafe5635SKent Overstreet 		if (!sb->nr_in_set ||
195cafe5635SKent Overstreet 		    sb->nr_in_set <= sb->nr_this_dev ||
196cafe5635SKent Overstreet 		    sb->nr_in_set > MAX_CACHES_PER_SET)
197cafe5635SKent Overstreet 			goto err;
198cafe5635SKent Overstreet 
199cafe5635SKent Overstreet 		err = "Journal buckets not sequential";
200cafe5635SKent Overstreet 		for (i = 0; i < sb->keys; i++)
201cafe5635SKent Overstreet 			if (sb->d[i] != sb->first_bucket + i)
202cafe5635SKent Overstreet 				goto err;
203cafe5635SKent Overstreet 
204cafe5635SKent Overstreet 		err = "Too many journal buckets";
205cafe5635SKent Overstreet 		if (sb->first_bucket + sb->keys > sb->nbuckets)
206cafe5635SKent Overstreet 			goto err;
207cafe5635SKent Overstreet 
208cafe5635SKent Overstreet 		err = "Invalid superblock: first bucket comes before end of super";
209cafe5635SKent Overstreet 		if (sb->first_bucket * sb->bucket_size < 16)
210cafe5635SKent Overstreet 			goto err;
2112903381fSKent Overstreet 
2122903381fSKent Overstreet 		break;
2132903381fSKent Overstreet 	default:
2142903381fSKent Overstreet 		err = "Unsupported superblock version";
2152903381fSKent Overstreet 		goto err;
2162903381fSKent Overstreet 	}
2172903381fSKent Overstreet 
218cafe5635SKent Overstreet 	sb->last_mount = get_seconds();
219cafe5635SKent Overstreet 	err = NULL;
220cafe5635SKent Overstreet 
221cafe5635SKent Overstreet 	get_page(bh->b_page);
222cafe5635SKent Overstreet 	*res = bh->b_page;
223cafe5635SKent Overstreet err:
224cafe5635SKent Overstreet 	put_bh(bh);
225cafe5635SKent Overstreet 	return err;
226cafe5635SKent Overstreet }
227cafe5635SKent Overstreet 
228cafe5635SKent Overstreet static void write_bdev_super_endio(struct bio *bio, int error)
229cafe5635SKent Overstreet {
230cafe5635SKent Overstreet 	struct cached_dev *dc = bio->bi_private;
231cafe5635SKent Overstreet 	/* XXX: error checking */
232cafe5635SKent Overstreet 
233cafe5635SKent Overstreet 	closure_put(&dc->sb_write.cl);
234cafe5635SKent Overstreet }
235cafe5635SKent Overstreet 
236cafe5635SKent Overstreet static void __write_super(struct cache_sb *sb, struct bio *bio)
237cafe5635SKent Overstreet {
238cafe5635SKent Overstreet 	struct cache_sb *out = page_address(bio->bi_io_vec[0].bv_page);
239cafe5635SKent Overstreet 	unsigned i;
240cafe5635SKent Overstreet 
241cafe5635SKent Overstreet 	bio->bi_sector	= SB_SECTOR;
242cafe5635SKent Overstreet 	bio->bi_rw	= REQ_SYNC|REQ_META;
243cafe5635SKent Overstreet 	bio->bi_size	= SB_SIZE;
244169ef1cfSKent Overstreet 	bch_bio_map(bio, NULL);
245cafe5635SKent Overstreet 
246cafe5635SKent Overstreet 	out->offset		= cpu_to_le64(sb->offset);
247cafe5635SKent Overstreet 	out->version		= cpu_to_le64(sb->version);
248cafe5635SKent Overstreet 
249cafe5635SKent Overstreet 	memcpy(out->uuid,	sb->uuid, 16);
250cafe5635SKent Overstreet 	memcpy(out->set_uuid,	sb->set_uuid, 16);
251cafe5635SKent Overstreet 	memcpy(out->label,	sb->label, SB_LABEL_SIZE);
252cafe5635SKent Overstreet 
253cafe5635SKent Overstreet 	out->flags		= cpu_to_le64(sb->flags);
254cafe5635SKent Overstreet 	out->seq		= cpu_to_le64(sb->seq);
255cafe5635SKent Overstreet 
256cafe5635SKent Overstreet 	out->last_mount		= cpu_to_le32(sb->last_mount);
257cafe5635SKent Overstreet 	out->first_bucket	= cpu_to_le16(sb->first_bucket);
258cafe5635SKent Overstreet 	out->keys		= cpu_to_le16(sb->keys);
259cafe5635SKent Overstreet 
260cafe5635SKent Overstreet 	for (i = 0; i < sb->keys; i++)
261cafe5635SKent Overstreet 		out->d[i] = cpu_to_le64(sb->d[i]);
262cafe5635SKent Overstreet 
263cafe5635SKent Overstreet 	out->csum = csum_set(out);
264cafe5635SKent Overstreet 
265cafe5635SKent Overstreet 	pr_debug("ver %llu, flags %llu, seq %llu",
266cafe5635SKent Overstreet 		 sb->version, sb->flags, sb->seq);
267cafe5635SKent Overstreet 
268cafe5635SKent Overstreet 	submit_bio(REQ_WRITE, bio);
269cafe5635SKent Overstreet }
270cafe5635SKent Overstreet 
271cafe5635SKent Overstreet void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)
272cafe5635SKent Overstreet {
273cafe5635SKent Overstreet 	struct closure *cl = &dc->sb_write.cl;
274cafe5635SKent Overstreet 	struct bio *bio = &dc->sb_bio;
275cafe5635SKent Overstreet 
276cafe5635SKent Overstreet 	closure_lock(&dc->sb_write, parent);
277cafe5635SKent Overstreet 
278cafe5635SKent Overstreet 	bio_reset(bio);
279cafe5635SKent Overstreet 	bio->bi_bdev	= dc->bdev;
280cafe5635SKent Overstreet 	bio->bi_end_io	= write_bdev_super_endio;
281cafe5635SKent Overstreet 	bio->bi_private = dc;
282cafe5635SKent Overstreet 
283cafe5635SKent Overstreet 	closure_get(cl);
284cafe5635SKent Overstreet 	__write_super(&dc->sb, bio);
285cafe5635SKent Overstreet 
286cafe5635SKent Overstreet 	closure_return(cl);
287cafe5635SKent Overstreet }
288cafe5635SKent Overstreet 
289cafe5635SKent Overstreet static void write_super_endio(struct bio *bio, int error)
290cafe5635SKent Overstreet {
291cafe5635SKent Overstreet 	struct cache *ca = bio->bi_private;
292cafe5635SKent Overstreet 
293cafe5635SKent Overstreet 	bch_count_io_errors(ca, error, "writing superblock");
294cafe5635SKent Overstreet 	closure_put(&ca->set->sb_write.cl);
295cafe5635SKent Overstreet }
296cafe5635SKent Overstreet 
297cafe5635SKent Overstreet void bcache_write_super(struct cache_set *c)
298cafe5635SKent Overstreet {
299cafe5635SKent Overstreet 	struct closure *cl = &c->sb_write.cl;
300cafe5635SKent Overstreet 	struct cache *ca;
301cafe5635SKent Overstreet 	unsigned i;
302cafe5635SKent Overstreet 
303cafe5635SKent Overstreet 	closure_lock(&c->sb_write, &c->cl);
304cafe5635SKent Overstreet 
305cafe5635SKent Overstreet 	c->sb.seq++;
306cafe5635SKent Overstreet 
307cafe5635SKent Overstreet 	for_each_cache(ca, c, i) {
308cafe5635SKent Overstreet 		struct bio *bio = &ca->sb_bio;
309cafe5635SKent Overstreet 
3102903381fSKent Overstreet 		ca->sb.version		= BCACHE_SB_VERSION_CDEV_WITH_UUID;
311cafe5635SKent Overstreet 		ca->sb.seq		= c->sb.seq;
312cafe5635SKent Overstreet 		ca->sb.last_mount	= c->sb.last_mount;
313cafe5635SKent Overstreet 
314cafe5635SKent Overstreet 		SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));
315cafe5635SKent Overstreet 
316cafe5635SKent Overstreet 		bio_reset(bio);
317cafe5635SKent Overstreet 		bio->bi_bdev	= ca->bdev;
318cafe5635SKent Overstreet 		bio->bi_end_io	= write_super_endio;
319cafe5635SKent Overstreet 		bio->bi_private = ca;
320cafe5635SKent Overstreet 
321cafe5635SKent Overstreet 		closure_get(cl);
322cafe5635SKent Overstreet 		__write_super(&ca->sb, bio);
323cafe5635SKent Overstreet 	}
324cafe5635SKent Overstreet 
325cafe5635SKent Overstreet 	closure_return(cl);
326cafe5635SKent Overstreet }
327cafe5635SKent Overstreet 
328cafe5635SKent Overstreet /* UUID io */
329cafe5635SKent Overstreet 
330cafe5635SKent Overstreet static void uuid_endio(struct bio *bio, int error)
331cafe5635SKent Overstreet {
332cafe5635SKent Overstreet 	struct closure *cl = bio->bi_private;
333cafe5635SKent Overstreet 	struct cache_set *c = container_of(cl, struct cache_set, uuid_write.cl);
334cafe5635SKent Overstreet 
335cafe5635SKent Overstreet 	cache_set_err_on(error, c, "accessing uuids");
336cafe5635SKent Overstreet 	bch_bbio_free(bio, c);
337cafe5635SKent Overstreet 	closure_put(cl);
338cafe5635SKent Overstreet }
339cafe5635SKent Overstreet 
340cafe5635SKent Overstreet static void uuid_io(struct cache_set *c, unsigned long rw,
341cafe5635SKent Overstreet 		    struct bkey *k, struct closure *parent)
342cafe5635SKent Overstreet {
343cafe5635SKent Overstreet 	struct closure *cl = &c->uuid_write.cl;
344cafe5635SKent Overstreet 	struct uuid_entry *u;
345cafe5635SKent Overstreet 	unsigned i;
34685b1492eSKent Overstreet 	char buf[80];
347cafe5635SKent Overstreet 
348cafe5635SKent Overstreet 	BUG_ON(!parent);
349cafe5635SKent Overstreet 	closure_lock(&c->uuid_write, parent);
350cafe5635SKent Overstreet 
351cafe5635SKent Overstreet 	for (i = 0; i < KEY_PTRS(k); i++) {
352cafe5635SKent Overstreet 		struct bio *bio = bch_bbio_alloc(c);
353cafe5635SKent Overstreet 
354cafe5635SKent Overstreet 		bio->bi_rw	= REQ_SYNC|REQ_META|rw;
355cafe5635SKent Overstreet 		bio->bi_size	= KEY_SIZE(k) << 9;
356cafe5635SKent Overstreet 
357cafe5635SKent Overstreet 		bio->bi_end_io	= uuid_endio;
358cafe5635SKent Overstreet 		bio->bi_private = cl;
359169ef1cfSKent Overstreet 		bch_bio_map(bio, c->uuids);
360cafe5635SKent Overstreet 
361cafe5635SKent Overstreet 		bch_submit_bbio(bio, c, k, i);
362cafe5635SKent Overstreet 
363cafe5635SKent Overstreet 		if (!(rw & WRITE))
364cafe5635SKent Overstreet 			break;
365cafe5635SKent Overstreet 	}
366cafe5635SKent Overstreet 
36785b1492eSKent Overstreet 	bch_bkey_to_text(buf, sizeof(buf), k);
36885b1492eSKent Overstreet 	pr_debug("%s UUIDs at %s", rw & REQ_WRITE ? "wrote" : "read", buf);
369cafe5635SKent Overstreet 
370cafe5635SKent Overstreet 	for (u = c->uuids; u < c->uuids + c->nr_uuids; u++)
371169ef1cfSKent Overstreet 		if (!bch_is_zero(u->uuid, 16))
372cafe5635SKent Overstreet 			pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u",
373cafe5635SKent Overstreet 				 u - c->uuids, u->uuid, u->label,
374cafe5635SKent Overstreet 				 u->first_reg, u->last_reg, u->invalidated);
375cafe5635SKent Overstreet 
376cafe5635SKent Overstreet 	closure_return(cl);
377cafe5635SKent Overstreet }
378cafe5635SKent Overstreet 
379cafe5635SKent Overstreet static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)
380cafe5635SKent Overstreet {
381cafe5635SKent Overstreet 	struct bkey *k = &j->uuid_bucket;
382cafe5635SKent Overstreet 
383cafe5635SKent Overstreet 	if (__bch_ptr_invalid(c, 1, k))
384cafe5635SKent Overstreet 		return "bad uuid pointer";
385cafe5635SKent Overstreet 
386cafe5635SKent Overstreet 	bkey_copy(&c->uuid_bucket, k);
387cafe5635SKent Overstreet 	uuid_io(c, READ_SYNC, k, cl);
388cafe5635SKent Overstreet 
389cafe5635SKent Overstreet 	if (j->version < BCACHE_JSET_VERSION_UUIDv1) {
390cafe5635SKent Overstreet 		struct uuid_entry_v0	*u0 = (void *) c->uuids;
391cafe5635SKent Overstreet 		struct uuid_entry	*u1 = (void *) c->uuids;
392cafe5635SKent Overstreet 		int i;
393cafe5635SKent Overstreet 
394cafe5635SKent Overstreet 		closure_sync(cl);
395cafe5635SKent Overstreet 
396cafe5635SKent Overstreet 		/*
397cafe5635SKent Overstreet 		 * Since the new uuid entry is bigger than the old, we have to
398cafe5635SKent Overstreet 		 * convert starting at the highest memory address and work down
399cafe5635SKent Overstreet 		 * in order to do it in place
400cafe5635SKent Overstreet 		 */
401cafe5635SKent Overstreet 
402cafe5635SKent Overstreet 		for (i = c->nr_uuids - 1;
403cafe5635SKent Overstreet 		     i >= 0;
404cafe5635SKent Overstreet 		     --i) {
405cafe5635SKent Overstreet 			memcpy(u1[i].uuid,	u0[i].uuid, 16);
406cafe5635SKent Overstreet 			memcpy(u1[i].label,	u0[i].label, 32);
407cafe5635SKent Overstreet 
408cafe5635SKent Overstreet 			u1[i].first_reg		= u0[i].first_reg;
409cafe5635SKent Overstreet 			u1[i].last_reg		= u0[i].last_reg;
410cafe5635SKent Overstreet 			u1[i].invalidated	= u0[i].invalidated;
411cafe5635SKent Overstreet 
412cafe5635SKent Overstreet 			u1[i].flags	= 0;
413cafe5635SKent Overstreet 			u1[i].sectors	= 0;
414cafe5635SKent Overstreet 		}
415cafe5635SKent Overstreet 	}
416cafe5635SKent Overstreet 
417cafe5635SKent Overstreet 	return NULL;
418cafe5635SKent Overstreet }
419cafe5635SKent Overstreet 
420cafe5635SKent Overstreet static int __uuid_write(struct cache_set *c)
421cafe5635SKent Overstreet {
422cafe5635SKent Overstreet 	BKEY_PADDED(key) k;
423cafe5635SKent Overstreet 	struct closure cl;
424cafe5635SKent Overstreet 	closure_init_stack(&cl);
425cafe5635SKent Overstreet 
426cafe5635SKent Overstreet 	lockdep_assert_held(&bch_register_lock);
427cafe5635SKent Overstreet 
428cafe5635SKent Overstreet 	if (bch_bucket_alloc_set(c, WATERMARK_METADATA, &k.key, 1, &cl))
429cafe5635SKent Overstreet 		return 1;
430cafe5635SKent Overstreet 
431cafe5635SKent Overstreet 	SET_KEY_SIZE(&k.key, c->sb.bucket_size);
432cafe5635SKent Overstreet 	uuid_io(c, REQ_WRITE, &k.key, &cl);
433cafe5635SKent Overstreet 	closure_sync(&cl);
434cafe5635SKent Overstreet 
435cafe5635SKent Overstreet 	bkey_copy(&c->uuid_bucket, &k.key);
436cafe5635SKent Overstreet 	__bkey_put(c, &k.key);
437cafe5635SKent Overstreet 	return 0;
438cafe5635SKent Overstreet }
439cafe5635SKent Overstreet 
440cafe5635SKent Overstreet int bch_uuid_write(struct cache_set *c)
441cafe5635SKent Overstreet {
442cafe5635SKent Overstreet 	int ret = __uuid_write(c);
443cafe5635SKent Overstreet 
444cafe5635SKent Overstreet 	if (!ret)
445cafe5635SKent Overstreet 		bch_journal_meta(c, NULL);
446cafe5635SKent Overstreet 
447cafe5635SKent Overstreet 	return ret;
448cafe5635SKent Overstreet }
449cafe5635SKent Overstreet 
450cafe5635SKent Overstreet static struct uuid_entry *uuid_find(struct cache_set *c, const char *uuid)
451cafe5635SKent Overstreet {
452cafe5635SKent Overstreet 	struct uuid_entry *u;
453cafe5635SKent Overstreet 
454cafe5635SKent Overstreet 	for (u = c->uuids;
455cafe5635SKent Overstreet 	     u < c->uuids + c->nr_uuids; u++)
456cafe5635SKent Overstreet 		if (!memcmp(u->uuid, uuid, 16))
457cafe5635SKent Overstreet 			return u;
458cafe5635SKent Overstreet 
459cafe5635SKent Overstreet 	return NULL;
460cafe5635SKent Overstreet }
461cafe5635SKent Overstreet 
462cafe5635SKent Overstreet static struct uuid_entry *uuid_find_empty(struct cache_set *c)
463cafe5635SKent Overstreet {
464cafe5635SKent Overstreet 	static const char zero_uuid[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
465cafe5635SKent Overstreet 	return uuid_find(c, zero_uuid);
466cafe5635SKent Overstreet }
467cafe5635SKent Overstreet 
468cafe5635SKent Overstreet /*
469cafe5635SKent Overstreet  * Bucket priorities/gens:
470cafe5635SKent Overstreet  *
471cafe5635SKent Overstreet  * For each bucket, we store on disk its
472cafe5635SKent Overstreet    * 8 bit gen
473cafe5635SKent Overstreet    * 16 bit priority
474cafe5635SKent Overstreet  *
475cafe5635SKent Overstreet  * See alloc.c for an explanation of the gen. The priority is used to implement
476cafe5635SKent Overstreet  * lru (and in the future other) cache replacement policies; for most purposes
477cafe5635SKent Overstreet  * it's just an opaque integer.
478cafe5635SKent Overstreet  *
479cafe5635SKent Overstreet  * The gens and the priorities don't have a whole lot to do with each other, and
480cafe5635SKent Overstreet  * it's actually the gens that must be written out at specific times - it's no
481cafe5635SKent Overstreet  * big deal if the priorities don't get written, if we lose them we just reuse
482cafe5635SKent Overstreet  * buckets in suboptimal order.
483cafe5635SKent Overstreet  *
484cafe5635SKent Overstreet  * On disk they're stored in a packed array, and in as many buckets are required
485cafe5635SKent Overstreet  * to fit them all. The buckets we use to store them form a list; the journal
486cafe5635SKent Overstreet  * header points to the first bucket, the first bucket points to the second
487cafe5635SKent Overstreet  * bucket, et cetera.
488cafe5635SKent Overstreet  *
489cafe5635SKent Overstreet  * This code is used by the allocation code; periodically (whenever it runs out
490cafe5635SKent Overstreet  * of buckets to allocate from) the allocation code will invalidate some
491cafe5635SKent Overstreet  * buckets, but it can't use those buckets until their new gens are safely on
492cafe5635SKent Overstreet  * disk.
493cafe5635SKent Overstreet  */
494cafe5635SKent Overstreet 
495cafe5635SKent Overstreet static void prio_endio(struct bio *bio, int error)
496cafe5635SKent Overstreet {
497cafe5635SKent Overstreet 	struct cache *ca = bio->bi_private;
498cafe5635SKent Overstreet 
499cafe5635SKent Overstreet 	cache_set_err_on(error, ca->set, "accessing priorities");
500cafe5635SKent Overstreet 	bch_bbio_free(bio, ca->set);
501cafe5635SKent Overstreet 	closure_put(&ca->prio);
502cafe5635SKent Overstreet }
503cafe5635SKent Overstreet 
504cafe5635SKent Overstreet static void prio_io(struct cache *ca, uint64_t bucket, unsigned long rw)
505cafe5635SKent Overstreet {
506cafe5635SKent Overstreet 	struct closure *cl = &ca->prio;
507cafe5635SKent Overstreet 	struct bio *bio = bch_bbio_alloc(ca->set);
508cafe5635SKent Overstreet 
509cafe5635SKent Overstreet 	closure_init_stack(cl);
510cafe5635SKent Overstreet 
511cafe5635SKent Overstreet 	bio->bi_sector	= bucket * ca->sb.bucket_size;
512cafe5635SKent Overstreet 	bio->bi_bdev	= ca->bdev;
513cafe5635SKent Overstreet 	bio->bi_rw	= REQ_SYNC|REQ_META|rw;
514cafe5635SKent Overstreet 	bio->bi_size	= bucket_bytes(ca);
515cafe5635SKent Overstreet 
516cafe5635SKent Overstreet 	bio->bi_end_io	= prio_endio;
517cafe5635SKent Overstreet 	bio->bi_private = ca;
518169ef1cfSKent Overstreet 	bch_bio_map(bio, ca->disk_buckets);
519cafe5635SKent Overstreet 
520cafe5635SKent Overstreet 	closure_bio_submit(bio, &ca->prio, ca);
521cafe5635SKent Overstreet 	closure_sync(cl);
522cafe5635SKent Overstreet }
523cafe5635SKent Overstreet 
524cafe5635SKent Overstreet #define buckets_free(c)	"free %zu, free_inc %zu, unused %zu",		\
525cafe5635SKent Overstreet 	fifo_used(&c->free), fifo_used(&c->free_inc), fifo_used(&c->unused)
526cafe5635SKent Overstreet 
527cafe5635SKent Overstreet void bch_prio_write(struct cache *ca)
528cafe5635SKent Overstreet {
529cafe5635SKent Overstreet 	int i;
530cafe5635SKent Overstreet 	struct bucket *b;
531cafe5635SKent Overstreet 	struct closure cl;
532cafe5635SKent Overstreet 
533cafe5635SKent Overstreet 	closure_init_stack(&cl);
534cafe5635SKent Overstreet 
535cafe5635SKent Overstreet 	lockdep_assert_held(&ca->set->bucket_lock);
536cafe5635SKent Overstreet 
537cafe5635SKent Overstreet 	for (b = ca->buckets;
538cafe5635SKent Overstreet 	     b < ca->buckets + ca->sb.nbuckets; b++)
539cafe5635SKent Overstreet 		b->disk_gen = b->gen;
540cafe5635SKent Overstreet 
541cafe5635SKent Overstreet 	ca->disk_buckets->seq++;
542cafe5635SKent Overstreet 
543cafe5635SKent Overstreet 	atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),
544cafe5635SKent Overstreet 			&ca->meta_sectors_written);
545cafe5635SKent Overstreet 
546cafe5635SKent Overstreet 	pr_debug("free %zu, free_inc %zu, unused %zu", fifo_used(&ca->free),
547cafe5635SKent Overstreet 		 fifo_used(&ca->free_inc), fifo_used(&ca->unused));
548cafe5635SKent Overstreet 
549cafe5635SKent Overstreet 	for (i = prio_buckets(ca) - 1; i >= 0; --i) {
550cafe5635SKent Overstreet 		long bucket;
551cafe5635SKent Overstreet 		struct prio_set *p = ca->disk_buckets;
552b1a67b0fSKent Overstreet 		struct bucket_disk *d = p->data;
553b1a67b0fSKent Overstreet 		struct bucket_disk *end = d + prios_per_bucket(ca);
554cafe5635SKent Overstreet 
555cafe5635SKent Overstreet 		for (b = ca->buckets + i * prios_per_bucket(ca);
556cafe5635SKent Overstreet 		     b < ca->buckets + ca->sb.nbuckets && d < end;
557cafe5635SKent Overstreet 		     b++, d++) {
558cafe5635SKent Overstreet 			d->prio = cpu_to_le16(b->prio);
559cafe5635SKent Overstreet 			d->gen = b->gen;
560cafe5635SKent Overstreet 		}
561cafe5635SKent Overstreet 
562cafe5635SKent Overstreet 		p->next_bucket	= ca->prio_buckets[i + 1];
563cafe5635SKent Overstreet 		p->magic	= pset_magic(ca);
564169ef1cfSKent Overstreet 		p->csum		= bch_crc64(&p->magic, bucket_bytes(ca) - 8);
565cafe5635SKent Overstreet 
566cafe5635SKent Overstreet 		bucket = bch_bucket_alloc(ca, WATERMARK_PRIO, &cl);
567cafe5635SKent Overstreet 		BUG_ON(bucket == -1);
568cafe5635SKent Overstreet 
569cafe5635SKent Overstreet 		mutex_unlock(&ca->set->bucket_lock);
570cafe5635SKent Overstreet 		prio_io(ca, bucket, REQ_WRITE);
571cafe5635SKent Overstreet 		mutex_lock(&ca->set->bucket_lock);
572cafe5635SKent Overstreet 
573cafe5635SKent Overstreet 		ca->prio_buckets[i] = bucket;
574cafe5635SKent Overstreet 		atomic_dec_bug(&ca->buckets[bucket].pin);
575cafe5635SKent Overstreet 	}
576cafe5635SKent Overstreet 
577cafe5635SKent Overstreet 	mutex_unlock(&ca->set->bucket_lock);
578cafe5635SKent Overstreet 
579cafe5635SKent Overstreet 	bch_journal_meta(ca->set, &cl);
580cafe5635SKent Overstreet 	closure_sync(&cl);
581cafe5635SKent Overstreet 
582cafe5635SKent Overstreet 	mutex_lock(&ca->set->bucket_lock);
583cafe5635SKent Overstreet 
584cafe5635SKent Overstreet 	ca->need_save_prio = 0;
585cafe5635SKent Overstreet 
586cafe5635SKent Overstreet 	/*
587cafe5635SKent Overstreet 	 * Don't want the old priorities to get garbage collected until after we
588cafe5635SKent Overstreet 	 * finish writing the new ones, and they're journalled
589cafe5635SKent Overstreet 	 */
590cafe5635SKent Overstreet 	for (i = 0; i < prio_buckets(ca); i++)
591cafe5635SKent Overstreet 		ca->prio_last_buckets[i] = ca->prio_buckets[i];
592cafe5635SKent Overstreet }
593cafe5635SKent Overstreet 
594cafe5635SKent Overstreet static void prio_read(struct cache *ca, uint64_t bucket)
595cafe5635SKent Overstreet {
596cafe5635SKent Overstreet 	struct prio_set *p = ca->disk_buckets;
597cafe5635SKent Overstreet 	struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;
598cafe5635SKent Overstreet 	struct bucket *b;
599cafe5635SKent Overstreet 	unsigned bucket_nr = 0;
600cafe5635SKent Overstreet 
601cafe5635SKent Overstreet 	for (b = ca->buckets;
602cafe5635SKent Overstreet 	     b < ca->buckets + ca->sb.nbuckets;
603cafe5635SKent Overstreet 	     b++, d++) {
604cafe5635SKent Overstreet 		if (d == end) {
605cafe5635SKent Overstreet 			ca->prio_buckets[bucket_nr] = bucket;
606cafe5635SKent Overstreet 			ca->prio_last_buckets[bucket_nr] = bucket;
607cafe5635SKent Overstreet 			bucket_nr++;
608cafe5635SKent Overstreet 
609cafe5635SKent Overstreet 			prio_io(ca, bucket, READ_SYNC);
610cafe5635SKent Overstreet 
611169ef1cfSKent Overstreet 			if (p->csum != bch_crc64(&p->magic, bucket_bytes(ca) - 8))
612cafe5635SKent Overstreet 				pr_warn("bad csum reading priorities");
613cafe5635SKent Overstreet 
614cafe5635SKent Overstreet 			if (p->magic != pset_magic(ca))
615cafe5635SKent Overstreet 				pr_warn("bad magic reading priorities");
616cafe5635SKent Overstreet 
617cafe5635SKent Overstreet 			bucket = p->next_bucket;
618cafe5635SKent Overstreet 			d = p->data;
619cafe5635SKent Overstreet 		}
620cafe5635SKent Overstreet 
621cafe5635SKent Overstreet 		b->prio = le16_to_cpu(d->prio);
622cafe5635SKent Overstreet 		b->gen = b->disk_gen = b->last_gc = b->gc_gen = d->gen;
623cafe5635SKent Overstreet 	}
624cafe5635SKent Overstreet }
625cafe5635SKent Overstreet 
626cafe5635SKent Overstreet /* Bcache device */
627cafe5635SKent Overstreet 
628cafe5635SKent Overstreet static int open_dev(struct block_device *b, fmode_t mode)
629cafe5635SKent Overstreet {
630cafe5635SKent Overstreet 	struct bcache_device *d = b->bd_disk->private_data;
631cafe5635SKent Overstreet 	if (atomic_read(&d->closing))
632cafe5635SKent Overstreet 		return -ENXIO;
633cafe5635SKent Overstreet 
634cafe5635SKent Overstreet 	closure_get(&d->cl);
635cafe5635SKent Overstreet 	return 0;
636cafe5635SKent Overstreet }
637cafe5635SKent Overstreet 
638867e1162SEmil Goode static void release_dev(struct gendisk *b, fmode_t mode)
639cafe5635SKent Overstreet {
640cafe5635SKent Overstreet 	struct bcache_device *d = b->private_data;
641cafe5635SKent Overstreet 	closure_put(&d->cl);
642cafe5635SKent Overstreet }
643cafe5635SKent Overstreet 
644cafe5635SKent Overstreet static int ioctl_dev(struct block_device *b, fmode_t mode,
645cafe5635SKent Overstreet 		     unsigned int cmd, unsigned long arg)
646cafe5635SKent Overstreet {
647cafe5635SKent Overstreet 	struct bcache_device *d = b->bd_disk->private_data;
648cafe5635SKent Overstreet 	return d->ioctl(d, mode, cmd, arg);
649cafe5635SKent Overstreet }
650cafe5635SKent Overstreet 
651cafe5635SKent Overstreet static const struct block_device_operations bcache_ops = {
652cafe5635SKent Overstreet 	.open		= open_dev,
653cafe5635SKent Overstreet 	.release	= release_dev,
654cafe5635SKent Overstreet 	.ioctl		= ioctl_dev,
655cafe5635SKent Overstreet 	.owner		= THIS_MODULE,
656cafe5635SKent Overstreet };
657cafe5635SKent Overstreet 
658cafe5635SKent Overstreet void bcache_device_stop(struct bcache_device *d)
659cafe5635SKent Overstreet {
660cafe5635SKent Overstreet 	if (!atomic_xchg(&d->closing, 1))
661cafe5635SKent Overstreet 		closure_queue(&d->cl);
662cafe5635SKent Overstreet }
663cafe5635SKent Overstreet 
664ee668506SKent Overstreet static void bcache_device_unlink(struct bcache_device *d)
665ee668506SKent Overstreet {
666ee668506SKent Overstreet 	unsigned i;
667ee668506SKent Overstreet 	struct cache *ca;
668ee668506SKent Overstreet 
669ee668506SKent Overstreet 	sysfs_remove_link(&d->c->kobj, d->name);
670ee668506SKent Overstreet 	sysfs_remove_link(&d->kobj, "cache");
671ee668506SKent Overstreet 
672ee668506SKent Overstreet 	for_each_cache(ca, d->c, i)
673ee668506SKent Overstreet 		bd_unlink_disk_holder(ca->bdev, d->disk);
674ee668506SKent Overstreet }
675ee668506SKent Overstreet 
676ee668506SKent Overstreet static void bcache_device_link(struct bcache_device *d, struct cache_set *c,
677ee668506SKent Overstreet 			       const char *name)
678ee668506SKent Overstreet {
679ee668506SKent Overstreet 	unsigned i;
680ee668506SKent Overstreet 	struct cache *ca;
681ee668506SKent Overstreet 
682ee668506SKent Overstreet 	for_each_cache(ca, d->c, i)
683ee668506SKent Overstreet 		bd_link_disk_holder(ca->bdev, d->disk);
684ee668506SKent Overstreet 
685ee668506SKent Overstreet 	snprintf(d->name, BCACHEDEVNAME_SIZE,
686ee668506SKent Overstreet 		 "%s%u", name, d->id);
687ee668506SKent Overstreet 
688ee668506SKent Overstreet 	WARN(sysfs_create_link(&d->kobj, &c->kobj, "cache") ||
689ee668506SKent Overstreet 	     sysfs_create_link(&c->kobj, &d->kobj, d->name),
690ee668506SKent Overstreet 	     "Couldn't create device <-> cache set symlinks");
691ee668506SKent Overstreet }
692ee668506SKent Overstreet 
693cafe5635SKent Overstreet static void bcache_device_detach(struct bcache_device *d)
694cafe5635SKent Overstreet {
695cafe5635SKent Overstreet 	lockdep_assert_held(&bch_register_lock);
696cafe5635SKent Overstreet 
697cafe5635SKent Overstreet 	if (atomic_read(&d->detaching)) {
698cafe5635SKent Overstreet 		struct uuid_entry *u = d->c->uuids + d->id;
699cafe5635SKent Overstreet 
700cafe5635SKent Overstreet 		SET_UUID_FLASH_ONLY(u, 0);
701cafe5635SKent Overstreet 		memcpy(u->uuid, invalid_uuid, 16);
702cafe5635SKent Overstreet 		u->invalidated = cpu_to_le32(get_seconds());
703cafe5635SKent Overstreet 		bch_uuid_write(d->c);
704cafe5635SKent Overstreet 
705cafe5635SKent Overstreet 		atomic_set(&d->detaching, 0);
706cafe5635SKent Overstreet 	}
707cafe5635SKent Overstreet 
708ee668506SKent Overstreet 	bcache_device_unlink(d);
709ee668506SKent Overstreet 
710cafe5635SKent Overstreet 	d->c->devices[d->id] = NULL;
711cafe5635SKent Overstreet 	closure_put(&d->c->caching);
712cafe5635SKent Overstreet 	d->c = NULL;
713cafe5635SKent Overstreet }
714cafe5635SKent Overstreet 
715cafe5635SKent Overstreet static void bcache_device_attach(struct bcache_device *d, struct cache_set *c,
716cafe5635SKent Overstreet 				 unsigned id)
717cafe5635SKent Overstreet {
718cafe5635SKent Overstreet 	BUG_ON(test_bit(CACHE_SET_STOPPING, &c->flags));
719cafe5635SKent Overstreet 
720cafe5635SKent Overstreet 	d->id = id;
721cafe5635SKent Overstreet 	d->c = c;
722cafe5635SKent Overstreet 	c->devices[id] = d;
723cafe5635SKent Overstreet 
724cafe5635SKent Overstreet 	closure_get(&c->caching);
725cafe5635SKent Overstreet }
726cafe5635SKent Overstreet 
727cafe5635SKent Overstreet static void bcache_device_free(struct bcache_device *d)
728cafe5635SKent Overstreet {
729cafe5635SKent Overstreet 	lockdep_assert_held(&bch_register_lock);
730cafe5635SKent Overstreet 
731cafe5635SKent Overstreet 	pr_info("%s stopped", d->disk->disk_name);
732cafe5635SKent Overstreet 
733cafe5635SKent Overstreet 	if (d->c)
734cafe5635SKent Overstreet 		bcache_device_detach(d);
735f59fce84SKent Overstreet 	if (d->disk && d->disk->flags & GENHD_FL_UP)
736cafe5635SKent Overstreet 		del_gendisk(d->disk);
737cafe5635SKent Overstreet 	if (d->disk && d->disk->queue)
738cafe5635SKent Overstreet 		blk_cleanup_queue(d->disk->queue);
739cafe5635SKent Overstreet 	if (d->disk)
740cafe5635SKent Overstreet 		put_disk(d->disk);
741cafe5635SKent Overstreet 
742cafe5635SKent Overstreet 	bio_split_pool_free(&d->bio_split_hook);
743cafe5635SKent Overstreet 	if (d->unaligned_bvec)
744cafe5635SKent Overstreet 		mempool_destroy(d->unaligned_bvec);
745cafe5635SKent Overstreet 	if (d->bio_split)
746cafe5635SKent Overstreet 		bioset_free(d->bio_split);
747cafe5635SKent Overstreet 
748cafe5635SKent Overstreet 	closure_debug_destroy(&d->cl);
749cafe5635SKent Overstreet }
750cafe5635SKent Overstreet 
751cafe5635SKent Overstreet static int bcache_device_init(struct bcache_device *d, unsigned block_size)
752cafe5635SKent Overstreet {
753cafe5635SKent Overstreet 	struct request_queue *q;
754cafe5635SKent Overstreet 
755cafe5635SKent Overstreet 	if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
756cafe5635SKent Overstreet 	    !(d->unaligned_bvec = mempool_create_kmalloc_pool(1,
757cafe5635SKent Overstreet 				sizeof(struct bio_vec) * BIO_MAX_PAGES)) ||
758f59fce84SKent Overstreet 	    bio_split_pool_init(&d->bio_split_hook) ||
759f59fce84SKent Overstreet 	    !(d->disk = alloc_disk(1)) ||
760f59fce84SKent Overstreet 	    !(q = blk_alloc_queue(GFP_KERNEL)))
761cafe5635SKent Overstreet 		return -ENOMEM;
762cafe5635SKent Overstreet 
763cafe5635SKent Overstreet 	snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", bcache_minor);
764cafe5635SKent Overstreet 
765cafe5635SKent Overstreet 	d->disk->major		= bcache_major;
766cafe5635SKent Overstreet 	d->disk->first_minor	= bcache_minor++;
767cafe5635SKent Overstreet 	d->disk->fops		= &bcache_ops;
768cafe5635SKent Overstreet 	d->disk->private_data	= d;
769cafe5635SKent Overstreet 
770cafe5635SKent Overstreet 	blk_queue_make_request(q, NULL);
771cafe5635SKent Overstreet 	d->disk->queue			= q;
772cafe5635SKent Overstreet 	q->queuedata			= d;
773cafe5635SKent Overstreet 	q->backing_dev_info.congested_data = d;
774cafe5635SKent Overstreet 	q->limits.max_hw_sectors	= UINT_MAX;
775cafe5635SKent Overstreet 	q->limits.max_sectors		= UINT_MAX;
776cafe5635SKent Overstreet 	q->limits.max_segment_size	= UINT_MAX;
777cafe5635SKent Overstreet 	q->limits.max_segments		= BIO_MAX_PAGES;
778cafe5635SKent Overstreet 	q->limits.max_discard_sectors	= UINT_MAX;
779cafe5635SKent Overstreet 	q->limits.io_min		= block_size;
780cafe5635SKent Overstreet 	q->limits.logical_block_size	= block_size;
781cafe5635SKent Overstreet 	q->limits.physical_block_size	= block_size;
782cafe5635SKent Overstreet 	set_bit(QUEUE_FLAG_NONROT,	&d->disk->queue->queue_flags);
783cafe5635SKent Overstreet 	set_bit(QUEUE_FLAG_DISCARD,	&d->disk->queue->queue_flags);
784cafe5635SKent Overstreet 
785cafe5635SKent Overstreet 	return 0;
786cafe5635SKent Overstreet }
787cafe5635SKent Overstreet 
788cafe5635SKent Overstreet /* Cached device */
789cafe5635SKent Overstreet 
790cafe5635SKent Overstreet static void calc_cached_dev_sectors(struct cache_set *c)
791cafe5635SKent Overstreet {
792cafe5635SKent Overstreet 	uint64_t sectors = 0;
793cafe5635SKent Overstreet 	struct cached_dev *dc;
794cafe5635SKent Overstreet 
795cafe5635SKent Overstreet 	list_for_each_entry(dc, &c->cached_devs, list)
796cafe5635SKent Overstreet 		sectors += bdev_sectors(dc->bdev);
797cafe5635SKent Overstreet 
798cafe5635SKent Overstreet 	c->cached_dev_sectors = sectors;
799cafe5635SKent Overstreet }
800cafe5635SKent Overstreet 
801cafe5635SKent Overstreet void bch_cached_dev_run(struct cached_dev *dc)
802cafe5635SKent Overstreet {
803cafe5635SKent Overstreet 	struct bcache_device *d = &dc->disk;
804cafe5635SKent Overstreet 
805cafe5635SKent Overstreet 	if (atomic_xchg(&dc->running, 1))
806cafe5635SKent Overstreet 		return;
807cafe5635SKent Overstreet 
808cafe5635SKent Overstreet 	if (!d->c &&
809cafe5635SKent Overstreet 	    BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) {
810cafe5635SKent Overstreet 		struct closure cl;
811cafe5635SKent Overstreet 		closure_init_stack(&cl);
812cafe5635SKent Overstreet 
813cafe5635SKent Overstreet 		SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE);
814cafe5635SKent Overstreet 		bch_write_bdev_super(dc, &cl);
815cafe5635SKent Overstreet 		closure_sync(&cl);
816cafe5635SKent Overstreet 	}
817cafe5635SKent Overstreet 
818cafe5635SKent Overstreet 	add_disk(d->disk);
819ee668506SKent Overstreet 	bd_link_disk_holder(dc->bdev, dc->disk.disk);
820cafe5635SKent Overstreet #if 0
821cafe5635SKent Overstreet 	char *env[] = { "SYMLINK=label" , NULL };
822cafe5635SKent Overstreet 	kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);
823cafe5635SKent Overstreet #endif
824cafe5635SKent Overstreet 	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
825cafe5635SKent Overstreet 	    sysfs_create_link(&disk_to_dev(d->disk)->kobj, &d->kobj, "bcache"))
826cafe5635SKent Overstreet 		pr_debug("error creating sysfs link");
827cafe5635SKent Overstreet }
828cafe5635SKent Overstreet 
829cafe5635SKent Overstreet static void cached_dev_detach_finish(struct work_struct *w)
830cafe5635SKent Overstreet {
831cafe5635SKent Overstreet 	struct cached_dev *dc = container_of(w, struct cached_dev, detach);
832cafe5635SKent Overstreet 	char buf[BDEVNAME_SIZE];
833cafe5635SKent Overstreet 	struct closure cl;
834cafe5635SKent Overstreet 	closure_init_stack(&cl);
835cafe5635SKent Overstreet 
836cafe5635SKent Overstreet 	BUG_ON(!atomic_read(&dc->disk.detaching));
837cafe5635SKent Overstreet 	BUG_ON(atomic_read(&dc->count));
838cafe5635SKent Overstreet 
839cafe5635SKent Overstreet 	mutex_lock(&bch_register_lock);
840cafe5635SKent Overstreet 
841cafe5635SKent Overstreet 	memset(&dc->sb.set_uuid, 0, 16);
842cafe5635SKent Overstreet 	SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);
843cafe5635SKent Overstreet 
844cafe5635SKent Overstreet 	bch_write_bdev_super(dc, &cl);
845cafe5635SKent Overstreet 	closure_sync(&cl);
846cafe5635SKent Overstreet 
847cafe5635SKent Overstreet 	bcache_device_detach(&dc->disk);
848cafe5635SKent Overstreet 	list_move(&dc->list, &uncached_devices);
849cafe5635SKent Overstreet 
850cafe5635SKent Overstreet 	mutex_unlock(&bch_register_lock);
851cafe5635SKent Overstreet 
852cafe5635SKent Overstreet 	pr_info("Caching disabled for %s", bdevname(dc->bdev, buf));
853cafe5635SKent Overstreet 
854cafe5635SKent Overstreet 	/* Drop ref we took in cached_dev_detach() */
855cafe5635SKent Overstreet 	closure_put(&dc->disk.cl);
856cafe5635SKent Overstreet }
857cafe5635SKent Overstreet 
858cafe5635SKent Overstreet void bch_cached_dev_detach(struct cached_dev *dc)
859cafe5635SKent Overstreet {
860cafe5635SKent Overstreet 	lockdep_assert_held(&bch_register_lock);
861cafe5635SKent Overstreet 
862cafe5635SKent Overstreet 	if (atomic_read(&dc->disk.closing))
863cafe5635SKent Overstreet 		return;
864cafe5635SKent Overstreet 
865cafe5635SKent Overstreet 	if (atomic_xchg(&dc->disk.detaching, 1))
866cafe5635SKent Overstreet 		return;
867cafe5635SKent Overstreet 
868cafe5635SKent Overstreet 	/*
869cafe5635SKent Overstreet 	 * Block the device from being closed and freed until we're finished
870cafe5635SKent Overstreet 	 * detaching
871cafe5635SKent Overstreet 	 */
872cafe5635SKent Overstreet 	closure_get(&dc->disk.cl);
873cafe5635SKent Overstreet 
874cafe5635SKent Overstreet 	bch_writeback_queue(dc);
875cafe5635SKent Overstreet 	cached_dev_put(dc);
876cafe5635SKent Overstreet }
877cafe5635SKent Overstreet 
878cafe5635SKent Overstreet int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c)
879cafe5635SKent Overstreet {
880cafe5635SKent Overstreet 	uint32_t rtime = cpu_to_le32(get_seconds());
881cafe5635SKent Overstreet 	struct uuid_entry *u;
882cafe5635SKent Overstreet 	char buf[BDEVNAME_SIZE];
883cafe5635SKent Overstreet 
884cafe5635SKent Overstreet 	bdevname(dc->bdev, buf);
885cafe5635SKent Overstreet 
886cafe5635SKent Overstreet 	if (memcmp(dc->sb.set_uuid, c->sb.set_uuid, 16))
887cafe5635SKent Overstreet 		return -ENOENT;
888cafe5635SKent Overstreet 
889cafe5635SKent Overstreet 	if (dc->disk.c) {
890cafe5635SKent Overstreet 		pr_err("Can't attach %s: already attached", buf);
891cafe5635SKent Overstreet 		return -EINVAL;
892cafe5635SKent Overstreet 	}
893cafe5635SKent Overstreet 
894cafe5635SKent Overstreet 	if (test_bit(CACHE_SET_STOPPING, &c->flags)) {
895cafe5635SKent Overstreet 		pr_err("Can't attach %s: shutting down", buf);
896cafe5635SKent Overstreet 		return -EINVAL;
897cafe5635SKent Overstreet 	}
898cafe5635SKent Overstreet 
899cafe5635SKent Overstreet 	if (dc->sb.block_size < c->sb.block_size) {
900cafe5635SKent Overstreet 		/* Will die */
901b1a67b0fSKent Overstreet 		pr_err("Couldn't attach %s: block size less than set's block size",
902b1a67b0fSKent Overstreet 		       buf);
903cafe5635SKent Overstreet 		return -EINVAL;
904cafe5635SKent Overstreet 	}
905cafe5635SKent Overstreet 
906cafe5635SKent Overstreet 	u = uuid_find(c, dc->sb.uuid);
907cafe5635SKent Overstreet 
908cafe5635SKent Overstreet 	if (u &&
909cafe5635SKent Overstreet 	    (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE ||
910cafe5635SKent Overstreet 	     BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) {
911cafe5635SKent Overstreet 		memcpy(u->uuid, invalid_uuid, 16);
912cafe5635SKent Overstreet 		u->invalidated = cpu_to_le32(get_seconds());
913cafe5635SKent Overstreet 		u = NULL;
914cafe5635SKent Overstreet 	}
915cafe5635SKent Overstreet 
916cafe5635SKent Overstreet 	if (!u) {
917cafe5635SKent Overstreet 		if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
918cafe5635SKent Overstreet 			pr_err("Couldn't find uuid for %s in set", buf);
919cafe5635SKent Overstreet 			return -ENOENT;
920cafe5635SKent Overstreet 		}
921cafe5635SKent Overstreet 
922cafe5635SKent Overstreet 		u = uuid_find_empty(c);
923cafe5635SKent Overstreet 		if (!u) {
924cafe5635SKent Overstreet 			pr_err("Not caching %s, no room for UUID", buf);
925cafe5635SKent Overstreet 			return -EINVAL;
926cafe5635SKent Overstreet 		}
927cafe5635SKent Overstreet 	}
928cafe5635SKent Overstreet 
929cafe5635SKent Overstreet 	/* Deadlocks since we're called via sysfs...
930cafe5635SKent Overstreet 	sysfs_remove_file(&dc->kobj, &sysfs_attach);
931cafe5635SKent Overstreet 	 */
932cafe5635SKent Overstreet 
933169ef1cfSKent Overstreet 	if (bch_is_zero(u->uuid, 16)) {
934cafe5635SKent Overstreet 		struct closure cl;
935cafe5635SKent Overstreet 		closure_init_stack(&cl);
936cafe5635SKent Overstreet 
937cafe5635SKent Overstreet 		memcpy(u->uuid, dc->sb.uuid, 16);
938cafe5635SKent Overstreet 		memcpy(u->label, dc->sb.label, SB_LABEL_SIZE);
939cafe5635SKent Overstreet 		u->first_reg = u->last_reg = rtime;
940cafe5635SKent Overstreet 		bch_uuid_write(c);
941cafe5635SKent Overstreet 
942cafe5635SKent Overstreet 		memcpy(dc->sb.set_uuid, c->sb.set_uuid, 16);
943cafe5635SKent Overstreet 		SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
944cafe5635SKent Overstreet 
945cafe5635SKent Overstreet 		bch_write_bdev_super(dc, &cl);
946cafe5635SKent Overstreet 		closure_sync(&cl);
947cafe5635SKent Overstreet 	} else {
948cafe5635SKent Overstreet 		u->last_reg = rtime;
949cafe5635SKent Overstreet 		bch_uuid_write(c);
950cafe5635SKent Overstreet 	}
951cafe5635SKent Overstreet 
952cafe5635SKent Overstreet 	bcache_device_attach(&dc->disk, c, u - c->uuids);
953cafe5635SKent Overstreet 	list_move(&dc->list, &c->cached_devs);
954cafe5635SKent Overstreet 	calc_cached_dev_sectors(c);
955cafe5635SKent Overstreet 
956cafe5635SKent Overstreet 	smp_wmb();
957cafe5635SKent Overstreet 	/*
958cafe5635SKent Overstreet 	 * dc->c must be set before dc->count != 0 - paired with the mb in
959cafe5635SKent Overstreet 	 * cached_dev_get()
960cafe5635SKent Overstreet 	 */
961cafe5635SKent Overstreet 	atomic_set(&dc->count, 1);
962cafe5635SKent Overstreet 
963cafe5635SKent Overstreet 	if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
964cafe5635SKent Overstreet 		atomic_set(&dc->has_dirty, 1);
965cafe5635SKent Overstreet 		atomic_inc(&dc->count);
966cafe5635SKent Overstreet 		bch_writeback_queue(dc);
967cafe5635SKent Overstreet 	}
968cafe5635SKent Overstreet 
969cafe5635SKent Overstreet 	bch_cached_dev_run(dc);
970ee668506SKent Overstreet 	bcache_device_link(&dc->disk, c, "bdev");
971cafe5635SKent Overstreet 
972cafe5635SKent Overstreet 	pr_info("Caching %s as %s on set %pU",
973cafe5635SKent Overstreet 		bdevname(dc->bdev, buf), dc->disk.disk->disk_name,
974cafe5635SKent Overstreet 		dc->disk.c->sb.set_uuid);
975cafe5635SKent Overstreet 	return 0;
976cafe5635SKent Overstreet }
977cafe5635SKent Overstreet 
978cafe5635SKent Overstreet void bch_cached_dev_release(struct kobject *kobj)
979cafe5635SKent Overstreet {
980cafe5635SKent Overstreet 	struct cached_dev *dc = container_of(kobj, struct cached_dev,
981cafe5635SKent Overstreet 					     disk.kobj);
982cafe5635SKent Overstreet 	kfree(dc);
983cafe5635SKent Overstreet 	module_put(THIS_MODULE);
984cafe5635SKent Overstreet }
985cafe5635SKent Overstreet 
986cafe5635SKent Overstreet static void cached_dev_free(struct closure *cl)
987cafe5635SKent Overstreet {
988cafe5635SKent Overstreet 	struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
989cafe5635SKent Overstreet 
990cafe5635SKent Overstreet 	cancel_delayed_work_sync(&dc->writeback_rate_update);
991cafe5635SKent Overstreet 
992cafe5635SKent Overstreet 	mutex_lock(&bch_register_lock);
993cafe5635SKent Overstreet 
994f59fce84SKent Overstreet 	if (atomic_read(&dc->running))
995ee668506SKent Overstreet 		bd_unlink_disk_holder(dc->bdev, dc->disk.disk);
996cafe5635SKent Overstreet 	bcache_device_free(&dc->disk);
997cafe5635SKent Overstreet 	list_del(&dc->list);
998cafe5635SKent Overstreet 
999cafe5635SKent Overstreet 	mutex_unlock(&bch_register_lock);
1000cafe5635SKent Overstreet 
1001cafe5635SKent Overstreet 	if (!IS_ERR_OR_NULL(dc->bdev)) {
1002f59fce84SKent Overstreet 		if (dc->bdev->bd_disk)
1003cafe5635SKent Overstreet 			blk_sync_queue(bdev_get_queue(dc->bdev));
1004f59fce84SKent Overstreet 
1005cafe5635SKent Overstreet 		blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
1006cafe5635SKent Overstreet 	}
1007cafe5635SKent Overstreet 
1008cafe5635SKent Overstreet 	wake_up(&unregister_wait);
1009cafe5635SKent Overstreet 
1010cafe5635SKent Overstreet 	kobject_put(&dc->disk.kobj);
1011cafe5635SKent Overstreet }
1012cafe5635SKent Overstreet 
1013cafe5635SKent Overstreet static void cached_dev_flush(struct closure *cl)
1014cafe5635SKent Overstreet {
1015cafe5635SKent Overstreet 	struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
1016cafe5635SKent Overstreet 	struct bcache_device *d = &dc->disk;
1017cafe5635SKent Overstreet 
1018cafe5635SKent Overstreet 	bch_cache_accounting_destroy(&dc->accounting);
1019cafe5635SKent Overstreet 	kobject_del(&d->kobj);
1020cafe5635SKent Overstreet 
1021cafe5635SKent Overstreet 	continue_at(cl, cached_dev_free, system_wq);
1022cafe5635SKent Overstreet }
1023cafe5635SKent Overstreet 
1024cafe5635SKent Overstreet static int cached_dev_init(struct cached_dev *dc, unsigned block_size)
1025cafe5635SKent Overstreet {
1026f59fce84SKent Overstreet 	int ret;
1027cafe5635SKent Overstreet 	struct io *io;
1028f59fce84SKent Overstreet 	struct request_queue *q = bdev_get_queue(dc->bdev);
1029cafe5635SKent Overstreet 
1030cafe5635SKent Overstreet 	__module_get(THIS_MODULE);
1031cafe5635SKent Overstreet 	INIT_LIST_HEAD(&dc->list);
1032f59fce84SKent Overstreet 	closure_init(&dc->disk.cl, NULL);
1033f59fce84SKent Overstreet 	set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq);
1034cafe5635SKent Overstreet 	kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype);
1035cafe5635SKent Overstreet 	INIT_WORK(&dc->detach, cached_dev_detach_finish);
1036f59fce84SKent Overstreet 	closure_init_unlocked(&dc->sb_write);
1037f59fce84SKent Overstreet 	INIT_LIST_HEAD(&dc->io_lru);
1038f59fce84SKent Overstreet 	spin_lock_init(&dc->io_lock);
1039f59fce84SKent Overstreet 	bch_cache_accounting_init(&dc->accounting, &dc->disk.cl);
1040cafe5635SKent Overstreet 
1041cafe5635SKent Overstreet 	dc->sequential_merge		= true;
1042cafe5635SKent Overstreet 	dc->sequential_cutoff		= 4 << 20;
1043cafe5635SKent Overstreet 
1044cafe5635SKent Overstreet 	for (io = dc->io; io < dc->io + RECENT_IO; io++) {
1045cafe5635SKent Overstreet 		list_add(&io->lru, &dc->io_lru);
1046cafe5635SKent Overstreet 		hlist_add_head(&io->hash, dc->io_hash + RECENT_IO);
1047cafe5635SKent Overstreet 	}
1048cafe5635SKent Overstreet 
1049f59fce84SKent Overstreet 	ret = bcache_device_init(&dc->disk, block_size);
1050f59fce84SKent Overstreet 	if (ret)
1051f59fce84SKent Overstreet 		return ret;
1052f59fce84SKent Overstreet 
1053f59fce84SKent Overstreet 	set_capacity(dc->disk.disk,
1054f59fce84SKent Overstreet 		     dc->bdev->bd_part->nr_sects - dc->sb.data_offset);
1055f59fce84SKent Overstreet 
1056f59fce84SKent Overstreet 	dc->disk.disk->queue->backing_dev_info.ra_pages =
1057f59fce84SKent Overstreet 		max(dc->disk.disk->queue->backing_dev_info.ra_pages,
1058f59fce84SKent Overstreet 		    q->backing_dev_info.ra_pages);
1059f59fce84SKent Overstreet 
1060f59fce84SKent Overstreet 	bch_cached_dev_request_init(dc);
1061f59fce84SKent Overstreet 	bch_cached_dev_writeback_init(dc);
1062cafe5635SKent Overstreet 	return 0;
1063cafe5635SKent Overstreet }
1064cafe5635SKent Overstreet 
1065cafe5635SKent Overstreet /* Cached device - bcache superblock */
1066cafe5635SKent Overstreet 
1067f59fce84SKent Overstreet static void register_bdev(struct cache_sb *sb, struct page *sb_page,
1068cafe5635SKent Overstreet 				 struct block_device *bdev,
1069cafe5635SKent Overstreet 				 struct cached_dev *dc)
1070cafe5635SKent Overstreet {
1071cafe5635SKent Overstreet 	char name[BDEVNAME_SIZE];
1072cafe5635SKent Overstreet 	const char *err = "cannot allocate memory";
1073cafe5635SKent Overstreet 	struct cache_set *c;
1074cafe5635SKent Overstreet 
1075cafe5635SKent Overstreet 	memcpy(&dc->sb, sb, sizeof(struct cache_sb));
1076cafe5635SKent Overstreet 	dc->bdev = bdev;
1077cafe5635SKent Overstreet 	dc->bdev->bd_holder = dc;
1078cafe5635SKent Overstreet 
1079f59fce84SKent Overstreet 	bio_init(&dc->sb_bio);
1080f59fce84SKent Overstreet 	dc->sb_bio.bi_max_vecs	= 1;
1081f59fce84SKent Overstreet 	dc->sb_bio.bi_io_vec	= dc->sb_bio.bi_inline_vecs;
1082f59fce84SKent Overstreet 	dc->sb_bio.bi_io_vec[0].bv_page = sb_page;
1083f59fce84SKent Overstreet 	get_page(sb_page);
1084cafe5635SKent Overstreet 
1085f59fce84SKent Overstreet 	if (cached_dev_init(dc, sb->block_size << 9))
1086f59fce84SKent Overstreet 		goto err;
1087cafe5635SKent Overstreet 
1088cafe5635SKent Overstreet 	err = "error creating kobject";
1089cafe5635SKent Overstreet 	if (kobject_add(&dc->disk.kobj, &part_to_dev(bdev->bd_part)->kobj,
1090cafe5635SKent Overstreet 			"bcache"))
1091cafe5635SKent Overstreet 		goto err;
1092cafe5635SKent Overstreet 	if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
1093cafe5635SKent Overstreet 		goto err;
1094cafe5635SKent Overstreet 
1095f59fce84SKent Overstreet 	pr_info("registered backing device %s", bdevname(bdev, name));
1096f59fce84SKent Overstreet 
1097cafe5635SKent Overstreet 	list_add(&dc->list, &uncached_devices);
1098cafe5635SKent Overstreet 	list_for_each_entry(c, &bch_cache_sets, list)
1099cafe5635SKent Overstreet 		bch_cached_dev_attach(dc, c);
1100cafe5635SKent Overstreet 
1101cafe5635SKent Overstreet 	if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
1102cafe5635SKent Overstreet 	    BDEV_STATE(&dc->sb) == BDEV_STATE_STALE)
1103cafe5635SKent Overstreet 		bch_cached_dev_run(dc);
1104cafe5635SKent Overstreet 
1105f59fce84SKent Overstreet 	return;
1106cafe5635SKent Overstreet err:
1107cafe5635SKent Overstreet 	pr_notice("error opening %s: %s", bdevname(bdev, name), err);
1108f59fce84SKent Overstreet 	bcache_device_stop(&dc->disk);
1109cafe5635SKent Overstreet }
1110cafe5635SKent Overstreet 
1111cafe5635SKent Overstreet /* Flash only volumes */
1112cafe5635SKent Overstreet 
1113cafe5635SKent Overstreet void bch_flash_dev_release(struct kobject *kobj)
1114cafe5635SKent Overstreet {
1115cafe5635SKent Overstreet 	struct bcache_device *d = container_of(kobj, struct bcache_device,
1116cafe5635SKent Overstreet 					       kobj);
1117cafe5635SKent Overstreet 	kfree(d);
1118cafe5635SKent Overstreet }
1119cafe5635SKent Overstreet 
1120cafe5635SKent Overstreet static void flash_dev_free(struct closure *cl)
1121cafe5635SKent Overstreet {
1122cafe5635SKent Overstreet 	struct bcache_device *d = container_of(cl, struct bcache_device, cl);
1123cafe5635SKent Overstreet 	bcache_device_free(d);
1124cafe5635SKent Overstreet 	kobject_put(&d->kobj);
1125cafe5635SKent Overstreet }
1126cafe5635SKent Overstreet 
1127cafe5635SKent Overstreet static void flash_dev_flush(struct closure *cl)
1128cafe5635SKent Overstreet {
1129cafe5635SKent Overstreet 	struct bcache_device *d = container_of(cl, struct bcache_device, cl);
1130cafe5635SKent Overstreet 
1131ee668506SKent Overstreet 	bcache_device_unlink(d);
1132cafe5635SKent Overstreet 	kobject_del(&d->kobj);
1133cafe5635SKent Overstreet 	continue_at(cl, flash_dev_free, system_wq);
1134cafe5635SKent Overstreet }
1135cafe5635SKent Overstreet 
1136cafe5635SKent Overstreet static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
1137cafe5635SKent Overstreet {
1138cafe5635SKent Overstreet 	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
1139cafe5635SKent Overstreet 					  GFP_KERNEL);
1140cafe5635SKent Overstreet 	if (!d)
1141cafe5635SKent Overstreet 		return -ENOMEM;
1142cafe5635SKent Overstreet 
1143cafe5635SKent Overstreet 	closure_init(&d->cl, NULL);
1144cafe5635SKent Overstreet 	set_closure_fn(&d->cl, flash_dev_flush, system_wq);
1145cafe5635SKent Overstreet 
1146cafe5635SKent Overstreet 	kobject_init(&d->kobj, &bch_flash_dev_ktype);
1147cafe5635SKent Overstreet 
1148cafe5635SKent Overstreet 	if (bcache_device_init(d, block_bytes(c)))
1149cafe5635SKent Overstreet 		goto err;
1150cafe5635SKent Overstreet 
1151cafe5635SKent Overstreet 	bcache_device_attach(d, c, u - c->uuids);
1152cafe5635SKent Overstreet 	set_capacity(d->disk, u->sectors);
1153cafe5635SKent Overstreet 	bch_flash_dev_request_init(d);
1154cafe5635SKent Overstreet 	add_disk(d->disk);
1155cafe5635SKent Overstreet 
1156cafe5635SKent Overstreet 	if (kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache"))
1157cafe5635SKent Overstreet 		goto err;
1158cafe5635SKent Overstreet 
1159cafe5635SKent Overstreet 	bcache_device_link(d, c, "volume");
1160cafe5635SKent Overstreet 
1161cafe5635SKent Overstreet 	return 0;
1162cafe5635SKent Overstreet err:
1163cafe5635SKent Overstreet 	kobject_put(&d->kobj);
1164cafe5635SKent Overstreet 	return -ENOMEM;
1165cafe5635SKent Overstreet }
1166cafe5635SKent Overstreet 
1167cafe5635SKent Overstreet static int flash_devs_run(struct cache_set *c)
1168cafe5635SKent Overstreet {
1169cafe5635SKent Overstreet 	int ret = 0;
1170cafe5635SKent Overstreet 	struct uuid_entry *u;
1171cafe5635SKent Overstreet 
1172cafe5635SKent Overstreet 	for (u = c->uuids;
1173cafe5635SKent Overstreet 	     u < c->uuids + c->nr_uuids && !ret;
1174cafe5635SKent Overstreet 	     u++)
1175cafe5635SKent Overstreet 		if (UUID_FLASH_ONLY(u))
1176cafe5635SKent Overstreet 			ret = flash_dev_run(c, u);
1177cafe5635SKent Overstreet 
1178cafe5635SKent Overstreet 	return ret;
1179cafe5635SKent Overstreet }
1180cafe5635SKent Overstreet 
1181cafe5635SKent Overstreet int bch_flash_dev_create(struct cache_set *c, uint64_t size)
1182cafe5635SKent Overstreet {
1183cafe5635SKent Overstreet 	struct uuid_entry *u;
1184cafe5635SKent Overstreet 
1185cafe5635SKent Overstreet 	if (test_bit(CACHE_SET_STOPPING, &c->flags))
1186cafe5635SKent Overstreet 		return -EINTR;
1187cafe5635SKent Overstreet 
1188cafe5635SKent Overstreet 	u = uuid_find_empty(c);
1189cafe5635SKent Overstreet 	if (!u) {
1190cafe5635SKent Overstreet 		pr_err("Can't create volume, no room for UUID");
1191cafe5635SKent Overstreet 		return -EINVAL;
1192cafe5635SKent Overstreet 	}
1193cafe5635SKent Overstreet 
1194cafe5635SKent Overstreet 	get_random_bytes(u->uuid, 16);
1195cafe5635SKent Overstreet 	memset(u->label, 0, 32);
1196cafe5635SKent Overstreet 	u->first_reg = u->last_reg = cpu_to_le32(get_seconds());
1197cafe5635SKent Overstreet 
1198cafe5635SKent Overstreet 	SET_UUID_FLASH_ONLY(u, 1);
1199cafe5635SKent Overstreet 	u->sectors = size >> 9;
1200cafe5635SKent Overstreet 
1201cafe5635SKent Overstreet 	bch_uuid_write(c);
1202cafe5635SKent Overstreet 
1203cafe5635SKent Overstreet 	return flash_dev_run(c, u);
1204cafe5635SKent Overstreet }
1205cafe5635SKent Overstreet 
1206cafe5635SKent Overstreet /* Cache set */
1207cafe5635SKent Overstreet 
1208cafe5635SKent Overstreet __printf(2, 3)
1209cafe5635SKent Overstreet bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)
1210cafe5635SKent Overstreet {
1211cafe5635SKent Overstreet 	va_list args;
1212cafe5635SKent Overstreet 
1213cafe5635SKent Overstreet 	if (test_bit(CACHE_SET_STOPPING, &c->flags))
1214cafe5635SKent Overstreet 		return false;
1215cafe5635SKent Overstreet 
1216cafe5635SKent Overstreet 	/* XXX: we can be called from atomic context
1217cafe5635SKent Overstreet 	acquire_console_sem();
1218cafe5635SKent Overstreet 	*/
1219cafe5635SKent Overstreet 
1220cafe5635SKent Overstreet 	printk(KERN_ERR "bcache: error on %pU: ", c->sb.set_uuid);
1221cafe5635SKent Overstreet 
1222cafe5635SKent Overstreet 	va_start(args, fmt);
1223cafe5635SKent Overstreet 	vprintk(fmt, args);
1224cafe5635SKent Overstreet 	va_end(args);
1225cafe5635SKent Overstreet 
1226cafe5635SKent Overstreet 	printk(", disabling caching\n");
1227cafe5635SKent Overstreet 
1228cafe5635SKent Overstreet 	bch_cache_set_unregister(c);
1229cafe5635SKent Overstreet 	return true;
1230cafe5635SKent Overstreet }
1231cafe5635SKent Overstreet 
1232cafe5635SKent Overstreet void bch_cache_set_release(struct kobject *kobj)
1233cafe5635SKent Overstreet {
1234cafe5635SKent Overstreet 	struct cache_set *c = container_of(kobj, struct cache_set, kobj);
1235cafe5635SKent Overstreet 	kfree(c);
1236cafe5635SKent Overstreet 	module_put(THIS_MODULE);
1237cafe5635SKent Overstreet }
1238cafe5635SKent Overstreet 
1239cafe5635SKent Overstreet static void cache_set_free(struct closure *cl)
1240cafe5635SKent Overstreet {
1241cafe5635SKent Overstreet 	struct cache_set *c = container_of(cl, struct cache_set, cl);
1242cafe5635SKent Overstreet 	struct cache *ca;
1243cafe5635SKent Overstreet 	unsigned i;
1244cafe5635SKent Overstreet 
1245cafe5635SKent Overstreet 	if (!IS_ERR_OR_NULL(c->debug))
1246cafe5635SKent Overstreet 		debugfs_remove(c->debug);
1247cafe5635SKent Overstreet 
1248cafe5635SKent Overstreet 	bch_open_buckets_free(c);
1249cafe5635SKent Overstreet 	bch_btree_cache_free(c);
1250cafe5635SKent Overstreet 	bch_journal_free(c);
1251cafe5635SKent Overstreet 
1252cafe5635SKent Overstreet 	for_each_cache(ca, c, i)
1253cafe5635SKent Overstreet 		if (ca)
1254cafe5635SKent Overstreet 			kobject_put(&ca->kobj);
1255cafe5635SKent Overstreet 
1256cafe5635SKent Overstreet 	free_pages((unsigned long) c->uuids, ilog2(bucket_pages(c)));
1257cafe5635SKent Overstreet 	free_pages((unsigned long) c->sort, ilog2(bucket_pages(c)));
1258cafe5635SKent Overstreet 
1259cafe5635SKent Overstreet 	if (c->bio_split)
1260cafe5635SKent Overstreet 		bioset_free(c->bio_split);
126157943511SKent Overstreet 	if (c->fill_iter)
126257943511SKent Overstreet 		mempool_destroy(c->fill_iter);
1263cafe5635SKent Overstreet 	if (c->bio_meta)
1264cafe5635SKent Overstreet 		mempool_destroy(c->bio_meta);
1265cafe5635SKent Overstreet 	if (c->search)
1266cafe5635SKent Overstreet 		mempool_destroy(c->search);
1267cafe5635SKent Overstreet 	kfree(c->devices);
1268cafe5635SKent Overstreet 
1269cafe5635SKent Overstreet 	mutex_lock(&bch_register_lock);
1270cafe5635SKent Overstreet 	list_del(&c->list);
1271cafe5635SKent Overstreet 	mutex_unlock(&bch_register_lock);
1272cafe5635SKent Overstreet 
1273cafe5635SKent Overstreet 	pr_info("Cache set %pU unregistered", c->sb.set_uuid);
1274cafe5635SKent Overstreet 	wake_up(&unregister_wait);
1275cafe5635SKent Overstreet 
1276cafe5635SKent Overstreet 	closure_debug_destroy(&c->cl);
1277cafe5635SKent Overstreet 	kobject_put(&c->kobj);
1278cafe5635SKent Overstreet }
1279cafe5635SKent Overstreet 
1280cafe5635SKent Overstreet static void cache_set_flush(struct closure *cl)
1281cafe5635SKent Overstreet {
1282cafe5635SKent Overstreet 	struct cache_set *c = container_of(cl, struct cache_set, caching);
1283cafe5635SKent Overstreet 	struct btree *b;
1284cafe5635SKent Overstreet 
1285cafe5635SKent Overstreet 	/* Shut down allocator threads */
1286cafe5635SKent Overstreet 	set_bit(CACHE_SET_STOPPING_2, &c->flags);
1287119ba0f8SKent Overstreet 	wake_up_allocators(c);
1288cafe5635SKent Overstreet 
1289cafe5635SKent Overstreet 	bch_cache_accounting_destroy(&c->accounting);
1290cafe5635SKent Overstreet 
1291cafe5635SKent Overstreet 	kobject_put(&c->internal);
1292cafe5635SKent Overstreet 	kobject_del(&c->kobj);
1293cafe5635SKent Overstreet 
1294cafe5635SKent Overstreet 	if (!IS_ERR_OR_NULL(c->root))
1295cafe5635SKent Overstreet 		list_add(&c->root->list, &c->btree_cache);
1296cafe5635SKent Overstreet 
1297cafe5635SKent Overstreet 	/* Should skip this if we're unregistering because of an error */
1298cafe5635SKent Overstreet 	list_for_each_entry(b, &c->btree_cache, list)
1299cafe5635SKent Overstreet 		if (btree_node_dirty(b))
130057943511SKent Overstreet 			bch_btree_node_write(b, NULL);
1301cafe5635SKent Overstreet 
1302cafe5635SKent Overstreet 	closure_return(cl);
1303cafe5635SKent Overstreet }
1304cafe5635SKent Overstreet 
1305cafe5635SKent Overstreet static void __cache_set_unregister(struct closure *cl)
1306cafe5635SKent Overstreet {
1307cafe5635SKent Overstreet 	struct cache_set *c = container_of(cl, struct cache_set, caching);
1308cafe5635SKent Overstreet 	struct cached_dev *dc, *t;
1309cafe5635SKent Overstreet 	size_t i;
1310cafe5635SKent Overstreet 
1311cafe5635SKent Overstreet 	mutex_lock(&bch_register_lock);
1312cafe5635SKent Overstreet 
1313cafe5635SKent Overstreet 	if (test_bit(CACHE_SET_UNREGISTERING, &c->flags))
1314cafe5635SKent Overstreet 		list_for_each_entry_safe(dc, t, &c->cached_devs, list)
1315cafe5635SKent Overstreet 			bch_cached_dev_detach(dc);
1316cafe5635SKent Overstreet 
1317cafe5635SKent Overstreet 	for (i = 0; i < c->nr_uuids; i++)
1318cafe5635SKent Overstreet 		if (c->devices[i] && UUID_FLASH_ONLY(&c->uuids[i]))
1319cafe5635SKent Overstreet 			bcache_device_stop(c->devices[i]);
1320cafe5635SKent Overstreet 
1321cafe5635SKent Overstreet 	mutex_unlock(&bch_register_lock);
1322cafe5635SKent Overstreet 
1323cafe5635SKent Overstreet 	continue_at(cl, cache_set_flush, system_wq);
1324cafe5635SKent Overstreet }
1325cafe5635SKent Overstreet 
1326cafe5635SKent Overstreet void bch_cache_set_stop(struct cache_set *c)
1327cafe5635SKent Overstreet {
1328cafe5635SKent Overstreet 	if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags))
1329cafe5635SKent Overstreet 		closure_queue(&c->caching);
1330cafe5635SKent Overstreet }
1331cafe5635SKent Overstreet 
1332cafe5635SKent Overstreet void bch_cache_set_unregister(struct cache_set *c)
1333cafe5635SKent Overstreet {
1334cafe5635SKent Overstreet 	set_bit(CACHE_SET_UNREGISTERING, &c->flags);
1335cafe5635SKent Overstreet 	bch_cache_set_stop(c);
1336cafe5635SKent Overstreet }
1337cafe5635SKent Overstreet 
1338cafe5635SKent Overstreet #define alloc_bucket_pages(gfp, c)			\
1339cafe5635SKent Overstreet 	((void *) __get_free_pages(__GFP_ZERO|gfp, ilog2(bucket_pages(c))))
1340cafe5635SKent Overstreet 
1341cafe5635SKent Overstreet struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
1342cafe5635SKent Overstreet {
1343cafe5635SKent Overstreet 	int iter_size;
1344cafe5635SKent Overstreet 	struct cache_set *c = kzalloc(sizeof(struct cache_set), GFP_KERNEL);
1345cafe5635SKent Overstreet 	if (!c)
1346cafe5635SKent Overstreet 		return NULL;
1347cafe5635SKent Overstreet 
1348cafe5635SKent Overstreet 	__module_get(THIS_MODULE);
1349cafe5635SKent Overstreet 	closure_init(&c->cl, NULL);
1350cafe5635SKent Overstreet 	set_closure_fn(&c->cl, cache_set_free, system_wq);
1351cafe5635SKent Overstreet 
1352cafe5635SKent Overstreet 	closure_init(&c->caching, &c->cl);
1353cafe5635SKent Overstreet 	set_closure_fn(&c->caching, __cache_set_unregister, system_wq);
1354cafe5635SKent Overstreet 
1355cafe5635SKent Overstreet 	/* Maybe create continue_at_noreturn() and use it here? */
1356cafe5635SKent Overstreet 	closure_set_stopped(&c->cl);
1357cafe5635SKent Overstreet 	closure_put(&c->cl);
1358cafe5635SKent Overstreet 
1359cafe5635SKent Overstreet 	kobject_init(&c->kobj, &bch_cache_set_ktype);
1360cafe5635SKent Overstreet 	kobject_init(&c->internal, &bch_cache_set_internal_ktype);
1361cafe5635SKent Overstreet 
1362cafe5635SKent Overstreet 	bch_cache_accounting_init(&c->accounting, &c->cl);
1363cafe5635SKent Overstreet 
1364cafe5635SKent Overstreet 	memcpy(c->sb.set_uuid, sb->set_uuid, 16);
1365cafe5635SKent Overstreet 	c->sb.block_size	= sb->block_size;
1366cafe5635SKent Overstreet 	c->sb.bucket_size	= sb->bucket_size;
1367cafe5635SKent Overstreet 	c->sb.nr_in_set		= sb->nr_in_set;
1368cafe5635SKent Overstreet 	c->sb.last_mount	= sb->last_mount;
1369cafe5635SKent Overstreet 	c->bucket_bits		= ilog2(sb->bucket_size);
1370cafe5635SKent Overstreet 	c->block_bits		= ilog2(sb->block_size);
1371cafe5635SKent Overstreet 	c->nr_uuids		= bucket_bytes(c) / sizeof(struct uuid_entry);
1372cafe5635SKent Overstreet 
1373cafe5635SKent Overstreet 	c->btree_pages		= c->sb.bucket_size / PAGE_SECTORS;
1374cafe5635SKent Overstreet 	if (c->btree_pages > BTREE_MAX_PAGES)
1375cafe5635SKent Overstreet 		c->btree_pages = max_t(int, c->btree_pages / 4,
1376cafe5635SKent Overstreet 				       BTREE_MAX_PAGES);
1377cafe5635SKent Overstreet 
1378*6ded34d1SKent Overstreet 	c->sort_crit_factor = int_sqrt(c->btree_pages);
1379*6ded34d1SKent Overstreet 
1380cafe5635SKent Overstreet 	mutex_init(&c->bucket_lock);
1381cafe5635SKent Overstreet 	mutex_init(&c->sort_lock);
1382cafe5635SKent Overstreet 	spin_lock_init(&c->sort_time_lock);
1383cafe5635SKent Overstreet 	closure_init_unlocked(&c->sb_write);
1384cafe5635SKent Overstreet 	closure_init_unlocked(&c->uuid_write);
1385cafe5635SKent Overstreet 	spin_lock_init(&c->btree_read_time_lock);
1386cafe5635SKent Overstreet 	bch_moving_init_cache_set(c);
1387cafe5635SKent Overstreet 
1388cafe5635SKent Overstreet 	INIT_LIST_HEAD(&c->list);
1389cafe5635SKent Overstreet 	INIT_LIST_HEAD(&c->cached_devs);
1390cafe5635SKent Overstreet 	INIT_LIST_HEAD(&c->btree_cache);
1391cafe5635SKent Overstreet 	INIT_LIST_HEAD(&c->btree_cache_freeable);
1392cafe5635SKent Overstreet 	INIT_LIST_HEAD(&c->btree_cache_freed);
1393cafe5635SKent Overstreet 	INIT_LIST_HEAD(&c->data_buckets);
1394cafe5635SKent Overstreet 
1395cafe5635SKent Overstreet 	c->search = mempool_create_slab_pool(32, bch_search_cache);
1396cafe5635SKent Overstreet 	if (!c->search)
1397cafe5635SKent Overstreet 		goto err;
1398cafe5635SKent Overstreet 
1399cafe5635SKent Overstreet 	iter_size = (sb->bucket_size / sb->block_size + 1) *
1400cafe5635SKent Overstreet 		sizeof(struct btree_iter_set);
1401cafe5635SKent Overstreet 
1402cafe5635SKent Overstreet 	if (!(c->devices = kzalloc(c->nr_uuids * sizeof(void *), GFP_KERNEL)) ||
1403cafe5635SKent Overstreet 	    !(c->bio_meta = mempool_create_kmalloc_pool(2,
1404cafe5635SKent Overstreet 				sizeof(struct bbio) + sizeof(struct bio_vec) *
1405cafe5635SKent Overstreet 				bucket_pages(c))) ||
140657943511SKent Overstreet 	    !(c->fill_iter = mempool_create_kmalloc_pool(1, iter_size)) ||
1407cafe5635SKent Overstreet 	    !(c->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
1408cafe5635SKent Overstreet 	    !(c->sort = alloc_bucket_pages(GFP_KERNEL, c)) ||
1409cafe5635SKent Overstreet 	    !(c->uuids = alloc_bucket_pages(GFP_KERNEL, c)) ||
1410cafe5635SKent Overstreet 	    bch_journal_alloc(c) ||
1411cafe5635SKent Overstreet 	    bch_btree_cache_alloc(c) ||
1412cafe5635SKent Overstreet 	    bch_open_buckets_alloc(c))
1413cafe5635SKent Overstreet 		goto err;
1414cafe5635SKent Overstreet 
1415cafe5635SKent Overstreet 	c->congested_read_threshold_us	= 2000;
1416cafe5635SKent Overstreet 	c->congested_write_threshold_us	= 20000;
1417cafe5635SKent Overstreet 	c->error_limit	= 8 << IO_ERROR_SHIFT;
1418cafe5635SKent Overstreet 
1419cafe5635SKent Overstreet 	return c;
1420cafe5635SKent Overstreet err:
1421cafe5635SKent Overstreet 	bch_cache_set_unregister(c);
1422cafe5635SKent Overstreet 	return NULL;
1423cafe5635SKent Overstreet }
1424cafe5635SKent Overstreet 
1425cafe5635SKent Overstreet static void run_cache_set(struct cache_set *c)
1426cafe5635SKent Overstreet {
1427cafe5635SKent Overstreet 	const char *err = "cannot allocate memory";
1428cafe5635SKent Overstreet 	struct cached_dev *dc, *t;
1429cafe5635SKent Overstreet 	struct cache *ca;
1430cafe5635SKent Overstreet 	unsigned i;
1431cafe5635SKent Overstreet 
1432cafe5635SKent Overstreet 	struct btree_op op;
1433cafe5635SKent Overstreet 	bch_btree_op_init_stack(&op);
1434cafe5635SKent Overstreet 	op.lock = SHRT_MAX;
1435cafe5635SKent Overstreet 
1436cafe5635SKent Overstreet 	for_each_cache(ca, c, i)
1437cafe5635SKent Overstreet 		c->nbuckets += ca->sb.nbuckets;
1438cafe5635SKent Overstreet 
1439cafe5635SKent Overstreet 	if (CACHE_SYNC(&c->sb)) {
1440cafe5635SKent Overstreet 		LIST_HEAD(journal);
1441cafe5635SKent Overstreet 		struct bkey *k;
1442cafe5635SKent Overstreet 		struct jset *j;
1443cafe5635SKent Overstreet 
1444cafe5635SKent Overstreet 		err = "cannot allocate memory for journal";
1445cafe5635SKent Overstreet 		if (bch_journal_read(c, &journal, &op))
1446cafe5635SKent Overstreet 			goto err;
1447cafe5635SKent Overstreet 
1448cafe5635SKent Overstreet 		pr_debug("btree_journal_read() done");
1449cafe5635SKent Overstreet 
1450cafe5635SKent Overstreet 		err = "no journal entries found";
1451cafe5635SKent Overstreet 		if (list_empty(&journal))
1452cafe5635SKent Overstreet 			goto err;
1453cafe5635SKent Overstreet 
1454cafe5635SKent Overstreet 		j = &list_entry(journal.prev, struct journal_replay, list)->j;
1455cafe5635SKent Overstreet 
1456cafe5635SKent Overstreet 		err = "IO error reading priorities";
1457cafe5635SKent Overstreet 		for_each_cache(ca, c, i)
1458cafe5635SKent Overstreet 			prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]);
1459cafe5635SKent Overstreet 
1460cafe5635SKent Overstreet 		/*
1461cafe5635SKent Overstreet 		 * If prio_read() fails it'll call cache_set_error and we'll
1462cafe5635SKent Overstreet 		 * tear everything down right away, but if we perhaps checked
1463cafe5635SKent Overstreet 		 * sooner we could avoid journal replay.
1464cafe5635SKent Overstreet 		 */
1465cafe5635SKent Overstreet 
1466cafe5635SKent Overstreet 		k = &j->btree_root;
1467cafe5635SKent Overstreet 
1468cafe5635SKent Overstreet 		err = "bad btree root";
1469cafe5635SKent Overstreet 		if (__bch_ptr_invalid(c, j->btree_level + 1, k))
1470cafe5635SKent Overstreet 			goto err;
1471cafe5635SKent Overstreet 
1472cafe5635SKent Overstreet 		err = "error reading btree root";
1473cafe5635SKent Overstreet 		c->root = bch_btree_node_get(c, k, j->btree_level, &op);
1474cafe5635SKent Overstreet 		if (IS_ERR_OR_NULL(c->root))
1475cafe5635SKent Overstreet 			goto err;
1476cafe5635SKent Overstreet 
1477cafe5635SKent Overstreet 		list_del_init(&c->root->list);
1478cafe5635SKent Overstreet 		rw_unlock(true, c->root);
1479cafe5635SKent Overstreet 
1480cafe5635SKent Overstreet 		err = uuid_read(c, j, &op.cl);
1481cafe5635SKent Overstreet 		if (err)
1482cafe5635SKent Overstreet 			goto err;
1483cafe5635SKent Overstreet 
1484cafe5635SKent Overstreet 		err = "error in recovery";
1485cafe5635SKent Overstreet 		if (bch_btree_check(c, &op))
1486cafe5635SKent Overstreet 			goto err;
1487cafe5635SKent Overstreet 
1488cafe5635SKent Overstreet 		bch_journal_mark(c, &journal);
1489cafe5635SKent Overstreet 		bch_btree_gc_finish(c);
1490cafe5635SKent Overstreet 		pr_debug("btree_check() done");
1491cafe5635SKent Overstreet 
1492cafe5635SKent Overstreet 		/*
1493cafe5635SKent Overstreet 		 * bcache_journal_next() can't happen sooner, or
1494cafe5635SKent Overstreet 		 * btree_gc_finish() will give spurious errors about last_gc >
1495cafe5635SKent Overstreet 		 * gc_gen - this is a hack but oh well.
1496cafe5635SKent Overstreet 		 */
1497cafe5635SKent Overstreet 		bch_journal_next(&c->journal);
1498cafe5635SKent Overstreet 
1499119ba0f8SKent Overstreet 		err = "error starting allocator thread";
1500cafe5635SKent Overstreet 		for_each_cache(ca, c, i)
1501119ba0f8SKent Overstreet 			if (bch_cache_allocator_start(ca))
1502119ba0f8SKent Overstreet 				goto err;
1503cafe5635SKent Overstreet 
1504cafe5635SKent Overstreet 		/*
1505cafe5635SKent Overstreet 		 * First place it's safe to allocate: btree_check() and
1506cafe5635SKent Overstreet 		 * btree_gc_finish() have to run before we have buckets to
1507cafe5635SKent Overstreet 		 * allocate, and bch_bucket_alloc_set() might cause a journal
1508cafe5635SKent Overstreet 		 * entry to be written so bcache_journal_next() has to be called
1509cafe5635SKent Overstreet 		 * first.
1510cafe5635SKent Overstreet 		 *
1511cafe5635SKent Overstreet 		 * If the uuids were in the old format we have to rewrite them
1512cafe5635SKent Overstreet 		 * before the next journal entry is written:
1513cafe5635SKent Overstreet 		 */
1514cafe5635SKent Overstreet 		if (j->version < BCACHE_JSET_VERSION_UUID)
1515cafe5635SKent Overstreet 			__uuid_write(c);
1516cafe5635SKent Overstreet 
1517cafe5635SKent Overstreet 		bch_journal_replay(c, &journal, &op);
1518cafe5635SKent Overstreet 	} else {
1519cafe5635SKent Overstreet 		pr_notice("invalidating existing data");
1520cafe5635SKent Overstreet 		/* Don't want invalidate_buckets() to queue a gc yet */
1521cafe5635SKent Overstreet 		closure_lock(&c->gc, NULL);
1522cafe5635SKent Overstreet 
1523cafe5635SKent Overstreet 		for_each_cache(ca, c, i) {
1524cafe5635SKent Overstreet 			unsigned j;
1525cafe5635SKent Overstreet 
1526cafe5635SKent Overstreet 			ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,
1527cafe5635SKent Overstreet 					      2, SB_JOURNAL_BUCKETS);
1528cafe5635SKent Overstreet 
1529cafe5635SKent Overstreet 			for (j = 0; j < ca->sb.keys; j++)
1530cafe5635SKent Overstreet 				ca->sb.d[j] = ca->sb.first_bucket + j;
1531cafe5635SKent Overstreet 		}
1532cafe5635SKent Overstreet 
1533cafe5635SKent Overstreet 		bch_btree_gc_finish(c);
1534cafe5635SKent Overstreet 
1535119ba0f8SKent Overstreet 		err = "error starting allocator thread";
1536cafe5635SKent Overstreet 		for_each_cache(ca, c, i)
1537119ba0f8SKent Overstreet 			if (bch_cache_allocator_start(ca))
1538119ba0f8SKent Overstreet 				goto err;
1539cafe5635SKent Overstreet 
1540cafe5635SKent Overstreet 		mutex_lock(&c->bucket_lock);
1541cafe5635SKent Overstreet 		for_each_cache(ca, c, i)
1542cafe5635SKent Overstreet 			bch_prio_write(ca);
1543cafe5635SKent Overstreet 		mutex_unlock(&c->bucket_lock);
1544cafe5635SKent Overstreet 
1545cafe5635SKent Overstreet 		err = "cannot allocate new UUID bucket";
1546cafe5635SKent Overstreet 		if (__uuid_write(c))
1547cafe5635SKent Overstreet 			goto err_unlock_gc;
1548cafe5635SKent Overstreet 
1549cafe5635SKent Overstreet 		err = "cannot allocate new btree root";
1550cafe5635SKent Overstreet 		c->root = bch_btree_node_alloc(c, 0, &op.cl);
1551cafe5635SKent Overstreet 		if (IS_ERR_OR_NULL(c->root))
1552cafe5635SKent Overstreet 			goto err_unlock_gc;
1553cafe5635SKent Overstreet 
1554cafe5635SKent Overstreet 		bkey_copy_key(&c->root->key, &MAX_KEY);
155557943511SKent Overstreet 		bch_btree_node_write(c->root, &op.cl);
1556cafe5635SKent Overstreet 
1557cafe5635SKent Overstreet 		bch_btree_set_root(c->root);
1558cafe5635SKent Overstreet 		rw_unlock(true, c->root);
1559cafe5635SKent Overstreet 
1560cafe5635SKent Overstreet 		/*
1561cafe5635SKent Overstreet 		 * We don't want to write the first journal entry until
1562cafe5635SKent Overstreet 		 * everything is set up - fortunately journal entries won't be
1563cafe5635SKent Overstreet 		 * written until the SET_CACHE_SYNC() here:
1564cafe5635SKent Overstreet 		 */
1565cafe5635SKent Overstreet 		SET_CACHE_SYNC(&c->sb, true);
1566cafe5635SKent Overstreet 
1567cafe5635SKent Overstreet 		bch_journal_next(&c->journal);
1568cafe5635SKent Overstreet 		bch_journal_meta(c, &op.cl);
1569cafe5635SKent Overstreet 
1570cafe5635SKent Overstreet 		/* Unlock */
1571cafe5635SKent Overstreet 		closure_set_stopped(&c->gc.cl);
1572cafe5635SKent Overstreet 		closure_put(&c->gc.cl);
1573cafe5635SKent Overstreet 	}
1574cafe5635SKent Overstreet 
1575cafe5635SKent Overstreet 	closure_sync(&op.cl);
1576cafe5635SKent Overstreet 	c->sb.last_mount = get_seconds();
1577cafe5635SKent Overstreet 	bcache_write_super(c);
1578cafe5635SKent Overstreet 
1579cafe5635SKent Overstreet 	list_for_each_entry_safe(dc, t, &uncached_devices, list)
1580cafe5635SKent Overstreet 		bch_cached_dev_attach(dc, c);
1581cafe5635SKent Overstreet 
1582cafe5635SKent Overstreet 	flash_devs_run(c);
1583cafe5635SKent Overstreet 
1584cafe5635SKent Overstreet 	return;
1585cafe5635SKent Overstreet err_unlock_gc:
1586cafe5635SKent Overstreet 	closure_set_stopped(&c->gc.cl);
1587cafe5635SKent Overstreet 	closure_put(&c->gc.cl);
1588cafe5635SKent Overstreet err:
1589cafe5635SKent Overstreet 	closure_sync(&op.cl);
1590cafe5635SKent Overstreet 	/* XXX: test this, it's broken */
1591cafe5635SKent Overstreet 	bch_cache_set_error(c, err);
1592cafe5635SKent Overstreet }
1593cafe5635SKent Overstreet 
1594cafe5635SKent Overstreet static bool can_attach_cache(struct cache *ca, struct cache_set *c)
1595cafe5635SKent Overstreet {
1596cafe5635SKent Overstreet 	return ca->sb.block_size	== c->sb.block_size &&
1597cafe5635SKent Overstreet 		ca->sb.bucket_size	== c->sb.block_size &&
1598cafe5635SKent Overstreet 		ca->sb.nr_in_set	== c->sb.nr_in_set;
1599cafe5635SKent Overstreet }
1600cafe5635SKent Overstreet 
1601cafe5635SKent Overstreet static const char *register_cache_set(struct cache *ca)
1602cafe5635SKent Overstreet {
1603cafe5635SKent Overstreet 	char buf[12];
1604cafe5635SKent Overstreet 	const char *err = "cannot allocate memory";
1605cafe5635SKent Overstreet 	struct cache_set *c;
1606cafe5635SKent Overstreet 
1607cafe5635SKent Overstreet 	list_for_each_entry(c, &bch_cache_sets, list)
1608cafe5635SKent Overstreet 		if (!memcmp(c->sb.set_uuid, ca->sb.set_uuid, 16)) {
1609cafe5635SKent Overstreet 			if (c->cache[ca->sb.nr_this_dev])
1610cafe5635SKent Overstreet 				return "duplicate cache set member";
1611cafe5635SKent Overstreet 
1612cafe5635SKent Overstreet 			if (!can_attach_cache(ca, c))
1613cafe5635SKent Overstreet 				return "cache sb does not match set";
1614cafe5635SKent Overstreet 
1615cafe5635SKent Overstreet 			if (!CACHE_SYNC(&ca->sb))
1616cafe5635SKent Overstreet 				SET_CACHE_SYNC(&c->sb, false);
1617cafe5635SKent Overstreet 
1618cafe5635SKent Overstreet 			goto found;
1619cafe5635SKent Overstreet 		}
1620cafe5635SKent Overstreet 
1621cafe5635SKent Overstreet 	c = bch_cache_set_alloc(&ca->sb);
1622cafe5635SKent Overstreet 	if (!c)
1623cafe5635SKent Overstreet 		return err;
1624cafe5635SKent Overstreet 
1625cafe5635SKent Overstreet 	err = "error creating kobject";
1626cafe5635SKent Overstreet 	if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->sb.set_uuid) ||
1627cafe5635SKent Overstreet 	    kobject_add(&c->internal, &c->kobj, "internal"))
1628cafe5635SKent Overstreet 		goto err;
1629cafe5635SKent Overstreet 
1630cafe5635SKent Overstreet 	if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj))
1631cafe5635SKent Overstreet 		goto err;
1632cafe5635SKent Overstreet 
1633cafe5635SKent Overstreet 	bch_debug_init_cache_set(c);
1634cafe5635SKent Overstreet 
1635cafe5635SKent Overstreet 	list_add(&c->list, &bch_cache_sets);
1636cafe5635SKent Overstreet found:
1637cafe5635SKent Overstreet 	sprintf(buf, "cache%i", ca->sb.nr_this_dev);
1638cafe5635SKent Overstreet 	if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||
1639cafe5635SKent Overstreet 	    sysfs_create_link(&c->kobj, &ca->kobj, buf))
1640cafe5635SKent Overstreet 		goto err;
1641cafe5635SKent Overstreet 
1642cafe5635SKent Overstreet 	if (ca->sb.seq > c->sb.seq) {
1643cafe5635SKent Overstreet 		c->sb.version		= ca->sb.version;
1644cafe5635SKent Overstreet 		memcpy(c->sb.set_uuid, ca->sb.set_uuid, 16);
1645cafe5635SKent Overstreet 		c->sb.flags             = ca->sb.flags;
1646cafe5635SKent Overstreet 		c->sb.seq		= ca->sb.seq;
1647cafe5635SKent Overstreet 		pr_debug("set version = %llu", c->sb.version);
1648cafe5635SKent Overstreet 	}
1649cafe5635SKent Overstreet 
1650cafe5635SKent Overstreet 	ca->set = c;
1651cafe5635SKent Overstreet 	ca->set->cache[ca->sb.nr_this_dev] = ca;
1652cafe5635SKent Overstreet 	c->cache_by_alloc[c->caches_loaded++] = ca;
1653cafe5635SKent Overstreet 
1654cafe5635SKent Overstreet 	if (c->caches_loaded == c->sb.nr_in_set)
1655cafe5635SKent Overstreet 		run_cache_set(c);
1656cafe5635SKent Overstreet 
1657cafe5635SKent Overstreet 	return NULL;
1658cafe5635SKent Overstreet err:
1659cafe5635SKent Overstreet 	bch_cache_set_unregister(c);
1660cafe5635SKent Overstreet 	return err;
1661cafe5635SKent Overstreet }
1662cafe5635SKent Overstreet 
1663cafe5635SKent Overstreet /* Cache device */
1664cafe5635SKent Overstreet 
1665cafe5635SKent Overstreet void bch_cache_release(struct kobject *kobj)
1666cafe5635SKent Overstreet {
1667cafe5635SKent Overstreet 	struct cache *ca = container_of(kobj, struct cache, kobj);
1668cafe5635SKent Overstreet 
1669cafe5635SKent Overstreet 	if (ca->set)
1670cafe5635SKent Overstreet 		ca->set->cache[ca->sb.nr_this_dev] = NULL;
1671cafe5635SKent Overstreet 
1672cafe5635SKent Overstreet 	bch_cache_allocator_exit(ca);
1673cafe5635SKent Overstreet 
1674cafe5635SKent Overstreet 	bio_split_pool_free(&ca->bio_split_hook);
1675cafe5635SKent Overstreet 
1676cafe5635SKent Overstreet 	free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));
1677cafe5635SKent Overstreet 	kfree(ca->prio_buckets);
1678cafe5635SKent Overstreet 	vfree(ca->buckets);
1679cafe5635SKent Overstreet 
1680cafe5635SKent Overstreet 	free_heap(&ca->heap);
1681cafe5635SKent Overstreet 	free_fifo(&ca->unused);
1682cafe5635SKent Overstreet 	free_fifo(&ca->free_inc);
1683cafe5635SKent Overstreet 	free_fifo(&ca->free);
1684cafe5635SKent Overstreet 
1685cafe5635SKent Overstreet 	if (ca->sb_bio.bi_inline_vecs[0].bv_page)
1686cafe5635SKent Overstreet 		put_page(ca->sb_bio.bi_io_vec[0].bv_page);
1687cafe5635SKent Overstreet 
1688cafe5635SKent Overstreet 	if (!IS_ERR_OR_NULL(ca->bdev)) {
1689cafe5635SKent Overstreet 		blk_sync_queue(bdev_get_queue(ca->bdev));
1690cafe5635SKent Overstreet 		blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
1691cafe5635SKent Overstreet 	}
1692cafe5635SKent Overstreet 
1693cafe5635SKent Overstreet 	kfree(ca);
1694cafe5635SKent Overstreet 	module_put(THIS_MODULE);
1695cafe5635SKent Overstreet }
1696cafe5635SKent Overstreet 
1697cafe5635SKent Overstreet static int cache_alloc(struct cache_sb *sb, struct cache *ca)
1698cafe5635SKent Overstreet {
1699cafe5635SKent Overstreet 	size_t free;
1700cafe5635SKent Overstreet 	struct bucket *b;
1701cafe5635SKent Overstreet 
1702cafe5635SKent Overstreet 	__module_get(THIS_MODULE);
1703cafe5635SKent Overstreet 	kobject_init(&ca->kobj, &bch_cache_ktype);
1704cafe5635SKent Overstreet 
1705cafe5635SKent Overstreet 	INIT_LIST_HEAD(&ca->discards);
1706cafe5635SKent Overstreet 
1707cafe5635SKent Overstreet 	bio_init(&ca->journal.bio);
1708cafe5635SKent Overstreet 	ca->journal.bio.bi_max_vecs = 8;
1709cafe5635SKent Overstreet 	ca->journal.bio.bi_io_vec = ca->journal.bio.bi_inline_vecs;
1710cafe5635SKent Overstreet 
1711cafe5635SKent Overstreet 	free = roundup_pow_of_two(ca->sb.nbuckets) >> 9;
1712cafe5635SKent Overstreet 	free = max_t(size_t, free, (prio_buckets(ca) + 8) * 2);
1713cafe5635SKent Overstreet 
1714cafe5635SKent Overstreet 	if (!init_fifo(&ca->free,	free, GFP_KERNEL) ||
1715cafe5635SKent Overstreet 	    !init_fifo(&ca->free_inc,	free << 2, GFP_KERNEL) ||
1716cafe5635SKent Overstreet 	    !init_fifo(&ca->unused,	free << 2, GFP_KERNEL) ||
1717cafe5635SKent Overstreet 	    !init_heap(&ca->heap,	free << 3, GFP_KERNEL) ||
1718f59fce84SKent Overstreet 	    !(ca->buckets	= vzalloc(sizeof(struct bucket) *
1719cafe5635SKent Overstreet 					  ca->sb.nbuckets)) ||
1720cafe5635SKent Overstreet 	    !(ca->prio_buckets	= kzalloc(sizeof(uint64_t) * prio_buckets(ca) *
1721cafe5635SKent Overstreet 					  2, GFP_KERNEL)) ||
1722cafe5635SKent Overstreet 	    !(ca->disk_buckets	= alloc_bucket_pages(GFP_KERNEL, ca)) ||
1723cafe5635SKent Overstreet 	    bio_split_pool_init(&ca->bio_split_hook))
1724f59fce84SKent Overstreet 		return -ENOMEM;
1725cafe5635SKent Overstreet 
1726cafe5635SKent Overstreet 	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
1727cafe5635SKent Overstreet 
1728cafe5635SKent Overstreet 	for_each_bucket(b, ca)
1729cafe5635SKent Overstreet 		atomic_set(&b->pin, 0);
1730cafe5635SKent Overstreet 
1731cafe5635SKent Overstreet 	if (bch_cache_allocator_init(ca))
1732cafe5635SKent Overstreet 		goto err;
1733cafe5635SKent Overstreet 
1734cafe5635SKent Overstreet 	return 0;
1735cafe5635SKent Overstreet err:
1736cafe5635SKent Overstreet 	kobject_put(&ca->kobj);
1737cafe5635SKent Overstreet 	return -ENOMEM;
1738cafe5635SKent Overstreet }
1739cafe5635SKent Overstreet 
1740f59fce84SKent Overstreet static void register_cache(struct cache_sb *sb, struct page *sb_page,
1741cafe5635SKent Overstreet 				  struct block_device *bdev, struct cache *ca)
1742cafe5635SKent Overstreet {
1743cafe5635SKent Overstreet 	char name[BDEVNAME_SIZE];
1744cafe5635SKent Overstreet 	const char *err = "cannot allocate memory";
1745cafe5635SKent Overstreet 
1746f59fce84SKent Overstreet 	memcpy(&ca->sb, sb, sizeof(struct cache_sb));
1747cafe5635SKent Overstreet 	ca->bdev = bdev;
1748cafe5635SKent Overstreet 	ca->bdev->bd_holder = ca;
1749cafe5635SKent Overstreet 
1750f59fce84SKent Overstreet 	bio_init(&ca->sb_bio);
1751f59fce84SKent Overstreet 	ca->sb_bio.bi_max_vecs	= 1;
1752f59fce84SKent Overstreet 	ca->sb_bio.bi_io_vec	= ca->sb_bio.bi_inline_vecs;
1753f59fce84SKent Overstreet 	ca->sb_bio.bi_io_vec[0].bv_page = sb_page;
1754f59fce84SKent Overstreet 	get_page(sb_page);
1755f59fce84SKent Overstreet 
1756cafe5635SKent Overstreet 	if (blk_queue_discard(bdev_get_queue(ca->bdev)))
1757cafe5635SKent Overstreet 		ca->discard = CACHE_DISCARD(&ca->sb);
1758cafe5635SKent Overstreet 
1759f59fce84SKent Overstreet 	if (cache_alloc(sb, ca) != 0)
1760f59fce84SKent Overstreet 		goto err;
1761f59fce84SKent Overstreet 
1762cafe5635SKent Overstreet 	err = "error creating kobject";
1763cafe5635SKent Overstreet 	if (kobject_add(&ca->kobj, &part_to_dev(bdev->bd_part)->kobj, "bcache"))
1764cafe5635SKent Overstreet 		goto err;
1765cafe5635SKent Overstreet 
1766cafe5635SKent Overstreet 	err = register_cache_set(ca);
1767cafe5635SKent Overstreet 	if (err)
1768cafe5635SKent Overstreet 		goto err;
1769cafe5635SKent Overstreet 
1770cafe5635SKent Overstreet 	pr_info("registered cache device %s", bdevname(bdev, name));
1771f59fce84SKent Overstreet 	return;
1772cafe5635SKent Overstreet err:
1773f59fce84SKent Overstreet 	pr_notice("error opening %s: %s", bdevname(bdev, name), err);
1774cafe5635SKent Overstreet 	kobject_put(&ca->kobj);
1775cafe5635SKent Overstreet }
1776cafe5635SKent Overstreet 
1777cafe5635SKent Overstreet /* Global interfaces/init */
1778cafe5635SKent Overstreet 
1779cafe5635SKent Overstreet static ssize_t register_bcache(struct kobject *, struct kobj_attribute *,
1780cafe5635SKent Overstreet 			       const char *, size_t);
1781cafe5635SKent Overstreet 
1782cafe5635SKent Overstreet kobj_attribute_write(register,		register_bcache);
1783cafe5635SKent Overstreet kobj_attribute_write(register_quiet,	register_bcache);
1784cafe5635SKent Overstreet 
1785a9dd53adSGabriel de Perthuis static bool bch_is_open_backing(struct block_device *bdev) {
1786a9dd53adSGabriel de Perthuis 	struct cache_set *c, *tc;
1787a9dd53adSGabriel de Perthuis 	struct cached_dev *dc, *t;
1788a9dd53adSGabriel de Perthuis 
1789a9dd53adSGabriel de Perthuis 	list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
1790a9dd53adSGabriel de Perthuis 		list_for_each_entry_safe(dc, t, &c->cached_devs, list)
1791a9dd53adSGabriel de Perthuis 			if (dc->bdev == bdev)
1792a9dd53adSGabriel de Perthuis 				return true;
1793a9dd53adSGabriel de Perthuis 	list_for_each_entry_safe(dc, t, &uncached_devices, list)
1794a9dd53adSGabriel de Perthuis 		if (dc->bdev == bdev)
1795a9dd53adSGabriel de Perthuis 			return true;
1796a9dd53adSGabriel de Perthuis 	return false;
1797a9dd53adSGabriel de Perthuis }
1798a9dd53adSGabriel de Perthuis 
1799a9dd53adSGabriel de Perthuis static bool bch_is_open_cache(struct block_device *bdev) {
1800a9dd53adSGabriel de Perthuis 	struct cache_set *c, *tc;
1801a9dd53adSGabriel de Perthuis 	struct cache *ca;
1802a9dd53adSGabriel de Perthuis 	unsigned i;
1803a9dd53adSGabriel de Perthuis 
1804a9dd53adSGabriel de Perthuis 	list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
1805a9dd53adSGabriel de Perthuis 		for_each_cache(ca, c, i)
1806a9dd53adSGabriel de Perthuis 			if (ca->bdev == bdev)
1807a9dd53adSGabriel de Perthuis 				return true;
1808a9dd53adSGabriel de Perthuis 	return false;
1809a9dd53adSGabriel de Perthuis }
1810a9dd53adSGabriel de Perthuis 
1811a9dd53adSGabriel de Perthuis static bool bch_is_open(struct block_device *bdev) {
1812a9dd53adSGabriel de Perthuis 	return bch_is_open_cache(bdev) || bch_is_open_backing(bdev);
1813a9dd53adSGabriel de Perthuis }
1814a9dd53adSGabriel de Perthuis 
1815cafe5635SKent Overstreet static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
1816cafe5635SKent Overstreet 			       const char *buffer, size_t size)
1817cafe5635SKent Overstreet {
1818cafe5635SKent Overstreet 	ssize_t ret = size;
1819cafe5635SKent Overstreet 	const char *err = "cannot allocate memory";
1820cafe5635SKent Overstreet 	char *path = NULL;
1821cafe5635SKent Overstreet 	struct cache_sb *sb = NULL;
1822cafe5635SKent Overstreet 	struct block_device *bdev = NULL;
1823cafe5635SKent Overstreet 	struct page *sb_page = NULL;
1824cafe5635SKent Overstreet 
1825cafe5635SKent Overstreet 	if (!try_module_get(THIS_MODULE))
1826cafe5635SKent Overstreet 		return -EBUSY;
1827cafe5635SKent Overstreet 
1828cafe5635SKent Overstreet 	mutex_lock(&bch_register_lock);
1829cafe5635SKent Overstreet 
1830cafe5635SKent Overstreet 	if (!(path = kstrndup(buffer, size, GFP_KERNEL)) ||
1831cafe5635SKent Overstreet 	    !(sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL)))
1832cafe5635SKent Overstreet 		goto err;
1833cafe5635SKent Overstreet 
1834cafe5635SKent Overstreet 	err = "failed to open device";
1835cafe5635SKent Overstreet 	bdev = blkdev_get_by_path(strim(path),
1836cafe5635SKent Overstreet 				  FMODE_READ|FMODE_WRITE|FMODE_EXCL,
1837cafe5635SKent Overstreet 				  sb);
1838f59fce84SKent Overstreet 	if (IS_ERR(bdev)) {
1839a9dd53adSGabriel de Perthuis 		if (bdev == ERR_PTR(-EBUSY)) {
1840a9dd53adSGabriel de Perthuis 			bdev = lookup_bdev(strim(path));
1841a9dd53adSGabriel de Perthuis 			if (!IS_ERR(bdev) && bch_is_open(bdev))
1842a9dd53adSGabriel de Perthuis 				err = "device already registered";
1843a9dd53adSGabriel de Perthuis 			else
1844cafe5635SKent Overstreet 				err = "device busy";
1845a9dd53adSGabriel de Perthuis 		}
1846cafe5635SKent Overstreet 		goto err;
1847f59fce84SKent Overstreet 	}
1848f59fce84SKent Overstreet 
1849f59fce84SKent Overstreet 	err = "failed to set blocksize";
1850f59fce84SKent Overstreet 	if (set_blocksize(bdev, 4096))
1851f59fce84SKent Overstreet 		goto err_close;
1852cafe5635SKent Overstreet 
1853cafe5635SKent Overstreet 	err = read_super(sb, bdev, &sb_page);
1854cafe5635SKent Overstreet 	if (err)
1855cafe5635SKent Overstreet 		goto err_close;
1856cafe5635SKent Overstreet 
18572903381fSKent Overstreet 	if (SB_IS_BDEV(sb)) {
1858cafe5635SKent Overstreet 		struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL);
1859f59fce84SKent Overstreet 		if (!dc)
1860f59fce84SKent Overstreet 			goto err_close;
1861cafe5635SKent Overstreet 
1862f59fce84SKent Overstreet 		register_bdev(sb, sb_page, bdev, dc);
1863cafe5635SKent Overstreet 	} else {
1864cafe5635SKent Overstreet 		struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
1865f59fce84SKent Overstreet 		if (!ca)
1866f59fce84SKent Overstreet 			goto err_close;
1867cafe5635SKent Overstreet 
1868f59fce84SKent Overstreet 		register_cache(sb, sb_page, bdev, ca);
1869cafe5635SKent Overstreet 	}
1870f59fce84SKent Overstreet out:
1871f59fce84SKent Overstreet 	if (sb_page)
1872cafe5635SKent Overstreet 		put_page(sb_page);
1873f59fce84SKent Overstreet 	kfree(sb);
1874f59fce84SKent Overstreet 	kfree(path);
1875f59fce84SKent Overstreet 	mutex_unlock(&bch_register_lock);
1876f59fce84SKent Overstreet 	module_put(THIS_MODULE);
1877f59fce84SKent Overstreet 	return ret;
1878f59fce84SKent Overstreet 
1879cafe5635SKent Overstreet err_close:
1880cafe5635SKent Overstreet 	blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
1881cafe5635SKent Overstreet err:
1882cafe5635SKent Overstreet 	if (attr != &ksysfs_register_quiet)
1883cafe5635SKent Overstreet 		pr_info("error opening %s: %s", path, err);
1884cafe5635SKent Overstreet 	ret = -EINVAL;
1885f59fce84SKent Overstreet 	goto out;
1886cafe5635SKent Overstreet }
1887cafe5635SKent Overstreet 
1888cafe5635SKent Overstreet static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x)
1889cafe5635SKent Overstreet {
1890cafe5635SKent Overstreet 	if (code == SYS_DOWN ||
1891cafe5635SKent Overstreet 	    code == SYS_HALT ||
1892cafe5635SKent Overstreet 	    code == SYS_POWER_OFF) {
1893cafe5635SKent Overstreet 		DEFINE_WAIT(wait);
1894cafe5635SKent Overstreet 		unsigned long start = jiffies;
1895cafe5635SKent Overstreet 		bool stopped = false;
1896cafe5635SKent Overstreet 
1897cafe5635SKent Overstreet 		struct cache_set *c, *tc;
1898cafe5635SKent Overstreet 		struct cached_dev *dc, *tdc;
1899cafe5635SKent Overstreet 
1900cafe5635SKent Overstreet 		mutex_lock(&bch_register_lock);
1901cafe5635SKent Overstreet 
1902cafe5635SKent Overstreet 		if (list_empty(&bch_cache_sets) &&
1903cafe5635SKent Overstreet 		    list_empty(&uncached_devices))
1904cafe5635SKent Overstreet 			goto out;
1905cafe5635SKent Overstreet 
1906cafe5635SKent Overstreet 		pr_info("Stopping all devices:");
1907cafe5635SKent Overstreet 
1908cafe5635SKent Overstreet 		list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
1909cafe5635SKent Overstreet 			bch_cache_set_stop(c);
1910cafe5635SKent Overstreet 
1911cafe5635SKent Overstreet 		list_for_each_entry_safe(dc, tdc, &uncached_devices, list)
1912cafe5635SKent Overstreet 			bcache_device_stop(&dc->disk);
1913cafe5635SKent Overstreet 
1914cafe5635SKent Overstreet 		/* What's a condition variable? */
1915cafe5635SKent Overstreet 		while (1) {
1916cafe5635SKent Overstreet 			long timeout = start + 2 * HZ - jiffies;
1917cafe5635SKent Overstreet 
1918cafe5635SKent Overstreet 			stopped = list_empty(&bch_cache_sets) &&
1919cafe5635SKent Overstreet 				list_empty(&uncached_devices);
1920cafe5635SKent Overstreet 
1921cafe5635SKent Overstreet 			if (timeout < 0 || stopped)
1922cafe5635SKent Overstreet 				break;
1923cafe5635SKent Overstreet 
1924cafe5635SKent Overstreet 			prepare_to_wait(&unregister_wait, &wait,
1925cafe5635SKent Overstreet 					TASK_UNINTERRUPTIBLE);
1926cafe5635SKent Overstreet 
1927cafe5635SKent Overstreet 			mutex_unlock(&bch_register_lock);
1928cafe5635SKent Overstreet 			schedule_timeout(timeout);
1929cafe5635SKent Overstreet 			mutex_lock(&bch_register_lock);
1930cafe5635SKent Overstreet 		}
1931cafe5635SKent Overstreet 
1932cafe5635SKent Overstreet 		finish_wait(&unregister_wait, &wait);
1933cafe5635SKent Overstreet 
1934cafe5635SKent Overstreet 		if (stopped)
1935cafe5635SKent Overstreet 			pr_info("All devices stopped");
1936cafe5635SKent Overstreet 		else
1937cafe5635SKent Overstreet 			pr_notice("Timeout waiting for devices to be closed");
1938cafe5635SKent Overstreet out:
1939cafe5635SKent Overstreet 		mutex_unlock(&bch_register_lock);
1940cafe5635SKent Overstreet 	}
1941cafe5635SKent Overstreet 
1942cafe5635SKent Overstreet 	return NOTIFY_DONE;
1943cafe5635SKent Overstreet }
1944cafe5635SKent Overstreet 
1945cafe5635SKent Overstreet static struct notifier_block reboot = {
1946cafe5635SKent Overstreet 	.notifier_call	= bcache_reboot,
1947cafe5635SKent Overstreet 	.priority	= INT_MAX, /* before any real devices */
1948cafe5635SKent Overstreet };
1949cafe5635SKent Overstreet 
1950cafe5635SKent Overstreet static void bcache_exit(void)
1951cafe5635SKent Overstreet {
1952cafe5635SKent Overstreet 	bch_debug_exit();
1953cafe5635SKent Overstreet 	bch_writeback_exit();
1954cafe5635SKent Overstreet 	bch_request_exit();
1955cafe5635SKent Overstreet 	bch_btree_exit();
1956cafe5635SKent Overstreet 	if (bcache_kobj)
1957cafe5635SKent Overstreet 		kobject_put(bcache_kobj);
1958cafe5635SKent Overstreet 	if (bcache_wq)
1959cafe5635SKent Overstreet 		destroy_workqueue(bcache_wq);
1960cafe5635SKent Overstreet 	unregister_blkdev(bcache_major, "bcache");
1961cafe5635SKent Overstreet 	unregister_reboot_notifier(&reboot);
1962cafe5635SKent Overstreet }
1963cafe5635SKent Overstreet 
1964cafe5635SKent Overstreet static int __init bcache_init(void)
1965cafe5635SKent Overstreet {
1966cafe5635SKent Overstreet 	static const struct attribute *files[] = {
1967cafe5635SKent Overstreet 		&ksysfs_register.attr,
1968cafe5635SKent Overstreet 		&ksysfs_register_quiet.attr,
1969cafe5635SKent Overstreet 		NULL
1970cafe5635SKent Overstreet 	};
1971cafe5635SKent Overstreet 
1972cafe5635SKent Overstreet 	mutex_init(&bch_register_lock);
1973cafe5635SKent Overstreet 	init_waitqueue_head(&unregister_wait);
1974cafe5635SKent Overstreet 	register_reboot_notifier(&reboot);
197507e86ccbSKent Overstreet 	closure_debug_init();
1976cafe5635SKent Overstreet 
1977cafe5635SKent Overstreet 	bcache_major = register_blkdev(0, "bcache");
1978cafe5635SKent Overstreet 	if (bcache_major < 0)
1979cafe5635SKent Overstreet 		return bcache_major;
1980cafe5635SKent Overstreet 
1981cafe5635SKent Overstreet 	if (!(bcache_wq = create_workqueue("bcache")) ||
1982cafe5635SKent Overstreet 	    !(bcache_kobj = kobject_create_and_add("bcache", fs_kobj)) ||
1983cafe5635SKent Overstreet 	    sysfs_create_files(bcache_kobj, files) ||
1984cafe5635SKent Overstreet 	    bch_btree_init() ||
1985cafe5635SKent Overstreet 	    bch_request_init() ||
1986cafe5635SKent Overstreet 	    bch_writeback_init() ||
1987cafe5635SKent Overstreet 	    bch_debug_init(bcache_kobj))
1988cafe5635SKent Overstreet 		goto err;
1989cafe5635SKent Overstreet 
1990cafe5635SKent Overstreet 	return 0;
1991cafe5635SKent Overstreet err:
1992cafe5635SKent Overstreet 	bcache_exit();
1993cafe5635SKent Overstreet 	return -ENOMEM;
1994cafe5635SKent Overstreet }
1995cafe5635SKent Overstreet 
1996cafe5635SKent Overstreet module_exit(bcache_exit);
1997cafe5635SKent Overstreet module_init(bcache_init);
1998