1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2cafe5635SKent Overstreet /*
3cafe5635SKent Overstreet * Some low level IO code, and hacks for various block layer limitations
4cafe5635SKent Overstreet *
5cafe5635SKent Overstreet * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6cafe5635SKent Overstreet * Copyright 2012 Google, Inc.
7cafe5635SKent Overstreet */
8cafe5635SKent Overstreet
9cafe5635SKent Overstreet #include "bcache.h"
10cafe5635SKent Overstreet #include "bset.h"
11cafe5635SKent Overstreet #include "debug.h"
12cafe5635SKent Overstreet
13c37511b8SKent Overstreet #include <linux/blkdev.h>
14c37511b8SKent Overstreet
15cafe5635SKent Overstreet /* Bios with headers */
16cafe5635SKent Overstreet
bch_bbio_free(struct bio * bio,struct cache_set * c)17cafe5635SKent Overstreet void bch_bbio_free(struct bio *bio, struct cache_set *c)
18cafe5635SKent Overstreet {
19cafe5635SKent Overstreet struct bbio *b = container_of(bio, struct bbio, bio);
201fae7cf0SColy Li
21d19936a2SKent Overstreet mempool_free(b, &c->bio_meta);
22cafe5635SKent Overstreet }
23cafe5635SKent Overstreet
bch_bbio_alloc(struct cache_set * c)24cafe5635SKent Overstreet struct bio *bch_bbio_alloc(struct cache_set *c)
25cafe5635SKent Overstreet {
26d19936a2SKent Overstreet struct bbio *b = mempool_alloc(&c->bio_meta, GFP_NOIO);
27cafe5635SKent Overstreet struct bio *bio = &b->bio;
28cafe5635SKent Overstreet
29*49add496SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs,
30*49add496SChristoph Hellwig meta_bucket_pages(&c->cache->sb), 0);
31cafe5635SKent Overstreet
32cafe5635SKent Overstreet return bio;
33cafe5635SKent Overstreet }
34cafe5635SKent Overstreet
__bch_submit_bbio(struct bio * bio,struct cache_set * c)35cafe5635SKent Overstreet void __bch_submit_bbio(struct bio *bio, struct cache_set *c)
36cafe5635SKent Overstreet {
37cafe5635SKent Overstreet struct bbio *b = container_of(bio, struct bbio, bio);
38cafe5635SKent Overstreet
394f024f37SKent Overstreet bio->bi_iter.bi_sector = PTR_OFFSET(&b->key, 0);
4011e9560eSChristoph Hellwig bio_set_dev(bio, c->cache->bdev);
41cafe5635SKent Overstreet
42cafe5635SKent Overstreet b->submit_time_us = local_clock_us();
43771f393eSColy Li closure_bio_submit(c, bio, bio->bi_private);
44cafe5635SKent Overstreet }
45cafe5635SKent Overstreet
bch_submit_bbio(struct bio * bio,struct cache_set * c,struct bkey * k,unsigned int ptr)46cafe5635SKent Overstreet void bch_submit_bbio(struct bio *bio, struct cache_set *c,
476f10f7d1SColy Li struct bkey *k, unsigned int ptr)
48cafe5635SKent Overstreet {
49cafe5635SKent Overstreet struct bbio *b = container_of(bio, struct bbio, bio);
501fae7cf0SColy Li
51cafe5635SKent Overstreet bch_bkey_copy_single_ptr(&b->key, k, ptr);
52cafe5635SKent Overstreet __bch_submit_bbio(bio, c);
53cafe5635SKent Overstreet }
54cafe5635SKent Overstreet
55cafe5635SKent Overstreet /* IO errors */
bch_count_backing_io_errors(struct cached_dev * dc,struct bio * bio)56c7b7bd07SColy Li void bch_count_backing_io_errors(struct cached_dev *dc, struct bio *bio)
57c7b7bd07SColy Li {
586f10f7d1SColy Li unsigned int errors;
59c7b7bd07SColy Li
60c7b7bd07SColy Li WARN_ONCE(!dc, "NULL pointer of struct cached_dev");
61c7b7bd07SColy Li
62578df99bSColy Li /*
63578df99bSColy Li * Read-ahead requests on a degrading and recovering md raid
64578df99bSColy Li * (e.g. raid6) device might be failured immediately by md
65578df99bSColy Li * raid code, which is not a real hardware media failure. So
66578df99bSColy Li * we shouldn't count failed REQ_RAHEAD bio to dc->io_errors.
67578df99bSColy Li */
68578df99bSColy Li if (bio->bi_opf & REQ_RAHEAD) {
690f5cd781SChristoph Hellwig pr_warn_ratelimited("%pg: Read-ahead I/O failed on backing device, ignore\n",
700f5cd781SChristoph Hellwig dc->bdev);
71578df99bSColy Li return;
72578df99bSColy Li }
73578df99bSColy Li
74c7b7bd07SColy Li errors = atomic_add_return(1, &dc->io_errors);
75c7b7bd07SColy Li if (errors < dc->error_limit)
760f5cd781SChristoph Hellwig pr_err("%pg: IO error on backing device, unrecoverable\n",
770f5cd781SChristoph Hellwig dc->bdev);
78c7b7bd07SColy Li else
79c7b7bd07SColy Li bch_cached_dev_error(dc);
80c7b7bd07SColy Li }
81cafe5635SKent Overstreet
bch_count_io_errors(struct cache * ca,blk_status_t error,int is_read,const char * m)825138ac67SColy Li void bch_count_io_errors(struct cache *ca,
835138ac67SColy Li blk_status_t error,
845138ac67SColy Li int is_read,
855138ac67SColy Li const char *m)
86cafe5635SKent Overstreet {
87cafe5635SKent Overstreet /*
88cafe5635SKent Overstreet * The halflife of an error is:
89cafe5635SKent Overstreet * log2(1/2)/log2(127/128) * refresh ~= 88 * refresh
90cafe5635SKent Overstreet */
91cafe5635SKent Overstreet
92cafe5635SKent Overstreet if (ca->set->error_decay) {
936f10f7d1SColy Li unsigned int count = atomic_inc_return(&ca->io_count);
94cafe5635SKent Overstreet
95cafe5635SKent Overstreet while (count > ca->set->error_decay) {
966f10f7d1SColy Li unsigned int errors;
976f10f7d1SColy Li unsigned int old = count;
986f10f7d1SColy Li unsigned int new = count - ca->set->error_decay;
99cafe5635SKent Overstreet
100cafe5635SKent Overstreet /*
101cafe5635SKent Overstreet * First we subtract refresh from count; each time we
1022b1edd23SColy Li * successfully do so, we rescale the errors once:
103cafe5635SKent Overstreet */
104cafe5635SKent Overstreet
105cafe5635SKent Overstreet count = atomic_cmpxchg(&ca->io_count, old, new);
106cafe5635SKent Overstreet
107cafe5635SKent Overstreet if (count == old) {
108cafe5635SKent Overstreet count = new;
109cafe5635SKent Overstreet
110cafe5635SKent Overstreet errors = atomic_read(&ca->io_errors);
111cafe5635SKent Overstreet do {
112cafe5635SKent Overstreet old = errors;
113cafe5635SKent Overstreet new = ((uint64_t) errors * 127) / 128;
114cafe5635SKent Overstreet errors = atomic_cmpxchg(&ca->io_errors,
115cafe5635SKent Overstreet old, new);
116cafe5635SKent Overstreet } while (old != errors);
117cafe5635SKent Overstreet }
118cafe5635SKent Overstreet }
119cafe5635SKent Overstreet }
120cafe5635SKent Overstreet
121cafe5635SKent Overstreet if (error) {
1226f10f7d1SColy Li unsigned int errors = atomic_add_return(1 << IO_ERROR_SHIFT,
123cafe5635SKent Overstreet &ca->io_errors);
124cafe5635SKent Overstreet errors >>= IO_ERROR_SHIFT;
125cafe5635SKent Overstreet
126cafe5635SKent Overstreet if (errors < ca->set->error_limit)
1277e84c215SChristoph Hellwig pr_err("%pg: IO error on %s%s\n",
1287e84c215SChristoph Hellwig ca->bdev, m,
1295138ac67SColy Li is_read ? ", recovering." : ".");
130cafe5635SKent Overstreet else
131cafe5635SKent Overstreet bch_cache_set_error(ca->set,
1327e84c215SChristoph Hellwig "%pg: too many IO errors %s\n",
1337e84c215SChristoph Hellwig ca->bdev, m);
134cafe5635SKent Overstreet }
135cafe5635SKent Overstreet }
136cafe5635SKent Overstreet
bch_bbio_count_io_errors(struct cache_set * c,struct bio * bio,blk_status_t error,const char * m)137cafe5635SKent Overstreet void bch_bbio_count_io_errors(struct cache_set *c, struct bio *bio,
1384e4cbee9SChristoph Hellwig blk_status_t error, const char *m)
139cafe5635SKent Overstreet {
140cafe5635SKent Overstreet struct bbio *b = container_of(bio, struct bbio, bio);
14111e9560eSChristoph Hellwig struct cache *ca = c->cache;
1425138ac67SColy Li int is_read = (bio_data_dir(bio) == READ ? 1 : 0);
143cafe5635SKent Overstreet
1446f10f7d1SColy Li unsigned int threshold = op_is_write(bio_op(bio))
145cafe5635SKent Overstreet ? c->congested_write_threshold_us
146cafe5635SKent Overstreet : c->congested_read_threshold_us;
147cafe5635SKent Overstreet
148cafe5635SKent Overstreet if (threshold) {
1496f10f7d1SColy Li unsigned int t = local_clock_us();
150cafe5635SKent Overstreet int us = t - b->submit_time_us;
151cafe5635SKent Overstreet int congested = atomic_read(&c->congested);
152cafe5635SKent Overstreet
153cafe5635SKent Overstreet if (us > (int) threshold) {
154cafe5635SKent Overstreet int ms = us / 1024;
1551fae7cf0SColy Li
156cafe5635SKent Overstreet c->congested_last_us = t;
157cafe5635SKent Overstreet
158cafe5635SKent Overstreet ms = min(ms, CONGESTED_MAX + congested);
159cafe5635SKent Overstreet atomic_sub(ms, &c->congested);
160cafe5635SKent Overstreet } else if (congested < 0)
161cafe5635SKent Overstreet atomic_inc(&c->congested);
162cafe5635SKent Overstreet }
163cafe5635SKent Overstreet
1645138ac67SColy Li bch_count_io_errors(ca, error, is_read, m);
165cafe5635SKent Overstreet }
166cafe5635SKent Overstreet
bch_bbio_endio(struct cache_set * c,struct bio * bio,blk_status_t error,const char * m)167cafe5635SKent Overstreet void bch_bbio_endio(struct cache_set *c, struct bio *bio,
1684e4cbee9SChristoph Hellwig blk_status_t error, const char *m)
169cafe5635SKent Overstreet {
170cafe5635SKent Overstreet struct closure *cl = bio->bi_private;
171cafe5635SKent Overstreet
172cafe5635SKent Overstreet bch_bbio_count_io_errors(c, bio, error, m);
173cafe5635SKent Overstreet bio_put(bio);
174cafe5635SKent Overstreet closure_put(cl);
175cafe5635SKent Overstreet }
176