request.c (27eb2c4b3d3e13f376a359e293c212a2e9407af5) request.c (54d12f2b4fd0f218590d1490b41a18d0e2328a9a)
1/*
2 * Main bcache entry point - handle a read or a write request and decide what to
3 * do with it; the make_request functions are called by the block layer.
4 *
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
7 */
8
9#include "bcache.h"
10#include "btree.h"
11#include "debug.h"
12#include "request.h"
1/*
2 * Main bcache entry point - handle a read or a write request and decide what to
3 * do with it; the make_request functions are called by the block layer.
4 *
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
7 */
8
9#include "bcache.h"
10#include "btree.h"
11#include "debug.h"
12#include "request.h"
13#include "writeback.h"
13
14#include <linux/cgroup.h>
15#include <linux/module.h>
16#include <linux/hash.h>
17#include <linux/random.h>
18#include "blk-cgroup.h"
19
20#include <trace/events/bcache.h>
21
22#define CUTOFF_CACHE_ADD 95
23#define CUTOFF_CACHE_READA 90
14
15#include <linux/cgroup.h>
16#include <linux/module.h>
17#include <linux/hash.h>
18#include <linux/random.h>
19#include "blk-cgroup.h"
20
21#include <trace/events/bcache.h>
22
23#define CUTOFF_CACHE_ADD 95
24#define CUTOFF_CACHE_READA 90
24#define CUTOFF_WRITEBACK 50
25#define CUTOFF_WRITEBACK_SYNC 75
26
27struct kmem_cache *bch_search_cache;
28
29static void check_should_skip(struct cached_dev *, struct search *);
30
31/* Cgroup interface */
32
33#ifdef CONFIG_CGROUP_BCACHE

--- 450 unchanged lines hidden (view full) ---

484 if (op->skip)
485 return bio_invalidate(cl);
486
487 if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) {
488 set_gc_sectors(op->c);
489 bch_queue_gc(op->c);
490 }
491
25
26struct kmem_cache *bch_search_cache;
27
28static void check_should_skip(struct cached_dev *, struct search *);
29
30/* Cgroup interface */
31
32#ifdef CONFIG_CGROUP_BCACHE

--- 450 unchanged lines hidden (view full) ---

483 if (op->skip)
484 return bio_invalidate(cl);
485
486 if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) {
487 set_gc_sectors(op->c);
488 bch_queue_gc(op->c);
489 }
490
491 /*
492 * Journal writes are marked REQ_FLUSH; if the original write was a
493 * flush, it'll wait on the journal write.
494 */
495 bio->bi_rw &= ~(REQ_FLUSH|REQ_FUA);
496
492 do {
493 unsigned i;
494 struct bkey *k;
495 struct bio_set *split = s->d
496 ? s->d->bio_split : op->c->bio_split;
497
498 /* 1 for the device pointer and 1 for the chksum */
499 if (bch_keylist_realloc(&op->keys,

--- 5 unchanged lines hidden (view full) ---

505 bkey_init(k);
506 SET_KEY_INODE(k, op->inode);
507 SET_KEY_OFFSET(k, bio->bi_sector);
508
509 if (!bch_alloc_sectors(k, bio_sectors(bio), s))
510 goto err;
511
512 n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split);
497 do {
498 unsigned i;
499 struct bkey *k;
500 struct bio_set *split = s->d
501 ? s->d->bio_split : op->c->bio_split;
502
503 /* 1 for the device pointer and 1 for the chksum */
504 if (bch_keylist_realloc(&op->keys,

--- 5 unchanged lines hidden (view full) ---

510 bkey_init(k);
511 SET_KEY_INODE(k, op->inode);
512 SET_KEY_OFFSET(k, bio->bi_sector);
513
514 if (!bch_alloc_sectors(k, bio_sectors(bio), s))
515 goto err;
516
517 n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split);
513 if (!n) {
514 __bkey_put(op->c, k);
515 continue_at(cl, bch_insert_data_loop, bcache_wq);
516 }
517
518 n->bi_end_io = bch_insert_data_endio;
519 n->bi_private = cl;
520
521 if (s->writeback) {
522 SET_KEY_DIRTY(k, true);
523
524 for (i = 0; i < KEY_PTRS(k); i++)
525 SET_GC_MARK(PTR_BUCKET(op->c, k, i),
526 GC_MARK_DIRTY);
527 }
528
529 SET_KEY_CSUM(k, op->csum);
530 if (KEY_CSUM(k))
531 bio_csum(n, k);
532
518
519 n->bi_end_io = bch_insert_data_endio;
520 n->bi_private = cl;
521
522 if (s->writeback) {
523 SET_KEY_DIRTY(k, true);
524
525 for (i = 0; i < KEY_PTRS(k); i++)
526 SET_GC_MARK(PTR_BUCKET(op->c, k, i),
527 GC_MARK_DIRTY);
528 }
529
530 SET_KEY_CSUM(k, op->csum);
531 if (KEY_CSUM(k))
532 bio_csum(n, k);
533
533 pr_debug("%s", pkey(k));
534 trace_bcache_cache_insert(k);
534 bch_keylist_push(&op->keys);
535
535 bch_keylist_push(&op->keys);
536
536 trace_bcache_cache_insert(n, n->bi_sector, n->bi_bdev);
537 n->bi_rw |= REQ_WRITE;
538 bch_submit_bbio(n, op->c, k, 0);
539 } while (n != bio);
540
541 op->insert_data_done = true;
542 continue_at(cl, bch_journal, bcache_wq);
543err:
544 /* bch_alloc_sectors() blocks if s->writeback = true */

--- 166 unchanged lines hidden (view full) ---

711
712 s->op.inode = d->id;
713 s->op.c = d->c;
714 s->d = d;
715 s->op.lock = -1;
716 s->task = current;
717 s->orig_bio = bio;
718 s->write = (bio->bi_rw & REQ_WRITE) != 0;
537 n->bi_rw |= REQ_WRITE;
538 bch_submit_bbio(n, op->c, k, 0);
539 } while (n != bio);
540
541 op->insert_data_done = true;
542 continue_at(cl, bch_journal, bcache_wq);
543err:
544 /* bch_alloc_sectors() blocks if s->writeback = true */

--- 166 unchanged lines hidden (view full) ---

711
712 s->op.inode = d->id;
713 s->op.c = d->c;
714 s->d = d;
715 s->op.lock = -1;
716 s->task = current;
717 s->orig_bio = bio;
718 s->write = (bio->bi_rw & REQ_WRITE) != 0;
719 s->op.flush_journal = (bio->bi_rw & REQ_FLUSH) != 0;
719 s->op.flush_journal = (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0;
720 s->op.skip = (bio->bi_rw & REQ_DISCARD) != 0;
721 s->recoverable = 1;
722 s->start_time = jiffies;
723 do_bio_hook(s);
724
725 if (bio->bi_size != bio_segments(bio) * PAGE_SIZE) {
726 bv = mempool_alloc(d->unaligned_bvec, GFP_NOIO);
727 memcpy(bv, bio_iovec(bio),

--- 51 unchanged lines hidden (view full) ---

779
780static void request_read_error(struct closure *cl)
781{
782 struct search *s = container_of(cl, struct search, cl);
783 struct bio_vec *bv;
784 int i;
785
786 if (s->recoverable) {
720 s->op.skip = (bio->bi_rw & REQ_DISCARD) != 0;
721 s->recoverable = 1;
722 s->start_time = jiffies;
723 do_bio_hook(s);
724
725 if (bio->bi_size != bio_segments(bio) * PAGE_SIZE) {
726 bv = mempool_alloc(d->unaligned_bvec, GFP_NOIO);
727 memcpy(bv, bio_iovec(bio),

--- 51 unchanged lines hidden (view full) ---

779
780static void request_read_error(struct closure *cl)
781{
782 struct search *s = container_of(cl, struct search, cl);
783 struct bio_vec *bv;
784 int i;
785
786 if (s->recoverable) {
787 /* The cache read failed, but we can retry from the backing
788 * device.
789 */
790 pr_debug("recovering at sector %llu",
791 (uint64_t) s->orig_bio->bi_sector);
787 /* Retry from the backing device: */
788 trace_bcache_read_retry(s->orig_bio);
792
793 s->error = 0;
794 bv = s->bio.bio.bi_io_vec;
795 do_bio_hook(s);
796 s->bio.bio.bi_io_vec = bv;
797
798 if (!s->unaligned_bvec)
799 bio_for_each_segment(bv, s->orig_bio, i)
800 bv->bv_offset = 0, bv->bv_len = PAGE_SIZE;
801 else
802 memcpy(s->bio.bio.bi_io_vec,
803 bio_iovec(s->orig_bio),
804 sizeof(struct bio_vec) *
805 bio_segments(s->orig_bio));
806
807 /* XXX: invalidate cache */
808
789
790 s->error = 0;
791 bv = s->bio.bio.bi_io_vec;
792 do_bio_hook(s);
793 s->bio.bio.bi_io_vec = bv;
794
795 if (!s->unaligned_bvec)
796 bio_for_each_segment(bv, s->orig_bio, i)
797 bv->bv_offset = 0, bv->bv_len = PAGE_SIZE;
798 else
799 memcpy(s->bio.bio.bi_io_vec,
800 bio_iovec(s->orig_bio),
801 sizeof(struct bio_vec) *
802 bio_segments(s->orig_bio));
803
804 /* XXX: invalidate cache */
805
809 trace_bcache_read_retry(&s->bio.bio);
810 closure_bio_submit(&s->bio.bio, &s->cl, s->d);
811 }
812
813 continue_at(cl, cached_dev_read_complete, NULL);
814}
815
816static void request_read_done(struct closure *cl)
817{

--- 4 unchanged lines hidden (view full) ---

822 * s->cache_bio != NULL implies that we had a cache miss; cache_bio now
823 * contains data ready to be inserted into the cache.
824 *
825 * First, we copy the data we just read from cache_bio's bounce buffers
826 * to the buffers the original bio pointed to:
827 */
828
829 if (s->op.cache_bio) {
806 closure_bio_submit(&s->bio.bio, &s->cl, s->d);
807 }
808
809 continue_at(cl, cached_dev_read_complete, NULL);
810}
811
812static void request_read_done(struct closure *cl)
813{

--- 4 unchanged lines hidden (view full) ---

818 * s->cache_bio != NULL implies that we had a cache miss; cache_bio now
819 * contains data ready to be inserted into the cache.
820 *
821 * First, we copy the data we just read from cache_bio's bounce buffers
822 * to the buffers the original bio pointed to:
823 */
824
825 if (s->op.cache_bio) {
830 struct bio_vec *src, *dst;
831 unsigned src_offset, dst_offset, bytes;
832 void *dst_ptr;
833
834 bio_reset(s->op.cache_bio);
835 s->op.cache_bio->bi_sector = s->cache_miss->bi_sector;
836 s->op.cache_bio->bi_bdev = s->cache_miss->bi_bdev;
837 s->op.cache_bio->bi_size = s->cache_bio_sectors << 9;
838 bch_bio_map(s->op.cache_bio, NULL);
839
826 bio_reset(s->op.cache_bio);
827 s->op.cache_bio->bi_sector = s->cache_miss->bi_sector;
828 s->op.cache_bio->bi_bdev = s->cache_miss->bi_bdev;
829 s->op.cache_bio->bi_size = s->cache_bio_sectors << 9;
830 bch_bio_map(s->op.cache_bio, NULL);
831
840 src = bio_iovec(s->op.cache_bio);
841 dst = bio_iovec(s->cache_miss);
842 src_offset = src->bv_offset;
843 dst_offset = dst->bv_offset;
844 dst_ptr = kmap(dst->bv_page);
832 bio_copy_data(s->cache_miss, s->op.cache_bio);
845
833
846 while (1) {
847 if (dst_offset == dst->bv_offset + dst->bv_len) {
848 kunmap(dst->bv_page);
849 dst++;
850 if (dst == bio_iovec_idx(s->cache_miss,
851 s->cache_miss->bi_vcnt))
852 break;
853
854 dst_offset = dst->bv_offset;
855 dst_ptr = kmap(dst->bv_page);
856 }
857
858 if (src_offset == src->bv_offset + src->bv_len) {
859 src++;
860 if (src == bio_iovec_idx(s->op.cache_bio,
861 s->op.cache_bio->bi_vcnt))
862 BUG();
863
864 src_offset = src->bv_offset;
865 }
866
867 bytes = min(dst->bv_offset + dst->bv_len - dst_offset,
868 src->bv_offset + src->bv_len - src_offset);
869
870 memcpy(dst_ptr + dst_offset,
871 page_address(src->bv_page) + src_offset,
872 bytes);
873
874 src_offset += bytes;
875 dst_offset += bytes;
876 }
877
878 bio_put(s->cache_miss);
879 s->cache_miss = NULL;
880 }
881
882 if (verify(dc, &s->bio.bio) && s->recoverable)
883 bch_data_verify(s);
884
885 bio_complete(s);

--- 8 unchanged lines hidden (view full) ---

894}
895
896static void request_read_done_bh(struct closure *cl)
897{
898 struct search *s = container_of(cl, struct search, cl);
899 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
900
901 bch_mark_cache_accounting(s, !s->cache_miss, s->op.skip);
834 bio_put(s->cache_miss);
835 s->cache_miss = NULL;
836 }
837
838 if (verify(dc, &s->bio.bio) && s->recoverable)
839 bch_data_verify(s);
840
841 bio_complete(s);

--- 8 unchanged lines hidden (view full) ---

850}
851
852static void request_read_done_bh(struct closure *cl)
853{
854 struct search *s = container_of(cl, struct search, cl);
855 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
856
857 bch_mark_cache_accounting(s, !s->cache_miss, s->op.skip);
858 trace_bcache_read(s->orig_bio, !s->cache_miss, s->op.skip);
902
903 if (s->error)
904 continue_at_nobarrier(cl, request_read_error, bcache_wq);
905 else if (s->op.cache_bio || verify(dc, &s->bio.bio))
906 continue_at_nobarrier(cl, request_read_done, bcache_wq);
907 else
908 continue_at_nobarrier(cl, cached_dev_read_complete, NULL);
909}
910
911static int cached_dev_cache_miss(struct btree *b, struct search *s,
912 struct bio *bio, unsigned sectors)
913{
914 int ret = 0;
915 unsigned reada;
916 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
917 struct bio *miss;
918
919 miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
859
860 if (s->error)
861 continue_at_nobarrier(cl, request_read_error, bcache_wq);
862 else if (s->op.cache_bio || verify(dc, &s->bio.bio))
863 continue_at_nobarrier(cl, request_read_done, bcache_wq);
864 else
865 continue_at_nobarrier(cl, cached_dev_read_complete, NULL);
866}
867
868static int cached_dev_cache_miss(struct btree *b, struct search *s,
869 struct bio *bio, unsigned sectors)
870{
871 int ret = 0;
872 unsigned reada;
873 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
874 struct bio *miss;
875
876 miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
920 if (!miss)
921 return -EAGAIN;
922
923 if (miss == bio)
924 s->op.lookup_done = true;
925
926 miss->bi_end_io = request_endio;
927 miss->bi_private = &s->cl;
928
929 if (s->cache_miss || s->op.skip)
930 goto out_submit;
931
932 if (miss != bio ||
933 (bio->bi_rw & REQ_RAHEAD) ||
934 (bio->bi_rw & REQ_META) ||
935 s->op.c->gc_stats.in_use >= CUTOFF_CACHE_READA)
936 reada = 0;
937 else {
938 reada = min(dc->readahead >> 9,
939 sectors - bio_sectors(miss));
940
877 if (miss == bio)
878 s->op.lookup_done = true;
879
880 miss->bi_end_io = request_endio;
881 miss->bi_private = &s->cl;
882
883 if (s->cache_miss || s->op.skip)
884 goto out_submit;
885
886 if (miss != bio ||
887 (bio->bi_rw & REQ_RAHEAD) ||
888 (bio->bi_rw & REQ_META) ||
889 s->op.c->gc_stats.in_use >= CUTOFF_CACHE_READA)
890 reada = 0;
891 else {
892 reada = min(dc->readahead >> 9,
893 sectors - bio_sectors(miss));
894
941 if (bio_end(miss) + reada > bdev_sectors(miss->bi_bdev))
942 reada = bdev_sectors(miss->bi_bdev) - bio_end(miss);
895 if (bio_end_sector(miss) + reada > bdev_sectors(miss->bi_bdev))
896 reada = bdev_sectors(miss->bi_bdev) -
897 bio_end_sector(miss);
943 }
944
945 s->cache_bio_sectors = bio_sectors(miss) + reada;
946 s->op.cache_bio = bio_alloc_bioset(GFP_NOWAIT,
947 DIV_ROUND_UP(s->cache_bio_sectors, PAGE_SECTORS),
948 dc->disk.bio_split);
949
950 if (!s->op.cache_bio)

--- 7 unchanged lines hidden (view full) ---

958 s->op.cache_bio->bi_private = &s->cl;
959
960 /* btree_search_recurse()'s btree iterator is no good anymore */
961 ret = -EINTR;
962 if (!bch_btree_insert_check_key(b, &s->op, s->op.cache_bio))
963 goto out_put;
964
965 bch_bio_map(s->op.cache_bio, NULL);
898 }
899
900 s->cache_bio_sectors = bio_sectors(miss) + reada;
901 s->op.cache_bio = bio_alloc_bioset(GFP_NOWAIT,
902 DIV_ROUND_UP(s->cache_bio_sectors, PAGE_SECTORS),
903 dc->disk.bio_split);
904
905 if (!s->op.cache_bio)

--- 7 unchanged lines hidden (view full) ---

913 s->op.cache_bio->bi_private = &s->cl;
914
915 /* btree_search_recurse()'s btree iterator is no good anymore */
916 ret = -EINTR;
917 if (!bch_btree_insert_check_key(b, &s->op, s->op.cache_bio))
918 goto out_put;
919
920 bch_bio_map(s->op.cache_bio, NULL);
966 if (bch_bio_alloc_pages(s->op.cache_bio, __GFP_NOWARN|GFP_NOIO))
921 if (bio_alloc_pages(s->op.cache_bio, __GFP_NOWARN|GFP_NOIO))
967 goto out_put;
968
969 s->cache_miss = miss;
970 bio_get(s->op.cache_bio);
971
922 goto out_put;
923
924 s->cache_miss = miss;
925 bio_get(s->op.cache_bio);
926
972 trace_bcache_cache_miss(s->orig_bio);
973 closure_bio_submit(s->op.cache_bio, &s->cl, s->d);
974
975 return ret;
976out_put:
977 bio_put(s->op.cache_bio);
978 s->op.cache_bio = NULL;
979out_submit:
980 closure_bio_submit(miss, &s->cl, s->d);

--- 16 unchanged lines hidden (view full) ---

997{
998 struct search *s = container_of(cl, struct search, cl);
999 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
1000
1001 up_read_non_owner(&dc->writeback_lock);
1002 cached_dev_bio_complete(cl);
1003}
1004
927 closure_bio_submit(s->op.cache_bio, &s->cl, s->d);
928
929 return ret;
930out_put:
931 bio_put(s->op.cache_bio);
932 s->op.cache_bio = NULL;
933out_submit:
934 closure_bio_submit(miss, &s->cl, s->d);

--- 16 unchanged lines hidden (view full) ---

951{
952 struct search *s = container_of(cl, struct search, cl);
953 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
954
955 up_read_non_owner(&dc->writeback_lock);
956 cached_dev_bio_complete(cl);
957}
958
1005static bool should_writeback(struct cached_dev *dc, struct bio *bio)
1006{
1007 unsigned threshold = (bio->bi_rw & REQ_SYNC)
1008 ? CUTOFF_WRITEBACK_SYNC
1009 : CUTOFF_WRITEBACK;
1010
1011 return !atomic_read(&dc->disk.detaching) &&
1012 cache_mode(dc, bio) == CACHE_MODE_WRITEBACK &&
1013 dc->disk.c->gc_stats.in_use < threshold;
1014}
1015
1016static void request_write(struct cached_dev *dc, struct search *s)
1017{
1018 struct closure *cl = &s->cl;
1019 struct bio *bio = &s->bio.bio;
1020 struct bkey start, end;
1021 start = KEY(dc->disk.id, bio->bi_sector, 0);
959static void request_write(struct cached_dev *dc, struct search *s)
960{
961 struct closure *cl = &s->cl;
962 struct bio *bio = &s->bio.bio;
963 struct bkey start, end;
964 start = KEY(dc->disk.id, bio->bi_sector, 0);
1022 end = KEY(dc->disk.id, bio_end(bio), 0);
965 end = KEY(dc->disk.id, bio_end_sector(bio), 0);
1023
1024 bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys, &start, &end);
1025
1026 check_should_skip(dc, s);
1027 down_read_non_owner(&dc->writeback_lock);
1028
1029 if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) {
1030 s->op.skip = false;
1031 s->writeback = true;
1032 }
1033
1034 if (bio->bi_rw & REQ_DISCARD)
1035 goto skip;
1036
966
967 bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys, &start, &end);
968
969 check_should_skip(dc, s);
970 down_read_non_owner(&dc->writeback_lock);
971
972 if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) {
973 s->op.skip = false;
974 s->writeback = true;
975 }
976
977 if (bio->bi_rw & REQ_DISCARD)
978 goto skip;
979
980 if (should_writeback(dc, s->orig_bio,
981 cache_mode(dc, bio),
982 s->op.skip)) {
983 s->op.skip = false;
984 s->writeback = true;
985 }
986
1037 if (s->op.skip)
1038 goto skip;
1039
987 if (s->op.skip)
988 goto skip;
989
1040 if (should_writeback(dc, s->orig_bio))
1041 s->writeback = true;
990 trace_bcache_write(s->orig_bio, s->writeback, s->op.skip);
1042
1043 if (!s->writeback) {
1044 s->op.cache_bio = bio_clone_bioset(bio, GFP_NOIO,
1045 dc->disk.bio_split);
1046
991
992 if (!s->writeback) {
993 s->op.cache_bio = bio_clone_bioset(bio, GFP_NOIO,
994 dc->disk.bio_split);
995
1047 trace_bcache_writethrough(s->orig_bio);
1048 closure_bio_submit(bio, cl, s->d);
1049 } else {
996 closure_bio_submit(bio, cl, s->d);
997 } else {
1050 s->op.cache_bio = bio;
1051 trace_bcache_writeback(s->orig_bio);
1052 bch_writeback_add(dc, bio_sectors(bio));
998 bch_writeback_add(dc);
999
1000 if (s->op.flush_journal) {
1001 /* Also need to send a flush to the backing device */
1002 s->op.cache_bio = bio_clone_bioset(bio, GFP_NOIO,
1003 dc->disk.bio_split);
1004
1005 bio->bi_size = 0;
1006 bio->bi_vcnt = 0;
1007 closure_bio_submit(bio, cl, s->d);
1008 } else {
1009 s->op.cache_bio = bio;
1010 }
1053 }
1054out:
1055 closure_call(&s->op.cl, bch_insert_data, NULL, cl);
1056 continue_at(cl, cached_dev_write_complete, NULL);
1057skip:
1058 s->op.skip = true;
1059 s->op.cache_bio = s->orig_bio;
1060 bio_get(s->op.cache_bio);
1011 }
1012out:
1013 closure_call(&s->op.cl, bch_insert_data, NULL, cl);
1014 continue_at(cl, cached_dev_write_complete, NULL);
1015skip:
1016 s->op.skip = true;
1017 s->op.cache_bio = s->orig_bio;
1018 bio_get(s->op.cache_bio);
1061 trace_bcache_write_skip(s->orig_bio);
1062
1063 if ((bio->bi_rw & REQ_DISCARD) &&
1064 !blk_queue_discard(bdev_get_queue(dc->bdev)))
1065 goto out;
1066
1067 closure_bio_submit(bio, cl, s->d);
1068 goto out;
1069}

--- 13 unchanged lines hidden (view full) ---

1083
1084 closure_bio_submit(bio, cl, s->d);
1085
1086 continue_at(cl, cached_dev_bio_complete, NULL);
1087}
1088
1089/* Cached devices - read & write stuff */
1090
1019
1020 if ((bio->bi_rw & REQ_DISCARD) &&
1021 !blk_queue_discard(bdev_get_queue(dc->bdev)))
1022 goto out;
1023
1024 closure_bio_submit(bio, cl, s->d);
1025 goto out;
1026}

--- 13 unchanged lines hidden (view full) ---

1040
1041 closure_bio_submit(bio, cl, s->d);
1042
1043 continue_at(cl, cached_dev_bio_complete, NULL);
1044}
1045
1046/* Cached devices - read & write stuff */
1047
1091int bch_get_congested(struct cache_set *c)
1048unsigned bch_get_congested(struct cache_set *c)
1092{
1093 int i;
1049{
1050 int i;
1051 long rand;
1094
1095 if (!c->congested_read_threshold_us &&
1096 !c->congested_write_threshold_us)
1097 return 0;
1098
1099 i = (local_clock_us() - c->congested_last_us) / 1024;
1100 if (i < 0)
1101 return 0;
1102
1103 i += atomic_read(&c->congested);
1104 if (i >= 0)
1105 return 0;
1106
1107 i += CONGESTED_MAX;
1108
1052
1053 if (!c->congested_read_threshold_us &&
1054 !c->congested_write_threshold_us)
1055 return 0;
1056
1057 i = (local_clock_us() - c->congested_last_us) / 1024;
1058 if (i < 0)
1059 return 0;
1060
1061 i += atomic_read(&c->congested);
1062 if (i >= 0)
1063 return 0;
1064
1065 i += CONGESTED_MAX;
1066
1109 return i <= 0 ? 1 : fract_exp_two(i, 6);
1067 if (i > 0)
1068 i = fract_exp_two(i, 6);
1069
1070 rand = get_random_int();
1071 i -= bitmap_weight(&rand, BITS_PER_LONG);
1072
1073 return i > 0 ? i : 1;
1110}
1111
1112static void add_sequential(struct task_struct *t)
1113{
1114 ewma_add(t->sequential_io_avg,
1115 t->sequential_io, 8, 0);
1116
1117 t->sequential_io = 0;
1118}
1119
1120static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k)
1121{
1122 return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];
1123}
1124
1125static void check_should_skip(struct cached_dev *dc, struct search *s)
1126{
1127 struct cache_set *c = s->op.c;
1128 struct bio *bio = &s->bio.bio;
1074}
1075
1076static void add_sequential(struct task_struct *t)
1077{
1078 ewma_add(t->sequential_io_avg,
1079 t->sequential_io, 8, 0);
1080
1081 t->sequential_io = 0;
1082}
1083
1084static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k)
1085{
1086 return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];
1087}
1088
1089static void check_should_skip(struct cached_dev *dc, struct search *s)
1090{
1091 struct cache_set *c = s->op.c;
1092 struct bio *bio = &s->bio.bio;
1129
1130 long rand;
1131 int cutoff = bch_get_congested(c);
1132 unsigned mode = cache_mode(dc, bio);
1093 unsigned mode = cache_mode(dc, bio);
1094 unsigned sectors, congested = bch_get_congested(c);
1133
1134 if (atomic_read(&dc->disk.detaching) ||
1135 c->gc_stats.in_use > CUTOFF_CACHE_ADD ||
1136 (bio->bi_rw & REQ_DISCARD))
1137 goto skip;
1138
1139 if (mode == CACHE_MODE_NONE ||
1140 (mode == CACHE_MODE_WRITEAROUND &&
1141 (bio->bi_rw & REQ_WRITE)))
1142 goto skip;
1143
1144 if (bio->bi_sector & (c->sb.block_size - 1) ||
1145 bio_sectors(bio) & (c->sb.block_size - 1)) {
1146 pr_debug("skipping unaligned io");
1147 goto skip;
1148 }
1149
1095
1096 if (atomic_read(&dc->disk.detaching) ||
1097 c->gc_stats.in_use > CUTOFF_CACHE_ADD ||
1098 (bio->bi_rw & REQ_DISCARD))
1099 goto skip;
1100
1101 if (mode == CACHE_MODE_NONE ||
1102 (mode == CACHE_MODE_WRITEAROUND &&
1103 (bio->bi_rw & REQ_WRITE)))
1104 goto skip;
1105
1106 if (bio->bi_sector & (c->sb.block_size - 1) ||
1107 bio_sectors(bio) & (c->sb.block_size - 1)) {
1108 pr_debug("skipping unaligned io");
1109 goto skip;
1110 }
1111
1150 if (!cutoff) {
1151 cutoff = dc->sequential_cutoff >> 9;
1112 if (!congested && !dc->sequential_cutoff)
1113 goto rescale;
1152
1114
1153 if (!cutoff)
1154 goto rescale;
1115 if (!congested &&
1116 mode == CACHE_MODE_WRITEBACK &&
1117 (bio->bi_rw & REQ_WRITE) &&
1118 (bio->bi_rw & REQ_SYNC))
1119 goto rescale;
1155
1120
1156 if (mode == CACHE_MODE_WRITEBACK &&
1157 (bio->bi_rw & REQ_WRITE) &&
1158 (bio->bi_rw & REQ_SYNC))
1159 goto rescale;
1160 }
1161
1162 if (dc->sequential_merge) {
1163 struct io *i;
1164
1165 spin_lock(&dc->io_lock);
1166
1167 hlist_for_each_entry(i, iohash(dc, bio->bi_sector), hash)
1168 if (i->last == bio->bi_sector &&
1169 time_before(jiffies, i->jiffies))
1170 goto found;
1171
1172 i = list_first_entry(&dc->io_lru, struct io, lru);
1173
1174 add_sequential(s->task);
1175 i->sequential = 0;
1176found:
1177 if (i->sequential + bio->bi_size > i->sequential)
1178 i->sequential += bio->bi_size;
1179
1121 if (dc->sequential_merge) {
1122 struct io *i;
1123
1124 spin_lock(&dc->io_lock);
1125
1126 hlist_for_each_entry(i, iohash(dc, bio->bi_sector), hash)
1127 if (i->last == bio->bi_sector &&
1128 time_before(jiffies, i->jiffies))
1129 goto found;
1130
1131 i = list_first_entry(&dc->io_lru, struct io, lru);
1132
1133 add_sequential(s->task);
1134 i->sequential = 0;
1135found:
1136 if (i->sequential + bio->bi_size > i->sequential)
1137 i->sequential += bio->bi_size;
1138
1180 i->last = bio_end(bio);
1139 i->last = bio_end_sector(bio);
1181 i->jiffies = jiffies + msecs_to_jiffies(5000);
1182 s->task->sequential_io = i->sequential;
1183
1184 hlist_del(&i->hash);
1185 hlist_add_head(&i->hash, iohash(dc, i->last));
1186 list_move_tail(&i->lru, &dc->io_lru);
1187
1188 spin_unlock(&dc->io_lock);
1189 } else {
1190 s->task->sequential_io = bio->bi_size;
1191
1192 add_sequential(s->task);
1193 }
1194
1140 i->jiffies = jiffies + msecs_to_jiffies(5000);
1141 s->task->sequential_io = i->sequential;
1142
1143 hlist_del(&i->hash);
1144 hlist_add_head(&i->hash, iohash(dc, i->last));
1145 list_move_tail(&i->lru, &dc->io_lru);
1146
1147 spin_unlock(&dc->io_lock);
1148 } else {
1149 s->task->sequential_io = bio->bi_size;
1150
1151 add_sequential(s->task);
1152 }
1153
1195 rand = get_random_int();
1196 cutoff -= bitmap_weight(&rand, BITS_PER_LONG);
1154 sectors = max(s->task->sequential_io,
1155 s->task->sequential_io_avg) >> 9;
1197
1156
1198 if (cutoff <= (int) (max(s->task->sequential_io,
1199 s->task->sequential_io_avg) >> 9))
1157 if (dc->sequential_cutoff &&
1158 sectors >= dc->sequential_cutoff >> 9) {
1159 trace_bcache_bypass_sequential(s->orig_bio);
1200 goto skip;
1160 goto skip;
1161 }
1201
1162
1163 if (congested && sectors >= congested) {
1164 trace_bcache_bypass_congested(s->orig_bio);
1165 goto skip;
1166 }
1167
1202rescale:
1203 bch_rescale_priorities(c, bio_sectors(bio));
1204 return;
1205skip:
1206 bch_mark_sectors_bypassed(s, bio_sectors(bio));
1207 s->op.skip = true;
1208}
1209

--- 73 unchanged lines hidden (view full) ---

1283 dc->disk.ioctl = cached_dev_ioctl;
1284}
1285
1286/* Flash backed devices */
1287
1288static int flash_dev_cache_miss(struct btree *b, struct search *s,
1289 struct bio *bio, unsigned sectors)
1290{
1168rescale:
1169 bch_rescale_priorities(c, bio_sectors(bio));
1170 return;
1171skip:
1172 bch_mark_sectors_bypassed(s, bio_sectors(bio));
1173 s->op.skip = true;
1174}
1175

--- 73 unchanged lines hidden (view full) ---

1249 dc->disk.ioctl = cached_dev_ioctl;
1250}
1251
1252/* Flash backed devices */
1253
1254static int flash_dev_cache_miss(struct btree *b, struct search *s,
1255 struct bio *bio, unsigned sectors)
1256{
1257 struct bio_vec *bv;
1258 int i;
1259
1291 /* Zero fill bio */
1292
1260 /* Zero fill bio */
1261
1293 while (bio->bi_idx != bio->bi_vcnt) {
1294 struct bio_vec *bv = bio_iovec(bio);
1262 bio_for_each_segment(bv, bio, i) {
1295 unsigned j = min(bv->bv_len >> 9, sectors);
1296
1297 void *p = kmap(bv->bv_page);
1298 memset(p + bv->bv_offset, 0, j << 9);
1299 kunmap(bv->bv_page);
1300
1263 unsigned j = min(bv->bv_len >> 9, sectors);
1264
1265 void *p = kmap(bv->bv_page);
1266 memset(p + bv->bv_offset, 0, j << 9);
1267 kunmap(bv->bv_page);
1268
1301 bv->bv_len -= j << 9;
1302 bv->bv_offset += j << 9;
1303
1304 if (bv->bv_len)
1305 return 0;
1306
1307 bio->bi_sector += j;
1308 bio->bi_size -= j << 9;
1309
1310 bio->bi_idx++;
1311 sectors -= j;
1269 sectors -= j;
1312 }
1313
1270 }
1271
1314 s->op.lookup_done = true;
1272 bio_advance(bio, min(sectors << 9, bio->bi_size));
1315
1273
1274 if (!bio->bi_size)
1275 s->op.lookup_done = true;
1276
1316 return 0;
1317}
1318
1319static void flash_dev_make_request(struct request_queue *q, struct bio *bio)
1320{
1321 struct search *s;
1322 struct closure *cl;
1323 struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;

--- 9 unchanged lines hidden (view full) ---

1333 bio = &s->bio.bio;
1334
1335 trace_bcache_request_start(s, bio);
1336
1337 if (bio_has_data(bio) && !rw) {
1338 closure_call(&s->op.cl, btree_read_async, NULL, cl);
1339 } else if (bio_has_data(bio) || s->op.skip) {
1340 bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys,
1277 return 0;
1278}
1279
1280static void flash_dev_make_request(struct request_queue *q, struct bio *bio)
1281{
1282 struct search *s;
1283 struct closure *cl;
1284 struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;

--- 9 unchanged lines hidden (view full) ---

1294 bio = &s->bio.bio;
1295
1296 trace_bcache_request_start(s, bio);
1297
1298 if (bio_has_data(bio) && !rw) {
1299 closure_call(&s->op.cl, btree_read_async, NULL, cl);
1300 } else if (bio_has_data(bio) || s->op.skip) {
1301 bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys,
1341 &KEY(d->id, bio->bi_sector, 0),
1342 &KEY(d->id, bio_end(bio), 0));
1302 &KEY(d->id, bio->bi_sector, 0),
1303 &KEY(d->id, bio_end_sector(bio), 0));
1343
1344 s->writeback = true;
1345 s->op.cache_bio = bio;
1346
1347 closure_call(&s->op.cl, bch_insert_data, NULL, cl);
1348 } else {
1349 /* No data - probably a cache flush */
1350 if (s->op.flush_journal)

--- 61 unchanged lines hidden --- 		1304
1305 s->writeback = true;
1306 s->op.cache_bio = bio;
1307
1308 closure_call(&s->op.cl, bch_insert_data, NULL, cl);
1309 } else {
1310 /* No data - probably a cache flush */
1311 if (s->op.flush_journal)

--- 61 unchanged lines hidden ---