segment.c (46f84c2c058784f42f2d021df79384ec66cdb256) segment.c (ba48a33ef6faa573257b2a4181329f2d1eaafed9)
1/*
2 * fs/f2fs/segment.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as

--- 663 unchanged lines hidden (view full) ---

672 mutex_unlock(&dirty_i->seglist_lock);
673}
674
675static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
676 struct block_device *bdev, block_t lstart,
677 block_t start, block_t len)
678{
679 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1/*
2 * fs/f2fs/segment.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as

--- 663 unchanged lines hidden (view full) ---

672 mutex_unlock(&dirty_i->seglist_lock);
673}
674
675static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
676 struct block_device *bdev, block_t lstart,
677 block_t start, block_t len)
678{
679 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
680 struct list_head *pend_list = &(dcc->pend_list);
680 struct list_head *pend_list;
681 struct discard_cmd *dc;
682
681 struct discard_cmd *dc;
682
683 f2fs_bug_on(sbi, !len);
684
685 pend_list = &dcc->pend_list[plist_idx(len)];
686
683 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
684 INIT_LIST_HEAD(&dc->list);
685 dc->bdev = bdev;
686 dc->lstart = lstart;
687 dc->start = start;
688 dc->len = len;
689 dc->state = D_PREP;
690 dc->error = 0;

--- 110 unchanged lines hidden (view full) ---

801do_insert:
802 dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, p);
803 if (!dc)
804 return NULL;
805
806 return dc;
807}
808
687 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
688 INIT_LIST_HEAD(&dc->list);
689 dc->bdev = bdev;
690 dc->lstart = lstart;
691 dc->start = start;
692 dc->len = len;
693 dc->state = D_PREP;
694 dc->error = 0;

--- 110 unchanged lines hidden (view full) ---

805do_insert:
806 dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, p);
807 if (!dc)
808 return NULL;
809
810 return dc;
811}
812
813static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
814 struct discard_cmd *dc)
815{
816 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
817}
818
809static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
810 struct discard_cmd *dc, block_t blkaddr)
811{
819static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
820 struct discard_cmd *dc, block_t blkaddr)
821{
822 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
812 struct discard_info di = dc->di;
813 bool modified = false;
814
815 if (dc->state == D_DONE || dc->len == 1) {
816 __remove_discard_cmd(sbi, dc);
817 return;
818 }
819
820 if (blkaddr > di.lstart) {
821 dc->len = blkaddr - dc->lstart;
823 struct discard_info di = dc->di;
824 bool modified = false;
825
826 if (dc->state == D_DONE || dc->len == 1) {
827 __remove_discard_cmd(sbi, dc);
828 return;
829 }
830
831 if (blkaddr > di.lstart) {
832 dc->len = blkaddr - dc->lstart;
833 __relocate_discard_cmd(dcc, dc);
822 modified = true;
823 }
824
825 if (blkaddr < di.lstart + di.len - 1) {
826 if (modified) {
827 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
828 di.start + blkaddr + 1 - di.lstart,
829 di.lstart + di.len - 1 - blkaddr,
830 NULL, NULL);
831 } else {
832 dc->lstart++;
833 dc->len--;
834 dc->start++;
834 modified = true;
835 }
836
837 if (blkaddr < di.lstart + di.len - 1) {
838 if (modified) {
839 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
840 di.start + blkaddr + 1 - di.lstart,
841 di.lstart + di.len - 1 - blkaddr,
842 NULL, NULL);
843 } else {
844 dc->lstart++;
845 dc->len--;
846 dc->start++;
847 __relocate_discard_cmd(dcc, dc);
835 }
836 }
837}
838
839static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
840 struct block_device *bdev, block_t lstart,
841 block_t start, block_t len)
842{

--- 42 unchanged lines hidden (view full) ---

885
886 if (!di.len)
887 goto next;
888
889 if (prev_dc && prev_dc->state == D_PREP &&
890 prev_dc->bdev == bdev &&
891 __is_discard_back_mergeable(&di, &prev_dc->di)) {
892 prev_dc->di.len += di.len;
848 }
849 }
850}
851
852static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
853 struct block_device *bdev, block_t lstart,
854 block_t start, block_t len)
855{

--- 42 unchanged lines hidden (view full) ---

898
899 if (!di.len)
900 goto next;
901
902 if (prev_dc && prev_dc->state == D_PREP &&
903 prev_dc->bdev == bdev &&
904 __is_discard_back_mergeable(&di, &prev_dc->di)) {
905 prev_dc->di.len += di.len;
906 __relocate_discard_cmd(dcc, prev_dc);
893 di = prev_dc->di;
894 tdc = prev_dc;
895 merged = true;
896 }
897
898 if (next_dc && next_dc->state == D_PREP &&
899 next_dc->bdev == bdev &&
900 __is_discard_front_mergeable(&di, &next_dc->di)) {
901 next_dc->di.lstart = di.lstart;
902 next_dc->di.len += di.len;
903 next_dc->di.start = di.start;
907 di = prev_dc->di;
908 tdc = prev_dc;
909 merged = true;
910 }
911
912 if (next_dc && next_dc->state == D_PREP &&
913 next_dc->bdev == bdev &&
914 __is_discard_front_mergeable(&di, &next_dc->di)) {
915 next_dc->di.lstart = di.lstart;
916 next_dc->di.len += di.len;
917 next_dc->di.start = di.start;
918 __relocate_discard_cmd(dcc, next_dc);
904 if (tdc)
905 __remove_discard_cmd(sbi, tdc);
906
907 merged = true;
908 }
909
910 if (!merged)
911 __insert_discard_tree(sbi, bdev, di.lstart, di.start,

--- 44 unchanged lines hidden (view full) ---

956
957 mutex_unlock(&dcc->cmd_lock);
958}
959
960/* This comes from f2fs_put_super */
961void f2fs_wait_discard_bios(struct f2fs_sb_info *sbi)
962{
963 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
919 if (tdc)
920 __remove_discard_cmd(sbi, tdc);
921
922 merged = true;
923 }
924
925 if (!merged)
926 __insert_discard_tree(sbi, bdev, di.lstart, di.start,

--- 44 unchanged lines hidden (view full) ---

971
972 mutex_unlock(&dcc->cmd_lock);
973}
974
975/* This comes from f2fs_put_super */
976void f2fs_wait_discard_bios(struct f2fs_sb_info *sbi)
977{
978 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
964 struct list_head *pend_list = &(dcc->pend_list);
979 struct list_head *pend_list;
965 struct list_head *wait_list = &(dcc->wait_list);
966 struct discard_cmd *dc, *tmp;
967 struct blk_plug plug;
980 struct list_head *wait_list = &(dcc->wait_list);
981 struct discard_cmd *dc, *tmp;
982 struct blk_plug plug;
983 int i;
968
969 mutex_lock(&dcc->cmd_lock);
970
971 blk_start_plug(&plug);
984
985 mutex_lock(&dcc->cmd_lock);
986
987 blk_start_plug(&plug);
972 list_for_each_entry_safe(dc, tmp, pend_list, list)
973 __submit_discard_cmd(sbi, dc);
988 for (i = 0; i < MAX_PLIST_NUM; i++) {
989 pend_list = &dcc->pend_list[i];
990 list_for_each_entry_safe(dc, tmp, pend_list, list)
991 __submit_discard_cmd(sbi, dc);
992 }
974 blk_finish_plug(&plug);
975
976 list_for_each_entry_safe(dc, tmp, wait_list, list) {
977 wait_for_completion_io(&dc->wait);
978 __remove_discard_cmd(sbi, dc);
979 }
980
981 mutex_unlock(&dcc->cmd_lock);
982}
983
984static int issue_discard_thread(void *data)
985{
986 struct f2fs_sb_info *sbi = data;
987 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
988 wait_queue_head_t *q = &dcc->discard_wait_queue;
993 blk_finish_plug(&plug);
994
995 list_for_each_entry_safe(dc, tmp, wait_list, list) {
996 wait_for_completion_io(&dc->wait);
997 __remove_discard_cmd(sbi, dc);
998 }
999
1000 mutex_unlock(&dcc->cmd_lock);
1001}
1002
1003static int issue_discard_thread(void *data)
1004{
1005 struct f2fs_sb_info *sbi = data;
1006 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1007 wait_queue_head_t *q = &dcc->discard_wait_queue;
989 struct list_head *pend_list = &dcc->pend_list;
1008 struct list_head *pend_list;
990 struct list_head *wait_list = &dcc->wait_list;
991 struct discard_cmd *dc, *tmp;
992 struct blk_plug plug;
1009 struct list_head *wait_list = &dcc->wait_list;
1010 struct discard_cmd *dc, *tmp;
1011 struct blk_plug plug;
993 int iter = 0;
1012 int iter = 0, i;
994repeat:
995 if (kthread_should_stop())
996 return 0;
997
998 mutex_lock(&dcc->cmd_lock);
999 blk_start_plug(&plug);
1013repeat:
1014 if (kthread_should_stop())
1015 return 0;
1016
1017 mutex_lock(&dcc->cmd_lock);
1018 blk_start_plug(&plug);
1000 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1001 f2fs_bug_on(sbi, dc->state != D_PREP);
1019 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1020 pend_list = &dcc->pend_list[i];
1021 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1022 f2fs_bug_on(sbi, dc->state != D_PREP);
1002
1023
1003 if (is_idle(sbi))
1004 __submit_discard_cmd(sbi, dc);
1024 if (is_idle(sbi))
1025 __submit_discard_cmd(sbi, dc);
1005
1026
1006 if (iter++ > DISCARD_ISSUE_RATE)
1007 break;
1027 if (iter++ > DISCARD_ISSUE_RATE)
1028 goto next_step;
1029 }
1008 }
1030 }
1031next_step:
1009 blk_finish_plug(&plug);
1010
1011 list_for_each_entry_safe(dc, tmp, wait_list, list) {
1012 if (dc->state == D_DONE) {
1013 wait_for_completion_io(&dc->wait);
1014 __remove_discard_cmd(sbi, dc);
1015 }
1016 }
1017 mutex_unlock(&dcc->cmd_lock);
1018
1019 iter = 0;
1020 congestion_wait(BLK_RW_SYNC, HZ/50);
1021
1022 wait_event_interruptible(*q, kthread_should_stop() ||
1032 blk_finish_plug(&plug);
1033
1034 list_for_each_entry_safe(dc, tmp, wait_list, list) {
1035 if (dc->state == D_DONE) {
1036 wait_for_completion_io(&dc->wait);
1037 __remove_discard_cmd(sbi, dc);
1038 }
1039 }
1040 mutex_unlock(&dcc->cmd_lock);
1041
1042 iter = 0;
1043 congestion_wait(BLK_RW_SYNC, HZ/50);
1044
1045 wait_event_interruptible(*q, kthread_should_stop() ||
1023 !list_empty(pend_list) || !list_empty(wait_list));
1046 atomic_read(&dcc->discard_cmd_cnt));
1024 goto repeat;
1025}
1026
1027#ifdef CONFIG_BLK_DEV_ZONED
1028static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1029 struct block_device *bdev, block_t blkstart, block_t blklen)
1030{
1031 sector_t sector, nr_sects;

--- 263 unchanged lines hidden (view full) ---

1295 kmem_cache_free(discard_entry_slab, entry);
1296 }
1297}
1298
1299static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
1300{
1301 dev_t dev = sbi->sb->s_bdev->bd_dev;
1302 struct discard_cmd_control *dcc;
1047 goto repeat;
1048}
1049
1050#ifdef CONFIG_BLK_DEV_ZONED
1051static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1052 struct block_device *bdev, block_t blkstart, block_t blklen)
1053{
1054 sector_t sector, nr_sects;

--- 263 unchanged lines hidden (view full) ---

1318 kmem_cache_free(discard_entry_slab, entry);
1319 }
1320}
1321
1322static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
1323{
1324 dev_t dev = sbi->sb->s_bdev->bd_dev;
1325 struct discard_cmd_control *dcc;
1303 int err = 0;
1326 int err = 0, i;
1304
1305 if (SM_I(sbi)->dcc_info) {
1306 dcc = SM_I(sbi)->dcc_info;
1307 goto init_thread;
1308 }
1309
1310 dcc = kzalloc(sizeof(struct discard_cmd_control), GFP_KERNEL);
1311 if (!dcc)
1312 return -ENOMEM;
1313
1314 INIT_LIST_HEAD(&dcc->entry_list);
1327
1328 if (SM_I(sbi)->dcc_info) {
1329 dcc = SM_I(sbi)->dcc_info;
1330 goto init_thread;
1331 }
1332
1333 dcc = kzalloc(sizeof(struct discard_cmd_control), GFP_KERNEL);
1334 if (!dcc)
1335 return -ENOMEM;
1336
1337 INIT_LIST_HEAD(&dcc->entry_list);
1315 INIT_LIST_HEAD(&dcc->pend_list);
1338 for (i = 0; i < MAX_PLIST_NUM; i++)
1339 INIT_LIST_HEAD(&dcc->pend_list[i]);
1316 INIT_LIST_HEAD(&dcc->wait_list);
1317 mutex_init(&dcc->cmd_lock);
1318 atomic_set(&dcc->issued_discard, 0);
1319 atomic_set(&dcc->issing_discard, 0);
1320 atomic_set(&dcc->discard_cmd_cnt, 0);
1321 dcc->nr_discards = 0;
1322 dcc->max_discards = 0;
1323 dcc->root = RB_ROOT;

--- 1971 unchanged lines hidden --- 		1340 INIT_LIST_HEAD(&dcc->wait_list);
1341 mutex_init(&dcc->cmd_lock);
1342 atomic_set(&dcc->issued_discard, 0);
1343 atomic_set(&dcc->issing_discard, 0);
1344 atomic_set(&dcc->discard_cmd_cnt, 0);
1345 dcc->nr_discards = 0;
1346 dcc->max_discards = 0;
1347 dcc->root = RB_ROOT;

--- 1971 unchanged lines hidden ---