segment.c (71af75b6929458d85f63c0649dc26d6f4c19729e) segment.c (4d67490498acb4ffcef5ba7bc44990d46e66a44c)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/segment.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 6 unchanged lines hidden (view full) ---

15#include <linux/timer.h>
16#include <linux/freezer.h>
17#include <linux/sched/signal.h>
18
19#include "f2fs.h"
20#include "segment.h"
21#include "node.h"
22#include "gc.h"
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/segment.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>

--- 6 unchanged lines hidden (view full) ---

15#include <linux/timer.h>
16#include <linux/freezer.h>
17#include <linux/sched/signal.h>
18
19#include "f2fs.h"
20#include "segment.h"
21#include "node.h"
22#include "gc.h"
23#include "iostat.h"
23#include <trace/events/f2fs.h>
24
25#define __reverse_ffz(x) __reverse_ffs(~(x))
26
27static struct kmem_cache *discard_entry_slab;
28static struct kmem_cache *discard_cmd_slab;
29static struct kmem_cache *sit_entry_set_slab;
30static struct kmem_cache *inmem_entry_slab;

--- 152 unchanged lines hidden (view full) ---

183}
184
185void f2fs_register_inmem_page(struct inode *inode, struct page *page)
186{
187 struct inmem_pages *new;
188
189 set_page_private_atomic(page);
190
24#include <trace/events/f2fs.h>
25
26#define __reverse_ffz(x) __reverse_ffs(~(x))
27
28static struct kmem_cache *discard_entry_slab;
29static struct kmem_cache *discard_cmd_slab;
30static struct kmem_cache *sit_entry_set_slab;
31static struct kmem_cache *inmem_entry_slab;

--- 152 unchanged lines hidden (view full) ---

184}
185
186void f2fs_register_inmem_page(struct inode *inode, struct page *page)
187{
188 struct inmem_pages *new;
189
190 set_page_private_atomic(page);
191
191 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
192 new = f2fs_kmem_cache_alloc(inmem_entry_slab,
193 GFP_NOFS, true, NULL);
192
193 /* add atomic page indices to the list */
194 new->page = page;
195 INIT_LIST_HEAD(&new->list);
196
197 /* increase reference count with clean state */
198 get_page(page);
199 mutex_lock(&F2FS_I(inode)->inmem_lock);

--- 571 unchanged lines hidden (view full) ---

771
772 if (!f2fs_is_multi_device(sbi))
773 return 0;
774
775 if (test_opt(sbi, NOBARRIER))
776 return 0;
777
778 for (i = 1; i < sbi->s_ndevs; i++) {
194
195 /* add atomic page indices to the list */
196 new->page = page;
197 INIT_LIST_HEAD(&new->list);
198
199 /* increase reference count with clean state */
200 get_page(page);
201 mutex_lock(&F2FS_I(inode)->inmem_lock);

--- 571 unchanged lines hidden (view full) ---

773
774 if (!f2fs_is_multi_device(sbi))
775 return 0;
776
777 if (test_opt(sbi, NOBARRIER))
778 return 0;
779
780 for (i = 1; i < sbi->s_ndevs; i++) {
781 int count = DEFAULT_RETRY_IO_COUNT;
782
779 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
780 continue;
783 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
784 continue;
781 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
782 if (ret)
785
786 do {
787 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
788 if (ret)
789 congestion_wait(BLK_RW_ASYNC,
790 DEFAULT_IO_TIMEOUT);
791 } while (ret && --count);
792
793 if (ret) {
794 f2fs_stop_checkpoint(sbi, false);
783 break;
795 break;
796 }
784
785 spin_lock(&sbi->dev_lock);
786 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
787 spin_unlock(&sbi->dev_lock);
788 }
789
790 return ret;
791}

--- 193 unchanged lines hidden (view full) ---

985 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
986 struct list_head *pend_list;
987 struct discard_cmd *dc;
988
989 f2fs_bug_on(sbi, !len);
990
991 pend_list = &dcc->pend_list[plist_idx(len)];
992
797
798 spin_lock(&sbi->dev_lock);
799 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
800 spin_unlock(&sbi->dev_lock);
801 }
802
803 return ret;
804}

--- 193 unchanged lines hidden (view full) ---

998 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
999 struct list_head *pend_list;
1000 struct discard_cmd *dc;
1001
1002 f2fs_bug_on(sbi, !len);
1003
1004 pend_list = &dcc->pend_list[plist_idx(len)];
1005
993 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
1006 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
994 INIT_LIST_HEAD(&dc->list);
995 dc->bdev = bdev;
996 dc->lstart = lstart;
997 dc->start = start;
998 dc->len = len;
999 dc->ref = 0;
1000 dc->state = D_PREP;
1001 dc->queued = 0;

--- 886 unchanged lines hidden (view full) ---

1888 start = i;
1889 len = 0;
1890 }
1891 }
1892
1893 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1894 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1895
1007 INIT_LIST_HEAD(&dc->list);
1008 dc->bdev = bdev;
1009 dc->lstart = lstart;
1010 dc->start = start;
1011 dc->len = len;
1012 dc->ref = 0;
1013 dc->state = D_PREP;
1014 dc->queued = 0;

--- 886 unchanged lines hidden (view full) ---

1901 start = i;
1902 len = 0;
1903 }
1904 }
1905
1906 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1907 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1908
1896 if (!f2fs_test_and_set_bit(offset, se->discard_map))
1909 if (f2fs_block_unit_discard(sbi) &&
1910 !f2fs_test_and_set_bit(offset, se->discard_map))
1897 sbi->discard_blks--;
1898 }
1899
1900 if (len)
1901 err = __issue_discard_async(sbi, bdev, start, len);
1902 return err;
1903}
1904

--- 8 unchanged lines hidden (view full) ---

1913 unsigned long *discard_map = (unsigned long *)se->discard_map;
1914 unsigned long *dmap = SIT_I(sbi)->tmp_map;
1915 unsigned int start = 0, end = -1;
1916 bool force = (cpc->reason & CP_DISCARD);
1917 struct discard_entry *de = NULL;
1918 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1919 int i;
1920
1911 sbi->discard_blks--;
1912 }
1913
1914 if (len)
1915 err = __issue_discard_async(sbi, bdev, start, len);
1916 return err;
1917}
1918

--- 8 unchanged lines hidden (view full) ---

1927 unsigned long *discard_map = (unsigned long *)se->discard_map;
1928 unsigned long *dmap = SIT_I(sbi)->tmp_map;
1929 unsigned int start = 0, end = -1;
1930 bool force = (cpc->reason & CP_DISCARD);
1931 struct discard_entry *de = NULL;
1932 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1933 int i;
1934
1921 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))
1935 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||
1936 !f2fs_block_unit_discard(sbi))
1922 return false;
1923
1924 if (!force) {
1925 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1926 SM_I(sbi)->dcc_info->nr_discards >=
1927 SM_I(sbi)->dcc_info->max_discards)
1928 return false;
1929 }

--- 14 unchanged lines hidden (view full) ---

1944 && (end - start) < cpc->trim_minlen)
1945 continue;
1946
1947 if (check_only)
1948 return true;
1949
1950 if (!de) {
1951 de = f2fs_kmem_cache_alloc(discard_entry_slab,
1937 return false;
1938
1939 if (!force) {
1940 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1941 SM_I(sbi)->dcc_info->nr_discards >=
1942 SM_I(sbi)->dcc_info->max_discards)
1943 return false;
1944 }

--- 14 unchanged lines hidden (view full) ---

1959 && (end - start) < cpc->trim_minlen)
1960 continue;
1961
1962 if (check_only)
1963 return true;
1964
1965 if (!de) {
1966 de = f2fs_kmem_cache_alloc(discard_entry_slab,
1952 GFP_F2FS_ZERO);
1967 GFP_F2FS_ZERO, true, NULL);
1953 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1954 list_add_tail(&de->list, head);
1955 }
1956
1957 for (i = start; i < end; i++)
1958 __set_bit_le(i, (void *)de->discard_map);
1959
1960 SM_I(sbi)->dcc_info->nr_discards += end - start;

--- 37 unchanged lines hidden (view full) ---

1998 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1999 struct list_head *head = &dcc->entry_list;
2000 struct discard_entry *entry, *this;
2001 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2002 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2003 unsigned int start = 0, end = -1;
2004 unsigned int secno, start_segno;
2005 bool force = (cpc->reason & CP_DISCARD);
1968 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1969 list_add_tail(&de->list, head);
1970 }
1971
1972 for (i = start; i < end; i++)
1973 __set_bit_le(i, (void *)de->discard_map);
1974
1975 SM_I(sbi)->dcc_info->nr_discards += end - start;

--- 37 unchanged lines hidden (view full) ---

2013 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2014 struct list_head *head = &dcc->entry_list;
2015 struct discard_entry *entry, *this;
2016 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2017 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2018 unsigned int start = 0, end = -1;
2019 unsigned int secno, start_segno;
2020 bool force = (cpc->reason & CP_DISCARD);
2006 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
2021 bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2022 DISCARD_UNIT_SECTION;
2007
2023
2024 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2025 section_alignment = true;
2026
2008 mutex_lock(&dirty_i->seglist_lock);
2009
2010 while (1) {
2011 int i;
2012
2027 mutex_lock(&dirty_i->seglist_lock);
2028
2029 while (1) {
2030 int i;
2031
2013 if (need_align && end != -1)
2032 if (section_alignment && end != -1)
2014 end--;
2015 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2016 if (start >= MAIN_SEGS(sbi))
2017 break;
2018 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2019 start + 1);
2020
2033 end--;
2034 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2035 if (start >= MAIN_SEGS(sbi))
2036 break;
2037 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2038 start + 1);
2039
2021 if (need_align) {
2040 if (section_alignment) {
2022 start = rounddown(start, sbi->segs_per_sec);
2023 end = roundup(end, sbi->segs_per_sec);
2024 }
2025
2026 for (i = start; i < end; i++) {
2027 if (test_and_clear_bit(i, prefree_map))
2028 dirty_i->nr_dirty[PRE]--;
2029 }

--- 21 unchanged lines hidden (view full) ---

2051 start = start_segno + sbi->segs_per_sec;
2052 if (start < end)
2053 goto next;
2054 else
2055 end = start - 1;
2056 }
2057 mutex_unlock(&dirty_i->seglist_lock);
2058
2041 start = rounddown(start, sbi->segs_per_sec);
2042 end = roundup(end, sbi->segs_per_sec);
2043 }
2044
2045 for (i = start; i < end; i++) {
2046 if (test_and_clear_bit(i, prefree_map))
2047 dirty_i->nr_dirty[PRE]--;
2048 }

--- 21 unchanged lines hidden (view full) ---

2070 start = start_segno + sbi->segs_per_sec;
2071 if (start < end)
2072 goto next;
2073 else
2074 end = start - 1;
2075 }
2076 mutex_unlock(&dirty_i->seglist_lock);
2077
2078 if (!f2fs_block_unit_discard(sbi))
2079 goto wakeup;
2080
2059 /* send small discards */
2060 list_for_each_entry_safe(entry, this, head, list) {
2061 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2062 bool is_valid = test_bit_le(0, entry->discard_map);
2063
2064find_next:
2065 if (is_valid) {
2066 next_pos = find_next_zero_bit_le(entry->discard_map,

--- 17 unchanged lines hidden (view full) ---

2084
2085 if (cur_pos < sbi->blocks_per_seg)
2086 goto find_next;
2087
2088 release_discard_addr(entry);
2089 dcc->nr_discards -= total_len;
2090 }
2091
2081 /* send small discards */
2082 list_for_each_entry_safe(entry, this, head, list) {
2083 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2084 bool is_valid = test_bit_le(0, entry->discard_map);
2085
2086find_next:
2087 if (is_valid) {
2088 next_pos = find_next_zero_bit_le(entry->discard_map,

--- 17 unchanged lines hidden (view full) ---

2106
2107 if (cur_pos < sbi->blocks_per_seg)
2108 goto find_next;
2109
2110 release_discard_addr(entry);
2111 dcc->nr_discards -= total_len;
2112 }
2113
2114wakeup:
2092 wake_up_discard_thread(sbi, false);
2093}
2094
2115 wake_up_discard_thread(sbi, false);
2116}
2117
2095static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2118int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2096{
2097 dev_t dev = sbi->sb->s_bdev->bd_dev;
2119{
2120 dev_t dev = sbi->sb->s_bdev->bd_dev;
2121 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2122 int err = 0;
2123
2124 if (!f2fs_realtime_discard_enable(sbi))
2125 return 0;
2126
2127 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2128 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2129 if (IS_ERR(dcc->f2fs_issue_discard))
2130 err = PTR_ERR(dcc->f2fs_issue_discard);
2131
2132 return err;
2133}
2134
2135static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2136{
2098 struct discard_cmd_control *dcc;
2099 int err = 0, i;
2100
2101 if (SM_I(sbi)->dcc_info) {
2102 dcc = SM_I(sbi)->dcc_info;
2103 goto init_thread;
2104 }
2105
2106 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2107 if (!dcc)
2108 return -ENOMEM;
2109
2110 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2137 struct discard_cmd_control *dcc;
2138 int err = 0, i;
2139
2140 if (SM_I(sbi)->dcc_info) {
2141 dcc = SM_I(sbi)->dcc_info;
2142 goto init_thread;
2143 }
2144
2145 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2146 if (!dcc)
2147 return -ENOMEM;
2148
2149 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2150 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2151 dcc->discard_granularity = sbi->blocks_per_seg;
2152 else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2153 dcc->discard_granularity = BLKS_PER_SEC(sbi);
2154
2111 INIT_LIST_HEAD(&dcc->entry_list);
2112 for (i = 0; i < MAX_PLIST_NUM; i++)
2113 INIT_LIST_HEAD(&dcc->pend_list[i]);
2114 INIT_LIST_HEAD(&dcc->wait_list);
2115 INIT_LIST_HEAD(&dcc->fstrim_list);
2116 mutex_init(&dcc->cmd_lock);
2117 atomic_set(&dcc->issued_discard, 0);
2118 atomic_set(&dcc->queued_discard, 0);
2119 atomic_set(&dcc->discard_cmd_cnt, 0);
2120 dcc->nr_discards = 0;
2121 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2122 dcc->undiscard_blks = 0;
2123 dcc->next_pos = 0;
2124 dcc->root = RB_ROOT_CACHED;
2125 dcc->rbtree_check = false;
2126
2127 init_waitqueue_head(&dcc->discard_wait_queue);
2128 SM_I(sbi)->dcc_info = dcc;
2129init_thread:
2155 INIT_LIST_HEAD(&dcc->entry_list);
2156 for (i = 0; i < MAX_PLIST_NUM; i++)
2157 INIT_LIST_HEAD(&dcc->pend_list[i]);
2158 INIT_LIST_HEAD(&dcc->wait_list);
2159 INIT_LIST_HEAD(&dcc->fstrim_list);
2160 mutex_init(&dcc->cmd_lock);
2161 atomic_set(&dcc->issued_discard, 0);
2162 atomic_set(&dcc->queued_discard, 0);
2163 atomic_set(&dcc->discard_cmd_cnt, 0);
2164 dcc->nr_discards = 0;
2165 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2166 dcc->undiscard_blks = 0;
2167 dcc->next_pos = 0;
2168 dcc->root = RB_ROOT_CACHED;
2169 dcc->rbtree_check = false;
2170
2171 init_waitqueue_head(&dcc->discard_wait_queue);
2172 SM_I(sbi)->dcc_info = dcc;
2173init_thread:
2130 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2131 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2132 if (IS_ERR(dcc->f2fs_issue_discard)) {
2133 err = PTR_ERR(dcc->f2fs_issue_discard);
2174 err = f2fs_start_discard_thread(sbi);
2175 if (err) {
2134 kfree(dcc);
2135 SM_I(sbi)->dcc_info = NULL;
2176 kfree(dcc);
2177 SM_I(sbi)->dcc_info = NULL;
2136 return err;
2137 }
2138
2139 return err;
2140}
2141
2142static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2143{
2144 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;

--- 105 unchanged lines hidden (view full) ---

2250 if (unlikely(exist)) {
2251 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2252 blkaddr);
2253 f2fs_bug_on(sbi, 1);
2254 se->valid_blocks--;
2255 del = 0;
2256 }
2257
2178 }
2179
2180 return err;
2181}
2182
2183static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2184{
2185 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;

--- 105 unchanged lines hidden (view full) ---

2291 if (unlikely(exist)) {
2292 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2293 blkaddr);
2294 f2fs_bug_on(sbi, 1);
2295 se->valid_blocks--;
2296 del = 0;
2297 }
2298
2258 if (!f2fs_test_and_set_bit(offset, se->discard_map))
2299 if (f2fs_block_unit_discard(sbi) &&
2300 !f2fs_test_and_set_bit(offset, se->discard_map))
2259 sbi->discard_blks--;
2260
2261 /*
2262 * SSR should never reuse block which is checkpointed
2263 * or newly invalidated.
2264 */
2265 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2266 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))

--- 25 unchanged lines hidden (view full) ---

2292 */
2293 if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2294 spin_lock(&sbi->stat_lock);
2295 sbi->unusable_block_count++;
2296 spin_unlock(&sbi->stat_lock);
2297 }
2298 }
2299
2301 sbi->discard_blks--;
2302
2303 /*
2304 * SSR should never reuse block which is checkpointed
2305 * or newly invalidated.
2306 */
2307 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2308 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))

--- 25 unchanged lines hidden (view full) ---

2334 */
2335 if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2336 spin_lock(&sbi->stat_lock);
2337 sbi->unusable_block_count++;
2338 spin_unlock(&sbi->stat_lock);
2339 }
2340 }
2341
2300 if (f2fs_test_and_clear_bit(offset, se->discard_map))
2342 if (f2fs_block_unit_discard(sbi) &&
2343 f2fs_test_and_clear_bit(offset, se->discard_map))
2301 sbi->discard_blks++;
2302 }
2303 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2304 se->ckpt_valid_blocks += del;
2305
2306 __mark_sit_entry_dirty(sbi, segno);
2307
2308 /* update total number of valid blocks to be written in ckpt area */

--- 1249 unchanged lines hidden (view full) ---

3558 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3559 set_sbi_flag(sbi, SBI_NEED_FSCK);
3560 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3561 __func__, segno);
3562 err = -EFSCORRUPTED;
3563 goto drop_bio;
3564 }
3565
2344 sbi->discard_blks++;
2345 }
2346 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2347 se->ckpt_valid_blocks += del;
2348
2349 __mark_sit_entry_dirty(sbi, segno);
2350
2351 /* update total number of valid blocks to be written in ckpt area */

--- 1249 unchanged lines hidden (view full) ---

3601 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3602 set_sbi_flag(sbi, SBI_NEED_FSCK);
3603 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3604 __func__, segno);
3605 err = -EFSCORRUPTED;
3606 goto drop_bio;
3607 }
3608
3566 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || f2fs_cp_error(sbi)) {
3609 if (f2fs_cp_error(sbi)) {
3567 err = -EIO;
3568 goto drop_bio;
3569 }
3570
3571 stat_inc_inplace_blocks(fio->sbi);
3572
3573 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3574 err = f2fs_merge_page_bio(fio);

--- 491 unchanged lines hidden (view full) ---

4066 set_to_next_sit(sit_i, start);
4067
4068 return page;
4069}
4070
4071static struct sit_entry_set *grab_sit_entry_set(void)
4072{
4073 struct sit_entry_set *ses =
3610 err = -EIO;
3611 goto drop_bio;
3612 }
3613
3614 stat_inc_inplace_blocks(fio->sbi);
3615
3616 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3617 err = f2fs_merge_page_bio(fio);

--- 491 unchanged lines hidden (view full) ---

4109 set_to_next_sit(sit_i, start);
4110
4111 return page;
4112}
4113
4114static struct sit_entry_set *grab_sit_entry_set(void)
4115{
4116 struct sit_entry_set *ses =
4074 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
4117 f2fs_kmem_cache_alloc(sit_entry_set_slab,
4118 GFP_NOFS, true, NULL);
4075
4076 ses->entry_cnt = 0;
4077 INIT_LIST_HEAD(&ses->set_list);
4078 return ses;
4079}
4080
4081static void release_sit_entry_set(struct sit_entry_set *ses)
4082{

--- 194 unchanged lines hidden (view full) ---

4277
4278static int build_sit_info(struct f2fs_sb_info *sbi)
4279{
4280 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4281 struct sit_info *sit_i;
4282 unsigned int sit_segs, start;
4283 char *src_bitmap, *bitmap;
4284 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4119
4120 ses->entry_cnt = 0;
4121 INIT_LIST_HEAD(&ses->set_list);
4122 return ses;
4123}
4124
4125static void release_sit_entry_set(struct sit_entry_set *ses)
4126{

--- 194 unchanged lines hidden (view full) ---

4321
4322static int build_sit_info(struct f2fs_sb_info *sbi)
4323{
4324 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4325 struct sit_info *sit_i;
4326 unsigned int sit_segs, start;
4327 char *src_bitmap, *bitmap;
4328 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4329 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4285
4286 /* allocate memory for SIT information */
4287 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4288 if (!sit_i)
4289 return -ENOMEM;
4290
4291 SM_I(sbi)->sit_info = sit_i;
4292

--- 6 unchanged lines hidden (view full) ---

4299
4300 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4301 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4302 GFP_KERNEL);
4303 if (!sit_i->dirty_sentries_bitmap)
4304 return -ENOMEM;
4305
4306#ifdef CONFIG_F2FS_CHECK_FS
4330
4331 /* allocate memory for SIT information */
4332 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4333 if (!sit_i)
4334 return -ENOMEM;
4335
4336 SM_I(sbi)->sit_info = sit_i;
4337

--- 6 unchanged lines hidden (view full) ---

4344
4345 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4346 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4347 GFP_KERNEL);
4348 if (!sit_i->dirty_sentries_bitmap)
4349 return -ENOMEM;
4350
4351#ifdef CONFIG_F2FS_CHECK_FS
4307 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4;
4352 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4308#else
4353#else
4309 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3;
4354 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4310#endif
4311 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4312 if (!sit_i->bitmap)
4313 return -ENOMEM;
4314
4315 bitmap = sit_i->bitmap;
4316
4317 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4318 sit_i->sentries[start].cur_valid_map = bitmap;
4319 bitmap += SIT_VBLOCK_MAP_SIZE;
4320
4321 sit_i->sentries[start].ckpt_valid_map = bitmap;
4322 bitmap += SIT_VBLOCK_MAP_SIZE;
4323
4324#ifdef CONFIG_F2FS_CHECK_FS
4325 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4326 bitmap += SIT_VBLOCK_MAP_SIZE;
4327#endif
4328
4355#endif
4356 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4357 if (!sit_i->bitmap)
4358 return -ENOMEM;
4359
4360 bitmap = sit_i->bitmap;
4361
4362 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4363 sit_i->sentries[start].cur_valid_map = bitmap;
4364 bitmap += SIT_VBLOCK_MAP_SIZE;
4365
4366 sit_i->sentries[start].ckpt_valid_map = bitmap;
4367 bitmap += SIT_VBLOCK_MAP_SIZE;
4368
4369#ifdef CONFIG_F2FS_CHECK_FS
4370 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4371 bitmap += SIT_VBLOCK_MAP_SIZE;
4372#endif
4373
4329 sit_i->sentries[start].discard_map = bitmap;
4330 bitmap += SIT_VBLOCK_MAP_SIZE;
4374 if (discard_map) {
4375 sit_i->sentries[start].discard_map = bitmap;
4376 bitmap += SIT_VBLOCK_MAP_SIZE;
4377 }
4331 }
4332
4333 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4334 if (!sit_i->tmp_map)
4335 return -ENOMEM;
4336
4337 if (__is_large_section(sbi)) {
4338 sit_i->sec_entries =

--- 145 unchanged lines hidden (view full) ---

4484
4485 err = check_block_count(sbi, start, &sit);
4486 if (err)
4487 return err;
4488 seg_info_from_raw_sit(se, &sit);
4489 if (IS_NODESEG(se->type))
4490 total_node_blocks += se->valid_blocks;
4491
4378 }
4379
4380 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4381 if (!sit_i->tmp_map)
4382 return -ENOMEM;
4383
4384 if (__is_large_section(sbi)) {
4385 sit_i->sec_entries =

--- 145 unchanged lines hidden (view full) ---

4531
4532 err = check_block_count(sbi, start, &sit);
4533 if (err)
4534 return err;
4535 seg_info_from_raw_sit(se, &sit);
4536 if (IS_NODESEG(se->type))
4537 total_node_blocks += se->valid_blocks;
4538
4492 /* build discard map only one time */
4493 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4494 memset(se->discard_map, 0xff,
4495 SIT_VBLOCK_MAP_SIZE);
4496 } else {
4497 memcpy(se->discard_map,
4498 se->cur_valid_map,
4499 SIT_VBLOCK_MAP_SIZE);
4500 sbi->discard_blks +=
4501 sbi->blocks_per_seg -
4502 se->valid_blocks;
4539 if (f2fs_block_unit_discard(sbi)) {
4540 /* build discard map only one time */
4541 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4542 memset(se->discard_map, 0xff,
4543 SIT_VBLOCK_MAP_SIZE);
4544 } else {
4545 memcpy(se->discard_map,
4546 se->cur_valid_map,
4547 SIT_VBLOCK_MAP_SIZE);
4548 sbi->discard_blks +=
4549 sbi->blocks_per_seg -
4550 se->valid_blocks;
4551 }
4503 }
4504
4505 if (__is_large_section(sbi))
4506 get_sec_entry(sbi, start)->valid_blocks +=
4507 se->valid_blocks;
4508 }
4509 start_blk += readed;
4510 } while (start_blk < sit_blk_cnt);

--- 19 unchanged lines hidden (view full) ---

4530
4531 err = check_block_count(sbi, start, &sit);
4532 if (err)
4533 break;
4534 seg_info_from_raw_sit(se, &sit);
4535 if (IS_NODESEG(se->type))
4536 total_node_blocks += se->valid_blocks;
4537
4552 }
4553
4554 if (__is_large_section(sbi))
4555 get_sec_entry(sbi, start)->valid_blocks +=
4556 se->valid_blocks;
4557 }
4558 start_blk += readed;
4559 } while (start_blk < sit_blk_cnt);

--- 19 unchanged lines hidden (view full) ---

4579
4580 err = check_block_count(sbi, start, &sit);
4581 if (err)
4582 break;
4583 seg_info_from_raw_sit(se, &sit);
4584 if (IS_NODESEG(se->type))
4585 total_node_blocks += se->valid_blocks;
4586
4538 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4539 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4540 } else {
4541 memcpy(se->discard_map, se->cur_valid_map,
4542 SIT_VBLOCK_MAP_SIZE);
4543 sbi->discard_blks += old_valid_blocks;
4544 sbi->discard_blks -= se->valid_blocks;
4587 if (f2fs_block_unit_discard(sbi)) {
4588 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4589 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4590 } else {
4591 memcpy(se->discard_map, se->cur_valid_map,
4592 SIT_VBLOCK_MAP_SIZE);
4593 sbi->discard_blks += old_valid_blocks;
4594 sbi->discard_blks -= se->valid_blocks;
4595 }
4545 }
4546
4547 if (__is_large_section(sbi)) {
4548 get_sec_entry(sbi, start)->valid_blocks +=
4549 se->valid_blocks;
4550 get_sec_entry(sbi, start)->valid_blocks -=
4551 old_valid_blocks;
4552 }

--- 601 unchanged lines hidden (view full) ---

5154 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5155 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5156 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5157
5158 if (!f2fs_lfs_mode(sbi))
5159 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5160 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5161 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
4596 }
4597
4598 if (__is_large_section(sbi)) {
4599 get_sec_entry(sbi, start)->valid_blocks +=
4600 se->valid_blocks;
4601 get_sec_entry(sbi, start)->valid_blocks -=
4602 old_valid_blocks;
4603 }

--- 601 unchanged lines hidden (view full) ---

5205 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5206 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5207 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5208
5209 if (!f2fs_lfs_mode(sbi))
5210 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5211 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5212 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5162 sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;
5213 sm_info->min_seq_blocks = sbi->blocks_per_seg;
5163 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5164 sm_info->min_ssr_sections = reserved_sections(sbi);
5165
5166 INIT_LIST_HEAD(&sm_info->sit_entry_set);
5167
5168 init_rwsem(&sm_info->curseg_lock);
5169
5170 if (!f2fs_readonly(sbi->sb)) {

--- 185 unchanged lines hidden --- 		5214 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5215 sm_info->min_ssr_sections = reserved_sections(sbi);
5216
5217 INIT_LIST_HEAD(&sm_info->sit_entry_set);
5218
5219 init_rwsem(&sm_info->curseg_lock);
5220
5221 if (!f2fs_readonly(sbi->sb)) {

--- 185 unchanged lines hidden ---