1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 */ 6 #include "internal.h" 7 8 struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp) 9 { 10 struct page *page = *pagepool; 11 12 if (page) { 13 DBG_BUGON(page_ref_count(page) != 1); 14 *pagepool = (struct page *)page_private(page); 15 } else { 16 page = alloc_page(gfp); 17 } 18 return page; 19 } 20 21 void erofs_release_pages(struct page **pagepool) 22 { 23 while (*pagepool) { 24 struct page *page = *pagepool; 25 26 *pagepool = (struct page *)page_private(page); 27 put_page(page); 28 } 29 } 30 31 #ifdef CONFIG_EROFS_FS_ZIP 32 /* global shrink count (for all mounted EROFS instances) */ 33 static atomic_long_t erofs_global_shrink_cnt; 34 35 static bool erofs_workgroup_get(struct erofs_workgroup *grp) 36 { 37 if (lockref_get_not_zero(&grp->lockref)) 38 return true; 39 40 spin_lock(&grp->lockref.lock); 41 if (__lockref_is_dead(&grp->lockref)) { 42 spin_unlock(&grp->lockref.lock); 43 return false; 44 } 45 46 if (!grp->lockref.count++) 47 atomic_long_dec(&erofs_global_shrink_cnt); 48 spin_unlock(&grp->lockref.lock); 49 return true; 50 } 51 52 struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb, 53 pgoff_t index) 54 { 55 struct erofs_sb_info *sbi = EROFS_SB(sb); 56 struct erofs_workgroup *grp; 57 58 repeat: 59 rcu_read_lock(); 60 grp = xa_load(&sbi->managed_pslots, index); 61 if (grp) { 62 if (!erofs_workgroup_get(grp)) { 63 /* prefer to relax rcu read side */ 64 rcu_read_unlock(); 65 goto repeat; 66 } 67 68 DBG_BUGON(index != grp->index); 69 } 70 rcu_read_unlock(); 71 return grp; 72 } 73 74 struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb, 75 struct erofs_workgroup *grp) 76 { 77 struct erofs_sb_info *const sbi = EROFS_SB(sb); 78 struct erofs_workgroup *pre; 79 80 DBG_BUGON(grp->lockref.count < 1); 81 repeat: 82 xa_lock(&sbi->managed_pslots); 83 pre = __xa_cmpxchg(&sbi->managed_pslots, grp->index, 84 NULL, grp, GFP_NOFS); 85 if (pre) { 86 if (xa_is_err(pre)) { 87 pre = ERR_PTR(xa_err(pre)); 88 } else if (!erofs_workgroup_get(pre)) { 89 /* try to legitimize the current in-tree one */ 90 xa_unlock(&sbi->managed_pslots); 91 cond_resched(); 92 goto repeat; 93 } 94 grp = pre; 95 } 96 xa_unlock(&sbi->managed_pslots); 97 return grp; 98 } 99 100 static void __erofs_workgroup_free(struct erofs_workgroup *grp) 101 { 102 atomic_long_dec(&erofs_global_shrink_cnt); 103 erofs_workgroup_free_rcu(grp); 104 } 105 106 void erofs_workgroup_put(struct erofs_workgroup *grp) 107 { 108 if (lockref_put_or_lock(&grp->lockref)) 109 return; 110 111 DBG_BUGON(__lockref_is_dead(&grp->lockref)); 112 if (grp->lockref.count == 1) 113 atomic_long_inc(&erofs_global_shrink_cnt); 114 --grp->lockref.count; 115 spin_unlock(&grp->lockref.lock); 116 } 117 118 static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, 119 struct erofs_workgroup *grp) 120 { 121 int free = false; 122 123 spin_lock(&grp->lockref.lock); 124 if (grp->lockref.count) 125 goto out; 126 127 /* 128 * Note that all cached pages should be detached before deleted from 129 * the XArray. Otherwise some cached pages could be still attached to 130 * the orphan old workgroup when the new one is available in the tree. 131 */ 132 if (erofs_try_to_free_all_cached_pages(sbi, grp)) 133 goto out; 134 135 /* 136 * It's impossible to fail after the workgroup is freezed, 137 * however in order to avoid some race conditions, add a 138 * DBG_BUGON to observe this in advance. 139 */ 140 DBG_BUGON(__xa_erase(&sbi->managed_pslots, grp->index) != grp); 141 142 lockref_mark_dead(&grp->lockref); 143 free = true; 144 out: 145 spin_unlock(&grp->lockref.lock); 146 if (free) 147 __erofs_workgroup_free(grp); 148 return free; 149 } 150 151 static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi, 152 unsigned long nr_shrink) 153 { 154 struct erofs_workgroup *grp; 155 unsigned int freed = 0; 156 unsigned long index; 157 158 xa_lock(&sbi->managed_pslots); 159 xa_for_each(&sbi->managed_pslots, index, grp) { 160 /* try to shrink each valid workgroup */ 161 if (!erofs_try_to_release_workgroup(sbi, grp)) 162 continue; 163 xa_unlock(&sbi->managed_pslots); 164 165 ++freed; 166 if (!--nr_shrink) 167 return freed; 168 xa_lock(&sbi->managed_pslots); 169 } 170 xa_unlock(&sbi->managed_pslots); 171 return freed; 172 } 173 174 /* protected by 'erofs_sb_list_lock' */ 175 static unsigned int shrinker_run_no; 176 177 /* protects the mounted 'erofs_sb_list' */ 178 static DEFINE_SPINLOCK(erofs_sb_list_lock); 179 static LIST_HEAD(erofs_sb_list); 180 181 void erofs_shrinker_register(struct super_block *sb) 182 { 183 struct erofs_sb_info *sbi = EROFS_SB(sb); 184 185 mutex_init(&sbi->umount_mutex); 186 187 spin_lock(&erofs_sb_list_lock); 188 list_add(&sbi->list, &erofs_sb_list); 189 spin_unlock(&erofs_sb_list_lock); 190 } 191 192 void erofs_shrinker_unregister(struct super_block *sb) 193 { 194 struct erofs_sb_info *const sbi = EROFS_SB(sb); 195 196 mutex_lock(&sbi->umount_mutex); 197 /* clean up all remaining workgroups in memory */ 198 erofs_shrink_workstation(sbi, ~0UL); 199 200 spin_lock(&erofs_sb_list_lock); 201 list_del(&sbi->list); 202 spin_unlock(&erofs_sb_list_lock); 203 mutex_unlock(&sbi->umount_mutex); 204 } 205 206 static unsigned long erofs_shrink_count(struct shrinker *shrink, 207 struct shrink_control *sc) 208 { 209 return atomic_long_read(&erofs_global_shrink_cnt); 210 } 211 212 static unsigned long erofs_shrink_scan(struct shrinker *shrink, 213 struct shrink_control *sc) 214 { 215 struct erofs_sb_info *sbi; 216 struct list_head *p; 217 218 unsigned long nr = sc->nr_to_scan; 219 unsigned int run_no; 220 unsigned long freed = 0; 221 222 spin_lock(&erofs_sb_list_lock); 223 do { 224 run_no = ++shrinker_run_no; 225 } while (run_no == 0); 226 227 /* Iterate over all mounted superblocks and try to shrink them */ 228 p = erofs_sb_list.next; 229 while (p != &erofs_sb_list) { 230 sbi = list_entry(p, struct erofs_sb_info, list); 231 232 /* 233 * We move the ones we do to the end of the list, so we stop 234 * when we see one we have already done. 235 */ 236 if (sbi->shrinker_run_no == run_no) 237 break; 238 239 if (!mutex_trylock(&sbi->umount_mutex)) { 240 p = p->next; 241 continue; 242 } 243 244 spin_unlock(&erofs_sb_list_lock); 245 sbi->shrinker_run_no = run_no; 246 247 freed += erofs_shrink_workstation(sbi, nr - freed); 248 249 spin_lock(&erofs_sb_list_lock); 250 /* Get the next list element before we move this one */ 251 p = p->next; 252 253 /* 254 * Move this one to the end of the list to provide some 255 * fairness. 256 */ 257 list_move_tail(&sbi->list, &erofs_sb_list); 258 mutex_unlock(&sbi->umount_mutex); 259 260 if (freed >= nr) 261 break; 262 } 263 spin_unlock(&erofs_sb_list_lock); 264 return freed; 265 } 266 267 static struct shrinker erofs_shrinker_info = { 268 .scan_objects = erofs_shrink_scan, 269 .count_objects = erofs_shrink_count, 270 .seeks = DEFAULT_SEEKS, 271 }; 272 273 int __init erofs_init_shrinker(void) 274 { 275 return register_shrinker(&erofs_shrinker_info, "erofs-shrinker"); 276 } 277 278 void erofs_exit_shrinker(void) 279 { 280 unregister_shrinker(&erofs_shrinker_info); 281 } 282 #endif /* !CONFIG_EROFS_FS_ZIP */ 283