1f5166768STheodore Ts'o // SPDX-License-Identifier: GPL-2.0 2c9de560dSAlex Tomas /* 3c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 4c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 5c9de560dSAlex Tomas */ 6c9de560dSAlex Tomas 7c9de560dSAlex Tomas 8c9de560dSAlex Tomas /* 9c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 10c9de560dSAlex Tomas */ 11c9de560dSAlex Tomas 1218aadd47SBobi Jam #include "ext4_jbd2.h" 138f6e39a7SMingming Cao #include "mballoc.h" 1428623c2fSTheodore Ts'o #include <linux/log2.h> 15a0b30c12STheodore Ts'o #include <linux/module.h> 165a0e3ad6STejun Heo #include <linux/slab.h> 171a5d5e5dSJeremy Cline #include <linux/nospec.h> 1866114cadSTejun Heo #include <linux/backing-dev.h> 199bffad1eSTheodore Ts'o #include <trace/events/ext4.h> 209bffad1eSTheodore Ts'o 21a0b30c12STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 22a0b30c12STheodore Ts'o ushort ext4_mballoc_debug __read_mostly; 23a0b30c12STheodore Ts'o 24a0b30c12STheodore Ts'o module_param_named(mballoc_debug, ext4_mballoc_debug, ushort, 0644); 25a0b30c12STheodore Ts'o MODULE_PARM_DESC(mballoc_debug, "Debugging level for ext4's mballoc"); 26a0b30c12STheodore Ts'o #endif 27a0b30c12STheodore Ts'o 28c9de560dSAlex Tomas /* 29c9de560dSAlex Tomas * MUSTDO: 30c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 31c9de560dSAlex Tomas * - search for metadata in few groups 32c9de560dSAlex Tomas * 33c9de560dSAlex Tomas * TODO v4: 34c9de560dSAlex Tomas * - normalization should take into account whether file is still open 35c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 36c9de560dSAlex Tomas * - don't normalize tails 37c9de560dSAlex Tomas * - quota 38c9de560dSAlex Tomas * - reservation for superuser 39c9de560dSAlex Tomas * 40c9de560dSAlex Tomas * TODO v3: 41c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 42c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 43c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 44c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 45c9de560dSAlex Tomas * - error handling 46c9de560dSAlex Tomas */ 47c9de560dSAlex Tomas 48c9de560dSAlex Tomas /* 49c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 50c9de560dSAlex Tomas * near to the goal(block) value specified. 51c9de560dSAlex Tomas * 52b713a5ecSTheodore Ts'o * During initialization phase of the allocator we decide to use the 53b713a5ecSTheodore Ts'o * group preallocation or inode preallocation depending on the size of 54b713a5ecSTheodore Ts'o * the file. The size of the file could be the resulting file size we 55b713a5ecSTheodore Ts'o * would have after allocation, or the current file size, which ever 56b713a5ecSTheodore Ts'o * is larger. If the size is less than sbi->s_mb_stream_request we 57b713a5ecSTheodore Ts'o * select to use the group preallocation. The default value of 58b713a5ecSTheodore Ts'o * s_mb_stream_request is 16 blocks. This can also be tuned via 59b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in 60b713a5ecSTheodore Ts'o * terms of number of blocks. 61c9de560dSAlex Tomas * 62c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 63b713a5ecSTheodore Ts'o * ensure that we have small files closer together on the disk. 64c9de560dSAlex Tomas * 65b713a5ecSTheodore Ts'o * First stage the allocator looks at the inode prealloc list, 66b713a5ecSTheodore Ts'o * ext4_inode_info->i_prealloc_list, which contains list of prealloc 67b713a5ecSTheodore Ts'o * spaces for this particular inode. The inode prealloc space is 68b713a5ecSTheodore Ts'o * represented as: 69c9de560dSAlex Tomas * 70c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 71c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 7253accfa9STheodore Ts'o * pa_len -> length for this prealloc space (in clusters) 7353accfa9STheodore Ts'o * pa_free -> free space available in this prealloc space (in clusters) 74c9de560dSAlex Tomas * 75c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 76c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 77caaf7a29STao Ma * space we will consume the particular prealloc space. This makes sure that 78caaf7a29STao Ma * we have contiguous physical blocks representing the file blocks 79c9de560dSAlex Tomas * 80c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 81c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 82c9de560dSAlex Tomas * pa_free. 83c9de560dSAlex Tomas * 84c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 85c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 86caaf7a29STao Ma * prealloc space. These are per CPU prealloc list represented as 87c9de560dSAlex Tomas * 88c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 89c9de560dSAlex Tomas * 90c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 91c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 92c9de560dSAlex Tomas * 93c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 9425985edcSLucas De Marchi * enough free space (pa_free) within the prealloc space. 95c9de560dSAlex Tomas * 96c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 97c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 98c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 99c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 100c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 101c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 102c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 103c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 104c9de560dSAlex Tomas * inode as: 105c9de560dSAlex Tomas * 106c9de560dSAlex Tomas * { page } 107c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 108c9de560dSAlex Tomas * 109c9de560dSAlex Tomas * 110c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 111ea1754a0SKirill A. Shutemov * take up 2 blocks. A page can contain blocks_per_page (PAGE_SIZE / 112c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 113c9de560dSAlex Tomas * which is blocks_per_page/2 114c9de560dSAlex Tomas * 115c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 116c9de560dSAlex Tomas * away when the filesystem is unmounted. 117c9de560dSAlex Tomas * 118c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 119c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 120c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 121c9de560dSAlex Tomas * 122c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 123c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 124c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 125c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 126c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 127c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 12827baebb8STheodore Ts'o * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is 12927baebb8STheodore Ts'o * dependent on the cluster size; for non-bigalloc file systems, it is 130c9de560dSAlex Tomas * 512 blocks. This can be tuned via 131d7a1fee1SDan Ehrenberg * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in 132c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 133c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 134d7a1fee1SDan Ehrenberg * the smallest multiple of the stripe value (sbi->s_stripe) which is 135d7a1fee1SDan Ehrenberg * greater than the default mb_group_prealloc. 136c9de560dSAlex Tomas * 137d7a1fee1SDan Ehrenberg * The regular allocator (using the buddy cache) supports a few tunables. 138c9de560dSAlex Tomas * 139b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_min_to_scan 140b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_max_to_scan 141b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req 142c9de560dSAlex Tomas * 143b713a5ecSTheodore Ts'o * The regular allocator uses buddy scan only if the request len is power of 144c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 145c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 146b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to 147af901ca1SAndré Goddard Rosa * stripe size (sbi->s_stripe), we try to search for contiguous block in 148b713a5ecSTheodore Ts'o * stripe size. This should result in better allocation on RAID setups. If 149b713a5ecSTheodore Ts'o * not, we search in the specific group using bitmap for best extents. The 150b713a5ecSTheodore Ts'o * tunable min_to_scan and max_to_scan control the behaviour here. 151c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 152b713a5ecSTheodore Ts'o * extent and max_to_scan indicates how long the mballoc __can__ look for a 153c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 154c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 155c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 156caaf7a29STao Ma * can be used for allocation. ext4_mb_good_group explains how the groups are 157c9de560dSAlex Tomas * checked. 158c9de560dSAlex Tomas * 159c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 160c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 161c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 162c9de560dSAlex Tomas * subsequent request. 163c9de560dSAlex Tomas */ 164c9de560dSAlex Tomas 165c9de560dSAlex Tomas /* 166c9de560dSAlex Tomas * mballoc operates on the following data: 167c9de560dSAlex Tomas * - on-disk bitmap 168c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 169c9de560dSAlex Tomas * - preallocation descriptors (PAs) 170c9de560dSAlex Tomas * 171c9de560dSAlex Tomas * there are two types of preallocations: 172c9de560dSAlex Tomas * - inode 173c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 174c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 175c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 176c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 177c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 178c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 179c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 180c9de560dSAlex Tomas * must discard all preallocated blocks. 181c9de560dSAlex Tomas * - locality group 182c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 183c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 184c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 185c9de560dSAlex Tomas * it's consumed from the beginning to the end. 186c9de560dSAlex Tomas * 187c9de560dSAlex Tomas * relation between them can be expressed as: 188c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 189c9de560dSAlex Tomas * 190c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 191c9de560dSAlex Tomas * - allocated blocks (persistent) 192c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 193c9de560dSAlex Tomas * 194c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 195c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 196c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 197c9de560dSAlex Tomas * 198c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 199c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 200c9de560dSAlex Tomas * 201c9de560dSAlex Tomas * all operations can be expressed as: 202c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 203c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 204c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 205c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 206c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 207c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 208c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 209c9de560dSAlex Tomas * is used in real operation because we can't know actual used 210c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 211c9de560dSAlex Tomas * 212c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 213c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 214c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 215c9de560dSAlex Tomas * the following knowledge: 216c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 217c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 218c9de560dSAlex Tomas * nobody can re-allocate that block 219c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 220c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 221c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 222c9de560dSAlex Tomas * block 223c9de560dSAlex Tomas * 224c9de560dSAlex Tomas * so, now we're building a concurrency table: 225c9de560dSAlex Tomas * - init buddy vs. 226c9de560dSAlex Tomas * - new PA 227c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 228c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 229c9de560dSAlex Tomas * - use inode PA 230c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 231c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 232c9de560dSAlex Tomas * - discard inode PA 233c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 234c9de560dSAlex Tomas * - use locality group PA 235c9de560dSAlex Tomas * again PA-=N must be serialized with init 236c9de560dSAlex Tomas * - discard locality group PA 237c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 238c9de560dSAlex Tomas * - new PA vs. 239c9de560dSAlex Tomas * - use inode PA 240c9de560dSAlex Tomas * i_data_sem serializes them 241c9de560dSAlex Tomas * - discard inode PA 242c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 243c9de560dSAlex Tomas * - use locality group PA 244c9de560dSAlex Tomas * some mutex should serialize them 245c9de560dSAlex Tomas * - discard locality group PA 246c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 247c9de560dSAlex Tomas * - use inode PA 248c9de560dSAlex Tomas * - use inode PA 249c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 250c9de560dSAlex Tomas * - discard inode PA 251c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 252c9de560dSAlex Tomas * - use locality group PA 253c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 254c9de560dSAlex Tomas * - discard locality group PA 255c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 256c9de560dSAlex Tomas * 257c9de560dSAlex Tomas * now we're ready to make few consequences: 258c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 259c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 260c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 261c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 262c9de560dSAlex Tomas * any discard or they're serialized somehow 263c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 264c9de560dSAlex Tomas * 265c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 266c9de560dSAlex Tomas * in this case, but we should care about concurrent init 267c9de560dSAlex Tomas * 268c9de560dSAlex Tomas */ 269c9de560dSAlex Tomas 270c9de560dSAlex Tomas /* 271c9de560dSAlex Tomas * Logic in few words: 272c9de560dSAlex Tomas * 273c9de560dSAlex Tomas * - allocation: 274c9de560dSAlex Tomas * load group 275c9de560dSAlex Tomas * find blocks 276c9de560dSAlex Tomas * mark bits in on-disk bitmap 277c9de560dSAlex Tomas * release group 278c9de560dSAlex Tomas * 279c9de560dSAlex Tomas * - use preallocation: 280c9de560dSAlex Tomas * find proper PA (per-inode or group) 281c9de560dSAlex Tomas * load group 282c9de560dSAlex Tomas * mark bits in on-disk bitmap 283c9de560dSAlex Tomas * release group 284c9de560dSAlex Tomas * release PA 285c9de560dSAlex Tomas * 286c9de560dSAlex Tomas * - free: 287c9de560dSAlex Tomas * load group 288c9de560dSAlex Tomas * mark bits in on-disk bitmap 289c9de560dSAlex Tomas * release group 290c9de560dSAlex Tomas * 291c9de560dSAlex Tomas * - discard preallocations in group: 292c9de560dSAlex Tomas * mark PAs deleted 293c9de560dSAlex Tomas * move them onto local list 294c9de560dSAlex Tomas * load on-disk bitmap 295c9de560dSAlex Tomas * load group 296c9de560dSAlex Tomas * remove PA from object (inode or locality group) 297c9de560dSAlex Tomas * mark free blocks in-core 298c9de560dSAlex Tomas * 299c9de560dSAlex Tomas * - discard inode's preallocations: 300c9de560dSAlex Tomas */ 301c9de560dSAlex Tomas 302c9de560dSAlex Tomas /* 303c9de560dSAlex Tomas * Locking rules 304c9de560dSAlex Tomas * 305c9de560dSAlex Tomas * Locks: 306c9de560dSAlex Tomas * - bitlock on a group (group) 307c9de560dSAlex Tomas * - object (inode/locality) (object) 308c9de560dSAlex Tomas * - per-pa lock (pa) 309c9de560dSAlex Tomas * 310c9de560dSAlex Tomas * Paths: 311c9de560dSAlex Tomas * - new pa 312c9de560dSAlex Tomas * object 313c9de560dSAlex Tomas * group 314c9de560dSAlex Tomas * 315c9de560dSAlex Tomas * - find and use pa: 316c9de560dSAlex Tomas * pa 317c9de560dSAlex Tomas * 318c9de560dSAlex Tomas * - release consumed pa: 319c9de560dSAlex Tomas * pa 320c9de560dSAlex Tomas * group 321c9de560dSAlex Tomas * object 322c9de560dSAlex Tomas * 323c9de560dSAlex Tomas * - generate in-core bitmap: 324c9de560dSAlex Tomas * group 325c9de560dSAlex Tomas * pa 326c9de560dSAlex Tomas * 327c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 328c9de560dSAlex Tomas * object 329c9de560dSAlex Tomas * pa 330c9de560dSAlex Tomas * group 331c9de560dSAlex Tomas * 332c9de560dSAlex Tomas * - discard all for given group: 333c9de560dSAlex Tomas * group 334c9de560dSAlex Tomas * pa 335c9de560dSAlex Tomas * group 336c9de560dSAlex Tomas * object 337c9de560dSAlex Tomas * 338c9de560dSAlex Tomas */ 339c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_pspace_cachep; 340c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_ac_cachep; 34118aadd47SBobi Jam static struct kmem_cache *ext4_free_data_cachep; 342fb1813f4SCurt Wohlgemuth 343fb1813f4SCurt Wohlgemuth /* We create slab caches for groupinfo data structures based on the 344fb1813f4SCurt Wohlgemuth * superblock block size. There will be one per mounted filesystem for 345fb1813f4SCurt Wohlgemuth * each unique s_blocksize_bits */ 3462892c15dSEric Sandeen #define NR_GRPINFO_CACHES 8 347fb1813f4SCurt Wohlgemuth static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES]; 348fb1813f4SCurt Wohlgemuth 349d6006186SEric Biggers static const char * const ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { 3502892c15dSEric Sandeen "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k", 3512892c15dSEric Sandeen "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k", 3522892c15dSEric Sandeen "ext4_groupinfo_64k", "ext4_groupinfo_128k" 3532892c15dSEric Sandeen }; 3542892c15dSEric Sandeen 355c3a326a6SAneesh Kumar K.V static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 356c3a326a6SAneesh Kumar K.V ext4_group_t group); 3577a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 3587a2fcbf7SAneesh Kumar K.V ext4_group_t group); 359c3a326a6SAneesh Kumar K.V 360ffad0a44SAneesh Kumar K.V static inline void *mb_correct_addr_and_bit(int *bit, void *addr) 361ffad0a44SAneesh Kumar K.V { 362c9de560dSAlex Tomas #if BITS_PER_LONG == 64 363ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 7UL) << 3; 364ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~7UL); 365c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 366ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 3UL) << 3; 367ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~3UL); 368c9de560dSAlex Tomas #else 369c9de560dSAlex Tomas #error "how many bits you are?!" 370c9de560dSAlex Tomas #endif 371ffad0a44SAneesh Kumar K.V return addr; 372ffad0a44SAneesh Kumar K.V } 373c9de560dSAlex Tomas 374c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 375c9de560dSAlex Tomas { 376c9de560dSAlex Tomas /* 377c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 378c9de560dSAlex Tomas * needs unsigned long aligned address 379c9de560dSAlex Tomas */ 380ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 381c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 382c9de560dSAlex Tomas } 383c9de560dSAlex Tomas 384c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 385c9de560dSAlex Tomas { 386ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 387c9de560dSAlex Tomas ext4_set_bit(bit, addr); 388c9de560dSAlex Tomas } 389c9de560dSAlex Tomas 390c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 391c9de560dSAlex Tomas { 392ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 393c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 394c9de560dSAlex Tomas } 395c9de560dSAlex Tomas 396eabe0444SAndrey Sidorov static inline int mb_test_and_clear_bit(int bit, void *addr) 397eabe0444SAndrey Sidorov { 398eabe0444SAndrey Sidorov addr = mb_correct_addr_and_bit(&bit, addr); 399eabe0444SAndrey Sidorov return ext4_test_and_clear_bit(bit, addr); 400eabe0444SAndrey Sidorov } 401eabe0444SAndrey Sidorov 402ffad0a44SAneesh Kumar K.V static inline int mb_find_next_zero_bit(void *addr, int max, int start) 403ffad0a44SAneesh Kumar K.V { 404e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 405ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 406e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 407ffad0a44SAneesh Kumar K.V start += fix; 408ffad0a44SAneesh Kumar K.V 409e7dfb246SAneesh Kumar K.V ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; 410e7dfb246SAneesh Kumar K.V if (ret > max) 411e7dfb246SAneesh Kumar K.V return max; 412e7dfb246SAneesh Kumar K.V return ret; 413ffad0a44SAneesh Kumar K.V } 414ffad0a44SAneesh Kumar K.V 415ffad0a44SAneesh Kumar K.V static inline int mb_find_next_bit(void *addr, int max, int start) 416ffad0a44SAneesh Kumar K.V { 417e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 418ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 419e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 420ffad0a44SAneesh Kumar K.V start += fix; 421ffad0a44SAneesh Kumar K.V 422e7dfb246SAneesh Kumar K.V ret = ext4_find_next_bit(addr, tmpmax, start) - fix; 423e7dfb246SAneesh Kumar K.V if (ret > max) 424e7dfb246SAneesh Kumar K.V return max; 425e7dfb246SAneesh Kumar K.V return ret; 426ffad0a44SAneesh Kumar K.V } 427ffad0a44SAneesh Kumar K.V 428c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 429c9de560dSAlex Tomas { 430c9de560dSAlex Tomas char *bb; 431c9de560dSAlex Tomas 432c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 433c9de560dSAlex Tomas BUG_ON(max == NULL); 434c9de560dSAlex Tomas 435c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 436c9de560dSAlex Tomas *max = 0; 437c9de560dSAlex Tomas return NULL; 438c9de560dSAlex Tomas } 439c9de560dSAlex Tomas 440c9de560dSAlex Tomas /* at order 0 we see each particular block */ 44184b775a3SColy Li if (order == 0) { 442c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 443c5e8f3f3STheodore Ts'o return e4b->bd_bitmap; 44484b775a3SColy Li } 445c9de560dSAlex Tomas 446c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 447c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 448c9de560dSAlex Tomas 449c9de560dSAlex Tomas return bb; 450c9de560dSAlex Tomas } 451c9de560dSAlex Tomas 452c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 453c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 454c9de560dSAlex Tomas int first, int count) 455c9de560dSAlex Tomas { 456c9de560dSAlex Tomas int i; 457c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 458c9de560dSAlex Tomas 459c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 460c9de560dSAlex Tomas return; 461bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 462c9de560dSAlex Tomas for (i = 0; i < count; i++) { 463c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 464c9de560dSAlex Tomas ext4_fsblk_t blocknr; 4655661bd68SAkinobu Mita 4665661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 46753accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), first + i); 4685d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 469e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 470e29136f8STheodore Ts'o blocknr, 471e29136f8STheodore Ts'o "freeing block already freed " 472e29136f8STheodore Ts'o "(bit %u)", 473e29136f8STheodore Ts'o first + i); 474736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 475736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 476c9de560dSAlex Tomas } 477c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 478c9de560dSAlex Tomas } 479c9de560dSAlex Tomas } 480c9de560dSAlex Tomas 481c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 482c9de560dSAlex Tomas { 483c9de560dSAlex Tomas int i; 484c9de560dSAlex Tomas 485c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 486c9de560dSAlex Tomas return; 487bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 488c9de560dSAlex Tomas for (i = 0; i < count; i++) { 489c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 490c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 491c9de560dSAlex Tomas } 492c9de560dSAlex Tomas } 493c9de560dSAlex Tomas 494c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 495c9de560dSAlex Tomas { 496c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 497c9de560dSAlex Tomas unsigned char *b1, *b2; 498c9de560dSAlex Tomas int i; 499c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 500c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 501c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 502c9de560dSAlex Tomas if (b1[i] != b2[i]) { 5039d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_ERR, 5049d8b9ec4STheodore Ts'o "corruption in group %u " 5054776004fSTheodore Ts'o "at byte %u(%u): %x in copy != %x " 5069d8b9ec4STheodore Ts'o "on disk/prealloc", 507c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 508c9de560dSAlex Tomas BUG(); 509c9de560dSAlex Tomas } 510c9de560dSAlex Tomas } 511c9de560dSAlex Tomas } 512c9de560dSAlex Tomas } 513c9de560dSAlex Tomas 514c9de560dSAlex Tomas #else 515c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 516c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 517c9de560dSAlex Tomas { 518c9de560dSAlex Tomas return; 519c9de560dSAlex Tomas } 520c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 521c9de560dSAlex Tomas int first, int count) 522c9de560dSAlex Tomas { 523c9de560dSAlex Tomas return; 524c9de560dSAlex Tomas } 525c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 526c9de560dSAlex Tomas { 527c9de560dSAlex Tomas return; 528c9de560dSAlex Tomas } 529c9de560dSAlex Tomas #endif 530c9de560dSAlex Tomas 531c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 532c9de560dSAlex Tomas 533c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 534c9de560dSAlex Tomas do { \ 535c9de560dSAlex Tomas if (!(assert)) { \ 536c9de560dSAlex Tomas printk(KERN_EMERG \ 537c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 538c9de560dSAlex Tomas function, file, line, # assert); \ 539c9de560dSAlex Tomas BUG(); \ 540c9de560dSAlex Tomas } \ 541c9de560dSAlex Tomas } while (0) 542c9de560dSAlex Tomas 543c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 544c9de560dSAlex Tomas const char *function, int line) 545c9de560dSAlex Tomas { 546c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 547c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 548c9de560dSAlex Tomas int max; 549c9de560dSAlex Tomas int max2; 550c9de560dSAlex Tomas int i; 551c9de560dSAlex Tomas int j; 552c9de560dSAlex Tomas int k; 553c9de560dSAlex Tomas int count; 554c9de560dSAlex Tomas struct ext4_group_info *grp; 555c9de560dSAlex Tomas int fragments = 0; 556c9de560dSAlex Tomas int fstart; 557c9de560dSAlex Tomas struct list_head *cur; 558c9de560dSAlex Tomas void *buddy; 559c9de560dSAlex Tomas void *buddy2; 560c9de560dSAlex Tomas 561c9de560dSAlex Tomas { 562c9de560dSAlex Tomas static int mb_check_counter; 563c9de560dSAlex Tomas if (mb_check_counter++ % 100 != 0) 564c9de560dSAlex Tomas return 0; 565c9de560dSAlex Tomas } 566c9de560dSAlex Tomas 567c9de560dSAlex Tomas while (order > 1) { 568c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 569c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 570c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 571c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 572c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 573c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 574c9de560dSAlex Tomas 575c9de560dSAlex Tomas count = 0; 576c9de560dSAlex Tomas for (i = 0; i < max; i++) { 577c9de560dSAlex Tomas 578c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 579c9de560dSAlex Tomas /* only single bit in buddy2 may be 1 */ 580c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 581c9de560dSAlex Tomas MB_CHECK_ASSERT( 582c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 583c9de560dSAlex Tomas } else if (!mb_test_bit((i << 1) + 1, buddy2)) { 584c9de560dSAlex Tomas MB_CHECK_ASSERT( 585c9de560dSAlex Tomas mb_test_bit(i << 1, buddy2)); 586c9de560dSAlex Tomas } 587c9de560dSAlex Tomas continue; 588c9de560dSAlex Tomas } 589c9de560dSAlex Tomas 5900a10da73SRobin Dong /* both bits in buddy2 must be 1 */ 591c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 592c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 593c9de560dSAlex Tomas 594c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 595c9de560dSAlex Tomas k = (i * (1 << order)) + j; 596c9de560dSAlex Tomas MB_CHECK_ASSERT( 597c5e8f3f3STheodore Ts'o !mb_test_bit(k, e4b->bd_bitmap)); 598c9de560dSAlex Tomas } 599c9de560dSAlex Tomas count++; 600c9de560dSAlex Tomas } 601c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 602c9de560dSAlex Tomas order--; 603c9de560dSAlex Tomas } 604c9de560dSAlex Tomas 605c9de560dSAlex Tomas fstart = -1; 606c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 607c9de560dSAlex Tomas for (i = 0; i < max; i++) { 608c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 609c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 610c9de560dSAlex Tomas if (fstart == -1) { 611c9de560dSAlex Tomas fragments++; 612c9de560dSAlex Tomas fstart = i; 613c9de560dSAlex Tomas } 614c9de560dSAlex Tomas continue; 615c9de560dSAlex Tomas } 616c9de560dSAlex Tomas fstart = -1; 617c9de560dSAlex Tomas /* check used bits only */ 618c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 619c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 620c9de560dSAlex Tomas k = i >> j; 621c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 622c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 623c9de560dSAlex Tomas } 624c9de560dSAlex Tomas } 625c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 626c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 627c9de560dSAlex Tomas 628c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 629c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 630c9de560dSAlex Tomas ext4_group_t groupnr; 631c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 63260bd63d1SSolofo Ramangalahy pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 63360bd63d1SSolofo Ramangalahy ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); 634c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 63560bd63d1SSolofo Ramangalahy for (i = 0; i < pa->pa_len; i++) 636c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 637c9de560dSAlex Tomas } 638c9de560dSAlex Tomas return 0; 639c9de560dSAlex Tomas } 640c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 641c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 64246e665e9SHarvey Harrison __FILE__, __func__, __LINE__) 643c9de560dSAlex Tomas #else 644c9de560dSAlex Tomas #define mb_check_buddy(e4b) 645c9de560dSAlex Tomas #endif 646c9de560dSAlex Tomas 6477c786059SColy Li /* 6487c786059SColy Li * Divide blocks started from @first with length @len into 6497c786059SColy Li * smaller chunks with power of 2 blocks. 6507c786059SColy Li * Clear the bits in bitmap which the blocks of the chunk(s) covered, 6517c786059SColy Li * then increase bb_counters[] for corresponded chunk size. 6527c786059SColy Li */ 653c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 654a36b4498SEric Sandeen void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, 655c9de560dSAlex Tomas struct ext4_group_info *grp) 656c9de560dSAlex Tomas { 657c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 658a36b4498SEric Sandeen ext4_grpblk_t min; 659a36b4498SEric Sandeen ext4_grpblk_t max; 660a36b4498SEric Sandeen ext4_grpblk_t chunk; 66169e43e8cSChandan Rajendra unsigned int border; 662c9de560dSAlex Tomas 6637137d7a4STheodore Ts'o BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); 664c9de560dSAlex Tomas 665c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 666c9de560dSAlex Tomas 667c9de560dSAlex Tomas while (len > 0) { 668c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 669c9de560dSAlex Tomas max = ffs(first | border) - 1; 670c9de560dSAlex Tomas 671c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 672c9de560dSAlex Tomas min = fls(len) - 1; 673c9de560dSAlex Tomas 674c9de560dSAlex Tomas if (max < min) 675c9de560dSAlex Tomas min = max; 676c9de560dSAlex Tomas chunk = 1 << min; 677c9de560dSAlex Tomas 678c9de560dSAlex Tomas /* mark multiblock chunks only */ 679c9de560dSAlex Tomas grp->bb_counters[min]++; 680c9de560dSAlex Tomas if (min > 0) 681c9de560dSAlex Tomas mb_clear_bit(first >> min, 682c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 683c9de560dSAlex Tomas 684c9de560dSAlex Tomas len -= chunk; 685c9de560dSAlex Tomas first += chunk; 686c9de560dSAlex Tomas } 687c9de560dSAlex Tomas } 688c9de560dSAlex Tomas 6898a57d9d6SCurt Wohlgemuth /* 6908a57d9d6SCurt Wohlgemuth * Cache the order of the largest free extent we have available in this block 6918a57d9d6SCurt Wohlgemuth * group. 6928a57d9d6SCurt Wohlgemuth */ 6938a57d9d6SCurt Wohlgemuth static void 6948a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) 6958a57d9d6SCurt Wohlgemuth { 6968a57d9d6SCurt Wohlgemuth int i; 6978a57d9d6SCurt Wohlgemuth int bits; 6988a57d9d6SCurt Wohlgemuth 6998a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = -1; /* uninit */ 7008a57d9d6SCurt Wohlgemuth 7018a57d9d6SCurt Wohlgemuth bits = sb->s_blocksize_bits + 1; 7028a57d9d6SCurt Wohlgemuth for (i = bits; i >= 0; i--) { 7038a57d9d6SCurt Wohlgemuth if (grp->bb_counters[i] > 0) { 7048a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = i; 7058a57d9d6SCurt Wohlgemuth break; 7068a57d9d6SCurt Wohlgemuth } 7078a57d9d6SCurt Wohlgemuth } 7088a57d9d6SCurt Wohlgemuth } 7098a57d9d6SCurt Wohlgemuth 710089ceeccSEric Sandeen static noinline_for_stack 711089ceeccSEric Sandeen void ext4_mb_generate_buddy(struct super_block *sb, 712c9de560dSAlex Tomas void *buddy, void *bitmap, ext4_group_t group) 713c9de560dSAlex Tomas { 714c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 715e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 7167137d7a4STheodore Ts'o ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); 717a36b4498SEric Sandeen ext4_grpblk_t i = 0; 718a36b4498SEric Sandeen ext4_grpblk_t first; 719a36b4498SEric Sandeen ext4_grpblk_t len; 720c9de560dSAlex Tomas unsigned free = 0; 721c9de560dSAlex Tomas unsigned fragments = 0; 722c9de560dSAlex Tomas unsigned long long period = get_cycles(); 723c9de560dSAlex Tomas 724c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 725c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 726ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, 0); 727c9de560dSAlex Tomas grp->bb_first_free = i; 728c9de560dSAlex Tomas while (i < max) { 729c9de560dSAlex Tomas fragments++; 730c9de560dSAlex Tomas first = i; 731ffad0a44SAneesh Kumar K.V i = mb_find_next_bit(bitmap, max, i); 732c9de560dSAlex Tomas len = i - first; 733c9de560dSAlex Tomas free += len; 734c9de560dSAlex Tomas if (len > 1) 735c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 736c9de560dSAlex Tomas else 737c9de560dSAlex Tomas grp->bb_counters[0]++; 738c9de560dSAlex Tomas if (i < max) 739ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, i); 740c9de560dSAlex Tomas } 741c9de560dSAlex Tomas grp->bb_fragments = fragments; 742c9de560dSAlex Tomas 743c9de560dSAlex Tomas if (free != grp->bb_free) { 744e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, 74594d4c066STheodore Ts'o "block bitmap and bg descriptor " 74694d4c066STheodore Ts'o "inconsistent: %u vs %u free clusters", 747e29136f8STheodore Ts'o free, grp->bb_free); 748e56eb659SAneesh Kumar K.V /* 749163a203dSDarrick J. Wong * If we intend to continue, we consider group descriptor 750e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 751e56eb659SAneesh Kumar K.V */ 752c9de560dSAlex Tomas grp->bb_free = free; 753db79e6d1SWang Shilong ext4_mark_group_bitmap_corrupted(sb, group, 754db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 755c9de560dSAlex Tomas } 7568a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, grp); 757c9de560dSAlex Tomas 758c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 759c9de560dSAlex Tomas 760c9de560dSAlex Tomas period = get_cycles() - period; 76149598e04SJun Piao spin_lock(&sbi->s_bal_lock); 76249598e04SJun Piao sbi->s_mb_buddies_generated++; 76349598e04SJun Piao sbi->s_mb_generation_time += period; 76449598e04SJun Piao spin_unlock(&sbi->s_bal_lock); 765c9de560dSAlex Tomas } 766c9de560dSAlex Tomas 767eabe0444SAndrey Sidorov static void mb_regenerate_buddy(struct ext4_buddy *e4b) 768eabe0444SAndrey Sidorov { 769eabe0444SAndrey Sidorov int count; 770eabe0444SAndrey Sidorov int order = 1; 771eabe0444SAndrey Sidorov void *buddy; 772eabe0444SAndrey Sidorov 773eabe0444SAndrey Sidorov while ((buddy = mb_find_buddy(e4b, order++, &count))) { 774eabe0444SAndrey Sidorov ext4_set_bits(buddy, 0, count); 775eabe0444SAndrey Sidorov } 776eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments = 0; 777eabe0444SAndrey Sidorov memset(e4b->bd_info->bb_counters, 0, 778eabe0444SAndrey Sidorov sizeof(*e4b->bd_info->bb_counters) * 779eabe0444SAndrey Sidorov (e4b->bd_sb->s_blocksize_bits + 2)); 780eabe0444SAndrey Sidorov 781eabe0444SAndrey Sidorov ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy, 782eabe0444SAndrey Sidorov e4b->bd_bitmap, e4b->bd_group); 783eabe0444SAndrey Sidorov } 784eabe0444SAndrey Sidorov 785c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 786c9de560dSAlex Tomas * for convenience. The information regarding each group 787c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 788c9de560dSAlex Tomas * block bitmap and buddy information. The information are 789c9de560dSAlex Tomas * stored in the inode as 790c9de560dSAlex Tomas * 791c9de560dSAlex Tomas * { page } 792c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 793c9de560dSAlex Tomas * 794c9de560dSAlex Tomas * 795c9de560dSAlex Tomas * one block each for bitmap and buddy information. 796c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 797ea1754a0SKirill A. Shutemov * contain blocks_per_page (PAGE_SIZE / blocksize) blocks. 798c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 799c9de560dSAlex Tomas * is blocks_per_page/2 8008a57d9d6SCurt Wohlgemuth * 8018a57d9d6SCurt Wohlgemuth * Locking note: This routine takes the block group lock of all groups 8028a57d9d6SCurt Wohlgemuth * for this page; do not hold this lock when calling this routine! 803c9de560dSAlex Tomas */ 804c9de560dSAlex Tomas 805adb7ef60SKonstantin Khlebnikov static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) 806c9de560dSAlex Tomas { 8078df9675fSTheodore Ts'o ext4_group_t ngroups; 808c9de560dSAlex Tomas int blocksize; 809c9de560dSAlex Tomas int blocks_per_page; 810c9de560dSAlex Tomas int groups_per_page; 811c9de560dSAlex Tomas int err = 0; 812c9de560dSAlex Tomas int i; 813813e5727STheodore Ts'o ext4_group_t first_group, group; 814c9de560dSAlex Tomas int first_block; 815c9de560dSAlex Tomas struct super_block *sb; 816c9de560dSAlex Tomas struct buffer_head *bhs; 817fa77dcfaSDarrick J. Wong struct buffer_head **bh = NULL; 818c9de560dSAlex Tomas struct inode *inode; 819c9de560dSAlex Tomas char *data; 820c9de560dSAlex Tomas char *bitmap; 8219b8b7d35SAmir Goldstein struct ext4_group_info *grinfo; 822c9de560dSAlex Tomas 8236ba495e9STheodore Ts'o mb_debug(1, "init page %lu\n", page->index); 824c9de560dSAlex Tomas 825c9de560dSAlex Tomas inode = page->mapping->host; 826c9de560dSAlex Tomas sb = inode->i_sb; 8278df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 82893407472SFabian Frederick blocksize = i_blocksize(inode); 82909cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / blocksize; 830c9de560dSAlex Tomas 831c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 832c9de560dSAlex Tomas if (groups_per_page == 0) 833c9de560dSAlex Tomas groups_per_page = 1; 834c9de560dSAlex Tomas 835c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 836c9de560dSAlex Tomas if (groups_per_page > 1) { 837c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 838adb7ef60SKonstantin Khlebnikov bh = kzalloc(i, gfp); 839813e5727STheodore Ts'o if (bh == NULL) { 840813e5727STheodore Ts'o err = -ENOMEM; 841c9de560dSAlex Tomas goto out; 842813e5727STheodore Ts'o } 843c9de560dSAlex Tomas } else 844c9de560dSAlex Tomas bh = &bhs; 845c9de560dSAlex Tomas 846c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 847c9de560dSAlex Tomas 848c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 849813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 850813e5727STheodore Ts'o if (group >= ngroups) 851c9de560dSAlex Tomas break; 852c9de560dSAlex Tomas 853813e5727STheodore Ts'o grinfo = ext4_get_group_info(sb, group); 8549b8b7d35SAmir Goldstein /* 8559b8b7d35SAmir Goldstein * If page is uptodate then we came here after online resize 8569b8b7d35SAmir Goldstein * which added some new uninitialized group info structs, so 8579b8b7d35SAmir Goldstein * we must skip all initialized uptodate buddies on the page, 8589b8b7d35SAmir Goldstein * which may be currently in use by an allocating task. 8599b8b7d35SAmir Goldstein */ 8609b8b7d35SAmir Goldstein if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) { 8619b8b7d35SAmir Goldstein bh[i] = NULL; 8629b8b7d35SAmir Goldstein continue; 8639b8b7d35SAmir Goldstein } 8649008a58eSDarrick J. Wong bh[i] = ext4_read_block_bitmap_nowait(sb, group); 8659008a58eSDarrick J. Wong if (IS_ERR(bh[i])) { 8669008a58eSDarrick J. Wong err = PTR_ERR(bh[i]); 8679008a58eSDarrick J. Wong bh[i] = NULL; 868c9de560dSAlex Tomas goto out; 8692ccb5fb9SAneesh Kumar K.V } 870813e5727STheodore Ts'o mb_debug(1, "read bitmap for group %u\n", group); 871c9de560dSAlex Tomas } 872c9de560dSAlex Tomas 873c9de560dSAlex Tomas /* wait for I/O completion */ 874813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 8759008a58eSDarrick J. Wong int err2; 8769008a58eSDarrick J. Wong 8779008a58eSDarrick J. Wong if (!bh[i]) 8789008a58eSDarrick J. Wong continue; 8799008a58eSDarrick J. Wong err2 = ext4_wait_block_bitmap(sb, group, bh[i]); 8809008a58eSDarrick J. Wong if (!err) 8819008a58eSDarrick J. Wong err = err2; 882813e5727STheodore Ts'o } 883c9de560dSAlex Tomas 884c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 885c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 886c9de560dSAlex Tomas group = (first_block + i) >> 1; 8878df9675fSTheodore Ts'o if (group >= ngroups) 888c9de560dSAlex Tomas break; 889c9de560dSAlex Tomas 8909b8b7d35SAmir Goldstein if (!bh[group - first_group]) 8919b8b7d35SAmir Goldstein /* skip initialized uptodate buddy */ 8929b8b7d35SAmir Goldstein continue; 8939b8b7d35SAmir Goldstein 894bbdc322fSLukas Czerner if (!buffer_verified(bh[group - first_group])) 895bbdc322fSLukas Czerner /* Skip faulty bitmaps */ 896bbdc322fSLukas Czerner continue; 897bbdc322fSLukas Czerner err = 0; 898bbdc322fSLukas Czerner 899c9de560dSAlex Tomas /* 900c9de560dSAlex Tomas * data carry information regarding this 901c9de560dSAlex Tomas * particular group in the format specified 902c9de560dSAlex Tomas * above 903c9de560dSAlex Tomas * 904c9de560dSAlex Tomas */ 905c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 906c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 907c9de560dSAlex Tomas 908c9de560dSAlex Tomas /* 909c9de560dSAlex Tomas * We place the buddy block and bitmap block 910c9de560dSAlex Tomas * close together 911c9de560dSAlex Tomas */ 912c9de560dSAlex Tomas if ((first_block + i) & 1) { 913c9de560dSAlex Tomas /* this is block of buddy */ 914c9de560dSAlex Tomas BUG_ON(incore == NULL); 9156ba495e9STheodore Ts'o mb_debug(1, "put buddy for group %u in page %lu/%x\n", 916c9de560dSAlex Tomas group, page->index, i * blocksize); 917f307333eSTheodore Ts'o trace_ext4_mb_buddy_bitmap_load(sb, group); 918c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 919c9de560dSAlex Tomas grinfo->bb_fragments = 0; 920c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 9211927805eSEric Sandeen sizeof(*grinfo->bb_counters) * 9221927805eSEric Sandeen (sb->s_blocksize_bits+2)); 923c9de560dSAlex Tomas /* 924c9de560dSAlex Tomas * incore got set to the group block bitmap below 925c9de560dSAlex Tomas */ 9267a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, group); 9279b8b7d35SAmir Goldstein /* init the buddy */ 9289b8b7d35SAmir Goldstein memset(data, 0xff, blocksize); 929c9de560dSAlex Tomas ext4_mb_generate_buddy(sb, data, incore, group); 9307a2fcbf7SAneesh Kumar K.V ext4_unlock_group(sb, group); 931c9de560dSAlex Tomas incore = NULL; 932c9de560dSAlex Tomas } else { 933c9de560dSAlex Tomas /* this is block of bitmap */ 934c9de560dSAlex Tomas BUG_ON(incore != NULL); 9356ba495e9STheodore Ts'o mb_debug(1, "put bitmap for group %u in page %lu/%x\n", 936c9de560dSAlex Tomas group, page->index, i * blocksize); 937f307333eSTheodore Ts'o trace_ext4_mb_bitmap_load(sb, group); 938c9de560dSAlex Tomas 939c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 940c9de560dSAlex Tomas ext4_lock_group(sb, group); 941c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 942c9de560dSAlex Tomas 943c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 944c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 9457a2fcbf7SAneesh Kumar K.V ext4_mb_generate_from_freelist(sb, data, group); 946c9de560dSAlex Tomas ext4_unlock_group(sb, group); 947c9de560dSAlex Tomas 948c9de560dSAlex Tomas /* set incore so that the buddy information can be 949c9de560dSAlex Tomas * generated using this 950c9de560dSAlex Tomas */ 951c9de560dSAlex Tomas incore = data; 952c9de560dSAlex Tomas } 953c9de560dSAlex Tomas } 954c9de560dSAlex Tomas SetPageUptodate(page); 955c9de560dSAlex Tomas 956c9de560dSAlex Tomas out: 957c9de560dSAlex Tomas if (bh) { 9589b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 959c9de560dSAlex Tomas brelse(bh[i]); 960c9de560dSAlex Tomas if (bh != &bhs) 961c9de560dSAlex Tomas kfree(bh); 962c9de560dSAlex Tomas } 963c9de560dSAlex Tomas return err; 964c9de560dSAlex Tomas } 965c9de560dSAlex Tomas 9668a57d9d6SCurt Wohlgemuth /* 9672de8807bSAmir Goldstein * Lock the buddy and bitmap pages. This make sure other parallel init_group 9682de8807bSAmir Goldstein * on the same buddy page doesn't happen whild holding the buddy page lock. 9692de8807bSAmir Goldstein * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap 9702de8807bSAmir Goldstein * are on the same page e4b->bd_buddy_page is NULL and return value is 0. 971eee4adc7SEric Sandeen */ 9722de8807bSAmir Goldstein static int ext4_mb_get_buddy_page_lock(struct super_block *sb, 973adb7ef60SKonstantin Khlebnikov ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp) 974eee4adc7SEric Sandeen { 9752de8807bSAmir Goldstein struct inode *inode = EXT4_SB(sb)->s_buddy_cache; 9762de8807bSAmir Goldstein int block, pnum, poff; 977eee4adc7SEric Sandeen int blocks_per_page; 9782de8807bSAmir Goldstein struct page *page; 9792de8807bSAmir Goldstein 9802de8807bSAmir Goldstein e4b->bd_buddy_page = NULL; 9812de8807bSAmir Goldstein e4b->bd_bitmap_page = NULL; 982eee4adc7SEric Sandeen 98309cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 984eee4adc7SEric Sandeen /* 985eee4adc7SEric Sandeen * the buddy cache inode stores the block bitmap 986eee4adc7SEric Sandeen * and buddy information in consecutive blocks. 987eee4adc7SEric Sandeen * So for each group we need two blocks. 988eee4adc7SEric Sandeen */ 989eee4adc7SEric Sandeen block = group * 2; 990eee4adc7SEric Sandeen pnum = block / blocks_per_page; 9912de8807bSAmir Goldstein poff = block % blocks_per_page; 992adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 9932de8807bSAmir Goldstein if (!page) 994c57ab39bSYounger Liu return -ENOMEM; 9952de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 9962de8807bSAmir Goldstein e4b->bd_bitmap_page = page; 9972de8807bSAmir Goldstein e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 998eee4adc7SEric Sandeen 9992de8807bSAmir Goldstein if (blocks_per_page >= 2) { 10002de8807bSAmir Goldstein /* buddy and bitmap are on the same page */ 10012de8807bSAmir Goldstein return 0; 1002eee4adc7SEric Sandeen } 1003eee4adc7SEric Sandeen 10042de8807bSAmir Goldstein block++; 1005eee4adc7SEric Sandeen pnum = block / blocks_per_page; 1006adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 10072de8807bSAmir Goldstein if (!page) 1008c57ab39bSYounger Liu return -ENOMEM; 10092de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 10102de8807bSAmir Goldstein e4b->bd_buddy_page = page; 10112de8807bSAmir Goldstein return 0; 1012eee4adc7SEric Sandeen } 1013eee4adc7SEric Sandeen 10142de8807bSAmir Goldstein static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) 10152de8807bSAmir Goldstein { 10162de8807bSAmir Goldstein if (e4b->bd_bitmap_page) { 10172de8807bSAmir Goldstein unlock_page(e4b->bd_bitmap_page); 101809cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 10192de8807bSAmir Goldstein } 10202de8807bSAmir Goldstein if (e4b->bd_buddy_page) { 10212de8807bSAmir Goldstein unlock_page(e4b->bd_buddy_page); 102209cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 10232de8807bSAmir Goldstein } 1024eee4adc7SEric Sandeen } 1025eee4adc7SEric Sandeen 1026eee4adc7SEric Sandeen /* 10278a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 10288a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 10298a57d9d6SCurt Wohlgemuth * calling this routine! 10308a57d9d6SCurt Wohlgemuth */ 1031b6a758ecSAneesh Kumar K.V static noinline_for_stack 1032adb7ef60SKonstantin Khlebnikov int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) 1033b6a758ecSAneesh Kumar K.V { 1034b6a758ecSAneesh Kumar K.V 1035b6a758ecSAneesh Kumar K.V struct ext4_group_info *this_grp; 10362de8807bSAmir Goldstein struct ext4_buddy e4b; 10372de8807bSAmir Goldstein struct page *page; 10382de8807bSAmir Goldstein int ret = 0; 1039b6a758ecSAneesh Kumar K.V 1040b10a44c3STheodore Ts'o might_sleep(); 1041b6a758ecSAneesh Kumar K.V mb_debug(1, "init group %u\n", group); 1042b6a758ecSAneesh Kumar K.V this_grp = ext4_get_group_info(sb, group); 1043b6a758ecSAneesh Kumar K.V /* 104408c3a813SAneesh Kumar K.V * This ensures that we don't reinit the buddy cache 104508c3a813SAneesh Kumar K.V * page which map to the group from which we are already 104608c3a813SAneesh Kumar K.V * allocating. If we are looking at the buddy cache we would 104708c3a813SAneesh Kumar K.V * have taken a reference using ext4_mb_load_buddy and that 10482de8807bSAmir Goldstein * would have pinned buddy page to page cache. 10492457aec6SMel Gorman * The call to ext4_mb_get_buddy_page_lock will mark the 10502457aec6SMel Gorman * page accessed. 1051b6a758ecSAneesh Kumar K.V */ 1052adb7ef60SKonstantin Khlebnikov ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp); 10532de8807bSAmir Goldstein if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) { 1054b6a758ecSAneesh Kumar K.V /* 1055b6a758ecSAneesh Kumar K.V * somebody initialized the group 1056b6a758ecSAneesh Kumar K.V * return without doing anything 1057b6a758ecSAneesh Kumar K.V */ 1058b6a758ecSAneesh Kumar K.V goto err; 1059b6a758ecSAneesh Kumar K.V } 10602de8807bSAmir Goldstein 10612de8807bSAmir Goldstein page = e4b.bd_bitmap_page; 1062adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 10632de8807bSAmir Goldstein if (ret) 1064b6a758ecSAneesh Kumar K.V goto err; 10652de8807bSAmir Goldstein if (!PageUptodate(page)) { 1066b6a758ecSAneesh Kumar K.V ret = -EIO; 1067b6a758ecSAneesh Kumar K.V goto err; 1068b6a758ecSAneesh Kumar K.V } 1069b6a758ecSAneesh Kumar K.V 10702de8807bSAmir Goldstein if (e4b.bd_buddy_page == NULL) { 1071b6a758ecSAneesh Kumar K.V /* 1072b6a758ecSAneesh Kumar K.V * If both the bitmap and buddy are in 1073b6a758ecSAneesh Kumar K.V * the same page we don't need to force 1074b6a758ecSAneesh Kumar K.V * init the buddy 1075b6a758ecSAneesh Kumar K.V */ 10762de8807bSAmir Goldstein ret = 0; 1077b6a758ecSAneesh Kumar K.V goto err; 1078b6a758ecSAneesh Kumar K.V } 10792de8807bSAmir Goldstein /* init buddy cache */ 10802de8807bSAmir Goldstein page = e4b.bd_buddy_page; 1081adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b.bd_bitmap, gfp); 10822de8807bSAmir Goldstein if (ret) 10832de8807bSAmir Goldstein goto err; 10842de8807bSAmir Goldstein if (!PageUptodate(page)) { 1085b6a758ecSAneesh Kumar K.V ret = -EIO; 1086b6a758ecSAneesh Kumar K.V goto err; 1087b6a758ecSAneesh Kumar K.V } 1088b6a758ecSAneesh Kumar K.V err: 10892de8807bSAmir Goldstein ext4_mb_put_buddy_page_lock(&e4b); 1090b6a758ecSAneesh Kumar K.V return ret; 1091b6a758ecSAneesh Kumar K.V } 1092b6a758ecSAneesh Kumar K.V 10938a57d9d6SCurt Wohlgemuth /* 10948a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 10958a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 10968a57d9d6SCurt Wohlgemuth * calling this routine! 10978a57d9d6SCurt Wohlgemuth */ 10984ddfef7bSEric Sandeen static noinline_for_stack int 1099adb7ef60SKonstantin Khlebnikov ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, 1100adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b, gfp_t gfp) 1101c9de560dSAlex Tomas { 1102c9de560dSAlex Tomas int blocks_per_page; 1103c9de560dSAlex Tomas int block; 1104c9de560dSAlex Tomas int pnum; 1105c9de560dSAlex Tomas int poff; 1106c9de560dSAlex Tomas struct page *page; 1107fdf6c7a7SShen Feng int ret; 1108920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1109920313a7SAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 1110920313a7SAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 1111c9de560dSAlex Tomas 1112b10a44c3STheodore Ts'o might_sleep(); 11136ba495e9STheodore Ts'o mb_debug(1, "load group %u\n", group); 1114c9de560dSAlex Tomas 111509cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 1116920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 1117c9de560dSAlex Tomas 1118c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1119529da704STao Ma e4b->bd_info = grp; 1120c9de560dSAlex Tomas e4b->bd_sb = sb; 1121c9de560dSAlex Tomas e4b->bd_group = group; 1122c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1123c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1124c9de560dSAlex Tomas 1125f41c0750SAneesh Kumar K.V if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 1126f41c0750SAneesh Kumar K.V /* 1127f41c0750SAneesh Kumar K.V * we need full data about the group 1128f41c0750SAneesh Kumar K.V * to make a good selection 1129f41c0750SAneesh Kumar K.V */ 1130adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, gfp); 1131f41c0750SAneesh Kumar K.V if (ret) 1132f41c0750SAneesh Kumar K.V return ret; 1133f41c0750SAneesh Kumar K.V } 1134f41c0750SAneesh Kumar K.V 1135c9de560dSAlex Tomas /* 1136c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1137c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1138c9de560dSAlex Tomas * So for each group we need two blocks. 1139c9de560dSAlex Tomas */ 1140c9de560dSAlex Tomas block = group * 2; 1141c9de560dSAlex Tomas pnum = block / blocks_per_page; 1142c9de560dSAlex Tomas poff = block % blocks_per_page; 1143c9de560dSAlex Tomas 1144c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1145c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 11462457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1147c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1148c9de560dSAlex Tomas if (page) 1149920313a7SAneesh Kumar K.V /* 1150920313a7SAneesh Kumar K.V * drop the page reference and try 1151920313a7SAneesh Kumar K.V * to get the page with lock. If we 1152920313a7SAneesh Kumar K.V * are not uptodate that implies 1153920313a7SAneesh Kumar K.V * somebody just created the page but 1154920313a7SAneesh Kumar K.V * is yet to initialize the same. So 1155920313a7SAneesh Kumar K.V * wait for it to initialize. 1156920313a7SAneesh Kumar K.V */ 115709cbfeafSKirill A. Shutemov put_page(page); 1158adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1159c9de560dSAlex Tomas if (page) { 1160c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1161c9de560dSAlex Tomas if (!PageUptodate(page)) { 1162adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 1163fdf6c7a7SShen Feng if (ret) { 1164fdf6c7a7SShen Feng unlock_page(page); 1165fdf6c7a7SShen Feng goto err; 1166fdf6c7a7SShen Feng } 1167c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1168c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1169c9de560dSAlex Tomas } 1170c9de560dSAlex Tomas unlock_page(page); 1171c9de560dSAlex Tomas } 1172c9de560dSAlex Tomas } 1173c57ab39bSYounger Liu if (page == NULL) { 1174c57ab39bSYounger Liu ret = -ENOMEM; 1175c57ab39bSYounger Liu goto err; 1176c57ab39bSYounger Liu } 1177c57ab39bSYounger Liu if (!PageUptodate(page)) { 1178fdf6c7a7SShen Feng ret = -EIO; 1179c9de560dSAlex Tomas goto err; 1180fdf6c7a7SShen Feng } 11812457aec6SMel Gorman 11822457aec6SMel Gorman /* Pages marked accessed already */ 1183c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1184c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1185c9de560dSAlex Tomas 1186c9de560dSAlex Tomas block++; 1187c9de560dSAlex Tomas pnum = block / blocks_per_page; 1188c9de560dSAlex Tomas poff = block % blocks_per_page; 1189c9de560dSAlex Tomas 11902457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1191c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1192c9de560dSAlex Tomas if (page) 119309cbfeafSKirill A. Shutemov put_page(page); 1194adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1195c9de560dSAlex Tomas if (page) { 1196c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1197fdf6c7a7SShen Feng if (!PageUptodate(page)) { 1198adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b->bd_bitmap, 1199adb7ef60SKonstantin Khlebnikov gfp); 1200fdf6c7a7SShen Feng if (ret) { 1201fdf6c7a7SShen Feng unlock_page(page); 1202fdf6c7a7SShen Feng goto err; 1203fdf6c7a7SShen Feng } 1204fdf6c7a7SShen Feng } 1205c9de560dSAlex Tomas unlock_page(page); 1206c9de560dSAlex Tomas } 1207c9de560dSAlex Tomas } 1208c57ab39bSYounger Liu if (page == NULL) { 1209c57ab39bSYounger Liu ret = -ENOMEM; 1210c57ab39bSYounger Liu goto err; 1211c57ab39bSYounger Liu } 1212c57ab39bSYounger Liu if (!PageUptodate(page)) { 1213fdf6c7a7SShen Feng ret = -EIO; 1214c9de560dSAlex Tomas goto err; 1215fdf6c7a7SShen Feng } 12162457aec6SMel Gorman 12172457aec6SMel Gorman /* Pages marked accessed already */ 1218c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1219c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1220c9de560dSAlex Tomas 1221c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 1222c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 1223c9de560dSAlex Tomas 1224c9de560dSAlex Tomas return 0; 1225c9de560dSAlex Tomas 1226c9de560dSAlex Tomas err: 122726626f11SYang Ruirui if (page) 122809cbfeafSKirill A. Shutemov put_page(page); 1229c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 123009cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1231c9de560dSAlex Tomas if (e4b->bd_buddy_page) 123209cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 1233c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1234c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1235fdf6c7a7SShen Feng return ret; 1236c9de560dSAlex Tomas } 1237c9de560dSAlex Tomas 1238adb7ef60SKonstantin Khlebnikov static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1239adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b) 1240adb7ef60SKonstantin Khlebnikov { 1241adb7ef60SKonstantin Khlebnikov return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS); 1242adb7ef60SKonstantin Khlebnikov } 1243adb7ef60SKonstantin Khlebnikov 1244e39e07fdSJing Zhang static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) 1245c9de560dSAlex Tomas { 1246c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 124709cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1248c9de560dSAlex Tomas if (e4b->bd_buddy_page) 124909cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 1250c9de560dSAlex Tomas } 1251c9de560dSAlex Tomas 1252c9de560dSAlex Tomas 1253c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1254c9de560dSAlex Tomas { 1255c9de560dSAlex Tomas int order = 1; 1256b5cb316cSNicolai Stange int bb_incr = 1 << (e4b->bd_blkbits - 1); 1257c9de560dSAlex Tomas void *bb; 1258c9de560dSAlex Tomas 1259c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 1260c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1261c9de560dSAlex Tomas 1262c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy; 1263c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1264c9de560dSAlex Tomas block = block >> 1; 1265c9de560dSAlex Tomas if (!mb_test_bit(block, bb)) { 1266c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1267c9de560dSAlex Tomas return order; 1268c9de560dSAlex Tomas } 1269b5cb316cSNicolai Stange bb += bb_incr; 1270b5cb316cSNicolai Stange bb_incr >>= 1; 1271c9de560dSAlex Tomas order++; 1272c9de560dSAlex Tomas } 1273c9de560dSAlex Tomas return 0; 1274c9de560dSAlex Tomas } 1275c9de560dSAlex Tomas 1276955ce5f5SAneesh Kumar K.V static void mb_clear_bits(void *bm, int cur, int len) 1277c9de560dSAlex Tomas { 1278c9de560dSAlex Tomas __u32 *addr; 1279c9de560dSAlex Tomas 1280c9de560dSAlex Tomas len = cur + len; 1281c9de560dSAlex Tomas while (cur < len) { 1282c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1283c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1284c9de560dSAlex Tomas addr = bm + (cur >> 3); 1285c9de560dSAlex Tomas *addr = 0; 1286c9de560dSAlex Tomas cur += 32; 1287c9de560dSAlex Tomas continue; 1288c9de560dSAlex Tomas } 1289e8134b27SAneesh Kumar K.V mb_clear_bit(cur, bm); 1290c9de560dSAlex Tomas cur++; 1291c9de560dSAlex Tomas } 1292c9de560dSAlex Tomas } 1293c9de560dSAlex Tomas 1294eabe0444SAndrey Sidorov /* clear bits in given range 1295eabe0444SAndrey Sidorov * will return first found zero bit if any, -1 otherwise 1296eabe0444SAndrey Sidorov */ 1297eabe0444SAndrey Sidorov static int mb_test_and_clear_bits(void *bm, int cur, int len) 1298eabe0444SAndrey Sidorov { 1299eabe0444SAndrey Sidorov __u32 *addr; 1300eabe0444SAndrey Sidorov int zero_bit = -1; 1301eabe0444SAndrey Sidorov 1302eabe0444SAndrey Sidorov len = cur + len; 1303eabe0444SAndrey Sidorov while (cur < len) { 1304eabe0444SAndrey Sidorov if ((cur & 31) == 0 && (len - cur) >= 32) { 1305eabe0444SAndrey Sidorov /* fast path: clear whole word at once */ 1306eabe0444SAndrey Sidorov addr = bm + (cur >> 3); 1307eabe0444SAndrey Sidorov if (*addr != (__u32)(-1) && zero_bit == -1) 1308eabe0444SAndrey Sidorov zero_bit = cur + mb_find_next_zero_bit(addr, 32, 0); 1309eabe0444SAndrey Sidorov *addr = 0; 1310eabe0444SAndrey Sidorov cur += 32; 1311eabe0444SAndrey Sidorov continue; 1312eabe0444SAndrey Sidorov } 1313eabe0444SAndrey Sidorov if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) 1314eabe0444SAndrey Sidorov zero_bit = cur; 1315eabe0444SAndrey Sidorov cur++; 1316eabe0444SAndrey Sidorov } 1317eabe0444SAndrey Sidorov 1318eabe0444SAndrey Sidorov return zero_bit; 1319eabe0444SAndrey Sidorov } 1320eabe0444SAndrey Sidorov 1321c3e94d1dSYongqiang Yang void ext4_set_bits(void *bm, int cur, int len) 1322c9de560dSAlex Tomas { 1323c9de560dSAlex Tomas __u32 *addr; 1324c9de560dSAlex Tomas 1325c9de560dSAlex Tomas len = cur + len; 1326c9de560dSAlex Tomas while (cur < len) { 1327c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1328c9de560dSAlex Tomas /* fast path: set whole word at once */ 1329c9de560dSAlex Tomas addr = bm + (cur >> 3); 1330c9de560dSAlex Tomas *addr = 0xffffffff; 1331c9de560dSAlex Tomas cur += 32; 1332c9de560dSAlex Tomas continue; 1333c9de560dSAlex Tomas } 1334e8134b27SAneesh Kumar K.V mb_set_bit(cur, bm); 1335c9de560dSAlex Tomas cur++; 1336c9de560dSAlex Tomas } 1337c9de560dSAlex Tomas } 1338c9de560dSAlex Tomas 1339eabe0444SAndrey Sidorov /* 1340eabe0444SAndrey Sidorov * _________________________________________________________________ */ 1341eabe0444SAndrey Sidorov 1342eabe0444SAndrey Sidorov static inline int mb_buddy_adjust_border(int* bit, void* bitmap, int side) 1343eabe0444SAndrey Sidorov { 1344eabe0444SAndrey Sidorov if (mb_test_bit(*bit + side, bitmap)) { 1345eabe0444SAndrey Sidorov mb_clear_bit(*bit, bitmap); 1346eabe0444SAndrey Sidorov (*bit) -= side; 1347eabe0444SAndrey Sidorov return 1; 1348eabe0444SAndrey Sidorov } 1349eabe0444SAndrey Sidorov else { 1350eabe0444SAndrey Sidorov (*bit) += side; 1351eabe0444SAndrey Sidorov mb_set_bit(*bit, bitmap); 1352eabe0444SAndrey Sidorov return -1; 1353eabe0444SAndrey Sidorov } 1354eabe0444SAndrey Sidorov } 1355eabe0444SAndrey Sidorov 1356eabe0444SAndrey Sidorov static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last) 1357eabe0444SAndrey Sidorov { 1358eabe0444SAndrey Sidorov int max; 1359eabe0444SAndrey Sidorov int order = 1; 1360eabe0444SAndrey Sidorov void *buddy = mb_find_buddy(e4b, order, &max); 1361eabe0444SAndrey Sidorov 1362eabe0444SAndrey Sidorov while (buddy) { 1363eabe0444SAndrey Sidorov void *buddy2; 1364eabe0444SAndrey Sidorov 1365eabe0444SAndrey Sidorov /* Bits in range [first; last] are known to be set since 1366eabe0444SAndrey Sidorov * corresponding blocks were allocated. Bits in range 1367eabe0444SAndrey Sidorov * (first; last) will stay set because they form buddies on 1368eabe0444SAndrey Sidorov * upper layer. We just deal with borders if they don't 1369eabe0444SAndrey Sidorov * align with upper layer and then go up. 1370eabe0444SAndrey Sidorov * Releasing entire group is all about clearing 1371eabe0444SAndrey Sidorov * single bit of highest order buddy. 1372eabe0444SAndrey Sidorov */ 1373eabe0444SAndrey Sidorov 1374eabe0444SAndrey Sidorov /* Example: 1375eabe0444SAndrey Sidorov * --------------------------------- 1376eabe0444SAndrey Sidorov * | 1 | 1 | 1 | 1 | 1377eabe0444SAndrey Sidorov * --------------------------------- 1378eabe0444SAndrey Sidorov * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1379eabe0444SAndrey Sidorov * --------------------------------- 1380eabe0444SAndrey Sidorov * 0 1 2 3 4 5 6 7 1381eabe0444SAndrey Sidorov * \_____________________/ 1382eabe0444SAndrey Sidorov * 1383eabe0444SAndrey Sidorov * Neither [1] nor [6] is aligned to above layer. 1384eabe0444SAndrey Sidorov * Left neighbour [0] is free, so mark it busy, 1385eabe0444SAndrey Sidorov * decrease bb_counters and extend range to 1386eabe0444SAndrey Sidorov * [0; 6] 1387eabe0444SAndrey Sidorov * Right neighbour [7] is busy. It can't be coaleasced with [6], so 1388eabe0444SAndrey Sidorov * mark [6] free, increase bb_counters and shrink range to 1389eabe0444SAndrey Sidorov * [0; 5]. 1390eabe0444SAndrey Sidorov * Then shift range to [0; 2], go up and do the same. 1391eabe0444SAndrey Sidorov */ 1392eabe0444SAndrey Sidorov 1393eabe0444SAndrey Sidorov 1394eabe0444SAndrey Sidorov if (first & 1) 1395eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); 1396eabe0444SAndrey Sidorov if (!(last & 1)) 1397eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); 1398eabe0444SAndrey Sidorov if (first > last) 1399eabe0444SAndrey Sidorov break; 1400eabe0444SAndrey Sidorov order++; 1401eabe0444SAndrey Sidorov 1402eabe0444SAndrey Sidorov if (first == last || !(buddy2 = mb_find_buddy(e4b, order, &max))) { 1403eabe0444SAndrey Sidorov mb_clear_bits(buddy, first, last - first + 1); 1404eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order - 1] += last - first + 1; 1405eabe0444SAndrey Sidorov break; 1406eabe0444SAndrey Sidorov } 1407eabe0444SAndrey Sidorov first >>= 1; 1408eabe0444SAndrey Sidorov last >>= 1; 1409eabe0444SAndrey Sidorov buddy = buddy2; 1410eabe0444SAndrey Sidorov } 1411eabe0444SAndrey Sidorov } 1412eabe0444SAndrey Sidorov 14137e5a8cddSShen Feng static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1414c9de560dSAlex Tomas int first, int count) 1415c9de560dSAlex Tomas { 1416eabe0444SAndrey Sidorov int left_is_free = 0; 1417eabe0444SAndrey Sidorov int right_is_free = 0; 1418eabe0444SAndrey Sidorov int block; 1419eabe0444SAndrey Sidorov int last = first + count - 1; 1420c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1421c9de560dSAlex Tomas 1422c99d1e6eSTheodore Ts'o if (WARN_ON(count == 0)) 1423c99d1e6eSTheodore Ts'o return; 1424eabe0444SAndrey Sidorov BUG_ON(last >= (sb->s_blocksize << 3)); 1425bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 1426163a203dSDarrick J. Wong /* Don't bother if the block group is corrupt. */ 1427163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) 1428163a203dSDarrick J. Wong return; 1429163a203dSDarrick J. Wong 1430c9de560dSAlex Tomas mb_check_buddy(e4b); 1431c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1432c9de560dSAlex Tomas 1433c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1434c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1435c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1436c9de560dSAlex Tomas 1437eabe0444SAndrey Sidorov /* access memory sequentially: check left neighbour, 1438eabe0444SAndrey Sidorov * clear range and then check right neighbour 1439eabe0444SAndrey Sidorov */ 1440c9de560dSAlex Tomas if (first != 0) 1441eabe0444SAndrey Sidorov left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); 1442eabe0444SAndrey Sidorov block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count); 1443eabe0444SAndrey Sidorov if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) 1444eabe0444SAndrey Sidorov right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); 1445c9de560dSAlex Tomas 1446eabe0444SAndrey Sidorov if (unlikely(block != -1)) { 1447e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 1448c9de560dSAlex Tomas ext4_fsblk_t blocknr; 14495661bd68SAkinobu Mita 14505661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 145149598e04SJun Piao blocknr += EXT4_C2B(sbi, block); 14525d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 1453e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 1454e29136f8STheodore Ts'o blocknr, 1455e29136f8STheodore Ts'o "freeing already freed block " 1456163a203dSDarrick J. Wong "(bit %u); block bitmap corrupt.", 1457163a203dSDarrick J. Wong block); 1458db79e6d1SWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 1459db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1460eabe0444SAndrey Sidorov mb_regenerate_buddy(e4b); 1461eabe0444SAndrey Sidorov goto done; 1462c9de560dSAlex Tomas } 1463c9de560dSAlex Tomas 1464eabe0444SAndrey Sidorov /* let's maintain fragments counter */ 1465eabe0444SAndrey Sidorov if (left_is_free && right_is_free) 1466eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments--; 1467eabe0444SAndrey Sidorov else if (!left_is_free && !right_is_free) 1468eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments++; 1469c9de560dSAlex Tomas 1470eabe0444SAndrey Sidorov /* buddy[0] == bd_bitmap is a special case, so handle 1471eabe0444SAndrey Sidorov * it right away and let mb_buddy_mark_free stay free of 1472eabe0444SAndrey Sidorov * zero order checks. 1473eabe0444SAndrey Sidorov * Check if neighbours are to be coaleasced, 1474eabe0444SAndrey Sidorov * adjust bitmap bb_counters and borders appropriately. 1475eabe0444SAndrey Sidorov */ 1476eabe0444SAndrey Sidorov if (first & 1) { 1477eabe0444SAndrey Sidorov first += !left_is_free; 1478eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; 1479c9de560dSAlex Tomas } 1480eabe0444SAndrey Sidorov if (!(last & 1)) { 1481eabe0444SAndrey Sidorov last -= !right_is_free; 1482eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; 1483c9de560dSAlex Tomas } 1484eabe0444SAndrey Sidorov 1485eabe0444SAndrey Sidorov if (first <= last) 1486eabe0444SAndrey Sidorov mb_buddy_mark_free(e4b, first >> 1, last >> 1); 1487eabe0444SAndrey Sidorov 1488eabe0444SAndrey Sidorov done: 14898a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, e4b->bd_info); 1490c9de560dSAlex Tomas mb_check_buddy(e4b); 1491c9de560dSAlex Tomas } 1492c9de560dSAlex Tomas 149315c006a2SRobin Dong static int mb_find_extent(struct ext4_buddy *e4b, int block, 1494c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1495c9de560dSAlex Tomas { 1496c9de560dSAlex Tomas int next = block; 149715c006a2SRobin Dong int max, order; 1498c9de560dSAlex Tomas void *buddy; 1499c9de560dSAlex Tomas 1500bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1501c9de560dSAlex Tomas BUG_ON(ex == NULL); 1502c9de560dSAlex Tomas 150315c006a2SRobin Dong buddy = mb_find_buddy(e4b, 0, &max); 1504c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1505c9de560dSAlex Tomas BUG_ON(block >= max); 1506c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1507c9de560dSAlex Tomas ex->fe_len = 0; 1508c9de560dSAlex Tomas ex->fe_start = 0; 1509c9de560dSAlex Tomas ex->fe_group = 0; 1510c9de560dSAlex Tomas return 0; 1511c9de560dSAlex Tomas } 1512c9de560dSAlex Tomas 1513c9de560dSAlex Tomas /* find actual order */ 1514c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1515c9de560dSAlex Tomas block = block >> order; 1516c9de560dSAlex Tomas 1517c9de560dSAlex Tomas ex->fe_len = 1 << order; 1518c9de560dSAlex Tomas ex->fe_start = block << order; 1519c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1520c9de560dSAlex Tomas 1521c9de560dSAlex Tomas /* calc difference from given start */ 1522c9de560dSAlex Tomas next = next - ex->fe_start; 1523c9de560dSAlex Tomas ex->fe_len -= next; 1524c9de560dSAlex Tomas ex->fe_start += next; 1525c9de560dSAlex Tomas 1526c9de560dSAlex Tomas while (needed > ex->fe_len && 1527d8ec0c39SAlan Cox mb_find_buddy(e4b, order, &max)) { 1528c9de560dSAlex Tomas 1529c9de560dSAlex Tomas if (block + 1 >= max) 1530c9de560dSAlex Tomas break; 1531c9de560dSAlex Tomas 1532c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1533c5e8f3f3STheodore Ts'o if (mb_test_bit(next, e4b->bd_bitmap)) 1534c9de560dSAlex Tomas break; 1535c9de560dSAlex Tomas 1536b051d8dcSRobin Dong order = mb_find_order_for_block(e4b, next); 1537c9de560dSAlex Tomas 1538c9de560dSAlex Tomas block = next >> order; 1539c9de560dSAlex Tomas ex->fe_len += 1 << order; 1540c9de560dSAlex Tomas } 1541c9de560dSAlex Tomas 154231562b95SJan Kara if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) { 154343c73221STheodore Ts'o /* Should never happen! (but apparently sometimes does?!?) */ 154443c73221STheodore Ts'o WARN_ON(1); 154543c73221STheodore Ts'o ext4_error(e4b->bd_sb, "corruption or bug in mb_find_extent " 154643c73221STheodore Ts'o "block=%d, order=%d needed=%d ex=%u/%d/%d@%u", 154743c73221STheodore Ts'o block, order, needed, ex->fe_group, ex->fe_start, 154843c73221STheodore Ts'o ex->fe_len, ex->fe_logical); 154943c73221STheodore Ts'o ex->fe_len = 0; 155043c73221STheodore Ts'o ex->fe_start = 0; 155143c73221STheodore Ts'o ex->fe_group = 0; 155243c73221STheodore Ts'o } 1553c9de560dSAlex Tomas return ex->fe_len; 1554c9de560dSAlex Tomas } 1555c9de560dSAlex Tomas 1556c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 1557c9de560dSAlex Tomas { 1558c9de560dSAlex Tomas int ord; 1559c9de560dSAlex Tomas int mlen = 0; 1560c9de560dSAlex Tomas int max = 0; 1561c9de560dSAlex Tomas int cur; 1562c9de560dSAlex Tomas int start = ex->fe_start; 1563c9de560dSAlex Tomas int len = ex->fe_len; 1564c9de560dSAlex Tomas unsigned ret = 0; 1565c9de560dSAlex Tomas int len0 = len; 1566c9de560dSAlex Tomas void *buddy; 1567c9de560dSAlex Tomas 1568c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 1569c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 1570bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1571c9de560dSAlex Tomas mb_check_buddy(e4b); 1572c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 1573c9de560dSAlex Tomas 1574c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 1575c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 1576c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 1577c9de560dSAlex Tomas 1578c9de560dSAlex Tomas /* let's maintain fragments counter */ 1579c9de560dSAlex Tomas if (start != 0) 1580c5e8f3f3STheodore Ts'o mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); 1581c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 1582c5e8f3f3STheodore Ts'o max = !mb_test_bit(start + len, e4b->bd_bitmap); 1583c9de560dSAlex Tomas if (mlen && max) 1584c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1585c9de560dSAlex Tomas else if (!mlen && !max) 1586c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1587c9de560dSAlex Tomas 1588c9de560dSAlex Tomas /* let's maintain buddy itself */ 1589c9de560dSAlex Tomas while (len) { 1590c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 1591c9de560dSAlex Tomas 1592c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 1593c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 1594c9de560dSAlex Tomas mlen = 1 << ord; 1595c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1596c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 1597c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1598c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1599c9de560dSAlex Tomas start += mlen; 1600c9de560dSAlex Tomas len -= mlen; 1601c9de560dSAlex Tomas BUG_ON(len < 0); 1602c9de560dSAlex Tomas continue; 1603c9de560dSAlex Tomas } 1604c9de560dSAlex Tomas 1605c9de560dSAlex Tomas /* store for history */ 1606c9de560dSAlex Tomas if (ret == 0) 1607c9de560dSAlex Tomas ret = len | (ord << 16); 1608c9de560dSAlex Tomas 1609c9de560dSAlex Tomas /* we have to split large buddy */ 1610c9de560dSAlex Tomas BUG_ON(ord <= 0); 1611c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1612c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1613c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1614c9de560dSAlex Tomas 1615c9de560dSAlex Tomas ord--; 1616c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 1617c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1618c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 1619c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 1620c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1621c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1622c9de560dSAlex Tomas } 16238a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); 1624c9de560dSAlex Tomas 1625c5e8f3f3STheodore Ts'o ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0); 1626c9de560dSAlex Tomas mb_check_buddy(e4b); 1627c9de560dSAlex Tomas 1628c9de560dSAlex Tomas return ret; 1629c9de560dSAlex Tomas } 1630c9de560dSAlex Tomas 1631c9de560dSAlex Tomas /* 1632c9de560dSAlex Tomas * Must be called under group lock! 1633c9de560dSAlex Tomas */ 1634c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 1635c9de560dSAlex Tomas struct ext4_buddy *e4b) 1636c9de560dSAlex Tomas { 1637c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1638c9de560dSAlex Tomas int ret; 1639c9de560dSAlex Tomas 1640c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 1641c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1642c9de560dSAlex Tomas 1643c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 1644c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 1645c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 1646c9de560dSAlex Tomas 1647c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 1648c9de560dSAlex Tomas * allocated blocks for history */ 1649c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 1650c9de560dSAlex Tomas 1651c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 1652c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 1653c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 1654c9de560dSAlex Tomas 1655c3a326a6SAneesh Kumar K.V /* 1656c3a326a6SAneesh Kumar K.V * take the page reference. We want the page to be pinned 1657c3a326a6SAneesh Kumar K.V * so that we don't get a ext4_mb_init_cache_call for this 1658c3a326a6SAneesh Kumar K.V * group until we update the bitmap. That would mean we 1659c3a326a6SAneesh Kumar K.V * double allocate blocks. The reference is dropped 1660c3a326a6SAneesh Kumar K.V * in ext4_mb_release_context 1661c3a326a6SAneesh Kumar K.V */ 1662c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 1663c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 1664c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 1665c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 1666c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 16674ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 1668c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 1669c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 1670c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 1671c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 1672c9de560dSAlex Tomas } 1673c9de560dSAlex Tomas } 1674c9de560dSAlex Tomas 1675c9de560dSAlex Tomas /* 1676c9de560dSAlex Tomas * regular allocator, for general purposes allocation 1677c9de560dSAlex Tomas */ 1678c9de560dSAlex Tomas 1679c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 1680c9de560dSAlex Tomas struct ext4_buddy *e4b, 1681c9de560dSAlex Tomas int finish_group) 1682c9de560dSAlex Tomas { 1683c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1684c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1685c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1686c9de560dSAlex Tomas struct ext4_free_extent ex; 1687c9de560dSAlex Tomas int max; 1688c9de560dSAlex Tomas 1689032115fcSAneesh Kumar K.V if (ac->ac_status == AC_STATUS_FOUND) 1690032115fcSAneesh Kumar K.V return; 1691c9de560dSAlex Tomas /* 1692c9de560dSAlex Tomas * We don't want to scan for a whole year 1693c9de560dSAlex Tomas */ 1694c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 1695c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1696c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 1697c9de560dSAlex Tomas return; 1698c9de560dSAlex Tomas } 1699c9de560dSAlex Tomas 1700c9de560dSAlex Tomas /* 1701c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 1702c9de560dSAlex Tomas */ 1703c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 1704c9de560dSAlex Tomas return; 1705c9de560dSAlex Tomas 1706c9de560dSAlex Tomas if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 1707c9de560dSAlex Tomas && bex->fe_group == e4b->bd_group) { 1708c9de560dSAlex Tomas /* recheck chunk's availability - we don't know 1709c9de560dSAlex Tomas * when it was found (within this lock-unlock 1710c9de560dSAlex Tomas * period or not) */ 171115c006a2SRobin Dong max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex); 1712c9de560dSAlex Tomas if (max >= gex->fe_len) { 1713c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1714c9de560dSAlex Tomas return; 1715c9de560dSAlex Tomas } 1716c9de560dSAlex Tomas } 1717c9de560dSAlex Tomas } 1718c9de560dSAlex Tomas 1719c9de560dSAlex Tomas /* 1720c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 1721c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 1722c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 1723c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 1724c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 1725c9de560dSAlex Tomas * mballoc can't find good enough extent. 1726c9de560dSAlex Tomas * 1727c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 1728c9de560dSAlex Tomas */ 1729c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 1730c9de560dSAlex Tomas struct ext4_free_extent *ex, 1731c9de560dSAlex Tomas struct ext4_buddy *e4b) 1732c9de560dSAlex Tomas { 1733c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1734c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1735c9de560dSAlex Tomas 1736c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 17377137d7a4STheodore Ts'o BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 17387137d7a4STheodore Ts'o BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 1739c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 1740c9de560dSAlex Tomas 1741c9de560dSAlex Tomas ac->ac_found++; 1742c9de560dSAlex Tomas 1743c9de560dSAlex Tomas /* 1744c9de560dSAlex Tomas * The special case - take what you catch first 1745c9de560dSAlex Tomas */ 1746c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1747c9de560dSAlex Tomas *bex = *ex; 1748c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1749c9de560dSAlex Tomas return; 1750c9de560dSAlex Tomas } 1751c9de560dSAlex Tomas 1752c9de560dSAlex Tomas /* 1753c9de560dSAlex Tomas * Let's check whether the chuck is good enough 1754c9de560dSAlex Tomas */ 1755c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 1756c9de560dSAlex Tomas *bex = *ex; 1757c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1758c9de560dSAlex Tomas return; 1759c9de560dSAlex Tomas } 1760c9de560dSAlex Tomas 1761c9de560dSAlex Tomas /* 1762c9de560dSAlex Tomas * If this is first found extent, just store it in the context 1763c9de560dSAlex Tomas */ 1764c9de560dSAlex Tomas if (bex->fe_len == 0) { 1765c9de560dSAlex Tomas *bex = *ex; 1766c9de560dSAlex Tomas return; 1767c9de560dSAlex Tomas } 1768c9de560dSAlex Tomas 1769c9de560dSAlex Tomas /* 1770c9de560dSAlex Tomas * If new found extent is better, store it in the context 1771c9de560dSAlex Tomas */ 1772c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 1773c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 1774c9de560dSAlex Tomas * larger than previous best one is better */ 1775c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 1776c9de560dSAlex Tomas *bex = *ex; 1777c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 1778c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 1779c9de560dSAlex Tomas * an extent that still satisfy the request, but is 1780c9de560dSAlex Tomas * smaller than previous one */ 1781c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 1782c9de560dSAlex Tomas *bex = *ex; 1783c9de560dSAlex Tomas } 1784c9de560dSAlex Tomas 1785c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 1786c9de560dSAlex Tomas } 1787c9de560dSAlex Tomas 1788089ceeccSEric Sandeen static noinline_for_stack 1789089ceeccSEric Sandeen int ext4_mb_try_best_found(struct ext4_allocation_context *ac, 1790c9de560dSAlex Tomas struct ext4_buddy *e4b) 1791c9de560dSAlex Tomas { 1792c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 1793c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 1794c9de560dSAlex Tomas int max; 1795c9de560dSAlex Tomas int err; 1796c9de560dSAlex Tomas 1797c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1798c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1799c9de560dSAlex Tomas if (err) 1800c9de560dSAlex Tomas return err; 1801c9de560dSAlex Tomas 1802c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 180315c006a2SRobin Dong max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex); 1804c9de560dSAlex Tomas 1805c9de560dSAlex Tomas if (max > 0) { 1806c9de560dSAlex Tomas ac->ac_b_ex = ex; 1807c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1808c9de560dSAlex Tomas } 1809c9de560dSAlex Tomas 1810c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1811e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1812c9de560dSAlex Tomas 1813c9de560dSAlex Tomas return 0; 1814c9de560dSAlex Tomas } 1815c9de560dSAlex Tomas 1816089ceeccSEric Sandeen static noinline_for_stack 1817089ceeccSEric Sandeen int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 1818c9de560dSAlex Tomas struct ext4_buddy *e4b) 1819c9de560dSAlex Tomas { 1820c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 1821c9de560dSAlex Tomas int max; 1822c9de560dSAlex Tomas int err; 1823c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1824838cd0cfSYongqiang Yang struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 1825c9de560dSAlex Tomas struct ext4_free_extent ex; 1826c9de560dSAlex Tomas 1827c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) 1828c9de560dSAlex Tomas return 0; 1829838cd0cfSYongqiang Yang if (grp->bb_free == 0) 1830838cd0cfSYongqiang Yang return 0; 1831c9de560dSAlex Tomas 1832c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1833c9de560dSAlex Tomas if (err) 1834c9de560dSAlex Tomas return err; 1835c9de560dSAlex Tomas 1836163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) { 1837163a203dSDarrick J. Wong ext4_mb_unload_buddy(e4b); 1838163a203dSDarrick J. Wong return 0; 1839163a203dSDarrick J. Wong } 1840163a203dSDarrick J. Wong 1841c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 184215c006a2SRobin Dong max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, 1843c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 1844ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADFA11; /* debug value */ 1845c9de560dSAlex Tomas 1846c9de560dSAlex Tomas if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { 1847c9de560dSAlex Tomas ext4_fsblk_t start; 1848c9de560dSAlex Tomas 18495661bd68SAkinobu Mita start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + 18505661bd68SAkinobu Mita ex.fe_start; 1851c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 1852c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 1853c9de560dSAlex Tomas ac->ac_found++; 1854c9de560dSAlex Tomas ac->ac_b_ex = ex; 1855c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1856c9de560dSAlex Tomas } 1857c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 1858c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1859c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1860c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1861c9de560dSAlex Tomas ac->ac_found++; 1862c9de560dSAlex Tomas ac->ac_b_ex = ex; 1863c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1864c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 1865c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 1866c9de560dSAlex Tomas * number of blocks to an existing extent */ 1867c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1868c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1869c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1870c9de560dSAlex Tomas ac->ac_found++; 1871c9de560dSAlex Tomas ac->ac_b_ex = ex; 1872c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1873c9de560dSAlex Tomas } 1874c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1875e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1876c9de560dSAlex Tomas 1877c9de560dSAlex Tomas return 0; 1878c9de560dSAlex Tomas } 1879c9de560dSAlex Tomas 1880c9de560dSAlex Tomas /* 1881c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 1882c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 1883c9de560dSAlex Tomas */ 1884089ceeccSEric Sandeen static noinline_for_stack 1885089ceeccSEric Sandeen void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 1886c9de560dSAlex Tomas struct ext4_buddy *e4b) 1887c9de560dSAlex Tomas { 1888c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1889c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 1890c9de560dSAlex Tomas void *buddy; 1891c9de560dSAlex Tomas int i; 1892c9de560dSAlex Tomas int k; 1893c9de560dSAlex Tomas int max; 1894c9de560dSAlex Tomas 1895c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 1896c9de560dSAlex Tomas for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { 1897c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 1898c9de560dSAlex Tomas continue; 1899c9de560dSAlex Tomas 1900c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 1901c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1902c9de560dSAlex Tomas 1903ffad0a44SAneesh Kumar K.V k = mb_find_next_zero_bit(buddy, max, 0); 1904eb576086SDmitry Monakhov if (k >= max) { 1905eb576086SDmitry Monakhov ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, 1906eb576086SDmitry Monakhov "%d free clusters of order %d. But found 0", 1907eb576086SDmitry Monakhov grp->bb_counters[i], i); 1908eb576086SDmitry Monakhov ext4_mark_group_bitmap_corrupted(ac->ac_sb, 1909eb576086SDmitry Monakhov e4b->bd_group, 1910eb576086SDmitry Monakhov EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1911eb576086SDmitry Monakhov break; 1912eb576086SDmitry Monakhov } 1913c9de560dSAlex Tomas ac->ac_found++; 1914c9de560dSAlex Tomas 1915c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 1916c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 1917c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 1918c9de560dSAlex Tomas 1919c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1920c9de560dSAlex Tomas 1921c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len); 1922c9de560dSAlex Tomas 1923c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 1924c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 1925c9de560dSAlex Tomas 1926c9de560dSAlex Tomas break; 1927c9de560dSAlex Tomas } 1928c9de560dSAlex Tomas } 1929c9de560dSAlex Tomas 1930c9de560dSAlex Tomas /* 1931c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 1932c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 1933c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 1934c9de560dSAlex Tomas */ 1935089ceeccSEric Sandeen static noinline_for_stack 1936089ceeccSEric Sandeen void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 1937c9de560dSAlex Tomas struct ext4_buddy *e4b) 1938c9de560dSAlex Tomas { 1939c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1940c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 1941c9de560dSAlex Tomas struct ext4_free_extent ex; 1942c9de560dSAlex Tomas int i; 1943c9de560dSAlex Tomas int free; 1944c9de560dSAlex Tomas 1945c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 1946907ea529STheodore Ts'o if (WARN_ON(free <= 0)) 1947907ea529STheodore Ts'o return; 1948c9de560dSAlex Tomas 1949c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 1950c9de560dSAlex Tomas 1951c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 1952ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, 19537137d7a4STheodore Ts'o EXT4_CLUSTERS_PER_GROUP(sb), i); 19547137d7a4STheodore Ts'o if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) { 195526346ff6SAneesh Kumar K.V /* 1956e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 195726346ff6SAneesh Kumar K.V * free blocks even though group info says we 195826346ff6SAneesh Kumar K.V * we have free blocks 195926346ff6SAneesh Kumar K.V */ 1960e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 196153accfa9STheodore Ts'o "%d free clusters as per " 1962fde4d95aSTheodore Ts'o "group info. But bitmap says 0", 196326346ff6SAneesh Kumar K.V free); 1964736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 1965736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1966c9de560dSAlex Tomas break; 1967c9de560dSAlex Tomas } 1968c9de560dSAlex Tomas 196915c006a2SRobin Dong mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); 1970907ea529STheodore Ts'o if (WARN_ON(ex.fe_len <= 0)) 1971907ea529STheodore Ts'o break; 197226346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 1973e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 197453accfa9STheodore Ts'o "%d free clusters as per " 1975fde4d95aSTheodore Ts'o "group info. But got %d blocks", 197626346ff6SAneesh Kumar K.V free, ex.fe_len); 1977736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 1978736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1979e56eb659SAneesh Kumar K.V /* 1980e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 1981e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 1982e56eb659SAneesh Kumar K.V * without claiming the space. 1983e56eb659SAneesh Kumar K.V */ 1984e56eb659SAneesh Kumar K.V break; 198526346ff6SAneesh Kumar K.V } 1986ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADC0DE; /* debug value */ 1987c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 1988c9de560dSAlex Tomas 1989c9de560dSAlex Tomas i += ex.fe_len; 1990c9de560dSAlex Tomas free -= ex.fe_len; 1991c9de560dSAlex Tomas } 1992c9de560dSAlex Tomas 1993c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 1994c9de560dSAlex Tomas } 1995c9de560dSAlex Tomas 1996c9de560dSAlex Tomas /* 1997c9de560dSAlex Tomas * This is a special case for storages like raid5 1998506bf2d8SEric Sandeen * we try to find stripe-aligned chunks for stripe-size-multiple requests 1999c9de560dSAlex Tomas */ 2000089ceeccSEric Sandeen static noinline_for_stack 2001089ceeccSEric Sandeen void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 2002c9de560dSAlex Tomas struct ext4_buddy *e4b) 2003c9de560dSAlex Tomas { 2004c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2005c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2006c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 2007c9de560dSAlex Tomas struct ext4_free_extent ex; 2008c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 2009c9de560dSAlex Tomas ext4_fsblk_t a; 2010c9de560dSAlex Tomas ext4_grpblk_t i; 2011c9de560dSAlex Tomas int max; 2012c9de560dSAlex Tomas 2013c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 2014c9de560dSAlex Tomas 2015c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 20165661bd68SAkinobu Mita first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); 20175661bd68SAkinobu Mita 2018c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 2019c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 2020c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 2021c9de560dSAlex Tomas 20227137d7a4STheodore Ts'o while (i < EXT4_CLUSTERS_PER_GROUP(sb)) { 2023c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 202415c006a2SRobin Dong max = mb_find_extent(e4b, i, sbi->s_stripe, &ex); 2025c9de560dSAlex Tomas if (max >= sbi->s_stripe) { 2026c9de560dSAlex Tomas ac->ac_found++; 2027ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADF00D; /* debug value */ 2028c9de560dSAlex Tomas ac->ac_b_ex = ex; 2029c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2030c9de560dSAlex Tomas break; 2031c9de560dSAlex Tomas } 2032c9de560dSAlex Tomas } 2033c9de560dSAlex Tomas i += sbi->s_stripe; 2034c9de560dSAlex Tomas } 2035c9de560dSAlex Tomas } 2036c9de560dSAlex Tomas 203742ac1848SLukas Czerner /* 203842ac1848SLukas Czerner * This is now called BEFORE we load the buddy bitmap. 203942ac1848SLukas Czerner * Returns either 1 or 0 indicating that the group is either suitable 204042ac1848SLukas Czerner * for the allocation or not. In addition it can also return negative 204142ac1848SLukas Czerner * error code when something goes wrong. 204242ac1848SLukas Czerner */ 2043c9de560dSAlex Tomas static int ext4_mb_good_group(struct ext4_allocation_context *ac, 2044c9de560dSAlex Tomas ext4_group_t group, int cr) 2045c9de560dSAlex Tomas { 2046c9de560dSAlex Tomas unsigned free, fragments; 2047a4912123STheodore Ts'o int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); 2048c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 2049c9de560dSAlex Tomas 2050c9de560dSAlex Tomas BUG_ON(cr < 0 || cr >= 4); 20518a57d9d6SCurt Wohlgemuth 205201fc48e8STheodore Ts'o free = grp->bb_free; 205301fc48e8STheodore Ts'o if (free == 0) 205401fc48e8STheodore Ts'o return 0; 205501fc48e8STheodore Ts'o if (cr <= 2 && free < ac->ac_g_ex.fe_len) 205601fc48e8STheodore Ts'o return 0; 205701fc48e8STheodore Ts'o 2058163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 2059163a203dSDarrick J. Wong return 0; 2060163a203dSDarrick J. Wong 20618a57d9d6SCurt Wohlgemuth /* We only do this if the grp has never been initialized */ 20628a57d9d6SCurt Wohlgemuth if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 2063adb7ef60SKonstantin Khlebnikov int ret = ext4_mb_init_group(ac->ac_sb, group, GFP_NOFS); 20648a57d9d6SCurt Wohlgemuth if (ret) 206542ac1848SLukas Czerner return ret; 20668a57d9d6SCurt Wohlgemuth } 2067c9de560dSAlex Tomas 2068c9de560dSAlex Tomas fragments = grp->bb_fragments; 2069c9de560dSAlex Tomas if (fragments == 0) 2070c9de560dSAlex Tomas return 0; 2071c9de560dSAlex Tomas 2072c9de560dSAlex Tomas switch (cr) { 2073c9de560dSAlex Tomas case 0: 2074c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 2075c9de560dSAlex Tomas 2076a4912123STheodore Ts'o /* Avoid using the first bg of a flexgroup for data files */ 2077a4912123STheodore Ts'o if ((ac->ac_flags & EXT4_MB_HINT_DATA) && 2078a4912123STheodore Ts'o (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && 2079a4912123STheodore Ts'o ((group % flex_size) == 0)) 2080a4912123STheodore Ts'o return 0; 2081a4912123STheodore Ts'o 208240ae3487STheodore Ts'o if ((ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) || 208340ae3487STheodore Ts'o (free / fragments) >= ac->ac_g_ex.fe_len) 208440ae3487STheodore Ts'o return 1; 208540ae3487STheodore Ts'o 208640ae3487STheodore Ts'o if (grp->bb_largest_free_order < ac->ac_2order) 208740ae3487STheodore Ts'o return 0; 208840ae3487STheodore Ts'o 2089c9de560dSAlex Tomas return 1; 2090c9de560dSAlex Tomas case 1: 2091c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 2092c9de560dSAlex Tomas return 1; 2093c9de560dSAlex Tomas break; 2094c9de560dSAlex Tomas case 2: 2095c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 2096c9de560dSAlex Tomas return 1; 2097c9de560dSAlex Tomas break; 2098c9de560dSAlex Tomas case 3: 2099c9de560dSAlex Tomas return 1; 2100c9de560dSAlex Tomas default: 2101c9de560dSAlex Tomas BUG(); 2102c9de560dSAlex Tomas } 2103c9de560dSAlex Tomas 2104c9de560dSAlex Tomas return 0; 2105c9de560dSAlex Tomas } 2106c9de560dSAlex Tomas 21074ddfef7bSEric Sandeen static noinline_for_stack int 21084ddfef7bSEric Sandeen ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 2109c9de560dSAlex Tomas { 21108df9675fSTheodore Ts'o ext4_group_t ngroups, group, i; 2111bbc4ec77SRitesh Harjani int cr = -1; 211242ac1848SLukas Czerner int err = 0, first_err = 0; 2113c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2114c9de560dSAlex Tomas struct super_block *sb; 2115c9de560dSAlex Tomas struct ext4_buddy e4b; 2116c9de560dSAlex Tomas 2117c9de560dSAlex Tomas sb = ac->ac_sb; 2118c9de560dSAlex Tomas sbi = EXT4_SB(sb); 21198df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 2120fb0a387dSEric Sandeen /* non-extent files are limited to low blocks/groups */ 212112e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) 2122fb0a387dSEric Sandeen ngroups = sbi->s_blockfile_groups; 2123fb0a387dSEric Sandeen 2124c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 2125c9de560dSAlex Tomas 2126c9de560dSAlex Tomas /* first, try the goal */ 2127c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 2128c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 2129c9de560dSAlex Tomas goto out; 2130c9de560dSAlex Tomas 2131c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 2132c9de560dSAlex Tomas goto out; 2133c9de560dSAlex Tomas 2134c9de560dSAlex Tomas /* 2135c9de560dSAlex Tomas * ac->ac2_order is set only if the fe_len is a power of 2 2136c9de560dSAlex Tomas * if ac2_order is set we also set criteria to 0 so that we 2137c9de560dSAlex Tomas * try exact allocation using buddy. 2138c9de560dSAlex Tomas */ 2139c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 2140c9de560dSAlex Tomas ac->ac_2order = 0; 2141c9de560dSAlex Tomas /* 2142c9de560dSAlex Tomas * We search using buddy data only if the order of the request 2143c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 2144b713a5ecSTheodore Ts'o * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req 2145d9b22cf9SJan Kara * We also support searching for power-of-two requests only for 2146d9b22cf9SJan Kara * requests upto maximum buddy size we have constructed. 2147c9de560dSAlex Tomas */ 2148d9b22cf9SJan Kara if (i >= sbi->s_mb_order2_reqs && i <= sb->s_blocksize_bits + 2) { 2149c9de560dSAlex Tomas /* 2150c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 2151c9de560dSAlex Tomas */ 2152c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 21531a5d5e5dSJeremy Cline ac->ac_2order = array_index_nospec(i - 1, 21541a5d5e5dSJeremy Cline sb->s_blocksize_bits + 2); 2155c9de560dSAlex Tomas } 2156c9de560dSAlex Tomas 21574ba74d00STheodore Ts'o /* if stream allocation is enabled, use global goal */ 21584ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2159c9de560dSAlex Tomas /* TBD: may be hot point */ 2160c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2161c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2162c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2163c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2164c9de560dSAlex Tomas } 21654ba74d00STheodore Ts'o 2166c9de560dSAlex Tomas /* Let's just scan groups to find more-less suitable blocks */ 2167c9de560dSAlex Tomas cr = ac->ac_2order ? 0 : 1; 2168c9de560dSAlex Tomas /* 2169c9de560dSAlex Tomas * cr == 0 try to get exact allocation, 2170c9de560dSAlex Tomas * cr == 3 try to get anything 2171c9de560dSAlex Tomas */ 2172c9de560dSAlex Tomas repeat: 2173c9de560dSAlex Tomas for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2174c9de560dSAlex Tomas ac->ac_criteria = cr; 2175ed8f9c75SAneesh Kumar K.V /* 2176ed8f9c75SAneesh Kumar K.V * searching for the right group start 2177ed8f9c75SAneesh Kumar K.V * from the goal value specified 2178ed8f9c75SAneesh Kumar K.V */ 2179ed8f9c75SAneesh Kumar K.V group = ac->ac_g_ex.fe_group; 2180ed8f9c75SAneesh Kumar K.V 21818df9675fSTheodore Ts'o for (i = 0; i < ngroups; group++, i++) { 218242ac1848SLukas Czerner int ret = 0; 21832ed5724dSTheodore Ts'o cond_resched(); 2184e6155736SLachlan McIlroy /* 2185e6155736SLachlan McIlroy * Artificially restricted ngroups for non-extent 2186e6155736SLachlan McIlroy * files makes group > ngroups possible on first loop. 2187e6155736SLachlan McIlroy */ 2188e6155736SLachlan McIlroy if (group >= ngroups) 2189c9de560dSAlex Tomas group = 0; 2190c9de560dSAlex Tomas 21918a57d9d6SCurt Wohlgemuth /* This now checks without needing the buddy page */ 219242ac1848SLukas Czerner ret = ext4_mb_good_group(ac, group, cr); 219342ac1848SLukas Czerner if (ret <= 0) { 219442ac1848SLukas Czerner if (!first_err) 219542ac1848SLukas Czerner first_err = ret; 2196c9de560dSAlex Tomas continue; 219742ac1848SLukas Czerner } 2198c9de560dSAlex Tomas 2199c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2200c9de560dSAlex Tomas if (err) 2201c9de560dSAlex Tomas goto out; 2202c9de560dSAlex Tomas 2203c9de560dSAlex Tomas ext4_lock_group(sb, group); 22048a57d9d6SCurt Wohlgemuth 22058a57d9d6SCurt Wohlgemuth /* 22068a57d9d6SCurt Wohlgemuth * We need to check again after locking the 22078a57d9d6SCurt Wohlgemuth * block group 22088a57d9d6SCurt Wohlgemuth */ 220942ac1848SLukas Czerner ret = ext4_mb_good_group(ac, group, cr); 221042ac1848SLukas Czerner if (ret <= 0) { 2211c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2212e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 221342ac1848SLukas Czerner if (!first_err) 221442ac1848SLukas Czerner first_err = ret; 2215c9de560dSAlex Tomas continue; 2216c9de560dSAlex Tomas } 2217c9de560dSAlex Tomas 2218c9de560dSAlex Tomas ac->ac_groups_scanned++; 2219d9b22cf9SJan Kara if (cr == 0) 2220c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2221506bf2d8SEric Sandeen else if (cr == 1 && sbi->s_stripe && 2222506bf2d8SEric Sandeen !(ac->ac_g_ex.fe_len % sbi->s_stripe)) 2223c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2224c9de560dSAlex Tomas else 2225c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2226c9de560dSAlex Tomas 2227c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2228e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2229c9de560dSAlex Tomas 2230c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2231c9de560dSAlex Tomas break; 2232c9de560dSAlex Tomas } 2233c9de560dSAlex Tomas } 2234c9de560dSAlex Tomas 2235c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2236c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2237c9de560dSAlex Tomas /* 2238c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2239c9de560dSAlex Tomas * the best chunk we've found so far 2240c9de560dSAlex Tomas */ 2241c9de560dSAlex Tomas 2242c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2243c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2244c9de560dSAlex Tomas /* 2245c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2246c9de560dSAlex Tomas * The only thing we can do is just take first 2247c9de560dSAlex Tomas * found block(s) 2248c9de560dSAlex Tomas printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n"); 2249c9de560dSAlex Tomas */ 2250c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2251c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2252c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2253c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2254c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2255c9de560dSAlex Tomas cr = 3; 2256c9de560dSAlex Tomas atomic_inc(&sbi->s_mb_lost_chunks); 2257c9de560dSAlex Tomas goto repeat; 2258c9de560dSAlex Tomas } 2259c9de560dSAlex Tomas } 2260c9de560dSAlex Tomas out: 226142ac1848SLukas Czerner if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) 226242ac1848SLukas Czerner err = first_err; 2263bbc4ec77SRitesh Harjani 2264bbc4ec77SRitesh Harjani mb_debug(1, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n", 2265bbc4ec77SRitesh Harjani ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, 2266bbc4ec77SRitesh Harjani ac->ac_flags, cr, err); 2267c9de560dSAlex Tomas return err; 2268c9de560dSAlex Tomas } 2269c9de560dSAlex Tomas 2270c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2271c9de560dSAlex Tomas { 2272247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2273c9de560dSAlex Tomas ext4_group_t group; 2274c9de560dSAlex Tomas 22758df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2276c9de560dSAlex Tomas return NULL; 2277c9de560dSAlex Tomas group = *pos + 1; 2278a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2279c9de560dSAlex Tomas } 2280c9de560dSAlex Tomas 2281c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2282c9de560dSAlex Tomas { 2283247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2284c9de560dSAlex Tomas ext4_group_t group; 2285c9de560dSAlex Tomas 2286c9de560dSAlex Tomas ++*pos; 22878df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2288c9de560dSAlex Tomas return NULL; 2289c9de560dSAlex Tomas group = *pos + 1; 2290a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2291c9de560dSAlex Tomas } 2292c9de560dSAlex Tomas 2293c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2294c9de560dSAlex Tomas { 2295247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2296a9df9a49STheodore Ts'o ext4_group_t group = (ext4_group_t) ((unsigned long) v); 2297c9de560dSAlex Tomas int i; 22981c8457caSAditya Kali int err, buddy_loaded = 0; 2299c9de560dSAlex Tomas struct ext4_buddy e4b; 23001c8457caSAditya Kali struct ext4_group_info *grinfo; 23012df2c340SArnd Bergmann unsigned char blocksize_bits = min_t(unsigned char, 23022df2c340SArnd Bergmann sb->s_blocksize_bits, 23032df2c340SArnd Bergmann EXT4_MAX_BLOCK_LOG_SIZE); 2304c9de560dSAlex Tomas struct sg { 2305c9de560dSAlex Tomas struct ext4_group_info info; 2306b80b32b6STheodore Ts'o ext4_grpblk_t counters[EXT4_MAX_BLOCK_LOG_SIZE + 2]; 2307c9de560dSAlex Tomas } sg; 2308c9de560dSAlex Tomas 2309c9de560dSAlex Tomas group--; 2310c9de560dSAlex Tomas if (group == 0) 231197b4af2fSRasmus Villemoes seq_puts(seq, "#group: free frags first [" 231297b4af2fSRasmus Villemoes " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " 2313802cf1f9SHuaitong Han " 2^7 2^8 2^9 2^10 2^11 2^12 2^13 ]\n"); 2314c9de560dSAlex Tomas 2315b80b32b6STheodore Ts'o i = (blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 2316b80b32b6STheodore Ts'o sizeof(struct ext4_group_info); 2317b80b32b6STheodore Ts'o 23181c8457caSAditya Kali grinfo = ext4_get_group_info(sb, group); 23191c8457caSAditya Kali /* Load the group info in memory only if not already loaded. */ 23201c8457caSAditya Kali if (unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) { 2321c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2322c9de560dSAlex Tomas if (err) { 2323a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: I/O error\n", group); 2324c9de560dSAlex Tomas return 0; 2325c9de560dSAlex Tomas } 23261c8457caSAditya Kali buddy_loaded = 1; 23271c8457caSAditya Kali } 23281c8457caSAditya Kali 2329b80b32b6STheodore Ts'o memcpy(&sg, ext4_get_group_info(sb, group), i); 23301c8457caSAditya Kali 23311c8457caSAditya Kali if (buddy_loaded) 2332e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2333c9de560dSAlex Tomas 2334a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, 2335c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 2336c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 23372df2c340SArnd Bergmann seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? 2338c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 2339c9de560dSAlex Tomas seq_printf(seq, " ]\n"); 2340c9de560dSAlex Tomas 2341c9de560dSAlex Tomas return 0; 2342c9de560dSAlex Tomas } 2343c9de560dSAlex Tomas 2344c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 2345c9de560dSAlex Tomas { 2346c9de560dSAlex Tomas } 2347c9de560dSAlex Tomas 2348247dbed8SChristoph Hellwig const struct seq_operations ext4_mb_seq_groups_ops = { 2349c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 2350c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 2351c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 2352c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 2353c9de560dSAlex Tomas }; 2354c9de560dSAlex Tomas 2355fb1813f4SCurt Wohlgemuth static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) 2356fb1813f4SCurt Wohlgemuth { 2357fb1813f4SCurt Wohlgemuth int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 2358fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index]; 2359fb1813f4SCurt Wohlgemuth 2360fb1813f4SCurt Wohlgemuth BUG_ON(!cachep); 2361fb1813f4SCurt Wohlgemuth return cachep; 2362fb1813f4SCurt Wohlgemuth } 23635f21b0e6SFrederic Bohe 236428623c2fSTheodore Ts'o /* 236528623c2fSTheodore Ts'o * Allocate the top-level s_group_info array for the specified number 236628623c2fSTheodore Ts'o * of groups 236728623c2fSTheodore Ts'o */ 236828623c2fSTheodore Ts'o int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups) 236928623c2fSTheodore Ts'o { 237028623c2fSTheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 237128623c2fSTheodore Ts'o unsigned size; 2372df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***old_groupinfo, ***new_groupinfo; 237328623c2fSTheodore Ts'o 237428623c2fSTheodore Ts'o size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> 237528623c2fSTheodore Ts'o EXT4_DESC_PER_BLOCK_BITS(sb); 237628623c2fSTheodore Ts'o if (size <= sbi->s_group_info_size) 237728623c2fSTheodore Ts'o return 0; 237828623c2fSTheodore Ts'o 237928623c2fSTheodore Ts'o size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); 2380a7c3e901SMichal Hocko new_groupinfo = kvzalloc(size, GFP_KERNEL); 238128623c2fSTheodore Ts'o if (!new_groupinfo) { 238228623c2fSTheodore Ts'o ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group"); 238328623c2fSTheodore Ts'o return -ENOMEM; 238428623c2fSTheodore Ts'o } 2385df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2386df3da4eaSSuraj Jitindar Singh old_groupinfo = rcu_dereference(sbi->s_group_info); 2387df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 2388df3da4eaSSuraj Jitindar Singh memcpy(new_groupinfo, old_groupinfo, 238928623c2fSTheodore Ts'o sbi->s_group_info_size * sizeof(*sbi->s_group_info)); 2390df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2391df3da4eaSSuraj Jitindar Singh rcu_assign_pointer(sbi->s_group_info, new_groupinfo); 239228623c2fSTheodore Ts'o sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); 2393df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 2394df3da4eaSSuraj Jitindar Singh ext4_kvfree_array_rcu(old_groupinfo); 239528623c2fSTheodore Ts'o ext4_debug("allocated s_groupinfo array for %d meta_bg's\n", 239628623c2fSTheodore Ts'o sbi->s_group_info_size); 239728623c2fSTheodore Ts'o return 0; 239828623c2fSTheodore Ts'o } 239928623c2fSTheodore Ts'o 24005f21b0e6SFrederic Bohe /* Create and initialize ext4_group_info data for the given group. */ 2401920313a7SAneesh Kumar K.V int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, 24025f21b0e6SFrederic Bohe struct ext4_group_desc *desc) 24035f21b0e6SFrederic Bohe { 2404fb1813f4SCurt Wohlgemuth int i; 24055f21b0e6SFrederic Bohe int metalen = 0; 2406df3da4eaSSuraj Jitindar Singh int idx = group >> EXT4_DESC_PER_BLOCK_BITS(sb); 24075f21b0e6SFrederic Bohe struct ext4_sb_info *sbi = EXT4_SB(sb); 24085f21b0e6SFrederic Bohe struct ext4_group_info **meta_group_info; 2409fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 24105f21b0e6SFrederic Bohe 24115f21b0e6SFrederic Bohe /* 24125f21b0e6SFrederic Bohe * First check if this group is the first of a reserved block. 24135f21b0e6SFrederic Bohe * If it's true, we have to allocate a new table of pointers 24145f21b0e6SFrederic Bohe * to ext4_group_info structures 24155f21b0e6SFrederic Bohe */ 24165f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 24175f21b0e6SFrederic Bohe metalen = sizeof(*meta_group_info) << 24185f21b0e6SFrederic Bohe EXT4_DESC_PER_BLOCK_BITS(sb); 24194fdb5543SDmitry Monakhov meta_group_info = kmalloc(metalen, GFP_NOFS); 24205f21b0e6SFrederic Bohe if (meta_group_info == NULL) { 24217f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate mem " 24229d8b9ec4STheodore Ts'o "for a buddy group"); 24235f21b0e6SFrederic Bohe goto exit_meta_group_info; 24245f21b0e6SFrederic Bohe } 2425df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2426df3da4eaSSuraj Jitindar Singh rcu_dereference(sbi->s_group_info)[idx] = meta_group_info; 2427df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 24285f21b0e6SFrederic Bohe } 24295f21b0e6SFrederic Bohe 2430df3da4eaSSuraj Jitindar Singh meta_group_info = sbi_array_rcu_deref(sbi, s_group_info, idx); 24315f21b0e6SFrederic Bohe i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); 24325f21b0e6SFrederic Bohe 24334fdb5543SDmitry Monakhov meta_group_info[i] = kmem_cache_zalloc(cachep, GFP_NOFS); 24345f21b0e6SFrederic Bohe if (meta_group_info[i] == NULL) { 24357f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate buddy mem"); 24365f21b0e6SFrederic Bohe goto exit_group_info; 24375f21b0e6SFrederic Bohe } 24385f21b0e6SFrederic Bohe set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 24395f21b0e6SFrederic Bohe &(meta_group_info[i]->bb_state)); 24405f21b0e6SFrederic Bohe 24415f21b0e6SFrederic Bohe /* 24425f21b0e6SFrederic Bohe * initialize bb_free to be able to skip 24435f21b0e6SFrederic Bohe * empty groups without initialization 24445f21b0e6SFrederic Bohe */ 24458844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 24468844618dSTheodore Ts'o (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 24475f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2448cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, group, desc); 24495f21b0e6SFrederic Bohe } else { 24505f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2451021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, desc); 24525f21b0e6SFrederic Bohe } 24535f21b0e6SFrederic Bohe 24545f21b0e6SFrederic Bohe INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); 2455920313a7SAneesh Kumar K.V init_rwsem(&meta_group_info[i]->alloc_sem); 245664e290ecSVenkatesh Pallipadi meta_group_info[i]->bb_free_root = RB_ROOT; 24578a57d9d6SCurt Wohlgemuth meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ 24585f21b0e6SFrederic Bohe 24595f21b0e6SFrederic Bohe #ifdef DOUBLE_CHECK 24605f21b0e6SFrederic Bohe { 24615f21b0e6SFrederic Bohe struct buffer_head *bh; 24625f21b0e6SFrederic Bohe meta_group_info[i]->bb_bitmap = 24634fdb5543SDmitry Monakhov kmalloc(sb->s_blocksize, GFP_NOFS); 24645f21b0e6SFrederic Bohe BUG_ON(meta_group_info[i]->bb_bitmap == NULL); 24655f21b0e6SFrederic Bohe bh = ext4_read_block_bitmap(sb, group); 24669008a58eSDarrick J. Wong BUG_ON(IS_ERR_OR_NULL(bh)); 24675f21b0e6SFrederic Bohe memcpy(meta_group_info[i]->bb_bitmap, bh->b_data, 24685f21b0e6SFrederic Bohe sb->s_blocksize); 24695f21b0e6SFrederic Bohe put_bh(bh); 24705f21b0e6SFrederic Bohe } 24715f21b0e6SFrederic Bohe #endif 24725f21b0e6SFrederic Bohe 24735f21b0e6SFrederic Bohe return 0; 24745f21b0e6SFrederic Bohe 24755f21b0e6SFrederic Bohe exit_group_info: 24765f21b0e6SFrederic Bohe /* If a meta_group_info table has been allocated, release it now */ 2477caaf7a29STao Ma if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 2478df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 2479df3da4eaSSuraj Jitindar Singh 2480df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2481df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 2482df3da4eaSSuraj Jitindar Singh kfree(group_info[idx]); 2483df3da4eaSSuraj Jitindar Singh group_info[idx] = NULL; 2484df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2485caaf7a29STao Ma } 24865f21b0e6SFrederic Bohe exit_meta_group_info: 24875f21b0e6SFrederic Bohe return -ENOMEM; 24885f21b0e6SFrederic Bohe } /* ext4_mb_add_groupinfo */ 24895f21b0e6SFrederic Bohe 2490c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 2491c9de560dSAlex Tomas { 24928df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2493c9de560dSAlex Tomas ext4_group_t i; 2494c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 249528623c2fSTheodore Ts'o int err; 24965f21b0e6SFrederic Bohe struct ext4_group_desc *desc; 2497df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 2498fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep; 2499c9de560dSAlex Tomas 250028623c2fSTheodore Ts'o err = ext4_mb_alloc_groupinfo(sb, ngroups); 250128623c2fSTheodore Ts'o if (err) 250228623c2fSTheodore Ts'o return err; 25035f21b0e6SFrederic Bohe 2504c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 2505c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 25069d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't get new inode"); 2507c9de560dSAlex Tomas goto err_freesgi; 2508c9de560dSAlex Tomas } 250948e6061bSYu Jian /* To avoid potentially colliding with an valid on-disk inode number, 251048e6061bSYu Jian * use EXT4_BAD_INO for the buddy cache inode number. This inode is 251148e6061bSYu Jian * not in the inode hash, so it should never be found by iget(), but 251248e6061bSYu Jian * this will avoid confusion if it ever shows up during debugging. */ 251348e6061bSYu Jian sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; 2514c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 25158df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 25164b99faa2SKhazhismel Kumykov cond_resched(); 2517c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 2518c9de560dSAlex Tomas if (desc == NULL) { 25199d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i); 2520c9de560dSAlex Tomas goto err_freebuddy; 2521c9de560dSAlex Tomas } 25225f21b0e6SFrederic Bohe if (ext4_mb_add_groupinfo(sb, i, desc) != 0) 25235f21b0e6SFrederic Bohe goto err_freebuddy; 2524c9de560dSAlex Tomas } 2525c9de560dSAlex Tomas 2526c9de560dSAlex Tomas return 0; 2527c9de560dSAlex Tomas 2528c9de560dSAlex Tomas err_freebuddy: 2529fb1813f4SCurt Wohlgemuth cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2530f1fa3342SRoel Kluin while (i-- > 0) 2531fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, ext4_get_group_info(sb, i)); 253228623c2fSTheodore Ts'o i = sbi->s_group_info_size; 2533df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2534df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 2535f1fa3342SRoel Kluin while (i-- > 0) 2536df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 2537df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2538c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2539c9de560dSAlex Tomas err_freesgi: 2540df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2541df3da4eaSSuraj Jitindar Singh kvfree(rcu_dereference(sbi->s_group_info)); 2542df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2543c9de560dSAlex Tomas return -ENOMEM; 2544c9de560dSAlex Tomas } 2545c9de560dSAlex Tomas 25462892c15dSEric Sandeen static void ext4_groupinfo_destroy_slabs(void) 25472892c15dSEric Sandeen { 25482892c15dSEric Sandeen int i; 25492892c15dSEric Sandeen 25502892c15dSEric Sandeen for (i = 0; i < NR_GRPINFO_CACHES; i++) { 25512892c15dSEric Sandeen kmem_cache_destroy(ext4_groupinfo_caches[i]); 25522892c15dSEric Sandeen ext4_groupinfo_caches[i] = NULL; 25532892c15dSEric Sandeen } 25542892c15dSEric Sandeen } 25552892c15dSEric Sandeen 25562892c15dSEric Sandeen static int ext4_groupinfo_create_slab(size_t size) 25572892c15dSEric Sandeen { 25582892c15dSEric Sandeen static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex); 25592892c15dSEric Sandeen int slab_size; 25602892c15dSEric Sandeen int blocksize_bits = order_base_2(size); 25612892c15dSEric Sandeen int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 25622892c15dSEric Sandeen struct kmem_cache *cachep; 25632892c15dSEric Sandeen 25642892c15dSEric Sandeen if (cache_index >= NR_GRPINFO_CACHES) 25652892c15dSEric Sandeen return -EINVAL; 25662892c15dSEric Sandeen 25672892c15dSEric Sandeen if (unlikely(cache_index < 0)) 25682892c15dSEric Sandeen cache_index = 0; 25692892c15dSEric Sandeen 25702892c15dSEric Sandeen mutex_lock(&ext4_grpinfo_slab_create_mutex); 25712892c15dSEric Sandeen if (ext4_groupinfo_caches[cache_index]) { 25722892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 25732892c15dSEric Sandeen return 0; /* Already created */ 25742892c15dSEric Sandeen } 25752892c15dSEric Sandeen 25762892c15dSEric Sandeen slab_size = offsetof(struct ext4_group_info, 25772892c15dSEric Sandeen bb_counters[blocksize_bits + 2]); 25782892c15dSEric Sandeen 25792892c15dSEric Sandeen cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index], 25802892c15dSEric Sandeen slab_size, 0, SLAB_RECLAIM_ACCOUNT, 25812892c15dSEric Sandeen NULL); 25822892c15dSEric Sandeen 2583823ba01fSTao Ma ext4_groupinfo_caches[cache_index] = cachep; 2584823ba01fSTao Ma 25852892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 25862892c15dSEric Sandeen if (!cachep) { 25879d8b9ec4STheodore Ts'o printk(KERN_EMERG 25889d8b9ec4STheodore Ts'o "EXT4-fs: no memory for groupinfo slab cache\n"); 25892892c15dSEric Sandeen return -ENOMEM; 25902892c15dSEric Sandeen } 25912892c15dSEric Sandeen 25922892c15dSEric Sandeen return 0; 25932892c15dSEric Sandeen } 25942892c15dSEric Sandeen 25959d99012fSAkira Fujita int ext4_mb_init(struct super_block *sb) 2596c9de560dSAlex Tomas { 2597c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 25986be2ded1SAneesh Kumar K.V unsigned i, j; 2599935244cdSNicolai Stange unsigned offset, offset_incr; 2600c9de560dSAlex Tomas unsigned max; 260174767c5aSShen Feng int ret; 2602c9de560dSAlex Tomas 26031927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets); 2604c9de560dSAlex Tomas 2605c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 2606c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 2607fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2608fb1813f4SCurt Wohlgemuth goto out; 2609c9de560dSAlex Tomas } 2610ff7ef329SYasunori Goto 26111927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs); 2612c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 2613c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 2614fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2615fb1813f4SCurt Wohlgemuth goto out; 2616fb1813f4SCurt Wohlgemuth } 2617fb1813f4SCurt Wohlgemuth 26182892c15dSEric Sandeen ret = ext4_groupinfo_create_slab(sb->s_blocksize); 26192892c15dSEric Sandeen if (ret < 0) 2620fb1813f4SCurt Wohlgemuth goto out; 2621c9de560dSAlex Tomas 2622c9de560dSAlex Tomas /* order 0 is regular bitmap */ 2623c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 2624c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 2625c9de560dSAlex Tomas 2626c9de560dSAlex Tomas i = 1; 2627c9de560dSAlex Tomas offset = 0; 2628935244cdSNicolai Stange offset_incr = 1 << (sb->s_blocksize_bits - 1); 2629c9de560dSAlex Tomas max = sb->s_blocksize << 2; 2630c9de560dSAlex Tomas do { 2631c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 2632c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 2633935244cdSNicolai Stange offset += offset_incr; 2634935244cdSNicolai Stange offset_incr = offset_incr >> 1; 2635c9de560dSAlex Tomas max = max >> 1; 2636c9de560dSAlex Tomas i++; 2637c9de560dSAlex Tomas } while (i <= sb->s_blocksize_bits + 1); 2638c9de560dSAlex Tomas 2639c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 2640c9de560dSAlex Tomas spin_lock_init(&sbi->s_bal_lock); 2641d08854f5STheodore Ts'o sbi->s_mb_free_pending = 0; 2642a0154344SDaeho Jeong INIT_LIST_HEAD(&sbi->s_freed_data_list); 2643c9de560dSAlex Tomas 2644c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 2645c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 2646c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 2647c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 2648c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 264927baebb8STheodore Ts'o /* 265027baebb8STheodore Ts'o * The default group preallocation is 512, which for 4k block 265127baebb8STheodore Ts'o * sizes translates to 2 megabytes. However for bigalloc file 265227baebb8STheodore Ts'o * systems, this is probably too big (i.e, if the cluster size 265327baebb8STheodore Ts'o * is 1 megabyte, then group preallocation size becomes half a 265427baebb8STheodore Ts'o * gigabyte!). As a default, we will keep a two megabyte 265527baebb8STheodore Ts'o * group pralloc size for cluster sizes up to 64k, and after 265627baebb8STheodore Ts'o * that, we will force a minimum group preallocation size of 265727baebb8STheodore Ts'o * 32 clusters. This translates to 8 megs when the cluster 265827baebb8STheodore Ts'o * size is 256k, and 32 megs when the cluster size is 1 meg, 265927baebb8STheodore Ts'o * which seems reasonable as a default. 266027baebb8STheodore Ts'o */ 266127baebb8STheodore Ts'o sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> 266227baebb8STheodore Ts'o sbi->s_cluster_bits, 32); 2663d7a1fee1SDan Ehrenberg /* 2664d7a1fee1SDan Ehrenberg * If there is a s_stripe > 1, then we set the s_mb_group_prealloc 2665d7a1fee1SDan Ehrenberg * to the lowest multiple of s_stripe which is bigger than 2666d7a1fee1SDan Ehrenberg * the s_mb_group_prealloc as determined above. We want 2667d7a1fee1SDan Ehrenberg * the preallocation size to be an exact multiple of the 2668d7a1fee1SDan Ehrenberg * RAID stripe size so that preallocations don't fragment 2669d7a1fee1SDan Ehrenberg * the stripes. 2670d7a1fee1SDan Ehrenberg */ 2671d7a1fee1SDan Ehrenberg if (sbi->s_stripe > 1) { 2672d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc = roundup( 2673d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc, sbi->s_stripe); 2674d7a1fee1SDan Ehrenberg } 2675c9de560dSAlex Tomas 2676730c213cSEric Sandeen sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); 2677c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 2678fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2679029b10c5SAndrey Tsyvarev goto out; 2680c9de560dSAlex Tomas } 2681730c213cSEric Sandeen for_each_possible_cpu(i) { 2682c9de560dSAlex Tomas struct ext4_locality_group *lg; 2683730c213cSEric Sandeen lg = per_cpu_ptr(sbi->s_locality_groups, i); 2684c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 26856be2ded1SAneesh Kumar K.V for (j = 0; j < PREALLOC_TB_SIZE; j++) 26866be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); 2687c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 2688c9de560dSAlex Tomas } 2689c9de560dSAlex Tomas 269079a77c5aSYu Jian /* init file for buddy data */ 269179a77c5aSYu Jian ret = ext4_mb_init_backend(sb); 26927aa0baeaSTao Ma if (ret != 0) 26937aa0baeaSTao Ma goto out_free_locality_groups; 269479a77c5aSYu Jian 26957aa0baeaSTao Ma return 0; 26967aa0baeaSTao Ma 26977aa0baeaSTao Ma out_free_locality_groups: 26987aa0baeaSTao Ma free_percpu(sbi->s_locality_groups); 26997aa0baeaSTao Ma sbi->s_locality_groups = NULL; 2700fb1813f4SCurt Wohlgemuth out: 2701fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_offsets); 27027aa0baeaSTao Ma sbi->s_mb_offsets = NULL; 2703fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_maxs); 27047aa0baeaSTao Ma sbi->s_mb_maxs = NULL; 2705fb1813f4SCurt Wohlgemuth return ret; 2706c9de560dSAlex Tomas } 2707c9de560dSAlex Tomas 2708955ce5f5SAneesh Kumar K.V /* need to called with the ext4 group lock held */ 2709c9de560dSAlex Tomas static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) 2710c9de560dSAlex Tomas { 2711c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 2712c9de560dSAlex Tomas struct list_head *cur, *tmp; 2713c9de560dSAlex Tomas int count = 0; 2714c9de560dSAlex Tomas 2715c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 2716c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 2717c9de560dSAlex Tomas list_del(&pa->pa_group_list); 2718c9de560dSAlex Tomas count++; 2719688f05a0SAneesh Kumar K.V kmem_cache_free(ext4_pspace_cachep, pa); 2720c9de560dSAlex Tomas } 2721c9de560dSAlex Tomas if (count) 27226ba495e9STheodore Ts'o mb_debug(1, "mballoc: %u PAs left\n", count); 2723c9de560dSAlex Tomas 2724c9de560dSAlex Tomas } 2725c9de560dSAlex Tomas 2726c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 2727c9de560dSAlex Tomas { 27288df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2729c9de560dSAlex Tomas ext4_group_t i; 2730c9de560dSAlex Tomas int num_meta_group_infos; 2731df3da4eaSSuraj Jitindar Singh struct ext4_group_info *grinfo, ***group_info; 2732c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2733fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2734c9de560dSAlex Tomas 2735c9de560dSAlex Tomas if (sbi->s_group_info) { 27368df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 27374b99faa2SKhazhismel Kumykov cond_resched(); 2738c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 2739c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 2740c9de560dSAlex Tomas kfree(grinfo->bb_bitmap); 2741c9de560dSAlex Tomas #endif 2742c9de560dSAlex Tomas ext4_lock_group(sb, i); 2743c9de560dSAlex Tomas ext4_mb_cleanup_pa(grinfo); 2744c9de560dSAlex Tomas ext4_unlock_group(sb, i); 2745fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, grinfo); 2746c9de560dSAlex Tomas } 27478df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + 2748c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 2749c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 2750df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2751df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 2752c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 2753df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 2754df3da4eaSSuraj Jitindar Singh kvfree(group_info); 2755df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2756c9de560dSAlex Tomas } 2757c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2758c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2759c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2760c9de560dSAlex Tomas if (sbi->s_mb_stats) { 27619d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27629d8b9ec4STheodore Ts'o "mballoc: %u blocks %u reqs (%u success)", 2763c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 2764c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 2765c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 27669d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27679d8b9ec4STheodore Ts'o "mballoc: %u extents scanned, %u goal hits, " 27689d8b9ec4STheodore Ts'o "%u 2^N hits, %u breaks, %u lost", 2769c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 2770c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 2771c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 2772c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 2773c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 27749d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27759d8b9ec4STheodore Ts'o "mballoc: %lu generated and it took %Lu", 2776ced156e4STao Ma sbi->s_mb_buddies_generated, 2777c9de560dSAlex Tomas sbi->s_mb_generation_time); 27789d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27799d8b9ec4STheodore Ts'o "mballoc: %u preallocated, %u discarded", 2780c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 2781c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 2782c9de560dSAlex Tomas } 2783c9de560dSAlex Tomas 2784730c213cSEric Sandeen free_percpu(sbi->s_locality_groups); 2785c9de560dSAlex Tomas 2786c9de560dSAlex Tomas return 0; 2787c9de560dSAlex Tomas } 2788c9de560dSAlex Tomas 278977ca6cdfSLukas Czerner static inline int ext4_issue_discard(struct super_block *sb, 2790a0154344SDaeho Jeong ext4_group_t block_group, ext4_grpblk_t cluster, int count, 2791a0154344SDaeho Jeong struct bio **biop) 27925c521830SJiaying Zhang { 27935c521830SJiaying Zhang ext4_fsblk_t discard_block; 27945c521830SJiaying Zhang 279584130193STheodore Ts'o discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) + 279684130193STheodore Ts'o ext4_group_first_block_no(sb, block_group)); 279784130193STheodore Ts'o count = EXT4_C2B(EXT4_SB(sb), count); 27985c521830SJiaying Zhang trace_ext4_discard_blocks(sb, 27995c521830SJiaying Zhang (unsigned long long) discard_block, count); 2800a0154344SDaeho Jeong if (biop) { 2801a0154344SDaeho Jeong return __blkdev_issue_discard(sb->s_bdev, 2802a0154344SDaeho Jeong (sector_t)discard_block << (sb->s_blocksize_bits - 9), 2803a0154344SDaeho Jeong (sector_t)count << (sb->s_blocksize_bits - 9), 2804a0154344SDaeho Jeong GFP_NOFS, 0, biop); 2805a0154344SDaeho Jeong } else 280693259636SLukas Czerner return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); 28075c521830SJiaying Zhang } 28085c521830SJiaying Zhang 2809a0154344SDaeho Jeong static void ext4_free_data_in_buddy(struct super_block *sb, 2810a0154344SDaeho Jeong struct ext4_free_data *entry) 2811c9de560dSAlex Tomas { 2812c9de560dSAlex Tomas struct ext4_buddy e4b; 2813c894058dSAneesh Kumar K.V struct ext4_group_info *db; 2814d9f34504STheodore Ts'o int err, count = 0, count2 = 0; 2815c9de560dSAlex Tomas 28166ba495e9STheodore Ts'o mb_debug(1, "gonna free %u blocks in group %u (0x%p):", 281718aadd47SBobi Jam entry->efd_count, entry->efd_group, entry); 2818c9de560dSAlex Tomas 281918aadd47SBobi Jam err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b); 2820c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 2821c9de560dSAlex Tomas BUG_ON(err != 0); 2822c9de560dSAlex Tomas 2823d08854f5STheodore Ts'o spin_lock(&EXT4_SB(sb)->s_md_lock); 2824d08854f5STheodore Ts'o EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count; 2825d08854f5STheodore Ts'o spin_unlock(&EXT4_SB(sb)->s_md_lock); 282618aadd47SBobi Jam 2827c894058dSAneesh Kumar K.V db = e4b.bd_info; 2828c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 282918aadd47SBobi Jam count += entry->efd_count; 2830c9de560dSAlex Tomas count2++; 283118aadd47SBobi Jam ext4_lock_group(sb, entry->efd_group); 2832c894058dSAneesh Kumar K.V /* Take it out of per group rb tree */ 283318aadd47SBobi Jam rb_erase(&entry->efd_node, &(db->bb_free_root)); 283418aadd47SBobi Jam mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count); 2835c9de560dSAlex Tomas 28363d56b8d2STao Ma /* 28373d56b8d2STao Ma * Clear the trimmed flag for the group so that the next 28383d56b8d2STao Ma * ext4_trim_fs can trim it. 28393d56b8d2STao Ma * If the volume is mounted with -o discard, online discard 28403d56b8d2STao Ma * is supported and the free blocks will be trimmed online. 28413d56b8d2STao Ma */ 28423d56b8d2STao Ma if (!test_opt(sb, DISCARD)) 28433d56b8d2STao Ma EXT4_MB_GRP_CLEAR_TRIMMED(db); 28443d56b8d2STao Ma 2845c894058dSAneesh Kumar K.V if (!db->bb_free_root.rb_node) { 2846c894058dSAneesh Kumar K.V /* No more items in the per group rb tree 2847c894058dSAneesh Kumar K.V * balance refcounts from ext4_mb_free_metadata() 2848c894058dSAneesh Kumar K.V */ 284909cbfeafSKirill A. Shutemov put_page(e4b.bd_buddy_page); 285009cbfeafSKirill A. Shutemov put_page(e4b.bd_bitmap_page); 2851c894058dSAneesh Kumar K.V } 285218aadd47SBobi Jam ext4_unlock_group(sb, entry->efd_group); 285318aadd47SBobi Jam kmem_cache_free(ext4_free_data_cachep, entry); 2854e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2855c9de560dSAlex Tomas 28566ba495e9STheodore Ts'o mb_debug(1, "freed %u blocks in %u structures\n", count, count2); 2857c9de560dSAlex Tomas } 2858c9de560dSAlex Tomas 2859a0154344SDaeho Jeong /* 2860a0154344SDaeho Jeong * This function is called by the jbd2 layer once the commit has finished, 2861a0154344SDaeho Jeong * so we know we can free the blocks that were released with that commit. 2862a0154344SDaeho Jeong */ 2863a0154344SDaeho Jeong void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid) 2864a0154344SDaeho Jeong { 2865a0154344SDaeho Jeong struct ext4_sb_info *sbi = EXT4_SB(sb); 2866a0154344SDaeho Jeong struct ext4_free_data *entry, *tmp; 2867a0154344SDaeho Jeong struct bio *discard_bio = NULL; 2868a0154344SDaeho Jeong struct list_head freed_data_list; 2869a0154344SDaeho Jeong struct list_head *cut_pos = NULL; 2870a0154344SDaeho Jeong int err; 2871a0154344SDaeho Jeong 2872a0154344SDaeho Jeong INIT_LIST_HEAD(&freed_data_list); 2873a0154344SDaeho Jeong 2874a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 2875a0154344SDaeho Jeong list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) { 2876a0154344SDaeho Jeong if (entry->efd_tid != commit_tid) 2877a0154344SDaeho Jeong break; 2878a0154344SDaeho Jeong cut_pos = &entry->efd_list; 2879a0154344SDaeho Jeong } 2880a0154344SDaeho Jeong if (cut_pos) 2881a0154344SDaeho Jeong list_cut_position(&freed_data_list, &sbi->s_freed_data_list, 2882a0154344SDaeho Jeong cut_pos); 2883a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 2884a0154344SDaeho Jeong 2885a0154344SDaeho Jeong if (test_opt(sb, DISCARD)) { 2886a0154344SDaeho Jeong list_for_each_entry(entry, &freed_data_list, efd_list) { 2887a0154344SDaeho Jeong err = ext4_issue_discard(sb, entry->efd_group, 2888a0154344SDaeho Jeong entry->efd_start_cluster, 2889a0154344SDaeho Jeong entry->efd_count, 2890a0154344SDaeho Jeong &discard_bio); 2891a0154344SDaeho Jeong if (err && err != -EOPNOTSUPP) { 2892a0154344SDaeho Jeong ext4_msg(sb, KERN_WARNING, "discard request in" 2893a0154344SDaeho Jeong " group:%d block:%d count:%d failed" 2894a0154344SDaeho Jeong " with %d", entry->efd_group, 2895a0154344SDaeho Jeong entry->efd_start_cluster, 2896a0154344SDaeho Jeong entry->efd_count, err); 2897a0154344SDaeho Jeong } else if (err == -EOPNOTSUPP) 2898a0154344SDaeho Jeong break; 2899a0154344SDaeho Jeong } 2900a0154344SDaeho Jeong 2901e4510577SDaeho Jeong if (discard_bio) { 2902a0154344SDaeho Jeong submit_bio_wait(discard_bio); 2903e4510577SDaeho Jeong bio_put(discard_bio); 2904e4510577SDaeho Jeong } 2905a0154344SDaeho Jeong } 2906a0154344SDaeho Jeong 2907a0154344SDaeho Jeong list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list) 2908a0154344SDaeho Jeong ext4_free_data_in_buddy(sb, entry); 2909a0154344SDaeho Jeong } 2910a0154344SDaeho Jeong 29115dabfc78STheodore Ts'o int __init ext4_init_mballoc(void) 2912c9de560dSAlex Tomas { 291316828088STheodore Ts'o ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space, 291416828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2915c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 2916*f283529aSRitesh Harjani goto out; 2917c9de560dSAlex Tomas 291816828088STheodore Ts'o ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context, 291916828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2920*f283529aSRitesh Harjani if (ext4_ac_cachep == NULL) 2921*f283529aSRitesh Harjani goto out_pa_free; 2922c894058dSAneesh Kumar K.V 292318aadd47SBobi Jam ext4_free_data_cachep = KMEM_CACHE(ext4_free_data, 292416828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2925*f283529aSRitesh Harjani if (ext4_free_data_cachep == NULL) 2926*f283529aSRitesh Harjani goto out_ac_free; 2927*f283529aSRitesh Harjani 2928c9de560dSAlex Tomas return 0; 2929*f283529aSRitesh Harjani 2930*f283529aSRitesh Harjani out_ac_free: 2931*f283529aSRitesh Harjani kmem_cache_destroy(ext4_ac_cachep); 2932*f283529aSRitesh Harjani out_pa_free: 2933*f283529aSRitesh Harjani kmem_cache_destroy(ext4_pspace_cachep); 2934*f283529aSRitesh Harjani out: 2935*f283529aSRitesh Harjani return -ENOMEM; 2936c9de560dSAlex Tomas } 2937c9de560dSAlex Tomas 29385dabfc78STheodore Ts'o void ext4_exit_mballoc(void) 2939c9de560dSAlex Tomas { 29403e03f9caSJesper Dangaard Brouer /* 29413e03f9caSJesper Dangaard Brouer * Wait for completion of call_rcu()'s on ext4_pspace_cachep 29423e03f9caSJesper Dangaard Brouer * before destroying the slab cache. 29433e03f9caSJesper Dangaard Brouer */ 29443e03f9caSJesper Dangaard Brouer rcu_barrier(); 2945c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 2946256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 294718aadd47SBobi Jam kmem_cache_destroy(ext4_free_data_cachep); 29482892c15dSEric Sandeen ext4_groupinfo_destroy_slabs(); 2949c9de560dSAlex Tomas } 2950c9de560dSAlex Tomas 2951c9de560dSAlex Tomas 2952c9de560dSAlex Tomas /* 295373b2c716SUwe Kleine-König * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps 2954c9de560dSAlex Tomas * Returns 0 if success or error code 2955c9de560dSAlex Tomas */ 29564ddfef7bSEric Sandeen static noinline_for_stack int 29574ddfef7bSEric Sandeen ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 295853accfa9STheodore Ts'o handle_t *handle, unsigned int reserv_clstrs) 2959c9de560dSAlex Tomas { 2960c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 2961c9de560dSAlex Tomas struct ext4_group_desc *gdp; 2962c9de560dSAlex Tomas struct buffer_head *gdp_bh; 2963c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2964c9de560dSAlex Tomas struct super_block *sb; 2965c9de560dSAlex Tomas ext4_fsblk_t block; 2966519deca0SAneesh Kumar K.V int err, len; 2967c9de560dSAlex Tomas 2968c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 2969c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 2970c9de560dSAlex Tomas 2971c9de560dSAlex Tomas sb = ac->ac_sb; 2972c9de560dSAlex Tomas sbi = EXT4_SB(sb); 2973c9de560dSAlex Tomas 2974574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); 29759008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 29769008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 29779008a58eSDarrick J. Wong bitmap_bh = NULL; 2978c9de560dSAlex Tomas goto out_err; 29799008a58eSDarrick J. Wong } 2980c9de560dSAlex Tomas 29815d601255Sliang xie BUFFER_TRACE(bitmap_bh, "getting write access"); 2982c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 2983c9de560dSAlex Tomas if (err) 2984c9de560dSAlex Tomas goto out_err; 2985c9de560dSAlex Tomas 2986c9de560dSAlex Tomas err = -EIO; 2987c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 2988c9de560dSAlex Tomas if (!gdp) 2989c9de560dSAlex Tomas goto out_err; 2990c9de560dSAlex Tomas 2991a9df9a49STheodore Ts'o ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, 2992021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, gdp)); 299303cddb80SAneesh Kumar K.V 29945d601255Sliang xie BUFFER_TRACE(gdp_bh, "get_write_access"); 2995c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gdp_bh); 2996c9de560dSAlex Tomas if (err) 2997c9de560dSAlex Tomas goto out_err; 2998c9de560dSAlex Tomas 2999bda00de7SAkinobu Mita block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3000c9de560dSAlex Tomas 300153accfa9STheodore Ts'o len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 30026fd058f7STheodore Ts'o if (!ext4_data_block_valid(sbi, block, len)) { 300312062dddSEric Sandeen ext4_error(sb, "Allocating blocks %llu-%llu which overlap " 30041084f252STheodore Ts'o "fs metadata", block, block+len); 3005519deca0SAneesh Kumar K.V /* File system mounted not to panic on error 3006554a5cccSVegard Nossum * Fix the bitmap and return EFSCORRUPTED 3007519deca0SAneesh Kumar K.V * We leak some of the blocks here. 3008519deca0SAneesh Kumar K.V */ 3009955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3010c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 3011519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len); 3012955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 30130390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 3014519deca0SAneesh Kumar K.V if (!err) 3015554a5cccSVegard Nossum err = -EFSCORRUPTED; 3016519deca0SAneesh Kumar K.V goto out_err; 3017c9de560dSAlex Tomas } 3018955ce5f5SAneesh Kumar K.V 3019955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3020c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 3021c9de560dSAlex Tomas { 3022c9de560dSAlex Tomas int i; 3023c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 3024c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 3025c9de560dSAlex Tomas bitmap_bh->b_data)); 3026c9de560dSAlex Tomas } 3027c9de560dSAlex Tomas } 3028c9de560dSAlex Tomas #endif 3029c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 3030c3e94d1dSYongqiang Yang ac->ac_b_ex.fe_len); 30318844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 30328844618dSTheodore Ts'o (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 3033c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 3034021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, 3035cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, 3036560671a0SAneesh Kumar K.V ac->ac_b_ex.fe_group, gdp)); 3037c9de560dSAlex Tomas } 3038021b65bbSTheodore Ts'o len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; 3039021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, len); 304079f1ba49STao Ma ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); 3041feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); 3042955ce5f5SAneesh Kumar K.V 3043955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 304457042651STheodore Ts'o percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); 3045d2a17637SMingming Cao /* 30466bc6e63fSAneesh Kumar K.V * Now reduce the dirty block count also. Should not go negative 3047d2a17637SMingming Cao */ 30486bc6e63fSAneesh Kumar K.V if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) 30496bc6e63fSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 305057042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 305157042651STheodore Ts'o reserv_clstrs); 3052c9de560dSAlex Tomas 3053772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 3054772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, 3055772cb7c8SJose R. Santos ac->ac_b_ex.fe_group); 305690ba983fSTheodore Ts'o atomic64_sub(ac->ac_b_ex.fe_len, 30577c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 30587c990728SSuraj Jitindar Singh flex_group)->free_clusters); 3059772cb7c8SJose R. Santos } 3060772cb7c8SJose R. Santos 30610390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 3062c9de560dSAlex Tomas if (err) 3063c9de560dSAlex Tomas goto out_err; 30640390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); 3065c9de560dSAlex Tomas 3066c9de560dSAlex Tomas out_err: 306742a10addSAneesh Kumar K.V brelse(bitmap_bh); 3068c9de560dSAlex Tomas return err; 3069c9de560dSAlex Tomas } 3070c9de560dSAlex Tomas 3071c9de560dSAlex Tomas /* 3072c9de560dSAlex Tomas * here we normalize request for locality group 3073d7a1fee1SDan Ehrenberg * Group request are normalized to s_mb_group_prealloc, which goes to 3074d7a1fee1SDan Ehrenberg * s_strip if we set the same via mount option. 3075d7a1fee1SDan Ehrenberg * s_mb_group_prealloc can be configured via 3076b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_group_prealloc 3077c9de560dSAlex Tomas * 3078c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 3079c9de560dSAlex Tomas */ 3080c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 3081c9de560dSAlex Tomas { 3082c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3083c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 3084c9de560dSAlex Tomas 3085c9de560dSAlex Tomas BUG_ON(lg == NULL); 3086c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 30876ba495e9STheodore Ts'o mb_debug(1, "#%u: goal %u blocks for locality group\n", 3088c9de560dSAlex Tomas current->pid, ac->ac_g_ex.fe_len); 3089c9de560dSAlex Tomas } 3090c9de560dSAlex Tomas 3091c9de560dSAlex Tomas /* 3092c9de560dSAlex Tomas * Normalization means making request better in terms of 3093c9de560dSAlex Tomas * size and alignment 3094c9de560dSAlex Tomas */ 30954ddfef7bSEric Sandeen static noinline_for_stack void 30964ddfef7bSEric Sandeen ext4_mb_normalize_request(struct ext4_allocation_context *ac, 3097c9de560dSAlex Tomas struct ext4_allocation_request *ar) 3098c9de560dSAlex Tomas { 309953accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3100c9de560dSAlex Tomas int bsbits, max; 3101c9de560dSAlex Tomas ext4_lblk_t end; 31021592d2c5SCurt Wohlgemuth loff_t size, start_off; 31031592d2c5SCurt Wohlgemuth loff_t orig_size __maybe_unused; 31045a0790c2SAndi Kleen ext4_lblk_t start; 3105c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 31069a0762c5SAneesh Kumar K.V struct ext4_prealloc_space *pa; 3107c9de560dSAlex Tomas 3108c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 3109c9de560dSAlex Tomas do not need preallocation */ 3110c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3111c9de560dSAlex Tomas return; 3112c9de560dSAlex Tomas 3113c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 3114c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 3115c9de560dSAlex Tomas return; 3116c9de560dSAlex Tomas 3117c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 3118c9de560dSAlex Tomas * required (it's a tail, for example) */ 3119c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 3120c9de560dSAlex Tomas return; 3121c9de560dSAlex Tomas 3122c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 3123c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 3124c9de560dSAlex Tomas return ; 3125c9de560dSAlex Tomas } 3126c9de560dSAlex Tomas 3127c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 3128c9de560dSAlex Tomas 3129c9de560dSAlex Tomas /* first, let's learn actual file size 3130c9de560dSAlex Tomas * given current request is allocated */ 313153accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 3132c9de560dSAlex Tomas size = size << bsbits; 3133c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 3134c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 31355a0790c2SAndi Kleen orig_size = size; 3136c9de560dSAlex Tomas 31371930479cSValerie Clement /* max size of free chunks */ 31381930479cSValerie Clement max = 2 << bsbits; 3139c9de560dSAlex Tomas 31401930479cSValerie Clement #define NRL_CHECK_SIZE(req, size, max, chunk_size) \ 31411930479cSValerie Clement (req <= (size) || max <= (chunk_size)) 3142c9de560dSAlex Tomas 3143c9de560dSAlex Tomas /* first, try to predict filesize */ 3144c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 3145c9de560dSAlex Tomas start_off = 0; 3146c9de560dSAlex Tomas if (size <= 16 * 1024) { 3147c9de560dSAlex Tomas size = 16 * 1024; 3148c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 3149c9de560dSAlex Tomas size = 32 * 1024; 3150c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 3151c9de560dSAlex Tomas size = 64 * 1024; 3152c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 3153c9de560dSAlex Tomas size = 128 * 1024; 3154c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 3155c9de560dSAlex Tomas size = 256 * 1024; 3156c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 3157c9de560dSAlex Tomas size = 512 * 1024; 3158c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 3159c9de560dSAlex Tomas size = 1024 * 1024; 31601930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { 3161c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 31621930479cSValerie Clement (21 - bsbits)) << 21; 31631930479cSValerie Clement size = 2 * 1024 * 1024; 31641930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { 3165c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3166c9de560dSAlex Tomas (22 - bsbits)) << 22; 3167c9de560dSAlex Tomas size = 4 * 1024 * 1024; 3168c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, 31691930479cSValerie Clement (8<<20)>>bsbits, max, 8 * 1024)) { 3170c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3171c9de560dSAlex Tomas (23 - bsbits)) << 23; 3172c9de560dSAlex Tomas size = 8 * 1024 * 1024; 3173c9de560dSAlex Tomas } else { 3174c9de560dSAlex Tomas start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; 3175b27b1535SXiaoguang Wang size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), 3176b27b1535SXiaoguang Wang ac->ac_o_ex.fe_len) << bsbits; 3177c9de560dSAlex Tomas } 31785a0790c2SAndi Kleen size = size >> bsbits; 31795a0790c2SAndi Kleen start = start_off >> bsbits; 3180c9de560dSAlex Tomas 3181c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 3182c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 3183c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 3184c9de560dSAlex Tomas start = ar->lleft + 1; 3185c9de560dSAlex Tomas } 3186c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 3187c9de560dSAlex Tomas size -= start + size - ar->lright; 3188c9de560dSAlex Tomas 3189cd648b8aSJan Kara /* 3190cd648b8aSJan Kara * Trim allocation request for filesystems with artificially small 3191cd648b8aSJan Kara * groups. 3192cd648b8aSJan Kara */ 3193cd648b8aSJan Kara if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) 3194cd648b8aSJan Kara size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); 3195cd648b8aSJan Kara 3196c9de560dSAlex Tomas end = start + size; 3197c9de560dSAlex Tomas 3198c9de560dSAlex Tomas /* check we don't cross already preallocated blocks */ 3199c9de560dSAlex Tomas rcu_read_lock(); 32009a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3201498e5f24STheodore Ts'o ext4_lblk_t pa_end; 3202c9de560dSAlex Tomas 3203c9de560dSAlex Tomas if (pa->pa_deleted) 3204c9de560dSAlex Tomas continue; 3205c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3206c9de560dSAlex Tomas if (pa->pa_deleted) { 3207c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3208c9de560dSAlex Tomas continue; 3209c9de560dSAlex Tomas } 3210c9de560dSAlex Tomas 321153accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 321253accfa9STheodore Ts'o pa->pa_len); 3213c9de560dSAlex Tomas 3214c9de560dSAlex Tomas /* PA must not overlap original request */ 3215c9de560dSAlex Tomas BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || 3216c9de560dSAlex Tomas ac->ac_o_ex.fe_logical < pa->pa_lstart)); 3217c9de560dSAlex Tomas 321838877f4eSEric Sandeen /* skip PAs this normalized request doesn't overlap with */ 321938877f4eSEric Sandeen if (pa->pa_lstart >= end || pa_end <= start) { 3220c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3221c9de560dSAlex Tomas continue; 3222c9de560dSAlex Tomas } 3223c9de560dSAlex Tomas BUG_ON(pa->pa_lstart <= start && pa_end >= end); 3224c9de560dSAlex Tomas 322538877f4eSEric Sandeen /* adjust start or end to be adjacent to this pa */ 3226c9de560dSAlex Tomas if (pa_end <= ac->ac_o_ex.fe_logical) { 3227c9de560dSAlex Tomas BUG_ON(pa_end < start); 3228c9de560dSAlex Tomas start = pa_end; 322938877f4eSEric Sandeen } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { 3230c9de560dSAlex Tomas BUG_ON(pa->pa_lstart > end); 3231c9de560dSAlex Tomas end = pa->pa_lstart; 3232c9de560dSAlex Tomas } 3233c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3234c9de560dSAlex Tomas } 3235c9de560dSAlex Tomas rcu_read_unlock(); 3236c9de560dSAlex Tomas size = end - start; 3237c9de560dSAlex Tomas 3238c9de560dSAlex Tomas /* XXX: extra loop to check we really don't overlap preallocations */ 3239c9de560dSAlex Tomas rcu_read_lock(); 32409a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3241498e5f24STheodore Ts'o ext4_lblk_t pa_end; 324253accfa9STheodore Ts'o 3243c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3244c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 324553accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 324653accfa9STheodore Ts'o pa->pa_len); 3247c9de560dSAlex Tomas BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); 3248c9de560dSAlex Tomas } 3249c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3250c9de560dSAlex Tomas } 3251c9de560dSAlex Tomas rcu_read_unlock(); 3252c9de560dSAlex Tomas 3253c9de560dSAlex Tomas if (start + size <= ac->ac_o_ex.fe_logical && 3254c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 32559d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, 32569d8b9ec4STheodore Ts'o "start %lu, size %lu, fe_logical %lu", 3257c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 3258c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 3259dfe076c1SDmitry Monakhov BUG(); 3260c9de560dSAlex Tomas } 3261b5b60778SMaurizio Lombardi BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 3262c9de560dSAlex Tomas 3263c9de560dSAlex Tomas /* now prepare goal request */ 3264c9de560dSAlex Tomas 3265c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 3266c9de560dSAlex Tomas * placement or satisfy big request as is */ 3267c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 326853accfa9STheodore Ts'o ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); 3269c9de560dSAlex Tomas 3270c9de560dSAlex Tomas /* define goal start in order to merge */ 3271c9de560dSAlex Tomas if (ar->pright && (ar->lright == (start + size))) { 3272c9de560dSAlex Tomas /* merge to the right */ 3273c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 3274c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3275c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3276c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3277c9de560dSAlex Tomas } 3278c9de560dSAlex Tomas if (ar->pleft && (ar->lleft + 1 == start)) { 3279c9de560dSAlex Tomas /* merge to the left */ 3280c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 3281c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3282c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3283c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3284c9de560dSAlex Tomas } 3285c9de560dSAlex Tomas 3286004379d0SRitesh Harjani mb_debug(1, "goal: %lld(was %lld) blocks at %u\n", size, orig_size, 3287004379d0SRitesh Harjani start); 3288c9de560dSAlex Tomas } 3289c9de560dSAlex Tomas 3290c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 3291c9de560dSAlex Tomas { 3292c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3293c9de560dSAlex Tomas 3294c9de560dSAlex Tomas if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { 3295c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 3296c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 3297291dae47SCurt Wohlgemuth if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) 3298c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 3299c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 3300c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 3301c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 3302c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 3303c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 3304c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 3305c9de560dSAlex Tomas } 3306c9de560dSAlex Tomas 3307296c355cSTheodore Ts'o if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) 3308296c355cSTheodore Ts'o trace_ext4_mballoc_alloc(ac); 3309296c355cSTheodore Ts'o else 3310296c355cSTheodore Ts'o trace_ext4_mballoc_prealloc(ac); 3311c9de560dSAlex Tomas } 3312c9de560dSAlex Tomas 3313c9de560dSAlex Tomas /* 3314b844167eSCurt Wohlgemuth * Called on failure; free up any blocks from the inode PA for this 3315b844167eSCurt Wohlgemuth * context. We don't need this for MB_GROUP_PA because we only change 3316b844167eSCurt Wohlgemuth * pa_free in ext4_mb_release_context(), but on failure, we've already 3317b844167eSCurt Wohlgemuth * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed. 3318b844167eSCurt Wohlgemuth */ 3319b844167eSCurt Wohlgemuth static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) 3320b844167eSCurt Wohlgemuth { 3321b844167eSCurt Wohlgemuth struct ext4_prealloc_space *pa = ac->ac_pa; 332286f0afd4STheodore Ts'o struct ext4_buddy e4b; 332386f0afd4STheodore Ts'o int err; 3324b844167eSCurt Wohlgemuth 332586f0afd4STheodore Ts'o if (pa == NULL) { 3326c99d1e6eSTheodore Ts'o if (ac->ac_f_ex.fe_len == 0) 3327c99d1e6eSTheodore Ts'o return; 332886f0afd4STheodore Ts'o err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); 332986f0afd4STheodore Ts'o if (err) { 333086f0afd4STheodore Ts'o /* 333186f0afd4STheodore Ts'o * This should never happen since we pin the 333286f0afd4STheodore Ts'o * pages in the ext4_allocation_context so 333386f0afd4STheodore Ts'o * ext4_mb_load_buddy() should never fail. 333486f0afd4STheodore Ts'o */ 333586f0afd4STheodore Ts'o WARN(1, "mb_load_buddy failed (%d)", err); 333686f0afd4STheodore Ts'o return; 333786f0afd4STheodore Ts'o } 333886f0afd4STheodore Ts'o ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 333986f0afd4STheodore Ts'o mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, 334086f0afd4STheodore Ts'o ac->ac_f_ex.fe_len); 334186f0afd4STheodore Ts'o ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 3342c99d1e6eSTheodore Ts'o ext4_mb_unload_buddy(&e4b); 334386f0afd4STheodore Ts'o return; 334486f0afd4STheodore Ts'o } 334586f0afd4STheodore Ts'o if (pa->pa_type == MB_INODE_PA) 3346400db9d3SZheng Liu pa->pa_free += ac->ac_b_ex.fe_len; 3347b844167eSCurt Wohlgemuth } 3348b844167eSCurt Wohlgemuth 3349b844167eSCurt Wohlgemuth /* 3350c9de560dSAlex Tomas * use blocks preallocated to inode 3351c9de560dSAlex Tomas */ 3352c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 3353c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3354c9de560dSAlex Tomas { 335553accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3356c9de560dSAlex Tomas ext4_fsblk_t start; 3357c9de560dSAlex Tomas ext4_fsblk_t end; 3358c9de560dSAlex Tomas int len; 3359c9de560dSAlex Tomas 3360c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3361c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 336253accfa9STheodore Ts'o end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), 336353accfa9STheodore Ts'o start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); 336453accfa9STheodore Ts'o len = EXT4_NUM_B2C(sbi, end - start); 3365c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 3366c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3367c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3368c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3369c9de560dSAlex Tomas ac->ac_pa = pa; 3370c9de560dSAlex Tomas 3371c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 337253accfa9STheodore Ts'o BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); 3373c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 3374c9de560dSAlex Tomas pa->pa_free -= len; 3375c9de560dSAlex Tomas 3376004379d0SRitesh Harjani mb_debug(1, "use %llu/%d from inode pa %p\n", start, len, pa); 3377c9de560dSAlex Tomas } 3378c9de560dSAlex Tomas 3379c9de560dSAlex Tomas /* 3380c9de560dSAlex Tomas * use blocks preallocated to locality group 3381c9de560dSAlex Tomas */ 3382c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 3383c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3384c9de560dSAlex Tomas { 338503cddb80SAneesh Kumar K.V unsigned int len = ac->ac_o_ex.fe_len; 33866be2ded1SAneesh Kumar K.V 3387c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 3388c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 3389c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3390c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3391c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3392c9de560dSAlex Tomas ac->ac_pa = pa; 3393c9de560dSAlex Tomas 3394c9de560dSAlex Tomas /* we don't correct pa_pstart or pa_plen here to avoid 339526346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 3396c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 339726346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 339826346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 3399c9de560dSAlex Tomas */ 34006ba495e9STheodore Ts'o mb_debug(1, "use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa); 3401c9de560dSAlex Tomas } 3402c9de560dSAlex Tomas 3403c9de560dSAlex Tomas /* 34045e745b04SAneesh Kumar K.V * Return the prealloc space that have minimal distance 34055e745b04SAneesh Kumar K.V * from the goal block. @cpa is the prealloc 34065e745b04SAneesh Kumar K.V * space that is having currently known minimal distance 34075e745b04SAneesh Kumar K.V * from the goal block. 34085e745b04SAneesh Kumar K.V */ 34095e745b04SAneesh Kumar K.V static struct ext4_prealloc_space * 34105e745b04SAneesh Kumar K.V ext4_mb_check_group_pa(ext4_fsblk_t goal_block, 34115e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, 34125e745b04SAneesh Kumar K.V struct ext4_prealloc_space *cpa) 34135e745b04SAneesh Kumar K.V { 34145e745b04SAneesh Kumar K.V ext4_fsblk_t cur_distance, new_distance; 34155e745b04SAneesh Kumar K.V 34165e745b04SAneesh Kumar K.V if (cpa == NULL) { 34175e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 34185e745b04SAneesh Kumar K.V return pa; 34195e745b04SAneesh Kumar K.V } 342079211c8eSAndrew Morton cur_distance = abs(goal_block - cpa->pa_pstart); 342179211c8eSAndrew Morton new_distance = abs(goal_block - pa->pa_pstart); 34225e745b04SAneesh Kumar K.V 34235a54b2f1SColy Li if (cur_distance <= new_distance) 34245e745b04SAneesh Kumar K.V return cpa; 34255e745b04SAneesh Kumar K.V 34265e745b04SAneesh Kumar K.V /* drop the previous reference */ 34275e745b04SAneesh Kumar K.V atomic_dec(&cpa->pa_count); 34285e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 34295e745b04SAneesh Kumar K.V return pa; 34305e745b04SAneesh Kumar K.V } 34315e745b04SAneesh Kumar K.V 34325e745b04SAneesh Kumar K.V /* 3433c9de560dSAlex Tomas * search goal blocks in preallocated space 3434c9de560dSAlex Tomas */ 34354ddfef7bSEric Sandeen static noinline_for_stack int 34364ddfef7bSEric Sandeen ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 3437c9de560dSAlex Tomas { 343853accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 34396be2ded1SAneesh Kumar K.V int order, i; 3440c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 3441c9de560dSAlex Tomas struct ext4_locality_group *lg; 34425e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, *cpa = NULL; 34435e745b04SAneesh Kumar K.V ext4_fsblk_t goal_block; 3444c9de560dSAlex Tomas 3445c9de560dSAlex Tomas /* only data can be preallocated */ 3446c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3447c9de560dSAlex Tomas return 0; 3448c9de560dSAlex Tomas 3449c9de560dSAlex Tomas /* first, try per-file preallocation */ 3450c9de560dSAlex Tomas rcu_read_lock(); 34519a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3452c9de560dSAlex Tomas 3453c9de560dSAlex Tomas /* all fields in this condition don't change, 3454c9de560dSAlex Tomas * so we can skip locking for them */ 3455c9de560dSAlex Tomas if (ac->ac_o_ex.fe_logical < pa->pa_lstart || 345653accfa9STheodore Ts'o ac->ac_o_ex.fe_logical >= (pa->pa_lstart + 345753accfa9STheodore Ts'o EXT4_C2B(sbi, pa->pa_len))) 3458c9de560dSAlex Tomas continue; 3459c9de560dSAlex Tomas 3460fb0a387dSEric Sandeen /* non-extent files can't have physical blocks past 2^32 */ 346112e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && 346253accfa9STheodore Ts'o (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) > 346353accfa9STheodore Ts'o EXT4_MAX_BLOCK_FILE_PHYS)) 3464fb0a387dSEric Sandeen continue; 3465fb0a387dSEric Sandeen 3466c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3467c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3468c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free) { 3469c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 3470c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3471c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3472c9de560dSAlex Tomas ac->ac_criteria = 10; 3473c9de560dSAlex Tomas rcu_read_unlock(); 3474c9de560dSAlex Tomas return 1; 3475c9de560dSAlex Tomas } 3476c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3477c9de560dSAlex Tomas } 3478c9de560dSAlex Tomas rcu_read_unlock(); 3479c9de560dSAlex Tomas 3480c9de560dSAlex Tomas /* can we use group allocation? */ 3481c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 3482c9de560dSAlex Tomas return 0; 3483c9de560dSAlex Tomas 3484c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 3485c9de560dSAlex Tomas lg = ac->ac_lg; 3486c9de560dSAlex Tomas if (lg == NULL) 3487c9de560dSAlex Tomas return 0; 34886be2ded1SAneesh Kumar K.V order = fls(ac->ac_o_ex.fe_len) - 1; 34896be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 34906be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 34916be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 3492c9de560dSAlex Tomas 3493bda00de7SAkinobu Mita goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); 34945e745b04SAneesh Kumar K.V /* 34955e745b04SAneesh Kumar K.V * search for the prealloc space that is having 34965e745b04SAneesh Kumar K.V * minimal distance from the goal block. 34975e745b04SAneesh Kumar K.V */ 34986be2ded1SAneesh Kumar K.V for (i = order; i < PREALLOC_TB_SIZE; i++) { 3499c9de560dSAlex Tomas rcu_read_lock(); 35006be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], 35016be2ded1SAneesh Kumar K.V pa_inode_list) { 3502c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 35036be2ded1SAneesh Kumar K.V if (pa->pa_deleted == 0 && 35046be2ded1SAneesh Kumar K.V pa->pa_free >= ac->ac_o_ex.fe_len) { 35055e745b04SAneesh Kumar K.V 35065e745b04SAneesh Kumar K.V cpa = ext4_mb_check_group_pa(goal_block, 35075e745b04SAneesh Kumar K.V pa, cpa); 35085e745b04SAneesh Kumar K.V } 3509c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 35105e745b04SAneesh Kumar K.V } 35115e745b04SAneesh Kumar K.V rcu_read_unlock(); 35125e745b04SAneesh Kumar K.V } 35135e745b04SAneesh Kumar K.V if (cpa) { 35145e745b04SAneesh Kumar K.V ext4_mb_use_group_pa(ac, cpa); 3515c9de560dSAlex Tomas ac->ac_criteria = 20; 3516c9de560dSAlex Tomas return 1; 3517c9de560dSAlex Tomas } 3518c9de560dSAlex Tomas return 0; 3519c9de560dSAlex Tomas } 3520c9de560dSAlex Tomas 3521c9de560dSAlex Tomas /* 35227a2fcbf7SAneesh Kumar K.V * the function goes through all block freed in the group 35237a2fcbf7SAneesh Kumar K.V * but not yet committed and marks them used in in-core bitmap. 35247a2fcbf7SAneesh Kumar K.V * buddy must be generated from this bitmap 3525955ce5f5SAneesh Kumar K.V * Need to be called with the ext4 group lock held 35267a2fcbf7SAneesh Kumar K.V */ 35277a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 35287a2fcbf7SAneesh Kumar K.V ext4_group_t group) 35297a2fcbf7SAneesh Kumar K.V { 35307a2fcbf7SAneesh Kumar K.V struct rb_node *n; 35317a2fcbf7SAneesh Kumar K.V struct ext4_group_info *grp; 35327a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 35337a2fcbf7SAneesh Kumar K.V 35347a2fcbf7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 35357a2fcbf7SAneesh Kumar K.V n = rb_first(&(grp->bb_free_root)); 35367a2fcbf7SAneesh Kumar K.V 35377a2fcbf7SAneesh Kumar K.V while (n) { 353818aadd47SBobi Jam entry = rb_entry(n, struct ext4_free_data, efd_node); 353918aadd47SBobi Jam ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count); 35407a2fcbf7SAneesh Kumar K.V n = rb_next(n); 35417a2fcbf7SAneesh Kumar K.V } 35427a2fcbf7SAneesh Kumar K.V return; 35437a2fcbf7SAneesh Kumar K.V } 35447a2fcbf7SAneesh Kumar K.V 35457a2fcbf7SAneesh Kumar K.V /* 3546c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 3547c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 3548955ce5f5SAneesh Kumar K.V * Need to be called with ext4 group lock held 3549c9de560dSAlex Tomas */ 3550089ceeccSEric Sandeen static noinline_for_stack 3551089ceeccSEric Sandeen void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 3552c9de560dSAlex Tomas ext4_group_t group) 3553c9de560dSAlex Tomas { 3554c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3555c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3556c9de560dSAlex Tomas struct list_head *cur; 3557c9de560dSAlex Tomas ext4_group_t groupnr; 3558c9de560dSAlex Tomas ext4_grpblk_t start; 3559c9de560dSAlex Tomas int preallocated = 0; 3560c9de560dSAlex Tomas int len; 3561c9de560dSAlex Tomas 3562c9de560dSAlex Tomas /* all form of preallocation discards first load group, 3563c9de560dSAlex Tomas * so the only competing code is preallocation use. 3564c9de560dSAlex Tomas * we don't need any locking here 3565c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 3566c9de560dSAlex Tomas * otherwise we could leave used blocks available for 3567c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 3568c9de560dSAlex Tomas * is dropping preallocation 3569c9de560dSAlex Tomas */ 3570c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 3571c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3572c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3573c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 3574c9de560dSAlex Tomas &groupnr, &start); 3575c9de560dSAlex Tomas len = pa->pa_len; 3576c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3577c9de560dSAlex Tomas if (unlikely(len == 0)) 3578c9de560dSAlex Tomas continue; 3579c9de560dSAlex Tomas BUG_ON(groupnr != group); 3580c3e94d1dSYongqiang Yang ext4_set_bits(bitmap, start, len); 3581c9de560dSAlex Tomas preallocated += len; 3582c9de560dSAlex Tomas } 3583004379d0SRitesh Harjani mb_debug(1, "preallocated %d for group %u\n", preallocated, group); 3584c9de560dSAlex Tomas } 3585c9de560dSAlex Tomas 3586c9de560dSAlex Tomas static void ext4_mb_pa_callback(struct rcu_head *head) 3587c9de560dSAlex Tomas { 3588c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3589c9de560dSAlex Tomas pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 35904e8d2139SJunho Ryu 35914e8d2139SJunho Ryu BUG_ON(atomic_read(&pa->pa_count)); 35924e8d2139SJunho Ryu BUG_ON(pa->pa_deleted == 0); 3593c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 3594c9de560dSAlex Tomas } 3595c9de560dSAlex Tomas 3596c9de560dSAlex Tomas /* 3597c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 3598c9de560dSAlex Tomas * if this was the last reference and the space is consumed 3599c9de560dSAlex Tomas */ 3600c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 3601c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 3602c9de560dSAlex Tomas { 3603a9df9a49STheodore Ts'o ext4_group_t grp; 3604d33a1976SEric Sandeen ext4_fsblk_t grp_blk; 3605c9de560dSAlex Tomas 3606c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 3607c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 36084e8d2139SJunho Ryu if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) { 36094e8d2139SJunho Ryu spin_unlock(&pa->pa_lock); 36104e8d2139SJunho Ryu return; 36114e8d2139SJunho Ryu } 36124e8d2139SJunho Ryu 3613c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 3614c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3615c9de560dSAlex Tomas return; 3616c9de560dSAlex Tomas } 3617c9de560dSAlex Tomas 3618c9de560dSAlex Tomas pa->pa_deleted = 1; 3619c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3620c9de560dSAlex Tomas 3621d33a1976SEric Sandeen grp_blk = pa->pa_pstart; 3622cc0fb9adSAneesh Kumar K.V /* 3623cc0fb9adSAneesh Kumar K.V * If doing group-based preallocation, pa_pstart may be in the 3624cc0fb9adSAneesh Kumar K.V * next group when pa is used up 3625cc0fb9adSAneesh Kumar K.V */ 3626cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 3627d33a1976SEric Sandeen grp_blk--; 3628d33a1976SEric Sandeen 3629bd86298eSLukas Czerner grp = ext4_get_group_number(sb, grp_blk); 3630c9de560dSAlex Tomas 3631c9de560dSAlex Tomas /* 3632c9de560dSAlex Tomas * possible race: 3633c9de560dSAlex Tomas * 3634c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 3635c9de560dSAlex Tomas * find block B in PA 3636c9de560dSAlex Tomas * copy on-disk bitmap to buddy 3637c9de560dSAlex Tomas * mark B in on-disk bitmap 3638c9de560dSAlex Tomas * drop PA from group 3639c9de560dSAlex Tomas * mark all PAs in buddy 3640c9de560dSAlex Tomas * 3641c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 3642c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 3643c9de560dSAlex Tomas * against that pair 3644c9de560dSAlex Tomas */ 3645c9de560dSAlex Tomas ext4_lock_group(sb, grp); 3646c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3647c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 3648c9de560dSAlex Tomas 3649c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3650c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3651c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3652c9de560dSAlex Tomas 3653c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3654c9de560dSAlex Tomas } 3655c9de560dSAlex Tomas 3656c9de560dSAlex Tomas /* 3657c9de560dSAlex Tomas * creates new preallocated space for given inode 3658c9de560dSAlex Tomas */ 36594ddfef7bSEric Sandeen static noinline_for_stack int 36604ddfef7bSEric Sandeen ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 3661c9de560dSAlex Tomas { 3662c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 366353accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 3664c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3665c9de560dSAlex Tomas struct ext4_group_info *grp; 3666c9de560dSAlex Tomas struct ext4_inode_info *ei; 3667c9de560dSAlex Tomas 3668c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3669c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3670c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3671c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3672c9de560dSAlex Tomas 3673c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3674c9de560dSAlex Tomas if (pa == NULL) 3675c9de560dSAlex Tomas return -ENOMEM; 3676c9de560dSAlex Tomas 3677c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { 3678c9de560dSAlex Tomas int winl; 3679c9de560dSAlex Tomas int wins; 3680c9de560dSAlex Tomas int win; 3681c9de560dSAlex Tomas int offs; 3682c9de560dSAlex Tomas 3683c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 3684c9de560dSAlex Tomas * so, found space must get proper lstart 3685c9de560dSAlex Tomas * to cover original request */ 3686c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 3687c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 3688c9de560dSAlex Tomas 3689c9de560dSAlex Tomas /* we're limited by original request in that 3690c9de560dSAlex Tomas * logical block must be covered any way 3691c9de560dSAlex Tomas * winl is window we can move our chunk within */ 3692c9de560dSAlex Tomas winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; 3693c9de560dSAlex Tomas 3694c9de560dSAlex Tomas /* also, we should cover whole original request */ 369553accfa9STheodore Ts'o wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); 3696c9de560dSAlex Tomas 3697c9de560dSAlex Tomas /* the smallest one defines real window */ 3698c9de560dSAlex Tomas win = min(winl, wins); 3699c9de560dSAlex Tomas 370053accfa9STheodore Ts'o offs = ac->ac_o_ex.fe_logical % 370153accfa9STheodore Ts'o EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 3702c9de560dSAlex Tomas if (offs && offs < win) 3703c9de560dSAlex Tomas win = offs; 3704c9de560dSAlex Tomas 370553accfa9STheodore Ts'o ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - 3706810da240SLukas Czerner EXT4_NUM_B2C(sbi, win); 3707c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 3708c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 3709c9de560dSAlex Tomas } 3710c9de560dSAlex Tomas 3711c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3712c9de560dSAlex Tomas * allocated blocks for history */ 3713c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3714c9de560dSAlex Tomas 3715c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 3716c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3717c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3718c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3719c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3720c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 3721d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3722d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3723c9de560dSAlex Tomas pa->pa_deleted = 0; 3724cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_INODE_PA; 3725c9de560dSAlex Tomas 372636bad423SRitesh Harjani mb_debug(1, "new inode pa %p: %llu/%d for %u\n", pa, 3727c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 37289bffad1eSTheodore Ts'o trace_ext4_mb_new_inode_pa(ac, pa); 3729c9de560dSAlex Tomas 3730c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 373153accfa9STheodore Ts'o atomic_add(pa->pa_free, &sbi->s_mb_preallocated); 3732c9de560dSAlex Tomas 3733c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 3734c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3735c9de560dSAlex Tomas 3736c9de560dSAlex Tomas pa->pa_obj_lock = &ei->i_prealloc_lock; 3737c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 3738c9de560dSAlex Tomas 3739c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3740c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3741c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3742c9de560dSAlex Tomas 3743c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3744c9de560dSAlex Tomas list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); 3745c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3746c9de560dSAlex Tomas 3747c9de560dSAlex Tomas return 0; 3748c9de560dSAlex Tomas } 3749c9de560dSAlex Tomas 3750c9de560dSAlex Tomas /* 3751c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 3752c9de560dSAlex Tomas */ 37534ddfef7bSEric Sandeen static noinline_for_stack int 37544ddfef7bSEric Sandeen ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 3755c9de560dSAlex Tomas { 3756c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3757c9de560dSAlex Tomas struct ext4_locality_group *lg; 3758c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3759c9de560dSAlex Tomas struct ext4_group_info *grp; 3760c9de560dSAlex Tomas 3761c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3762c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3763c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3764c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3765c9de560dSAlex Tomas 3766c9de560dSAlex Tomas BUG_ON(ext4_pspace_cachep == NULL); 3767c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3768c9de560dSAlex Tomas if (pa == NULL) 3769c9de560dSAlex Tomas return -ENOMEM; 3770c9de560dSAlex Tomas 3771c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3772c9de560dSAlex Tomas * allocated blocks for history */ 3773c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3774c9de560dSAlex Tomas 3775c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3776c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 3777c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3778c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3779c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3780c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 37816be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3782d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3783c9de560dSAlex Tomas pa->pa_deleted = 0; 3784cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_GROUP_PA; 3785c9de560dSAlex Tomas 378636bad423SRitesh Harjani mb_debug(1, "new group pa %p: %llu/%d for %u\n", pa, 3787c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 37889bffad1eSTheodore Ts'o trace_ext4_mb_new_group_pa(ac, pa); 3789c9de560dSAlex Tomas 3790c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 3791c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 3792c9de560dSAlex Tomas 3793c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3794c9de560dSAlex Tomas lg = ac->ac_lg; 3795c9de560dSAlex Tomas BUG_ON(lg == NULL); 3796c9de560dSAlex Tomas 3797c9de560dSAlex Tomas pa->pa_obj_lock = &lg->lg_prealloc_lock; 3798c9de560dSAlex Tomas pa->pa_inode = NULL; 3799c9de560dSAlex Tomas 3800c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3801c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3802c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3803c9de560dSAlex Tomas 38046be2ded1SAneesh Kumar K.V /* 38056be2ded1SAneesh Kumar K.V * We will later add the new pa to the right bucket 38066be2ded1SAneesh Kumar K.V * after updating the pa_free in ext4_mb_release_context 38076be2ded1SAneesh Kumar K.V */ 3808c9de560dSAlex Tomas return 0; 3809c9de560dSAlex Tomas } 3810c9de560dSAlex Tomas 3811c9de560dSAlex Tomas static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 3812c9de560dSAlex Tomas { 3813c9de560dSAlex Tomas int err; 3814c9de560dSAlex Tomas 3815c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 3816c9de560dSAlex Tomas err = ext4_mb_new_group_pa(ac); 3817c9de560dSAlex Tomas else 3818c9de560dSAlex Tomas err = ext4_mb_new_inode_pa(ac); 3819c9de560dSAlex Tomas return err; 3820c9de560dSAlex Tomas } 3821c9de560dSAlex Tomas 3822c9de560dSAlex Tomas /* 3823c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 3824c9de560dSAlex Tomas * in-core bitmap and buddy. 3825c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 3826c9de560dSAlex Tomas * nobody else can find/use it. 3827c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 3828c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 3829c9de560dSAlex Tomas */ 38304ddfef7bSEric Sandeen static noinline_for_stack int 38314ddfef7bSEric Sandeen ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, 38323e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 3833c9de560dSAlex Tomas { 3834c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3835c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 3836498e5f24STheodore Ts'o unsigned int end; 3837498e5f24STheodore Ts'o unsigned int next; 3838c9de560dSAlex Tomas ext4_group_t group; 3839c9de560dSAlex Tomas ext4_grpblk_t bit; 3840ba80b101STheodore Ts'o unsigned long long grp_blk_start; 3841c9de560dSAlex Tomas int free = 0; 3842c9de560dSAlex Tomas 3843c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3844c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 384553accfa9STheodore Ts'o grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); 3846c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3847c9de560dSAlex Tomas end = bit + pa->pa_len; 3848c9de560dSAlex Tomas 3849c9de560dSAlex Tomas while (bit < end) { 3850ffad0a44SAneesh Kumar K.V bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); 3851c9de560dSAlex Tomas if (bit >= end) 3852c9de560dSAlex Tomas break; 3853ffad0a44SAneesh Kumar K.V next = mb_find_next_bit(bitmap_bh->b_data, end, bit); 38546ba495e9STheodore Ts'o mb_debug(1, " free preallocated %u/%u in group %u\n", 38555a0790c2SAndi Kleen (unsigned) ext4_group_first_block_no(sb, group) + bit, 38565a0790c2SAndi Kleen (unsigned) next - bit, (unsigned) group); 3857c9de560dSAlex Tomas free += next - bit; 3858c9de560dSAlex Tomas 38593e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); 386053accfa9STheodore Ts'o trace_ext4_mb_release_inode_pa(pa, (grp_blk_start + 386153accfa9STheodore Ts'o EXT4_C2B(sbi, bit)), 3862a9c667f8SLukas Czerner next - bit); 3863c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 3864c9de560dSAlex Tomas bit = next + 1; 3865c9de560dSAlex Tomas } 3866c9de560dSAlex Tomas if (free != pa->pa_free) { 38679d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_CRIT, 386836bad423SRitesh Harjani "pa %p: logic %lu, phys. %lu, len %d", 3869c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 3870c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 387136bad423SRitesh Harjani pa->pa_len); 3872e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", 387326346ff6SAneesh Kumar K.V free, pa->pa_free); 3874e56eb659SAneesh Kumar K.V /* 3875e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 3876e56eb659SAneesh Kumar K.V * from the bitmap and continue. 3877e56eb659SAneesh Kumar K.V */ 3878c9de560dSAlex Tomas } 3879c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 3880c9de560dSAlex Tomas 3881863c37fcSzhong jiang return 0; 3882c9de560dSAlex Tomas } 3883c9de560dSAlex Tomas 38844ddfef7bSEric Sandeen static noinline_for_stack int 38854ddfef7bSEric Sandeen ext4_mb_release_group_pa(struct ext4_buddy *e4b, 38863e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 3887c9de560dSAlex Tomas { 3888c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3889c9de560dSAlex Tomas ext4_group_t group; 3890c9de560dSAlex Tomas ext4_grpblk_t bit; 3891c9de560dSAlex Tomas 389260e07cf5SYongqiang Yang trace_ext4_mb_release_group_pa(sb, pa); 3893c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3894c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3895c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3896c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 3897c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 38983e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); 3899c9de560dSAlex Tomas 3900c9de560dSAlex Tomas return 0; 3901c9de560dSAlex Tomas } 3902c9de560dSAlex Tomas 3903c9de560dSAlex Tomas /* 3904c9de560dSAlex Tomas * releases all preallocations in given group 3905c9de560dSAlex Tomas * 3906c9de560dSAlex Tomas * first, we need to decide discard policy: 3907c9de560dSAlex Tomas * - when do we discard 3908c9de560dSAlex Tomas * 1) ENOSPC 3909c9de560dSAlex Tomas * - how many do we discard 3910c9de560dSAlex Tomas * 1) how many requested 3911c9de560dSAlex Tomas */ 39124ddfef7bSEric Sandeen static noinline_for_stack int 39134ddfef7bSEric Sandeen ext4_mb_discard_group_preallocations(struct super_block *sb, 3914c9de560dSAlex Tomas ext4_group_t group, int needed) 3915c9de560dSAlex Tomas { 3916c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3917c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3918c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3919c9de560dSAlex Tomas struct list_head list; 3920c9de560dSAlex Tomas struct ext4_buddy e4b; 3921c9de560dSAlex Tomas int err; 3922c9de560dSAlex Tomas int busy = 0; 3923c9de560dSAlex Tomas int free = 0; 3924c9de560dSAlex Tomas 39256ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for group %u\n", group); 3926c9de560dSAlex Tomas 3927c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 3928bbc4ec77SRitesh Harjani goto out_dbg; 3929c9de560dSAlex Tomas 3930574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 39319008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 39329008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 393354d3adbcSTheodore Ts'o ext4_error_err(sb, -err, 393454d3adbcSTheodore Ts'o "Error %d reading block bitmap for %u", 39359008a58eSDarrick J. Wong err, group); 3936bbc4ec77SRitesh Harjani goto out_dbg; 3937c9de560dSAlex Tomas } 3938c9de560dSAlex Tomas 3939c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3940ce89f46cSAneesh Kumar K.V if (err) { 39419651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 39429651e6b2SKonstantin Khlebnikov err, group); 3943ce89f46cSAneesh Kumar K.V put_bh(bitmap_bh); 3944bbc4ec77SRitesh Harjani goto out_dbg; 3945ce89f46cSAneesh Kumar K.V } 3946c9de560dSAlex Tomas 3947c9de560dSAlex Tomas if (needed == 0) 39487137d7a4STheodore Ts'o needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; 3949c9de560dSAlex Tomas 3950c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3951c9de560dSAlex Tomas repeat: 3952c9de560dSAlex Tomas ext4_lock_group(sb, group); 3953c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 3954c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 3955c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3956c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3957c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3958c9de560dSAlex Tomas busy = 1; 3959c9de560dSAlex Tomas continue; 3960c9de560dSAlex Tomas } 3961c9de560dSAlex Tomas if (pa->pa_deleted) { 3962c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3963c9de560dSAlex Tomas continue; 3964c9de560dSAlex Tomas } 3965c9de560dSAlex Tomas 3966c9de560dSAlex Tomas /* seems this one can be freed ... */ 3967c9de560dSAlex Tomas pa->pa_deleted = 1; 3968c9de560dSAlex Tomas 3969c9de560dSAlex Tomas /* we can trust pa_free ... */ 3970c9de560dSAlex Tomas free += pa->pa_free; 3971c9de560dSAlex Tomas 3972c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3973c9de560dSAlex Tomas 3974c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3975c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3976c9de560dSAlex Tomas } 3977c9de560dSAlex Tomas 3978c9de560dSAlex Tomas /* if we still need more blocks and some PAs were used, try again */ 3979c9de560dSAlex Tomas if (free < needed && busy) { 3980c9de560dSAlex Tomas busy = 0; 3981c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3982bb8b20edSLukas Czerner cond_resched(); 3983c9de560dSAlex Tomas goto repeat; 3984c9de560dSAlex Tomas } 3985c9de560dSAlex Tomas 3986c9de560dSAlex Tomas /* found anything to free? */ 3987c9de560dSAlex Tomas if (list_empty(&list)) { 3988c9de560dSAlex Tomas BUG_ON(free != 0); 3989bbc4ec77SRitesh Harjani mb_debug(1, "Someone else may have freed PA for this group %u\n", 3990bbc4ec77SRitesh Harjani group); 3991c9de560dSAlex Tomas goto out; 3992c9de560dSAlex Tomas } 3993c9de560dSAlex Tomas 3994c9de560dSAlex Tomas /* now free all selected PAs */ 3995c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 3996c9de560dSAlex Tomas 3997c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 3998c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3999c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4000c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 4001c9de560dSAlex Tomas 4002cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 40033e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 4004c9de560dSAlex Tomas else 40053e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 4006c9de560dSAlex Tomas 4007c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4008c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4009c9de560dSAlex Tomas } 4010c9de560dSAlex Tomas 4011c9de560dSAlex Tomas out: 4012c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4013e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4014c9de560dSAlex Tomas put_bh(bitmap_bh); 4015bbc4ec77SRitesh Harjani out_dbg: 4016bbc4ec77SRitesh Harjani mb_debug(1, "discarded (%d) blocks preallocated for group %u bb_free (%d)\n", 4017bbc4ec77SRitesh Harjani free, group, grp->bb_free); 4018c9de560dSAlex Tomas return free; 4019c9de560dSAlex Tomas } 4020c9de560dSAlex Tomas 4021c9de560dSAlex Tomas /* 4022c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 4023c9de560dSAlex Tomas * 4024c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 4025c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 4026c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 4027c9de560dSAlex Tomas * 4028c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 4029c9de560dSAlex Tomas */ 4030c2ea3fdeSTheodore Ts'o void ext4_discard_preallocations(struct inode *inode) 4031c9de560dSAlex Tomas { 4032c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 4033c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4034c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 4035c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 4036c9de560dSAlex Tomas ext4_group_t group = 0; 4037c9de560dSAlex Tomas struct list_head list; 4038c9de560dSAlex Tomas struct ext4_buddy e4b; 4039c9de560dSAlex Tomas int err; 4040c9de560dSAlex Tomas 4041c2ea3fdeSTheodore Ts'o if (!S_ISREG(inode->i_mode)) { 4042c9de560dSAlex Tomas /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ 4043c9de560dSAlex Tomas return; 4044c9de560dSAlex Tomas } 4045c9de560dSAlex Tomas 40466ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for inode %lu\n", inode->i_ino); 40479bffad1eSTheodore Ts'o trace_ext4_discard_preallocations(inode); 4048c9de560dSAlex Tomas 4049c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 4050c9de560dSAlex Tomas 4051c9de560dSAlex Tomas repeat: 4052c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 4053c9de560dSAlex Tomas spin_lock(&ei->i_prealloc_lock); 4054c9de560dSAlex Tomas while (!list_empty(&ei->i_prealloc_list)) { 4055c9de560dSAlex Tomas pa = list_entry(ei->i_prealloc_list.next, 4056c9de560dSAlex Tomas struct ext4_prealloc_space, pa_inode_list); 4057c9de560dSAlex Tomas BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); 4058c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4059c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 4060c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 4061c9de560dSAlex Tomas * use preallocation while we're discarding it */ 4062c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4063c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 40649d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, 40659d8b9ec4STheodore Ts'o "uh-oh! used pa while discarding"); 4066c9de560dSAlex Tomas WARN_ON(1); 4067c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4068c9de560dSAlex Tomas goto repeat; 4069c9de560dSAlex Tomas 4070c9de560dSAlex Tomas } 4071c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 4072c9de560dSAlex Tomas pa->pa_deleted = 1; 4073c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4074c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4075c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 4076c9de560dSAlex Tomas continue; 4077c9de560dSAlex Tomas } 4078c9de560dSAlex Tomas 4079c9de560dSAlex Tomas /* someone is deleting pa right now */ 4080c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4081c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4082c9de560dSAlex Tomas 4083c9de560dSAlex Tomas /* we have to wait here because pa_deleted 4084c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 4085c9de560dSAlex Tomas * the list. as we might be called from 4086c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 4087c9de560dSAlex Tomas * and concurrent thread which is unlinking 4088c9de560dSAlex Tomas * pa from inode's list may access already 4089c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 4090c9de560dSAlex Tomas 4091c9de560dSAlex Tomas /* XXX: if this happens too often, we can 4092c9de560dSAlex Tomas * add a flag to force wait only in case 4093c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 4094c9de560dSAlex Tomas * regular truncate */ 4095c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4096c9de560dSAlex Tomas goto repeat; 4097c9de560dSAlex Tomas } 4098c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4099c9de560dSAlex Tomas 4100c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 4101cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_INODE_PA); 4102bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 4103c9de560dSAlex Tomas 41049651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 41059651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 4106ce89f46cSAneesh Kumar K.V if (err) { 410754d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 41089651e6b2SKonstantin Khlebnikov err, group); 4109ce89f46cSAneesh Kumar K.V continue; 4110ce89f46cSAneesh Kumar K.V } 4111c9de560dSAlex Tomas 4112574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 41139008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 41149008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 411554d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d reading block bitmap for %u", 41169008a58eSDarrick J. Wong err, group); 4117e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4118ce89f46cSAneesh Kumar K.V continue; 4119c9de560dSAlex Tomas } 4120c9de560dSAlex Tomas 4121c9de560dSAlex Tomas ext4_lock_group(sb, group); 4122c9de560dSAlex Tomas list_del(&pa->pa_group_list); 41233e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 4124c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4125c9de560dSAlex Tomas 4126e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4127c9de560dSAlex Tomas put_bh(bitmap_bh); 4128c9de560dSAlex Tomas 4129c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4130c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4131c9de560dSAlex Tomas } 4132c9de560dSAlex Tomas } 4133c9de560dSAlex Tomas 41346ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 4135e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 4136c9de560dSAlex Tomas { 4137e68cf40cSRitesh Harjani ext4_group_t i, ngroups; 4138c9de560dSAlex Tomas 4139a0b30c12STheodore Ts'o if (!ext4_mballoc_debug || 41404dd89fc6STheodore Ts'o (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)) 4141e3570639SEric Sandeen return; 4142e3570639SEric Sandeen 41438df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 4144e68cf40cSRitesh Harjani ext4_msg(sb, KERN_ERR, "groups: "); 41458df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 4146c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 4147c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4148c9de560dSAlex Tomas ext4_grpblk_t start; 4149c9de560dSAlex Tomas struct list_head *cur; 4150c9de560dSAlex Tomas ext4_lock_group(sb, i); 4151c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 4152c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 4153c9de560dSAlex Tomas pa_group_list); 4154c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4155c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 4156c9de560dSAlex Tomas NULL, &start); 4157c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 415836bad423SRitesh Harjani printk(KERN_ERR "PA:%u:%d:%d \n", i, 4159c9de560dSAlex Tomas start, pa->pa_len); 4160c9de560dSAlex Tomas } 416160bd63d1SSolofo Ramangalahy ext4_unlock_group(sb, i); 4162c9de560dSAlex Tomas 41631c718505SAkira Fujita printk(KERN_ERR "%u: %d/%d \n", 4164c9de560dSAlex Tomas i, grp->bb_free, grp->bb_fragments); 4165c9de560dSAlex Tomas } 4166c9de560dSAlex Tomas printk(KERN_ERR "\n"); 4167c9de560dSAlex Tomas } 4168e68cf40cSRitesh Harjani 4169e68cf40cSRitesh Harjani static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4170e68cf40cSRitesh Harjani { 4171e68cf40cSRitesh Harjani struct super_block *sb = ac->ac_sb; 4172e68cf40cSRitesh Harjani 4173e68cf40cSRitesh Harjani if (!ext4_mballoc_debug || 4174e68cf40cSRitesh Harjani (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)) 4175e68cf40cSRitesh Harjani return; 4176e68cf40cSRitesh Harjani 4177e68cf40cSRitesh Harjani ext4_msg(sb, KERN_ERR, "Can't allocate:" 4178e68cf40cSRitesh Harjani " Allocation context details:"); 4179004379d0SRitesh Harjani ext4_msg(sb, KERN_ERR, "status %u flags 0x%x", 4180e68cf40cSRitesh Harjani ac->ac_status, ac->ac_flags); 4181e68cf40cSRitesh Harjani ext4_msg(sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, " 4182e68cf40cSRitesh Harjani "goal %lu/%lu/%lu@%lu, " 4183e68cf40cSRitesh Harjani "best %lu/%lu/%lu@%lu cr %d", 4184e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_group, 4185e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_start, 4186e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_len, 4187e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_logical, 4188e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_group, 4189e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_start, 4190e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_len, 4191e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_logical, 4192e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_group, 4193e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_start, 4194e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_len, 4195e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_logical, 4196e68cf40cSRitesh Harjani (int)ac->ac_criteria); 4197004379d0SRitesh Harjani ext4_msg(sb, KERN_ERR, "%u found", ac->ac_found); 4198e68cf40cSRitesh Harjani ext4_mb_show_pa(sb); 4199e68cf40cSRitesh Harjani } 4200c9de560dSAlex Tomas #else 4201e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 4202e68cf40cSRitesh Harjani { 4203e68cf40cSRitesh Harjani return; 4204e68cf40cSRitesh Harjani } 4205c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4206c9de560dSAlex Tomas { 4207e68cf40cSRitesh Harjani ext4_mb_show_pa(ac->ac_sb); 4208c9de560dSAlex Tomas return; 4209c9de560dSAlex Tomas } 4210c9de560dSAlex Tomas #endif 4211c9de560dSAlex Tomas 4212c9de560dSAlex Tomas /* 4213c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 4214c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 4215c9de560dSAlex Tomas * allocation which ever is larger 4216c9de560dSAlex Tomas * 4217b713a5ecSTheodore Ts'o * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req 4218c9de560dSAlex Tomas */ 4219c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 4220c9de560dSAlex Tomas { 4221c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4222c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 4223c9de560dSAlex Tomas loff_t size, isize; 4224c9de560dSAlex Tomas 4225c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 4226c9de560dSAlex Tomas return; 4227c9de560dSAlex Tomas 42284ba74d00STheodore Ts'o if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 42294ba74d00STheodore Ts'o return; 42304ba74d00STheodore Ts'o 423153accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 423250797481STheodore Ts'o isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) 423350797481STheodore Ts'o >> bsbits; 4234c9de560dSAlex Tomas 423582dd124cSNikolay Borisov if ((size == isize) && !ext4_fs_is_busy(sbi) && 423682dd124cSNikolay Borisov !inode_is_open_for_write(ac->ac_inode)) { 423750797481STheodore Ts'o ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; 423850797481STheodore Ts'o return; 423950797481STheodore Ts'o } 424050797481STheodore Ts'o 4241ebbe0277SRobin Dong if (sbi->s_mb_group_prealloc <= 0) { 4242ebbe0277SRobin Dong ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 4243ebbe0277SRobin Dong return; 4244ebbe0277SRobin Dong } 4245ebbe0277SRobin Dong 4246c9de560dSAlex Tomas /* don't use group allocation for large files */ 424771780577STheodore Ts'o size = max(size, isize); 4248cc483f10STao Ma if (size > sbi->s_mb_stream_request) { 42494ba74d00STheodore Ts'o ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 4250c9de560dSAlex Tomas return; 42514ba74d00STheodore Ts'o } 4252c9de560dSAlex Tomas 4253c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 4254c9de560dSAlex Tomas /* 4255c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 4256c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 4257c9de560dSAlex Tomas * request from multiple CPUs. 4258c9de560dSAlex Tomas */ 4259a0b6bc63SChristoph Lameter ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); 4260c9de560dSAlex Tomas 4261c9de560dSAlex Tomas /* we're going to use group allocation */ 4262c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 4263c9de560dSAlex Tomas 4264c9de560dSAlex Tomas /* serialize all allocations in the group */ 4265c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 4266c9de560dSAlex Tomas } 4267c9de560dSAlex Tomas 42684ddfef7bSEric Sandeen static noinline_for_stack int 42694ddfef7bSEric Sandeen ext4_mb_initialize_context(struct ext4_allocation_context *ac, 4270c9de560dSAlex Tomas struct ext4_allocation_request *ar) 4271c9de560dSAlex Tomas { 4272c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 4273c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4274c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 4275c9de560dSAlex Tomas ext4_group_t group; 4276498e5f24STheodore Ts'o unsigned int len; 4277498e5f24STheodore Ts'o ext4_fsblk_t goal; 4278c9de560dSAlex Tomas ext4_grpblk_t block; 4279c9de560dSAlex Tomas 4280c9de560dSAlex Tomas /* we can't allocate > group size */ 4281c9de560dSAlex Tomas len = ar->len; 4282c9de560dSAlex Tomas 4283c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 428440ae3487STheodore Ts'o if (len >= EXT4_CLUSTERS_PER_GROUP(sb)) 428540ae3487STheodore Ts'o len = EXT4_CLUSTERS_PER_GROUP(sb); 4286c9de560dSAlex Tomas 4287c9de560dSAlex Tomas /* start searching from the goal */ 4288c9de560dSAlex Tomas goal = ar->goal; 4289c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 4290c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 4291c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 4292c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 4293c9de560dSAlex Tomas 4294c9de560dSAlex Tomas /* set up allocation goals */ 4295f5a44db5STheodore Ts'o ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); 4296c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 4297c9de560dSAlex Tomas ac->ac_sb = sb; 4298c9de560dSAlex Tomas ac->ac_inode = ar->inode; 429953accfa9STheodore Ts'o ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; 4300c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 4301c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 4302c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 430353accfa9STheodore Ts'o ac->ac_g_ex = ac->ac_o_ex; 4304c9de560dSAlex Tomas ac->ac_flags = ar->flags; 4305c9de560dSAlex Tomas 4306c9de560dSAlex Tomas /* we have to define context: we'll we work with a file or 4307c9de560dSAlex Tomas * locality group. this is a policy, actually */ 4308c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 4309c9de560dSAlex Tomas 43106ba495e9STheodore Ts'o mb_debug(1, "init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, " 4311c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 4312c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 4313c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 4314c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 4315c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 431682dd124cSNikolay Borisov inode_is_open_for_write(ar->inode) ? "" : "non-"); 4317c9de560dSAlex Tomas return 0; 4318c9de560dSAlex Tomas 4319c9de560dSAlex Tomas } 4320c9de560dSAlex Tomas 43216be2ded1SAneesh Kumar K.V static noinline_for_stack void 43226be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(struct super_block *sb, 43236be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg, 43246be2ded1SAneesh Kumar K.V int order, int total_entries) 43256be2ded1SAneesh Kumar K.V { 43266be2ded1SAneesh Kumar K.V ext4_group_t group = 0; 43276be2ded1SAneesh Kumar K.V struct ext4_buddy e4b; 43286be2ded1SAneesh Kumar K.V struct list_head discard_list; 43296be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa, *tmp; 43306be2ded1SAneesh Kumar K.V 43316ba495e9STheodore Ts'o mb_debug(1, "discard locality group preallocation\n"); 43326be2ded1SAneesh Kumar K.V 43336be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&discard_list); 43346be2ded1SAneesh Kumar K.V 43356be2ded1SAneesh Kumar K.V spin_lock(&lg->lg_prealloc_lock); 43366be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], 433792e9c58cSMadhuparna Bhowmik pa_inode_list, 433892e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 43396be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 43406be2ded1SAneesh Kumar K.V if (atomic_read(&pa->pa_count)) { 43416be2ded1SAneesh Kumar K.V /* 43426be2ded1SAneesh Kumar K.V * This is the pa that we just used 43436be2ded1SAneesh Kumar K.V * for block allocation. So don't 43446be2ded1SAneesh Kumar K.V * free that 43456be2ded1SAneesh Kumar K.V */ 43466be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 43476be2ded1SAneesh Kumar K.V continue; 43486be2ded1SAneesh Kumar K.V } 43496be2ded1SAneesh Kumar K.V if (pa->pa_deleted) { 43506be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 43516be2ded1SAneesh Kumar K.V continue; 43526be2ded1SAneesh Kumar K.V } 43536be2ded1SAneesh Kumar K.V /* only lg prealloc space */ 4354cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_GROUP_PA); 43556be2ded1SAneesh Kumar K.V 43566be2ded1SAneesh Kumar K.V /* seems this one can be freed ... */ 43576be2ded1SAneesh Kumar K.V pa->pa_deleted = 1; 43586be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 43596be2ded1SAneesh Kumar K.V 43606be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 43616be2ded1SAneesh Kumar K.V list_add(&pa->u.pa_tmp_list, &discard_list); 43626be2ded1SAneesh Kumar K.V 43636be2ded1SAneesh Kumar K.V total_entries--; 43646be2ded1SAneesh Kumar K.V if (total_entries <= 5) { 43656be2ded1SAneesh Kumar K.V /* 43666be2ded1SAneesh Kumar K.V * we want to keep only 5 entries 43676be2ded1SAneesh Kumar K.V * allowing it to grow to 8. This 43686be2ded1SAneesh Kumar K.V * mak sure we don't call discard 43696be2ded1SAneesh Kumar K.V * soon for this list. 43706be2ded1SAneesh Kumar K.V */ 43716be2ded1SAneesh Kumar K.V break; 43726be2ded1SAneesh Kumar K.V } 43736be2ded1SAneesh Kumar K.V } 43746be2ded1SAneesh Kumar K.V spin_unlock(&lg->lg_prealloc_lock); 43756be2ded1SAneesh Kumar K.V 43766be2ded1SAneesh Kumar K.V list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { 43779651e6b2SKonstantin Khlebnikov int err; 43786be2ded1SAneesh Kumar K.V 4379bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 43809651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 43819651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 43829651e6b2SKonstantin Khlebnikov if (err) { 438354d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 43849651e6b2SKonstantin Khlebnikov err, group); 43856be2ded1SAneesh Kumar K.V continue; 43866be2ded1SAneesh Kumar K.V } 43876be2ded1SAneesh Kumar K.V ext4_lock_group(sb, group); 43886be2ded1SAneesh Kumar K.V list_del(&pa->pa_group_list); 43893e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 43906be2ded1SAneesh Kumar K.V ext4_unlock_group(sb, group); 43916be2ded1SAneesh Kumar K.V 4392e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 43936be2ded1SAneesh Kumar K.V list_del(&pa->u.pa_tmp_list); 43946be2ded1SAneesh Kumar K.V call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 43956be2ded1SAneesh Kumar K.V } 43966be2ded1SAneesh Kumar K.V } 43976be2ded1SAneesh Kumar K.V 43986be2ded1SAneesh Kumar K.V /* 43996be2ded1SAneesh Kumar K.V * We have incremented pa_count. So it cannot be freed at this 44006be2ded1SAneesh Kumar K.V * point. Also we hold lg_mutex. So no parallel allocation is 44016be2ded1SAneesh Kumar K.V * possible from this lg. That means pa_free cannot be updated. 44026be2ded1SAneesh Kumar K.V * 44036be2ded1SAneesh Kumar K.V * A parallel ext4_mb_discard_group_preallocations is possible. 44046be2ded1SAneesh Kumar K.V * which can cause the lg_prealloc_list to be updated. 44056be2ded1SAneesh Kumar K.V */ 44066be2ded1SAneesh Kumar K.V 44076be2ded1SAneesh Kumar K.V static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) 44086be2ded1SAneesh Kumar K.V { 44096be2ded1SAneesh Kumar K.V int order, added = 0, lg_prealloc_count = 1; 44106be2ded1SAneesh Kumar K.V struct super_block *sb = ac->ac_sb; 44116be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg = ac->ac_lg; 44126be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; 44136be2ded1SAneesh Kumar K.V 44146be2ded1SAneesh Kumar K.V order = fls(pa->pa_free) - 1; 44156be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 44166be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 44176be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 44186be2ded1SAneesh Kumar K.V /* Add the prealloc space to lg */ 4419f1167009SNiu Yawei spin_lock(&lg->lg_prealloc_lock); 44206be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], 442192e9c58cSMadhuparna Bhowmik pa_inode_list, 442292e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 44236be2ded1SAneesh Kumar K.V spin_lock(&tmp_pa->pa_lock); 44246be2ded1SAneesh Kumar K.V if (tmp_pa->pa_deleted) { 4425e7c9e3e9STheodore Ts'o spin_unlock(&tmp_pa->pa_lock); 44266be2ded1SAneesh Kumar K.V continue; 44276be2ded1SAneesh Kumar K.V } 44286be2ded1SAneesh Kumar K.V if (!added && pa->pa_free < tmp_pa->pa_free) { 44296be2ded1SAneesh Kumar K.V /* Add to the tail of the previous entry */ 44306be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 44316be2ded1SAneesh Kumar K.V &tmp_pa->pa_inode_list); 44326be2ded1SAneesh Kumar K.V added = 1; 44336be2ded1SAneesh Kumar K.V /* 44346be2ded1SAneesh Kumar K.V * we want to count the total 44356be2ded1SAneesh Kumar K.V * number of entries in the list 44366be2ded1SAneesh Kumar K.V */ 44376be2ded1SAneesh Kumar K.V } 44386be2ded1SAneesh Kumar K.V spin_unlock(&tmp_pa->pa_lock); 44396be2ded1SAneesh Kumar K.V lg_prealloc_count++; 44406be2ded1SAneesh Kumar K.V } 44416be2ded1SAneesh Kumar K.V if (!added) 44426be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 44436be2ded1SAneesh Kumar K.V &lg->lg_prealloc_list[order]); 4444f1167009SNiu Yawei spin_unlock(&lg->lg_prealloc_lock); 44456be2ded1SAneesh Kumar K.V 44466be2ded1SAneesh Kumar K.V /* Now trim the list to be not more than 8 elements */ 44476be2ded1SAneesh Kumar K.V if (lg_prealloc_count > 8) { 44486be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(sb, lg, 44496be2ded1SAneesh Kumar K.V order, lg_prealloc_count); 44506be2ded1SAneesh Kumar K.V return; 44516be2ded1SAneesh Kumar K.V } 44526be2ded1SAneesh Kumar K.V return ; 44536be2ded1SAneesh Kumar K.V } 44546be2ded1SAneesh Kumar K.V 4455c9de560dSAlex Tomas /* 4456c9de560dSAlex Tomas * release all resource we used in allocation 4457c9de560dSAlex Tomas */ 4458c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 4459c9de560dSAlex Tomas { 446053accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 44616be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa = ac->ac_pa; 44626be2ded1SAneesh Kumar K.V if (pa) { 4463cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) { 4464c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 44656be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 446653accfa9STheodore Ts'o pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 446753accfa9STheodore Ts'o pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 44686be2ded1SAneesh Kumar K.V pa->pa_free -= ac->ac_b_ex.fe_len; 44696be2ded1SAneesh Kumar K.V pa->pa_len -= ac->ac_b_ex.fe_len; 44706be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 4471ba443916SAneesh Kumar K.V } 4472ba443916SAneesh Kumar K.V } 4473ba443916SAneesh Kumar K.V if (pa) { 44746be2ded1SAneesh Kumar K.V /* 44756be2ded1SAneesh Kumar K.V * We want to add the pa to the right bucket. 44766be2ded1SAneesh Kumar K.V * Remove it from the list and while adding 44776be2ded1SAneesh Kumar K.V * make sure the list to which we are adding 447844183d42SAmir Goldstein * doesn't grow big. 44796be2ded1SAneesh Kumar K.V */ 4480cc0fb9adSAneesh Kumar K.V if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) { 44816be2ded1SAneesh Kumar K.V spin_lock(pa->pa_obj_lock); 44826be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 44836be2ded1SAneesh Kumar K.V spin_unlock(pa->pa_obj_lock); 44846be2ded1SAneesh Kumar K.V ext4_mb_add_n_trim(ac); 4485c9de560dSAlex Tomas } 44866be2ded1SAneesh Kumar K.V ext4_mb_put_pa(ac, ac->ac_sb, pa); 4487c9de560dSAlex Tomas } 4488c9de560dSAlex Tomas if (ac->ac_bitmap_page) 448909cbfeafSKirill A. Shutemov put_page(ac->ac_bitmap_page); 4490c9de560dSAlex Tomas if (ac->ac_buddy_page) 449109cbfeafSKirill A. Shutemov put_page(ac->ac_buddy_page); 4492c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 4493c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 4494c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 4495c9de560dSAlex Tomas return 0; 4496c9de560dSAlex Tomas } 4497c9de560dSAlex Tomas 4498c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 4499c9de560dSAlex Tomas { 45008df9675fSTheodore Ts'o ext4_group_t i, ngroups = ext4_get_groups_count(sb); 4501c9de560dSAlex Tomas int ret; 4502c9de560dSAlex Tomas int freed = 0; 4503c9de560dSAlex Tomas 45049bffad1eSTheodore Ts'o trace_ext4_mb_discard_preallocations(sb, needed); 45058df9675fSTheodore Ts'o for (i = 0; i < ngroups && needed > 0; i++) { 4506c9de560dSAlex Tomas ret = ext4_mb_discard_group_preallocations(sb, i, needed); 4507c9de560dSAlex Tomas freed += ret; 4508c9de560dSAlex Tomas needed -= ret; 4509c9de560dSAlex Tomas } 4510c9de560dSAlex Tomas 4511c9de560dSAlex Tomas return freed; 4512c9de560dSAlex Tomas } 4513c9de560dSAlex Tomas 4514c9de560dSAlex Tomas /* 4515c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 4516c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 4517c9de560dSAlex Tomas * to usual allocation 4518c9de560dSAlex Tomas */ 4519c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 4520c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 4521c9de560dSAlex Tomas { 45226bc6e63fSAneesh Kumar K.V int freed; 4523256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4524c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4525c9de560dSAlex Tomas struct super_block *sb; 4526c9de560dSAlex Tomas ext4_fsblk_t block = 0; 452760e58e0fSMingming Cao unsigned int inquota = 0; 452853accfa9STheodore Ts'o unsigned int reserv_clstrs = 0; 4529c9de560dSAlex Tomas 4530b10a44c3STheodore Ts'o might_sleep(); 4531c9de560dSAlex Tomas sb = ar->inode->i_sb; 4532c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4533c9de560dSAlex Tomas 45349bffad1eSTheodore Ts'o trace_ext4_request_blocks(ar); 4535ba80b101STheodore Ts'o 453645dc63e7SDmitry Monakhov /* Allow to use superuser reservation for quota file */ 453702749a4cSTahsin Erdogan if (ext4_is_quota_file(ar->inode)) 453845dc63e7SDmitry Monakhov ar->flags |= EXT4_MB_USE_ROOT_BLOCKS; 453945dc63e7SDmitry Monakhov 4540e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) { 454160e58e0fSMingming Cao /* Without delayed allocation we need to verify 454260e58e0fSMingming Cao * there is enough free blocks to do block allocation 454360e58e0fSMingming Cao * and verify allocation doesn't exceed the quota limits. 4544d2a17637SMingming Cao */ 454555f020dbSAllison Henderson while (ar->len && 4546e7d5f315STheodore Ts'o ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { 454755f020dbSAllison Henderson 4548030ba6bcSAneesh Kumar K.V /* let others to free the space */ 4549bb8b20edSLukas Czerner cond_resched(); 4550030ba6bcSAneesh Kumar K.V ar->len = ar->len >> 1; 4551030ba6bcSAneesh Kumar K.V } 4552030ba6bcSAneesh Kumar K.V if (!ar->len) { 4553bbc4ec77SRitesh Harjani ext4_mb_show_pa(sb); 455407031431SMingming Cao *errp = -ENOSPC; 455507031431SMingming Cao return 0; 455607031431SMingming Cao } 455753accfa9STheodore Ts'o reserv_clstrs = ar->len; 455855f020dbSAllison Henderson if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { 455953accfa9STheodore Ts'o dquot_alloc_block_nofail(ar->inode, 456053accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len)); 456155f020dbSAllison Henderson } else { 456255f020dbSAllison Henderson while (ar->len && 456353accfa9STheodore Ts'o dquot_alloc_block(ar->inode, 456453accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len))) { 456555f020dbSAllison Henderson 4566c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 4567c9de560dSAlex Tomas ar->len--; 4568c9de560dSAlex Tomas } 456955f020dbSAllison Henderson } 457060e58e0fSMingming Cao inquota = ar->len; 4571c9de560dSAlex Tomas if (ar->len == 0) { 4572c9de560dSAlex Tomas *errp = -EDQUOT; 45736c7a120aSAditya Kali goto out; 4574c9de560dSAlex Tomas } 457560e58e0fSMingming Cao } 4576d2a17637SMingming Cao 457785556c9aSWei Yongjun ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); 4578833576b3STheodore Ts'o if (!ac) { 4579363d4251SShen Feng ar->len = 0; 4580256bdb49SEric Sandeen *errp = -ENOMEM; 45816c7a120aSAditya Kali goto out; 4582256bdb49SEric Sandeen } 4583256bdb49SEric Sandeen 4584256bdb49SEric Sandeen *errp = ext4_mb_initialize_context(ac, ar); 4585c9de560dSAlex Tomas if (*errp) { 4586c9de560dSAlex Tomas ar->len = 0; 45876c7a120aSAditya Kali goto out; 4588c9de560dSAlex Tomas } 4589c9de560dSAlex Tomas 4590256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 4591256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 4592256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 4593256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 4594c9de560dSAlex Tomas repeat: 4595c9de560dSAlex Tomas /* allocate space in core */ 45966c7a120aSAditya Kali *errp = ext4_mb_regular_allocator(ac); 45972c00ef3eSAlexey Khoroshilov if (*errp) 45982c00ef3eSAlexey Khoroshilov goto discard_and_exit; 4599c9de560dSAlex Tomas 4600c9de560dSAlex Tomas /* as we've just preallocated more space than 46012c00ef3eSAlexey Khoroshilov * user requested originally, we store allocated 4602c9de560dSAlex Tomas * space in a special descriptor */ 4603256bdb49SEric Sandeen if (ac->ac_status == AC_STATUS_FOUND && 4604256bdb49SEric Sandeen ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 46052c00ef3eSAlexey Khoroshilov *errp = ext4_mb_new_preallocation(ac); 46062c00ef3eSAlexey Khoroshilov if (*errp) { 46072c00ef3eSAlexey Khoroshilov discard_and_exit: 46082c00ef3eSAlexey Khoroshilov ext4_discard_allocated_blocks(ac); 46092c00ef3eSAlexey Khoroshilov goto errout; 46102c00ef3eSAlexey Khoroshilov } 4611c9de560dSAlex Tomas } 4612256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 461353accfa9STheodore Ts'o *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); 4614554a5cccSVegard Nossum if (*errp) { 4615b844167eSCurt Wohlgemuth ext4_discard_allocated_blocks(ac); 46166d138cedSEric Sandeen goto errout; 46176d138cedSEric Sandeen } else { 4618256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4619256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 4620519deca0SAneesh Kumar K.V } 4621c9de560dSAlex Tomas } else { 4622256bdb49SEric Sandeen freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 4623c9de560dSAlex Tomas if (freed) 4624c9de560dSAlex Tomas goto repeat; 4625c9de560dSAlex Tomas *errp = -ENOSPC; 46266c7a120aSAditya Kali } 46276c7a120aSAditya Kali 46286d138cedSEric Sandeen errout: 46296c7a120aSAditya Kali if (*errp) { 4630256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 4631c9de560dSAlex Tomas ar->len = 0; 4632256bdb49SEric Sandeen ext4_mb_show_ac(ac); 4633c9de560dSAlex Tomas } 4634256bdb49SEric Sandeen ext4_mb_release_context(ac); 46356c7a120aSAditya Kali out: 46366c7a120aSAditya Kali if (ac) 4637363d4251SShen Feng kmem_cache_free(ext4_ac_cachep, ac); 463860e58e0fSMingming Cao if (inquota && ar->len < inquota) 463953accfa9STheodore Ts'o dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); 46400087d9fbSAneesh Kumar K.V if (!ar->len) { 4641e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) 46420087d9fbSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 464357042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 464453accfa9STheodore Ts'o reserv_clstrs); 46450087d9fbSAneesh Kumar K.V } 4646c9de560dSAlex Tomas 46479bffad1eSTheodore Ts'o trace_ext4_allocate_blocks(ar, (unsigned long long)block); 4648ba80b101STheodore Ts'o 4649c9de560dSAlex Tomas return block; 4650c9de560dSAlex Tomas } 4651c9de560dSAlex Tomas 4652c894058dSAneesh Kumar K.V /* 4653c894058dSAneesh Kumar K.V * We can merge two free data extents only if the physical blocks 4654c894058dSAneesh Kumar K.V * are contiguous, AND the extents were freed by the same transaction, 4655c894058dSAneesh Kumar K.V * AND the blocks are associated with the same group. 4656c894058dSAneesh Kumar K.V */ 4657a0154344SDaeho Jeong static void ext4_try_merge_freed_extent(struct ext4_sb_info *sbi, 4658a0154344SDaeho Jeong struct ext4_free_data *entry, 4659a0154344SDaeho Jeong struct ext4_free_data *new_entry, 4660a0154344SDaeho Jeong struct rb_root *entry_rb_root) 4661c894058dSAneesh Kumar K.V { 4662a0154344SDaeho Jeong if ((entry->efd_tid != new_entry->efd_tid) || 4663a0154344SDaeho Jeong (entry->efd_group != new_entry->efd_group)) 4664a0154344SDaeho Jeong return; 4665a0154344SDaeho Jeong if (entry->efd_start_cluster + entry->efd_count == 4666a0154344SDaeho Jeong new_entry->efd_start_cluster) { 4667a0154344SDaeho Jeong new_entry->efd_start_cluster = entry->efd_start_cluster; 4668a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 4669a0154344SDaeho Jeong } else if (new_entry->efd_start_cluster + new_entry->efd_count == 4670a0154344SDaeho Jeong entry->efd_start_cluster) { 4671a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 4672a0154344SDaeho Jeong } else 4673a0154344SDaeho Jeong return; 4674a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 4675a0154344SDaeho Jeong list_del(&entry->efd_list); 4676a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 4677a0154344SDaeho Jeong rb_erase(&entry->efd_node, entry_rb_root); 4678a0154344SDaeho Jeong kmem_cache_free(ext4_free_data_cachep, entry); 4679c894058dSAneesh Kumar K.V } 4680c894058dSAneesh Kumar K.V 46814ddfef7bSEric Sandeen static noinline_for_stack int 46824ddfef7bSEric Sandeen ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 46837a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry) 4684c9de560dSAlex Tomas { 4685e29136f8STheodore Ts'o ext4_group_t group = e4b->bd_group; 468684130193STheodore Ts'o ext4_grpblk_t cluster; 4687d08854f5STheodore Ts'o ext4_grpblk_t clusters = new_entry->efd_count; 46887a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 4689c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 4690c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4691c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4692c894058dSAneesh Kumar K.V struct rb_node **n = &db->bb_free_root.rb_node, *node; 4693c894058dSAneesh Kumar K.V struct rb_node *parent = NULL, *new_node; 4694c894058dSAneesh Kumar K.V 46950390131bSFrank Mayhar BUG_ON(!ext4_handle_valid(handle)); 4696c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 4697c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 4698c9de560dSAlex Tomas 469918aadd47SBobi Jam new_node = &new_entry->efd_node; 470018aadd47SBobi Jam cluster = new_entry->efd_start_cluster; 4701c9de560dSAlex Tomas 4702c894058dSAneesh Kumar K.V if (!*n) { 4703c894058dSAneesh Kumar K.V /* first free block exent. We need to 4704c894058dSAneesh Kumar K.V protect buddy cache from being freed, 4705c9de560dSAlex Tomas * otherwise we'll refresh it from 4706c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 4707c9de560dSAlex Tomas * blocks */ 470809cbfeafSKirill A. Shutemov get_page(e4b->bd_buddy_page); 470909cbfeafSKirill A. Shutemov get_page(e4b->bd_bitmap_page); 4710c894058dSAneesh Kumar K.V } 4711c894058dSAneesh Kumar K.V while (*n) { 4712c894058dSAneesh Kumar K.V parent = *n; 471318aadd47SBobi Jam entry = rb_entry(parent, struct ext4_free_data, efd_node); 471418aadd47SBobi Jam if (cluster < entry->efd_start_cluster) 4715c894058dSAneesh Kumar K.V n = &(*n)->rb_left; 471618aadd47SBobi Jam else if (cluster >= (entry->efd_start_cluster + entry->efd_count)) 4717c894058dSAneesh Kumar K.V n = &(*n)->rb_right; 4718c894058dSAneesh Kumar K.V else { 4719e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 472084130193STheodore Ts'o ext4_group_first_block_no(sb, group) + 472184130193STheodore Ts'o EXT4_C2B(sbi, cluster), 4722e29136f8STheodore Ts'o "Block already on to-be-freed list"); 4723c894058dSAneesh Kumar K.V return 0; 4724c9de560dSAlex Tomas } 4725c9de560dSAlex Tomas } 4726c9de560dSAlex Tomas 4727c894058dSAneesh Kumar K.V rb_link_node(new_node, parent, n); 4728c894058dSAneesh Kumar K.V rb_insert_color(new_node, &db->bb_free_root); 4729c894058dSAneesh Kumar K.V 4730c894058dSAneesh Kumar K.V /* Now try to see the extent can be merged to left and right */ 4731c894058dSAneesh Kumar K.V node = rb_prev(new_node); 4732c894058dSAneesh Kumar K.V if (node) { 473318aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 4734a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 4735a0154344SDaeho Jeong &(db->bb_free_root)); 4736c9de560dSAlex Tomas } 4737c894058dSAneesh Kumar K.V 4738c894058dSAneesh Kumar K.V node = rb_next(new_node); 4739c894058dSAneesh Kumar K.V if (node) { 474018aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 4741a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 4742a0154344SDaeho Jeong &(db->bb_free_root)); 4743c894058dSAneesh Kumar K.V } 4744a0154344SDaeho Jeong 4745d08854f5STheodore Ts'o spin_lock(&sbi->s_md_lock); 4746a0154344SDaeho Jeong list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list); 4747d08854f5STheodore Ts'o sbi->s_mb_free_pending += clusters; 4748d08854f5STheodore Ts'o spin_unlock(&sbi->s_md_lock); 4749c9de560dSAlex Tomas return 0; 4750c9de560dSAlex Tomas } 4751c9de560dSAlex Tomas 475244338711STheodore Ts'o /** 475344338711STheodore Ts'o * ext4_free_blocks() -- Free given blocks and update quota 475444338711STheodore Ts'o * @handle: handle for this transaction 475544338711STheodore Ts'o * @inode: inode 4756c60990b3STheodore Ts'o * @bh: optional buffer of the block to be freed 4757c60990b3STheodore Ts'o * @block: starting physical block to be freed 4758c60990b3STheodore Ts'o * @count: number of blocks to be freed 47595def1360SYongqiang Yang * @flags: flags used by ext4_free_blocks 4760c9de560dSAlex Tomas */ 476144338711STheodore Ts'o void ext4_free_blocks(handle_t *handle, struct inode *inode, 4762e6362609STheodore Ts'o struct buffer_head *bh, ext4_fsblk_t block, 4763e6362609STheodore Ts'o unsigned long count, int flags) 4764c9de560dSAlex Tomas { 476526346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 4766c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4767c9de560dSAlex Tomas struct ext4_group_desc *gdp; 4768498e5f24STheodore Ts'o unsigned int overflow; 4769c9de560dSAlex Tomas ext4_grpblk_t bit; 4770c9de560dSAlex Tomas struct buffer_head *gd_bh; 4771c9de560dSAlex Tomas ext4_group_t block_group; 4772c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4773c9de560dSAlex Tomas struct ext4_buddy e4b; 477484130193STheodore Ts'o unsigned int count_clusters; 4775c9de560dSAlex Tomas int err = 0; 4776c9de560dSAlex Tomas int ret; 4777c9de560dSAlex Tomas 4778b10a44c3STheodore Ts'o might_sleep(); 4779e6362609STheodore Ts'o if (bh) { 4780e6362609STheodore Ts'o if (block) 4781e6362609STheodore Ts'o BUG_ON(block != bh->b_blocknr); 4782e6362609STheodore Ts'o else 4783e6362609STheodore Ts'o block = bh->b_blocknr; 4784e6362609STheodore Ts'o } 4785c9de560dSAlex Tomas 4786c9de560dSAlex Tomas sbi = EXT4_SB(sb); 47871f2acb60STheodore Ts'o if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 47881f2acb60STheodore Ts'o !ext4_data_block_valid(sbi, block, count)) { 478912062dddSEric Sandeen ext4_error(sb, "Freeing blocks not in datazone - " 47900610b6e9STheodore Ts'o "block = %llu, count = %lu", block, count); 4791c9de560dSAlex Tomas goto error_return; 4792c9de560dSAlex Tomas } 4793c9de560dSAlex Tomas 47940610b6e9STheodore Ts'o ext4_debug("freeing block %llu\n", block); 4795e6362609STheodore Ts'o trace_ext4_free_blocks(inode, block, count, flags); 4796e6362609STheodore Ts'o 47979c02ac97SDaeho Jeong if (bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 47989c02ac97SDaeho Jeong BUG_ON(count > 1); 4799e6362609STheodore Ts'o 4800e6362609STheodore Ts'o ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA, 48019c02ac97SDaeho Jeong inode, bh, block); 4802e6362609STheodore Ts'o } 4803e6362609STheodore Ts'o 4804e6362609STheodore Ts'o /* 480584130193STheodore Ts'o * If the extent to be freed does not begin on a cluster 480684130193STheodore Ts'o * boundary, we need to deal with partial clusters at the 480784130193STheodore Ts'o * beginning and end of the extent. Normally we will free 480884130193STheodore Ts'o * blocks at the beginning or the end unless we are explicitly 480984130193STheodore Ts'o * requested to avoid doing so. 481084130193STheodore Ts'o */ 4811f5a44db5STheodore Ts'o overflow = EXT4_PBLK_COFF(sbi, block); 481284130193STheodore Ts'o if (overflow) { 481384130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER) { 481484130193STheodore Ts'o overflow = sbi->s_cluster_ratio - overflow; 481584130193STheodore Ts'o block += overflow; 481684130193STheodore Ts'o if (count > overflow) 481784130193STheodore Ts'o count -= overflow; 481884130193STheodore Ts'o else 481984130193STheodore Ts'o return; 482084130193STheodore Ts'o } else { 482184130193STheodore Ts'o block -= overflow; 482284130193STheodore Ts'o count += overflow; 482384130193STheodore Ts'o } 482484130193STheodore Ts'o } 4825f5a44db5STheodore Ts'o overflow = EXT4_LBLK_COFF(sbi, count); 482684130193STheodore Ts'o if (overflow) { 482784130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER) { 482884130193STheodore Ts'o if (count > overflow) 482984130193STheodore Ts'o count -= overflow; 483084130193STheodore Ts'o else 483184130193STheodore Ts'o return; 483284130193STheodore Ts'o } else 483384130193STheodore Ts'o count += sbi->s_cluster_ratio - overflow; 483484130193STheodore Ts'o } 483584130193STheodore Ts'o 48369c02ac97SDaeho Jeong if (!bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 48379c02ac97SDaeho Jeong int i; 4838f96c450dSDaeho Jeong int is_metadata = flags & EXT4_FREE_BLOCKS_METADATA; 48399c02ac97SDaeho Jeong 48409c02ac97SDaeho Jeong for (i = 0; i < count; i++) { 48419c02ac97SDaeho Jeong cond_resched(); 4842f96c450dSDaeho Jeong if (is_metadata) 48439c02ac97SDaeho Jeong bh = sb_find_get_block(inode->i_sb, block + i); 4844f96c450dSDaeho Jeong ext4_forget(handle, is_metadata, inode, bh, block + i); 48459c02ac97SDaeho Jeong } 48469c02ac97SDaeho Jeong } 48479c02ac97SDaeho Jeong 4848c9de560dSAlex Tomas do_more: 4849c9de560dSAlex Tomas overflow = 0; 4850c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 4851c9de560dSAlex Tomas 4852163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT( 4853163a203dSDarrick J. Wong ext4_get_group_info(sb, block_group)))) 4854163a203dSDarrick J. Wong return; 4855163a203dSDarrick J. Wong 4856c9de560dSAlex Tomas /* 4857c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 4858c9de560dSAlex Tomas * boundary. 4859c9de560dSAlex Tomas */ 486084130193STheodore Ts'o if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) { 486184130193STheodore Ts'o overflow = EXT4_C2B(sbi, bit) + count - 486284130193STheodore Ts'o EXT4_BLOCKS_PER_GROUP(sb); 4863c9de560dSAlex Tomas count -= overflow; 4864c9de560dSAlex Tomas } 4865810da240SLukas Czerner count_clusters = EXT4_NUM_B2C(sbi, count); 4866574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, block_group); 48679008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 48689008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 48699008a58eSDarrick J. Wong bitmap_bh = NULL; 4870c9de560dSAlex Tomas goto error_return; 4871ce89f46cSAneesh Kumar K.V } 4872c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 4873ce89f46cSAneesh Kumar K.V if (!gdp) { 4874ce89f46cSAneesh Kumar K.V err = -EIO; 4875c9de560dSAlex Tomas goto error_return; 4876ce89f46cSAneesh Kumar K.V } 4877c9de560dSAlex Tomas 4878c9de560dSAlex Tomas if (in_range(ext4_block_bitmap(sb, gdp), block, count) || 4879c9de560dSAlex Tomas in_range(ext4_inode_bitmap(sb, gdp), block, count) || 4880c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 488149598e04SJun Piao sbi->s_itb_per_group) || 4882c9de560dSAlex Tomas in_range(block + count - 1, ext4_inode_table(sb, gdp), 488349598e04SJun Piao sbi->s_itb_per_group)) { 4884c9de560dSAlex Tomas 488512062dddSEric Sandeen ext4_error(sb, "Freeing blocks in system zone - " 48860610b6e9STheodore Ts'o "Block = %llu, count = %lu", block, count); 4887519deca0SAneesh Kumar K.V /* err = 0. ext4_std_error should be a no op */ 4888519deca0SAneesh Kumar K.V goto error_return; 4889c9de560dSAlex Tomas } 4890c9de560dSAlex Tomas 4891c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 4892c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 4893c9de560dSAlex Tomas if (err) 4894c9de560dSAlex Tomas goto error_return; 4895c9de560dSAlex Tomas 4896c9de560dSAlex Tomas /* 4897c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 4898c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 4899c9de560dSAlex Tomas * using it 4900c9de560dSAlex Tomas */ 4901c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 4902c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gd_bh); 4903c9de560dSAlex Tomas if (err) 4904c9de560dSAlex Tomas goto error_return; 4905c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 4906c9de560dSAlex Tomas { 4907c9de560dSAlex Tomas int i; 490884130193STheodore Ts'o for (i = 0; i < count_clusters; i++) 4909c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 4910c9de560dSAlex Tomas } 4911c9de560dSAlex Tomas #endif 491284130193STheodore Ts'o trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters); 4913c9de560dSAlex Tomas 4914adb7ef60SKonstantin Khlebnikov /* __GFP_NOFAIL: retry infinitely, ignore TIF_MEMDIE and memcg limit. */ 4915adb7ef60SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b, 4916adb7ef60SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 4917920313a7SAneesh Kumar K.V if (err) 4918920313a7SAneesh Kumar K.V goto error_return; 4919e6362609STheodore Ts'o 4920f96c450dSDaeho Jeong /* 4921f96c450dSDaeho Jeong * We need to make sure we don't reuse the freed block until after the 4922f96c450dSDaeho Jeong * transaction is committed. We make an exception if the inode is to be 4923f96c450dSDaeho Jeong * written in writeback mode since writeback mode has weak data 4924f96c450dSDaeho Jeong * consistency guarantees. 4925f96c450dSDaeho Jeong */ 4926f96c450dSDaeho Jeong if (ext4_handle_valid(handle) && 4927f96c450dSDaeho Jeong ((flags & EXT4_FREE_BLOCKS_METADATA) || 4928f96c450dSDaeho Jeong !ext4_should_writeback_data(inode))) { 49297a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry; 49307a2fcbf7SAneesh Kumar K.V /* 49317444a072SMichal Hocko * We use __GFP_NOFAIL because ext4_free_blocks() is not allowed 49327444a072SMichal Hocko * to fail. 49337a2fcbf7SAneesh Kumar K.V */ 49347444a072SMichal Hocko new_entry = kmem_cache_alloc(ext4_free_data_cachep, 49357444a072SMichal Hocko GFP_NOFS|__GFP_NOFAIL); 493618aadd47SBobi Jam new_entry->efd_start_cluster = bit; 493718aadd47SBobi Jam new_entry->efd_group = block_group; 493818aadd47SBobi Jam new_entry->efd_count = count_clusters; 493918aadd47SBobi Jam new_entry->efd_tid = handle->h_transaction->t_tid; 4940955ce5f5SAneesh Kumar K.V 49417a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, block_group); 494284130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 49437a2fcbf7SAneesh Kumar K.V ext4_mb_free_metadata(handle, &e4b, new_entry); 4944c9de560dSAlex Tomas } else { 49457a2fcbf7SAneesh Kumar K.V /* need to update group_info->bb_free and bitmap 49467a2fcbf7SAneesh Kumar K.V * with group lock held. generate_buddy look at 49477a2fcbf7SAneesh Kumar K.V * them with group lock_held 49487a2fcbf7SAneesh Kumar K.V */ 4949d71c1ae2SLukas Czerner if (test_opt(sb, DISCARD)) { 4950a0154344SDaeho Jeong err = ext4_issue_discard(sb, block_group, bit, count, 4951a0154344SDaeho Jeong NULL); 4952d71c1ae2SLukas Czerner if (err && err != -EOPNOTSUPP) 4953d71c1ae2SLukas Czerner ext4_msg(sb, KERN_WARNING, "discard request in" 4954d71c1ae2SLukas Czerner " group:%d block:%d count:%lu failed" 4955d71c1ae2SLukas Czerner " with %d", block_group, bit, count, 4956d71c1ae2SLukas Czerner err); 49578f9ff189SLukas Czerner } else 49588f9ff189SLukas Czerner EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info); 4959d71c1ae2SLukas Czerner 4960955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, block_group); 496184130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 496284130193STheodore Ts'o mb_free_blocks(inode, &e4b, bit, count_clusters); 4963c9de560dSAlex Tomas } 4964c9de560dSAlex Tomas 4965021b65bbSTheodore Ts'o ret = ext4_free_group_clusters(sb, gdp) + count_clusters; 4966021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, ret); 496779f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, gdp, bitmap_bh); 4968feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, gdp); 4969955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, block_group); 4970c9de560dSAlex Tomas 4971772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 4972772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 497390ba983fSTheodore Ts'o atomic64_add(count_clusters, 49747c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 49757c990728SSuraj Jitindar Singh flex_group)->free_clusters); 4976772cb7c8SJose R. Santos } 4977772cb7c8SJose R. Santos 49789fe67149SEric Whitney /* 49799fe67149SEric Whitney * on a bigalloc file system, defer the s_freeclusters_counter 49809fe67149SEric Whitney * update to the caller (ext4_remove_space and friends) so they 49819fe67149SEric Whitney * can determine if a cluster freed here should be rereserved 49829fe67149SEric Whitney */ 49839fe67149SEric Whitney if (!(flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)) { 49847b415bf6SAditya Kali if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) 49857b415bf6SAditya Kali dquot_free_block(inode, EXT4_C2B(sbi, count_clusters)); 49869fe67149SEric Whitney percpu_counter_add(&sbi->s_freeclusters_counter, 49879fe67149SEric Whitney count_clusters); 49889fe67149SEric Whitney } 49897d734532SJan Kara 49907d734532SJan Kara ext4_mb_unload_buddy(&e4b); 49917b415bf6SAditya Kali 49927a2fcbf7SAneesh Kumar K.V /* We dirtied the bitmap block */ 49937a2fcbf7SAneesh Kumar K.V BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 49947a2fcbf7SAneesh Kumar K.V err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 49957a2fcbf7SAneesh Kumar K.V 4996c9de560dSAlex Tomas /* And the group descriptor block */ 4997c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 49980390131bSFrank Mayhar ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 4999c9de560dSAlex Tomas if (!err) 5000c9de560dSAlex Tomas err = ret; 5001c9de560dSAlex Tomas 5002c9de560dSAlex Tomas if (overflow && !err) { 5003c9de560dSAlex Tomas block += count; 5004c9de560dSAlex Tomas count = overflow; 5005c9de560dSAlex Tomas put_bh(bitmap_bh); 5006c9de560dSAlex Tomas goto do_more; 5007c9de560dSAlex Tomas } 5008c9de560dSAlex Tomas error_return: 5009c9de560dSAlex Tomas brelse(bitmap_bh); 5010c9de560dSAlex Tomas ext4_std_error(sb, err); 5011c9de560dSAlex Tomas return; 5012c9de560dSAlex Tomas } 50137360d173SLukas Czerner 50147360d173SLukas Czerner /** 50150529155eSYongqiang Yang * ext4_group_add_blocks() -- Add given blocks to an existing group 50162846e820SAmir Goldstein * @handle: handle to this transaction 50172846e820SAmir Goldstein * @sb: super block 50184907cb7bSAnatol Pomozov * @block: start physical block to add to the block group 50192846e820SAmir Goldstein * @count: number of blocks to free 50202846e820SAmir Goldstein * 5021e73a347bSAmir Goldstein * This marks the blocks as free in the bitmap and buddy. 50222846e820SAmir Goldstein */ 5023cc7365dfSYongqiang Yang int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, 50242846e820SAmir Goldstein ext4_fsblk_t block, unsigned long count) 50252846e820SAmir Goldstein { 50262846e820SAmir Goldstein struct buffer_head *bitmap_bh = NULL; 50272846e820SAmir Goldstein struct buffer_head *gd_bh; 50282846e820SAmir Goldstein ext4_group_t block_group; 50292846e820SAmir Goldstein ext4_grpblk_t bit; 50302846e820SAmir Goldstein unsigned int i; 50312846e820SAmir Goldstein struct ext4_group_desc *desc; 50322846e820SAmir Goldstein struct ext4_sb_info *sbi = EXT4_SB(sb); 5033e73a347bSAmir Goldstein struct ext4_buddy e4b; 5034d77147ffSharshads int err = 0, ret, free_clusters_count; 5035d77147ffSharshads ext4_grpblk_t clusters_freed; 5036d77147ffSharshads ext4_fsblk_t first_cluster = EXT4_B2C(sbi, block); 5037d77147ffSharshads ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); 5038d77147ffSharshads unsigned long cluster_count = last_cluster - first_cluster + 1; 50392846e820SAmir Goldstein 50402846e820SAmir Goldstein ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); 50412846e820SAmir Goldstein 50424740b830SYongqiang Yang if (count == 0) 50434740b830SYongqiang Yang return 0; 50444740b830SYongqiang Yang 50452846e820SAmir Goldstein ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 50462846e820SAmir Goldstein /* 50472846e820SAmir Goldstein * Check to see if we are freeing blocks across a group 50482846e820SAmir Goldstein * boundary. 50492846e820SAmir Goldstein */ 5050d77147ffSharshads if (bit + cluster_count > EXT4_CLUSTERS_PER_GROUP(sb)) { 5051d77147ffSharshads ext4_warning(sb, "too many blocks added to group %u", 5052cc7365dfSYongqiang Yang block_group); 5053cc7365dfSYongqiang Yang err = -EINVAL; 50542846e820SAmir Goldstein goto error_return; 5055cc7365dfSYongqiang Yang } 50562cd05cc3STheodore Ts'o 50572846e820SAmir Goldstein bitmap_bh = ext4_read_block_bitmap(sb, block_group); 50589008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 50599008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 50609008a58eSDarrick J. Wong bitmap_bh = NULL; 50612846e820SAmir Goldstein goto error_return; 5062cc7365dfSYongqiang Yang } 5063cc7365dfSYongqiang Yang 50642846e820SAmir Goldstein desc = ext4_get_group_desc(sb, block_group, &gd_bh); 5065cc7365dfSYongqiang Yang if (!desc) { 5066cc7365dfSYongqiang Yang err = -EIO; 50672846e820SAmir Goldstein goto error_return; 5068cc7365dfSYongqiang Yang } 50692846e820SAmir Goldstein 50702846e820SAmir Goldstein if (in_range(ext4_block_bitmap(sb, desc), block, count) || 50712846e820SAmir Goldstein in_range(ext4_inode_bitmap(sb, desc), block, count) || 50722846e820SAmir Goldstein in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) || 50732846e820SAmir Goldstein in_range(block + count - 1, ext4_inode_table(sb, desc), 50742846e820SAmir Goldstein sbi->s_itb_per_group)) { 50752846e820SAmir Goldstein ext4_error(sb, "Adding blocks in system zones - " 50762846e820SAmir Goldstein "Block = %llu, count = %lu", 50772846e820SAmir Goldstein block, count); 5078cc7365dfSYongqiang Yang err = -EINVAL; 50792846e820SAmir Goldstein goto error_return; 50802846e820SAmir Goldstein } 50812846e820SAmir Goldstein 50822cd05cc3STheodore Ts'o BUFFER_TRACE(bitmap_bh, "getting write access"); 50832cd05cc3STheodore Ts'o err = ext4_journal_get_write_access(handle, bitmap_bh); 50842846e820SAmir Goldstein if (err) 50852846e820SAmir Goldstein goto error_return; 50862846e820SAmir Goldstein 50872846e820SAmir Goldstein /* 50882846e820SAmir Goldstein * We are about to modify some metadata. Call the journal APIs 50892846e820SAmir Goldstein * to unshare ->b_data if a currently-committing transaction is 50902846e820SAmir Goldstein * using it 50912846e820SAmir Goldstein */ 50922846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "get_write_access"); 50932846e820SAmir Goldstein err = ext4_journal_get_write_access(handle, gd_bh); 50942846e820SAmir Goldstein if (err) 50952846e820SAmir Goldstein goto error_return; 5096e73a347bSAmir Goldstein 5097d77147ffSharshads for (i = 0, clusters_freed = 0; i < cluster_count; i++) { 50982846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "clear bit"); 5099e73a347bSAmir Goldstein if (!mb_test_bit(bit + i, bitmap_bh->b_data)) { 51002846e820SAmir Goldstein ext4_error(sb, "bit already cleared for block %llu", 51012846e820SAmir Goldstein (ext4_fsblk_t)(block + i)); 51022846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "bit already cleared"); 51032846e820SAmir Goldstein } else { 5104d77147ffSharshads clusters_freed++; 51052846e820SAmir Goldstein } 51062846e820SAmir Goldstein } 5107e73a347bSAmir Goldstein 5108e73a347bSAmir Goldstein err = ext4_mb_load_buddy(sb, block_group, &e4b); 5109e73a347bSAmir Goldstein if (err) 5110e73a347bSAmir Goldstein goto error_return; 5111e73a347bSAmir Goldstein 5112e73a347bSAmir Goldstein /* 5113e73a347bSAmir Goldstein * need to update group_info->bb_free and bitmap 5114e73a347bSAmir Goldstein * with group lock held. generate_buddy look at 5115e73a347bSAmir Goldstein * them with group lock_held 5116e73a347bSAmir Goldstein */ 51172846e820SAmir Goldstein ext4_lock_group(sb, block_group); 5118d77147ffSharshads mb_clear_bits(bitmap_bh->b_data, bit, cluster_count); 5119d77147ffSharshads mb_free_blocks(NULL, &e4b, bit, cluster_count); 5120d77147ffSharshads free_clusters_count = clusters_freed + 5121d77147ffSharshads ext4_free_group_clusters(sb, desc); 5122d77147ffSharshads ext4_free_group_clusters_set(sb, desc, free_clusters_count); 512379f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, desc, bitmap_bh); 5124feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, desc); 51252846e820SAmir Goldstein ext4_unlock_group(sb, block_group); 512657042651STheodore Ts'o percpu_counter_add(&sbi->s_freeclusters_counter, 5127d77147ffSharshads clusters_freed); 51282846e820SAmir Goldstein 51292846e820SAmir Goldstein if (sbi->s_log_groups_per_flex) { 51302846e820SAmir Goldstein ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 5131d77147ffSharshads atomic64_add(clusters_freed, 51327c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 51337c990728SSuraj Jitindar Singh flex_group)->free_clusters); 51342846e820SAmir Goldstein } 5135e73a347bSAmir Goldstein 5136e73a347bSAmir Goldstein ext4_mb_unload_buddy(&e4b); 51372846e820SAmir Goldstein 51382846e820SAmir Goldstein /* We dirtied the bitmap block */ 51392846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 51402846e820SAmir Goldstein err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 51412846e820SAmir Goldstein 51422846e820SAmir Goldstein /* And the group descriptor block */ 51432846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 51442846e820SAmir Goldstein ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 51452846e820SAmir Goldstein if (!err) 51462846e820SAmir Goldstein err = ret; 51472846e820SAmir Goldstein 51482846e820SAmir Goldstein error_return: 51492846e820SAmir Goldstein brelse(bitmap_bh); 51502846e820SAmir Goldstein ext4_std_error(sb, err); 5151cc7365dfSYongqiang Yang return err; 51522846e820SAmir Goldstein } 51532846e820SAmir Goldstein 51542846e820SAmir Goldstein /** 51557360d173SLukas Czerner * ext4_trim_extent -- function to TRIM one single free extent in the group 51567360d173SLukas Czerner * @sb: super block for the file system 51577360d173SLukas Czerner * @start: starting block of the free extent in the alloc. group 51587360d173SLukas Czerner * @count: number of blocks to TRIM 51597360d173SLukas Czerner * @group: alloc. group we are working with 51607360d173SLukas Czerner * @e4b: ext4 buddy for the group 51617360d173SLukas Czerner * 51627360d173SLukas Czerner * Trim "count" blocks starting at "start" in the "group". To assure that no 51637360d173SLukas Czerner * one will allocate those blocks, mark it as used in buddy bitmap. This must 51647360d173SLukas Czerner * be called with under the group lock. 51657360d173SLukas Czerner */ 5166d71c1ae2SLukas Czerner static int ext4_trim_extent(struct super_block *sb, int start, int count, 51677360d173SLukas Czerner ext4_group_t group, struct ext4_buddy *e4b) 5168e2cbd587Sjon ernst __releases(bitlock) 5169e2cbd587Sjon ernst __acquires(bitlock) 51707360d173SLukas Czerner { 51717360d173SLukas Czerner struct ext4_free_extent ex; 5172d71c1ae2SLukas Czerner int ret = 0; 51737360d173SLukas Czerner 5174b3d4c2b1STao Ma trace_ext4_trim_extent(sb, group, start, count); 5175b3d4c2b1STao Ma 51767360d173SLukas Czerner assert_spin_locked(ext4_group_lock_ptr(sb, group)); 51777360d173SLukas Czerner 51787360d173SLukas Czerner ex.fe_start = start; 51797360d173SLukas Czerner ex.fe_group = group; 51807360d173SLukas Czerner ex.fe_len = count; 51817360d173SLukas Czerner 51827360d173SLukas Czerner /* 51837360d173SLukas Czerner * Mark blocks used, so no one can reuse them while 51847360d173SLukas Czerner * being trimmed. 51857360d173SLukas Czerner */ 51867360d173SLukas Czerner mb_mark_used(e4b, &ex); 51877360d173SLukas Czerner ext4_unlock_group(sb, group); 5188a0154344SDaeho Jeong ret = ext4_issue_discard(sb, group, start, count, NULL); 51897360d173SLukas Czerner ext4_lock_group(sb, group); 51907360d173SLukas Czerner mb_free_blocks(NULL, e4b, start, ex.fe_len); 5191d71c1ae2SLukas Czerner return ret; 51927360d173SLukas Czerner } 51937360d173SLukas Czerner 51947360d173SLukas Czerner /** 51957360d173SLukas Czerner * ext4_trim_all_free -- function to trim all free space in alloc. group 51967360d173SLukas Czerner * @sb: super block for file system 519722612283STao Ma * @group: group to be trimmed 51987360d173SLukas Czerner * @start: first group block to examine 51997360d173SLukas Czerner * @max: last group block to examine 52007360d173SLukas Czerner * @minblocks: minimum extent block count 52017360d173SLukas Czerner * 52027360d173SLukas Czerner * ext4_trim_all_free walks through group's buddy bitmap searching for free 52037360d173SLukas Czerner * extents. When the free block is found, ext4_trim_extent is called to TRIM 52047360d173SLukas Czerner * the extent. 52057360d173SLukas Czerner * 52067360d173SLukas Czerner * 52077360d173SLukas Czerner * ext4_trim_all_free walks through group's block bitmap searching for free 52087360d173SLukas Czerner * extents. When the free extent is found, mark it as used in group buddy 52097360d173SLukas Czerner * bitmap. Then issue a TRIM command on this extent and free the extent in 52107360d173SLukas Czerner * the group buddy bitmap. This is done until whole group is scanned. 52117360d173SLukas Czerner */ 52120b75a840SLukas Czerner static ext4_grpblk_t 521378944086SLukas Czerner ext4_trim_all_free(struct super_block *sb, ext4_group_t group, 521478944086SLukas Czerner ext4_grpblk_t start, ext4_grpblk_t max, 521578944086SLukas Czerner ext4_grpblk_t minblocks) 52167360d173SLukas Czerner { 52177360d173SLukas Czerner void *bitmap; 5218169ddc3eSTao Ma ext4_grpblk_t next, count = 0, free_count = 0; 521978944086SLukas Czerner struct ext4_buddy e4b; 5220d71c1ae2SLukas Czerner int ret = 0; 52217360d173SLukas Czerner 5222b3d4c2b1STao Ma trace_ext4_trim_all_free(sb, group, start, max); 5223b3d4c2b1STao Ma 522478944086SLukas Czerner ret = ext4_mb_load_buddy(sb, group, &e4b); 522578944086SLukas Czerner if (ret) { 52269651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 52279651e6b2SKonstantin Khlebnikov ret, group); 522878944086SLukas Czerner return ret; 522978944086SLukas Czerner } 523078944086SLukas Czerner bitmap = e4b.bd_bitmap; 523128739eeaSLukas Czerner 523228739eeaSLukas Czerner ext4_lock_group(sb, group); 52333d56b8d2STao Ma if (EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) && 52343d56b8d2STao Ma minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks)) 52353d56b8d2STao Ma goto out; 52363d56b8d2STao Ma 523778944086SLukas Czerner start = (e4b.bd_info->bb_first_free > start) ? 523878944086SLukas Czerner e4b.bd_info->bb_first_free : start; 52397360d173SLukas Czerner 5240913eed83SLukas Czerner while (start <= max) { 5241913eed83SLukas Czerner start = mb_find_next_zero_bit(bitmap, max + 1, start); 5242913eed83SLukas Czerner if (start > max) 52437360d173SLukas Czerner break; 5244913eed83SLukas Czerner next = mb_find_next_bit(bitmap, max + 1, start); 52457360d173SLukas Czerner 52467360d173SLukas Czerner if ((next - start) >= minblocks) { 5247d71c1ae2SLukas Czerner ret = ext4_trim_extent(sb, start, 524878944086SLukas Czerner next - start, group, &e4b); 5249d71c1ae2SLukas Czerner if (ret && ret != -EOPNOTSUPP) 5250d71c1ae2SLukas Czerner break; 5251d71c1ae2SLukas Czerner ret = 0; 52527360d173SLukas Czerner count += next - start; 52537360d173SLukas Czerner } 5254169ddc3eSTao Ma free_count += next - start; 52557360d173SLukas Czerner start = next + 1; 52567360d173SLukas Czerner 52577360d173SLukas Czerner if (fatal_signal_pending(current)) { 52587360d173SLukas Czerner count = -ERESTARTSYS; 52597360d173SLukas Czerner break; 52607360d173SLukas Czerner } 52617360d173SLukas Czerner 52627360d173SLukas Czerner if (need_resched()) { 52637360d173SLukas Czerner ext4_unlock_group(sb, group); 52647360d173SLukas Czerner cond_resched(); 52657360d173SLukas Czerner ext4_lock_group(sb, group); 52667360d173SLukas Czerner } 52677360d173SLukas Czerner 5268169ddc3eSTao Ma if ((e4b.bd_info->bb_free - free_count) < minblocks) 52697360d173SLukas Czerner break; 52707360d173SLukas Czerner } 52713d56b8d2STao Ma 5272d71c1ae2SLukas Czerner if (!ret) { 5273d71c1ae2SLukas Czerner ret = count; 52743d56b8d2STao Ma EXT4_MB_GRP_SET_TRIMMED(e4b.bd_info); 5275d71c1ae2SLukas Czerner } 52763d56b8d2STao Ma out: 52777360d173SLukas Czerner ext4_unlock_group(sb, group); 527878944086SLukas Czerner ext4_mb_unload_buddy(&e4b); 52797360d173SLukas Czerner 52807360d173SLukas Czerner ext4_debug("trimmed %d blocks in the group %d\n", 52817360d173SLukas Czerner count, group); 52827360d173SLukas Czerner 5283d71c1ae2SLukas Czerner return ret; 52847360d173SLukas Czerner } 52857360d173SLukas Czerner 52867360d173SLukas Czerner /** 52877360d173SLukas Czerner * ext4_trim_fs() -- trim ioctl handle function 52887360d173SLukas Czerner * @sb: superblock for filesystem 52897360d173SLukas Czerner * @range: fstrim_range structure 52907360d173SLukas Czerner * 52917360d173SLukas Czerner * start: First Byte to trim 52927360d173SLukas Czerner * len: number of Bytes to trim from start 52937360d173SLukas Czerner * minlen: minimum extent length in Bytes 52947360d173SLukas Czerner * ext4_trim_fs goes through all allocation groups containing Bytes from 52957360d173SLukas Czerner * start to start+len. For each such a group ext4_trim_all_free function 52967360d173SLukas Czerner * is invoked to trim all free space. 52977360d173SLukas Czerner */ 52987360d173SLukas Czerner int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) 52997360d173SLukas Czerner { 530078944086SLukas Czerner struct ext4_group_info *grp; 5301913eed83SLukas Czerner ext4_group_t group, first_group, last_group; 53027137d7a4STheodore Ts'o ext4_grpblk_t cnt = 0, first_cluster, last_cluster; 5303913eed83SLukas Czerner uint64_t start, end, minlen, trimmed = 0; 53040f0a25bfSJan Kara ext4_fsblk_t first_data_blk = 53050f0a25bfSJan Kara le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 5306913eed83SLukas Czerner ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es); 53077360d173SLukas Czerner int ret = 0; 53087360d173SLukas Czerner 53097360d173SLukas Czerner start = range->start >> sb->s_blocksize_bits; 5310913eed83SLukas Czerner end = start + (range->len >> sb->s_blocksize_bits) - 1; 5311aaf7d73eSLukas Czerner minlen = EXT4_NUM_B2C(EXT4_SB(sb), 5312aaf7d73eSLukas Czerner range->minlen >> sb->s_blocksize_bits); 53137360d173SLukas Czerner 53145de35e8dSLukas Czerner if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) || 53155de35e8dSLukas Czerner start >= max_blks || 53165de35e8dSLukas Czerner range->len < sb->s_blocksize) 53177360d173SLukas Czerner return -EINVAL; 5318913eed83SLukas Czerner if (end >= max_blks) 5319913eed83SLukas Czerner end = max_blks - 1; 5320913eed83SLukas Czerner if (end <= first_data_blk) 532122f10457STao Ma goto out; 5322913eed83SLukas Czerner if (start < first_data_blk) 53230f0a25bfSJan Kara start = first_data_blk; 53247360d173SLukas Czerner 5325913eed83SLukas Czerner /* Determine first and last group to examine based on start and end */ 53267360d173SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start, 53277137d7a4STheodore Ts'o &first_group, &first_cluster); 5328913eed83SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end, 53297137d7a4STheodore Ts'o &last_group, &last_cluster); 53307360d173SLukas Czerner 5331913eed83SLukas Czerner /* end now represents the last cluster to discard in this group */ 5332913eed83SLukas Czerner end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 53337360d173SLukas Czerner 53347360d173SLukas Czerner for (group = first_group; group <= last_group; group++) { 533578944086SLukas Czerner grp = ext4_get_group_info(sb, group); 533678944086SLukas Czerner /* We only do this if the grp has never been initialized */ 533778944086SLukas Czerner if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 5338adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, GFP_NOFS); 533978944086SLukas Czerner if (ret) 53407360d173SLukas Czerner break; 53417360d173SLukas Czerner } 53427360d173SLukas Czerner 53430ba08517STao Ma /* 5344913eed83SLukas Czerner * For all the groups except the last one, last cluster will 5345913eed83SLukas Czerner * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to 5346913eed83SLukas Czerner * change it for the last group, note that last_cluster is 5347913eed83SLukas Czerner * already computed earlier by ext4_get_group_no_and_offset() 53480ba08517STao Ma */ 5349913eed83SLukas Czerner if (group == last_group) 5350913eed83SLukas Czerner end = last_cluster; 53517360d173SLukas Czerner 535278944086SLukas Czerner if (grp->bb_free >= minlen) { 53537137d7a4STheodore Ts'o cnt = ext4_trim_all_free(sb, group, first_cluster, 5354913eed83SLukas Czerner end, minlen); 53557360d173SLukas Czerner if (cnt < 0) { 53567360d173SLukas Czerner ret = cnt; 53577360d173SLukas Czerner break; 53587360d173SLukas Czerner } 53597360d173SLukas Czerner trimmed += cnt; 536021e7fd22SLukas Czerner } 5361913eed83SLukas Czerner 5362913eed83SLukas Czerner /* 5363913eed83SLukas Czerner * For every group except the first one, we are sure 5364913eed83SLukas Czerner * that the first cluster to discard will be cluster #0. 5365913eed83SLukas Czerner */ 53667137d7a4STheodore Ts'o first_cluster = 0; 53677360d173SLukas Czerner } 53687360d173SLukas Czerner 53693d56b8d2STao Ma if (!ret) 53703d56b8d2STao Ma atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen); 53713d56b8d2STao Ma 537222f10457STao Ma out: 5373aaf7d73eSLukas Czerner range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits; 53747360d173SLukas Czerner return ret; 53757360d173SLukas Czerner } 53760c9ec4beSDarrick J. Wong 53770c9ec4beSDarrick J. Wong /* Iterate all the free extents in the group. */ 53780c9ec4beSDarrick J. Wong int 53790c9ec4beSDarrick J. Wong ext4_mballoc_query_range( 53800c9ec4beSDarrick J. Wong struct super_block *sb, 53810c9ec4beSDarrick J. Wong ext4_group_t group, 53820c9ec4beSDarrick J. Wong ext4_grpblk_t start, 53830c9ec4beSDarrick J. Wong ext4_grpblk_t end, 53840c9ec4beSDarrick J. Wong ext4_mballoc_query_range_fn formatter, 53850c9ec4beSDarrick J. Wong void *priv) 53860c9ec4beSDarrick J. Wong { 53870c9ec4beSDarrick J. Wong void *bitmap; 53880c9ec4beSDarrick J. Wong ext4_grpblk_t next; 53890c9ec4beSDarrick J. Wong struct ext4_buddy e4b; 53900c9ec4beSDarrick J. Wong int error; 53910c9ec4beSDarrick J. Wong 53920c9ec4beSDarrick J. Wong error = ext4_mb_load_buddy(sb, group, &e4b); 53930c9ec4beSDarrick J. Wong if (error) 53940c9ec4beSDarrick J. Wong return error; 53950c9ec4beSDarrick J. Wong bitmap = e4b.bd_bitmap; 53960c9ec4beSDarrick J. Wong 53970c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 53980c9ec4beSDarrick J. Wong 53990c9ec4beSDarrick J. Wong start = (e4b.bd_info->bb_first_free > start) ? 54000c9ec4beSDarrick J. Wong e4b.bd_info->bb_first_free : start; 54010c9ec4beSDarrick J. Wong if (end >= EXT4_CLUSTERS_PER_GROUP(sb)) 54020c9ec4beSDarrick J. Wong end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 54030c9ec4beSDarrick J. Wong 54040c9ec4beSDarrick J. Wong while (start <= end) { 54050c9ec4beSDarrick J. Wong start = mb_find_next_zero_bit(bitmap, end + 1, start); 54060c9ec4beSDarrick J. Wong if (start > end) 54070c9ec4beSDarrick J. Wong break; 54080c9ec4beSDarrick J. Wong next = mb_find_next_bit(bitmap, end + 1, start); 54090c9ec4beSDarrick J. Wong 54100c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 54110c9ec4beSDarrick J. Wong error = formatter(sb, group, start, next - start, priv); 54120c9ec4beSDarrick J. Wong if (error) 54130c9ec4beSDarrick J. Wong goto out_unload; 54140c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 54150c9ec4beSDarrick J. Wong 54160c9ec4beSDarrick J. Wong start = next + 1; 54170c9ec4beSDarrick J. Wong } 54180c9ec4beSDarrick J. Wong 54190c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 54200c9ec4beSDarrick J. Wong out_unload: 54210c9ec4beSDarrick J. Wong ext4_mb_unload_buddy(&e4b); 54220c9ec4beSDarrick J. Wong 54230c9ec4beSDarrick J. Wong return error; 54240c9ec4beSDarrick J. Wong } 5425